In #29792, once a provider and model had accepted `drop_block`, any 400 on a request with it retried once without it, since I didn't know what error a provider returns for an unsupported beta. That turned out to hide real errors. On dogfood, a chat replaying some very tall screenshots hit Anthropic's many-image limit: ``` messages.9.content.43.image.source.base64.data: At least one of the image dimensions exceed max allowed size for many-image requests: 3000 pixels ``` The retry without `drop_block` then hit the stale thinking block, so the chat failed with the binding error instead, and the image error only showed up in a warn log. To fix, we'll only retry without `drop_block` when the 400 is a known rejection of the field or its beta, and otherwise return the error as is and keep the hint. The catch is that if a provider rejects the beta with an error we don't recognise, chats on that model will see it until a restart clears the hint, rather than us recovering. Claude Code (as of 2.1.300) does the same on Anthropic's own API, Claude Platform on AWS included: it only retries without the beta when the 400 names it or the `block_binding` field. On Bedrock, Vertex and Mantle it also retries on any 400 it doesn't otherwise recognise, which we could copy if that catch turns out to matter. The oversized screenshots are a separate problem, since chatd doesn't resize images at all.
212 lines
6.9 KiB
Go
212 lines
6.9 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"maps"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
|
|
"charm.land/fantasy"
|
|
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
|
"github.com/google/uuid"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
)
|
|
|
|
const (
|
|
thinkingBindingBeta = "thinking-binding-controls-2026-08-01"
|
|
thinkingBindingDropBlock = "drop_block"
|
|
thinkingBindingDropBlockPath = "thinking.block_binding.prefix_mismatch_behavior"
|
|
)
|
|
|
|
// Anthropic reports "block_binding: Extra inputs are not permitted" or names
|
|
// the rejected beta in an "Unexpected value(s)" error. Bedrock reports
|
|
// unsupported betas as "invalid beta flag".
|
|
var dropBlockRejections = []string{"block_binding", thinkingBindingBeta, "invalid beta flag"}
|
|
|
|
// isThinkingBindingError reports whether Anthropic rejected a replayed
|
|
// signed thinking block because the system prompt or tools changed since
|
|
// the block was produced.
|
|
func isThinkingBindingError(err error) bool {
|
|
text, ok := badRequestText(err)
|
|
return ok && strings.Contains(text, "bound to a different conversation")
|
|
}
|
|
|
|
func isDropBlockRejectedError(err error) bool {
|
|
text, ok := badRequestText(err)
|
|
return ok && slices.ContainsFunc(dropBlockRejections, func(pattern string) bool {
|
|
return strings.Contains(text, pattern)
|
|
})
|
|
}
|
|
|
|
func badRequestText(err error) (string, bool) {
|
|
var pe *fantasy.ProviderError
|
|
if !errors.As(err, &pe) || pe.StatusCode != http.StatusBadRequest {
|
|
return "", false
|
|
}
|
|
return strings.ToLower(pe.Message + string(pe.ResponseBody)), true
|
|
}
|
|
|
|
type thinkingDropBlockKey struct {
|
|
providerID uuid.UUID
|
|
model string
|
|
}
|
|
|
|
// withThinkingDropBlock wraps Anthropic models in a thinkingDropBlockModel.
|
|
func (p *Server) withThinkingDropBlock(
|
|
model chatprovider.Model,
|
|
providerID uuid.UUID,
|
|
callConfig codersdk.ChatModelCallConfig,
|
|
chatID uuid.UUID,
|
|
) chatprovider.Model {
|
|
if chatprovider.NormalizeProvider(model.Provider()) != fantasyanthropic.Name {
|
|
return model
|
|
}
|
|
// Call headers replace client headers of the same name, so the value
|
|
// must repeat the betas the client already sends.
|
|
betas := []string{thinkingBindingBeta}
|
|
if existing := chatprovider.BetaHeadersFromCallConfig(fantasyanthropic.Name, &callConfig)[chatprovider.HeaderAnthropicBeta]; existing != "" {
|
|
betas = []string{existing, thinkingBindingBeta}
|
|
}
|
|
return model.WithLanguageModel(&thinkingDropBlockModel{
|
|
LanguageModel: model.LanguageModel(),
|
|
logger: p.logger.With(slog.F("chat_id", chatID)),
|
|
hints: &p.thinkingDropBlock,
|
|
key: thinkingDropBlockKey{providerID: providerID, model: model.ModelID()},
|
|
betas: strings.Join(betas, ","),
|
|
})
|
|
}
|
|
|
|
// thinkingDropBlockModel makes Anthropic drop stale signed thinking blocks.
|
|
// A hint for each provider and model picks whether to send drop_block first.
|
|
// When hinted, only a drop_block rejection retries, so other errors are not
|
|
// hidden behind a binding error.
|
|
type thinkingDropBlockModel struct {
|
|
fantasy.LanguageModel
|
|
logger slog.Logger
|
|
hints *sync.Map
|
|
key thinkingDropBlockKey
|
|
betas string
|
|
}
|
|
|
|
// Stream retries only when the rejection arrives before any output.
|
|
// fantasy reports the HTTP 400 as a stream part, not as the Stream error.
|
|
func (m *thinkingDropBlockModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
|
|
first, fallback := call, m.withDropBlock(call)
|
|
retryable, updateHint := isThinkingBindingError, func() { m.hints.Store(m.key, struct{}{}) }
|
|
_, hinted := m.hints.Load(m.key)
|
|
if hinted {
|
|
first, fallback = fallback, first
|
|
retryable, updateHint = isDropBlockRejectedError, func() { m.hints.Delete(m.key) }
|
|
}
|
|
stream, err := m.LanguageModel.Stream(ctx, first)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
var rejection error
|
|
started := false
|
|
for part := range stream {
|
|
if !started && part.Type == fantasy.StreamPartTypeError && retryable(part.Error) {
|
|
rejection = part.Error
|
|
break
|
|
}
|
|
started = started || part.Type != fantasy.StreamPartTypeWarnings
|
|
if !yield(part) {
|
|
return
|
|
}
|
|
}
|
|
if rejection == nil {
|
|
return
|
|
}
|
|
m.logger.Warn(ctx, "retrying model call with thinking drop_block toggled",
|
|
slog.F("provider_id", m.key.providerID),
|
|
slog.F("model", m.key.model),
|
|
slog.F("drop_block", !hinted),
|
|
slogError(rejection),
|
|
)
|
|
m.retryStream(ctx, fallback, updateHint, yield)
|
|
}, nil
|
|
}
|
|
|
|
// retryStream streams the fallback request and calls updateHint when the
|
|
// fallback starts without an error.
|
|
func (m *thinkingDropBlockModel) retryStream(ctx context.Context, fallback fantasy.Call, updateHint func(), yield func(fantasy.StreamPart) bool) {
|
|
stream, err := m.LanguageModel.Stream(ctx, fallback)
|
|
if err != nil {
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: err})
|
|
return
|
|
}
|
|
decided := false
|
|
for part := range stream {
|
|
if !decided && part.Type != fantasy.StreamPartTypeWarnings {
|
|
decided = true
|
|
if part.Type != fantasy.StreamPartTypeError {
|
|
updateHint()
|
|
}
|
|
}
|
|
if !yield(part) {
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// withDropBlock adds the drop_block body field and the beta header, which
|
|
// must travel together.
|
|
func (m *thinkingDropBlockModel) withDropBlock(call fantasy.Call) fantasy.Call {
|
|
options := maps.Clone(call.ProviderOptions)
|
|
if options == nil {
|
|
options = fantasy.ProviderOptions{}
|
|
}
|
|
anthropicOptions := &fantasyanthropic.ProviderOptions{}
|
|
if existing, ok := options[fantasyanthropic.Name].(*fantasyanthropic.ProviderOptions); ok && existing != nil {
|
|
copied := *existing
|
|
anthropicOptions = &copied
|
|
}
|
|
anthropicOptions.ExtraBody = maps.Clone(anthropicOptions.ExtraBody)
|
|
if anthropicOptions.ExtraBody == nil {
|
|
anthropicOptions.ExtraBody = map[string]any{}
|
|
}
|
|
if anthropicThinkingOmitted(anthropicOptions, m.Model()) {
|
|
// Claude 5+ models think by default when the request omits the
|
|
// thinking field, so an explicit adaptive config keeps behavior.
|
|
anthropicOptions.ExtraBody["thinking"] = map[string]any{
|
|
"type": "adaptive",
|
|
"block_binding": map[string]any{
|
|
"prefix_mismatch_behavior": thinkingBindingDropBlock,
|
|
},
|
|
}
|
|
} else {
|
|
anthropicOptions.ExtraBody[thinkingBindingDropBlockPath] = thinkingBindingDropBlock
|
|
}
|
|
options[fantasyanthropic.Name] = anthropicOptions
|
|
call.ProviderOptions = options
|
|
|
|
headers := maps.Clone(call.Headers)
|
|
if headers == nil {
|
|
headers = map[string]string{}
|
|
}
|
|
headers[chatprovider.HeaderAnthropicBeta] = m.betas
|
|
call.Headers = headers
|
|
return call
|
|
}
|
|
|
|
// anthropicThinkingOmitted mirrors when fantasy leaves the thinking field
|
|
// out of an Anthropic request for a model that thinks by default.
|
|
func anthropicThinkingOmitted(options *fantasyanthropic.ProviderOptions, modelID string) bool {
|
|
if options.Thinking != nil {
|
|
return false
|
|
}
|
|
model := strings.ToLower(modelID)
|
|
if options.Effort == nil {
|
|
return !strings.Contains(model, "claude-mythos-preview")
|
|
}
|
|
return *options.Effort == fantasyanthropic.EffortNone &&
|
|
(strings.Contains(model, "claude-mythos") || strings.Contains(model, "claude-fable"))
|
|
}
|