Files
Dean Sheather acbcd0e807 fix(coderd/x/chatd): remember quickgen parameter rejections per model (#31493)
On Claude Opus 5.5, chat title generation and the turn-status label each
made four upstream requests, three of them rejected with a 400. The
temperature retry in `generateQuickgenObject` called `generateObject`
again, which restarted from the forced tool call the model had already
rejected. Each turn left about six `bad_request` rows in
`aibridge_interceptions` and added 1.5 to 3.5 seconds of latency.

This replaces the per-call fallbacks with `quickgenParamModel`, a
wrapper applied in `resolveModelCall`. It falls back to plain JSON text
when the model rejects the forced `tool_choice`, drops `temperature`
when the model rejects it, and records each rejection in a replica-wide
`sync.Map` keyed by AI provider ID, model, and parameter, following the
existing `thinkingDropBlock` map. Entries expire after 6 hours so a
provider that starts accepting a parameter again is picked up without a
restart. Only quickgen calls `GenerateObject`, so chat turns are
unaffected.

| Opus 5.5 | Before | First job per replica (and after expiry) | Later
jobs |
|---|---|---|---|
| Title / status label | 4 (3 rejected) | 3 (2 rejected) | 1 |
| Summary | 2 (1 rejected) | 2 (1 rejected) | 1 |

Rejections are still detected by matching `tool_choice` or `temperature`
in a 400 body, as before. A false match now persists for the TTL on that
replica rather than one job; the worst case is text mode or no
temperature for that model.

Closes
[CODAGT-1411](https://linear.app/codercom/issue/CODAGT-1411/quickgen-side-jobs-make-3-rejected-requests-per-run-on-claude-opus-55)

<details>
<summary>Decision log</summary>

- Opus 5.5 reports one rejected parameter per 400, so a job that knows
nothing yet needs 3 requests; the issue's "2 requests" target for the
minimal fix isn't reachable.
- A per-job flag (first iteration) only got jobs to 3 requests. Sharing
rejections across jobs gets steady state to 1 without a hardcoded model
list, which wouldn't match gateway aliases such as Bedrock IDs anyway.
- Per-replica memory over a database column or admin setting: no
migration, no request-path writes, self-healing via TTL. Each replica
pays the learning cost once per model per TTL window.
- Keyed by provider ID plus model so the same model name behind
different providers is learned independently.

</details>

> Generated by Coder Agents on behalf of @deansheather.
2026-11-09 22:39:29 +11:00

276 lines
9.6 KiB
Go

package chatd
import (
"context"
"sync/atomic"
"charm.land/fantasy"
fantasyopenai "charm.land/fantasy/providers/openai"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/codersdk"
)
const defaultChatMaxOutputTokens = int64(32_000)
type modelCallSpec struct {
// purpose labels resolver logs only; it does not affect call behavior.
purpose string
chat database.Chat
explicitConfig *database.ChatModelConfig
// requestedEffort overrides the config's default reasoning effort,
// clamped to the config's max.
requestedEffort *string
// maxOutputTokens overrides the config's output-token cap and the
// package default.
maxOutputTokens *int64
// chatdScopedRoute resolves the route with chatd scope. Deployment-wide
// override models must route for user-owned chats regardless of the
// caller's actor.
chatdScopedRoute bool
// requireOpenAIResponses forces the Responses API when the route uses an
// OpenAI client, and fails if the config explicitly disables it.
requireOpenAIResponses bool
buildOptions modelBuildOptions
}
func chatRequestedEffort(chat database.Chat) *string {
if !chat.LastReasoningEffort.Valid {
return nil
}
return new(string(chat.LastReasoningEffort.ChatReasoningEffort))
}
// modelCallConfigParseError lets the advisor distinguish malformed options
// from route and client failures when deciding whether to fall back.
type modelCallConfigParseError struct{ err error }
func (e modelCallConfigParseError) Error() string {
return "parse model call config: " + e.err.Error()
}
func (e modelCallConfigParseError) Unwrap() error { return e.err }
type resolvedModelCall struct {
model chatprovider.Model
dbConfig database.ChatModelConfig
callConfig codersdk.ChatModelCallConfig
providerOptions fantasy.ProviderOptions
resolvedProvider string
resolvedModel string
resolvedEffort string
route aiGatewayModelRoute
debugEnabled bool
// reasoningRejected reports whether the provider rejected encrypted
// reasoning replayed into a call to model.
reasoningRejected *atomic.Bool
}
// stageModel uses the configured provider type rather than
// resolvedProvider, which is inferred from the model name for types such
// as copilot.
func (r resolvedModelCall) stageModel() chatloop.StageModel {
stageModel := chatloop.StageModel{
ProviderType: string(r.route.Provider.Type),
Model: r.resolvedModel,
Effort: r.resolvedEffort,
}
if r.model.Valid() {
stageModel.Provider = r.model.Provider()
}
return stageModel
}
// resolveModelCall is the single pipeline from a spec to a ready model
// client plus the call metadata flows need.
func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (resolvedModelCall, error) {
out := resolvedModelCall{}
var modelName string
var configOptions []byte
switch {
case spec.explicitConfig != nil:
out.dbConfig = *spec.explicitConfig
modelName = out.dbConfig.Model
configOptions = out.dbConfig.Options
default:
dbConfig, err := p.resolveModelConfig(ctx, spec.chat)
if err != nil {
return resolvedModelCall{}, xerrors.Errorf("resolve model config: %w", err)
}
if !dbConfig.Enabled {
return resolvedModelCall{}, xerrors.Errorf("chat model config %s is disabled", dbConfig.ID)
}
out.dbConfig = dbConfig
modelName = dbConfig.Model
configOptions = dbConfig.Options
}
// clientCallConfig drives client construction; out.callConfig drives
// per-call option derivation.
clientCallConfig, err := parseModelConfigOptions(configOptions)
if err != nil {
return resolvedModelCall{}, modelCallConfigParseError{err: err}
}
out.callConfig = clientCallConfig
switch {
case spec.maxOutputTokens != nil:
out.callConfig.MaxOutputTokens = spec.maxOutputTokens
case out.callConfig.MaxOutputTokens == nil:
out.callConfig.MaxOutputTokens = ptr.Ref(defaultChatMaxOutputTokens)
}
routeCtx := ctx
if spec.chatdScopedRoute {
//nolint:gocritic // Deployment-wide override models need chatd-scoped provider reads for user-owned chats.
routeCtx = dbauthz.AsChatd(ctx)
}
out.route, err = p.resolveModelRouteForConfig(routeCtx, spec.chat.OwnerID, out.dbConfig)
if err != nil {
return resolvedModelCall{}, err
}
if spec.requireOpenAIResponses &&
chatprovider.NormalizeProvider(string(out.route.Provider.Type)) == fantasyopenai.Name {
clientCallConfig.OpenAIConfig, err = requireResponsesAPI(clientCallConfig.OpenAIConfig, modelName)
if err != nil {
return resolvedModelCall{}, err
}
out.callConfig.OpenAIConfig = clientCallConfig.OpenAIConfig
}
// The resolved identity feeds metadata, logs, and debug labels. The
// client is constructed with the configured model string so gateway
// validation sees the name exactly as configured.
out.resolvedProvider, out.resolvedModel, err = chatprovider.ResolveModelWithProviderHint(
modelName,
out.route.ModelProviderHint,
)
if err != nil {
return resolvedModelCall{}, xerrors.Errorf("resolve model metadata: %w", err)
}
debugSvc := p.debugService()
out.debugEnabled = debugSvc != nil && debugSvc.IsEnabled(ctx, spec.chat.ID, spec.chat.OwnerID)
// Coder-scale effort, before provider mapping can change or drop it.
if effectiveEffort := chatprovider.ResolveReasoningEffort(spec.requestedEffort, out.callConfig.ReasoningEffort); effectiveEffort != nil {
out.resolvedEffort = *effectiveEffort
}
buildOpts := spec.buildOptions
buildOpts.RecordHTTP = out.debugEnabled
buildOpts.StageModel = out.stageModel()
model, err := p.newModel(ctx, modelClientRequest{
Chat: spec.chat,
ModelName: modelName,
UserAgent: chatprovider.UserAgent(),
ExtraHeaders: chatprovider.CoderHeaders(spec.chat),
CallConfig: clientCallConfig,
}, out.route, buildOpts)
if err != nil {
return resolvedModelCall{}, xerrors.Errorf("create model: %w", err)
}
model = p.withThinkingDropBlock(model, out.route.Provider.ID, clientCallConfig, spec.chat.ID)
model = p.withQuickgenParamFallback(model, out.route.Provider.ID)
out.reasoningRejected = &atomic.Bool{}
model = p.withEncryptedReasoningFallback(model, spec.chat.ID, out.reasoningRejected)
if out.debugEnabled {
model = model.WithLanguageModel(chatdebug.WrapModel(model.LanguageModel(), debugSvc, chatdebug.RecorderOptions{
ChatID: spec.chat.ID,
OwnerID: spec.chat.OwnerID,
}))
}
out.model = model
// Only the derived provider options see the coerced config; callConfig
// stays as configured.
out.providerOptions = chatprovider.ProviderOptionsForCall(
out.model,
coerceBedrockReasoningSummary(out.route.Provider.Type, modelName, out.callConfig),
spec.requestedEffort,
)
p.logger.Debug(ctx, "resolved model call",
slog.F("purpose", spec.purpose),
slog.F("chat_id", spec.chat.ID),
slog.F("provider", out.resolvedProvider),
slog.F("model", out.resolvedModel),
slog.F("reasoning_effort", out.resolvedEffort),
slog.F("debug_enabled", out.debugEnabled),
)
return out, nil
}
// requireResponsesAPI returns an OpenAI config that selects the Responses API.
// It never mutates config.
func requireResponsesAPI(
config *codersdk.ChatModelOpenAIConfig,
modelName string,
) (*codersdk.ChatModelOpenAIConfig, error) {
var out codersdk.ChatModelOpenAIConfig
if config != nil {
if config.UseResponsesAPI != nil && !*config.UseResponsesAPI {
return nil, chaterror.WithClassification(
xerrors.Errorf("model %q disables the OpenAI Responses API, which computer use requires", modelName),
chaterror.ClassifiedError{
Kind: codersdk.ChatErrorKindConfig,
Retryable: false,
Detail: "Ask an administrator to enable the Responses API for this model or choose another computer use model.",
},
)
}
out = *config
}
out.UseResponsesAPI = ptr.Ref(true)
return &out, nil
}
func (r resolvedModelCall) newCall() fantasy.Call {
return fantasy.Call{
ProviderOptions: r.providerOptions,
MaxOutputTokens: r.callConfig.MaxOutputTokens,
Temperature: r.callConfig.Temperature,
TopP: r.callConfig.TopP,
TopK: r.callConfig.TopK,
PresencePenalty: r.callConfig.PresencePenalty,
FrequencyPenalty: r.callConfig.FrequencyPenalty,
}
}
// compactionSummaryCall follows the resolved call template, except summaries
// must not call tools. Streaming avoids the Anthropic SDK's non-streaming
// duration limit, while the explicit cap prevents adaptive thinking from
// exhausting fantasy's smaller provider default before producing summary text.
// The cap is not raised: the configured cap is the provider ceiling chatd
// knows, and providers reject a larger one. GenerateCompaction can only
// lower it, to the remaining context window.
func compactionSummaryCall(resolved resolvedModelCall) fantasy.Call {
call := resolved.newCall()
toolChoiceNone := fantasy.ToolChoiceNone
call.ToolChoice = &toolChoiceNone
return call
}
func (r resolvedModelCall) newObjectCall(schemaName, schemaDescription string, maxOutputTokens int64) fantasy.ObjectCall {
return fantasy.ObjectCall{
SchemaName: schemaName,
SchemaDescription: schemaDescription,
MaxOutputTokens: ptr.Ref(maxOutputTokens),
Temperature: r.callConfig.Temperature,
TopP: r.callConfig.TopP,
TopK: r.callConfig.TopK,
PresencePenalty: r.callConfig.PresencePenalty,
FrequencyPenalty: r.callConfig.FrequencyPenalty,
ProviderOptions: r.providerOptions,
}
}