On Claude Opus 5.5, chat title generation and the turn-status label each made four upstream requests, three of them rejected with a 400. The temperature retry in `generateQuickgenObject` called `generateObject` again, which restarted from the forced tool call the model had already rejected. Each turn left about six `bad_request` rows in `aibridge_interceptions` and added 1.5 to 3.5 seconds of latency. This replaces the per-call fallbacks with `quickgenParamModel`, a wrapper applied in `resolveModelCall`. It falls back to plain JSON text when the model rejects the forced `tool_choice`, drops `temperature` when the model rejects it, and records each rejection in a replica-wide `sync.Map` keyed by AI provider ID, model, and parameter, following the existing `thinkingDropBlock` map. Entries expire after 6 hours so a provider that starts accepting a parameter again is picked up without a restart. Only quickgen calls `GenerateObject`, so chat turns are unaffected. | Opus 5.5 | Before | First job per replica (and after expiry) | Later jobs | |---|---|---|---| | Title / status label | 4 (3 rejected) | 3 (2 rejected) | 1 | | Summary | 2 (1 rejected) | 2 (1 rejected) | 1 | Rejections are still detected by matching `tool_choice` or `temperature` in a 400 body, as before. A false match now persists for the TTL on that replica rather than one job; the worst case is text mode or no temperature for that model. Closes [CODAGT-1411](https://linear.app/codercom/issue/CODAGT-1411/quickgen-side-jobs-make-3-rejected-requests-per-run-on-claude-opus-55) <details> <summary>Decision log</summary> - Opus 5.5 reports one rejected parameter per 400, so a job that knows nothing yet needs 3 requests; the issue's "2 requests" target for the minimal fix isn't reachable. - A per-job flag (first iteration) only got jobs to 3 requests. Sharing rejections across jobs gets steady state to 1 without a hardcoded model list, which wouldn't match gateway aliases such as Bedrock IDs anyway. - Per-replica memory over a database column or admin setting: no migration, no request-path writes, self-healing via TTL. Each replica pays the learning cost once per model per TTL window. - Keyed by provider ID plus model so the same model name behind different providers is learned independently. </details> > Generated by Coder Agents on behalf of @deansheather.
276 lines
9.6 KiB
Go
276 lines
9.6 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"sync/atomic"
|
|
|
|
"charm.land/fantasy"
|
|
fantasyopenai "charm.land/fantasy/providers/openai"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbauthz"
|
|
"github.com/coder/coder/v2/coderd/util/ptr"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
)
|
|
|
|
const defaultChatMaxOutputTokens = int64(32_000)
|
|
|
|
type modelCallSpec struct {
|
|
// purpose labels resolver logs only; it does not affect call behavior.
|
|
purpose string
|
|
chat database.Chat
|
|
explicitConfig *database.ChatModelConfig
|
|
// requestedEffort overrides the config's default reasoning effort,
|
|
// clamped to the config's max.
|
|
requestedEffort *string
|
|
// maxOutputTokens overrides the config's output-token cap and the
|
|
// package default.
|
|
maxOutputTokens *int64
|
|
// chatdScopedRoute resolves the route with chatd scope. Deployment-wide
|
|
// override models must route for user-owned chats regardless of the
|
|
// caller's actor.
|
|
chatdScopedRoute bool
|
|
// requireOpenAIResponses forces the Responses API when the route uses an
|
|
// OpenAI client, and fails if the config explicitly disables it.
|
|
requireOpenAIResponses bool
|
|
buildOptions modelBuildOptions
|
|
}
|
|
|
|
func chatRequestedEffort(chat database.Chat) *string {
|
|
if !chat.LastReasoningEffort.Valid {
|
|
return nil
|
|
}
|
|
return new(string(chat.LastReasoningEffort.ChatReasoningEffort))
|
|
}
|
|
|
|
// modelCallConfigParseError lets the advisor distinguish malformed options
|
|
// from route and client failures when deciding whether to fall back.
|
|
type modelCallConfigParseError struct{ err error }
|
|
|
|
func (e modelCallConfigParseError) Error() string {
|
|
return "parse model call config: " + e.err.Error()
|
|
}
|
|
|
|
func (e modelCallConfigParseError) Unwrap() error { return e.err }
|
|
|
|
type resolvedModelCall struct {
|
|
model chatprovider.Model
|
|
dbConfig database.ChatModelConfig
|
|
callConfig codersdk.ChatModelCallConfig
|
|
providerOptions fantasy.ProviderOptions
|
|
resolvedProvider string
|
|
resolvedModel string
|
|
resolvedEffort string
|
|
route aiGatewayModelRoute
|
|
debugEnabled bool
|
|
// reasoningRejected reports whether the provider rejected encrypted
|
|
// reasoning replayed into a call to model.
|
|
reasoningRejected *atomic.Bool
|
|
}
|
|
|
|
// stageModel uses the configured provider type rather than
|
|
// resolvedProvider, which is inferred from the model name for types such
|
|
// as copilot.
|
|
func (r resolvedModelCall) stageModel() chatloop.StageModel {
|
|
stageModel := chatloop.StageModel{
|
|
ProviderType: string(r.route.Provider.Type),
|
|
Model: r.resolvedModel,
|
|
Effort: r.resolvedEffort,
|
|
}
|
|
if r.model.Valid() {
|
|
stageModel.Provider = r.model.Provider()
|
|
}
|
|
return stageModel
|
|
}
|
|
|
|
// resolveModelCall is the single pipeline from a spec to a ready model
|
|
// client plus the call metadata flows need.
|
|
func (p *Server) resolveModelCall(ctx context.Context, spec modelCallSpec) (resolvedModelCall, error) {
|
|
out := resolvedModelCall{}
|
|
|
|
var modelName string
|
|
var configOptions []byte
|
|
switch {
|
|
case spec.explicitConfig != nil:
|
|
out.dbConfig = *spec.explicitConfig
|
|
modelName = out.dbConfig.Model
|
|
configOptions = out.dbConfig.Options
|
|
default:
|
|
dbConfig, err := p.resolveModelConfig(ctx, spec.chat)
|
|
if err != nil {
|
|
return resolvedModelCall{}, xerrors.Errorf("resolve model config: %w", err)
|
|
}
|
|
if !dbConfig.Enabled {
|
|
return resolvedModelCall{}, xerrors.Errorf("chat model config %s is disabled", dbConfig.ID)
|
|
}
|
|
out.dbConfig = dbConfig
|
|
modelName = dbConfig.Model
|
|
configOptions = dbConfig.Options
|
|
}
|
|
|
|
// clientCallConfig drives client construction; out.callConfig drives
|
|
// per-call option derivation.
|
|
clientCallConfig, err := parseModelConfigOptions(configOptions)
|
|
if err != nil {
|
|
return resolvedModelCall{}, modelCallConfigParseError{err: err}
|
|
}
|
|
out.callConfig = clientCallConfig
|
|
switch {
|
|
case spec.maxOutputTokens != nil:
|
|
out.callConfig.MaxOutputTokens = spec.maxOutputTokens
|
|
case out.callConfig.MaxOutputTokens == nil:
|
|
out.callConfig.MaxOutputTokens = ptr.Ref(defaultChatMaxOutputTokens)
|
|
}
|
|
|
|
routeCtx := ctx
|
|
if spec.chatdScopedRoute {
|
|
//nolint:gocritic // Deployment-wide override models need chatd-scoped provider reads for user-owned chats.
|
|
routeCtx = dbauthz.AsChatd(ctx)
|
|
}
|
|
out.route, err = p.resolveModelRouteForConfig(routeCtx, spec.chat.OwnerID, out.dbConfig)
|
|
if err != nil {
|
|
return resolvedModelCall{}, err
|
|
}
|
|
if spec.requireOpenAIResponses &&
|
|
chatprovider.NormalizeProvider(string(out.route.Provider.Type)) == fantasyopenai.Name {
|
|
clientCallConfig.OpenAIConfig, err = requireResponsesAPI(clientCallConfig.OpenAIConfig, modelName)
|
|
if err != nil {
|
|
return resolvedModelCall{}, err
|
|
}
|
|
out.callConfig.OpenAIConfig = clientCallConfig.OpenAIConfig
|
|
}
|
|
|
|
// The resolved identity feeds metadata, logs, and debug labels. The
|
|
// client is constructed with the configured model string so gateway
|
|
// validation sees the name exactly as configured.
|
|
out.resolvedProvider, out.resolvedModel, err = chatprovider.ResolveModelWithProviderHint(
|
|
modelName,
|
|
out.route.ModelProviderHint,
|
|
)
|
|
if err != nil {
|
|
return resolvedModelCall{}, xerrors.Errorf("resolve model metadata: %w", err)
|
|
}
|
|
|
|
debugSvc := p.debugService()
|
|
out.debugEnabled = debugSvc != nil && debugSvc.IsEnabled(ctx, spec.chat.ID, spec.chat.OwnerID)
|
|
|
|
// Coder-scale effort, before provider mapping can change or drop it.
|
|
if effectiveEffort := chatprovider.ResolveReasoningEffort(spec.requestedEffort, out.callConfig.ReasoningEffort); effectiveEffort != nil {
|
|
out.resolvedEffort = *effectiveEffort
|
|
}
|
|
|
|
buildOpts := spec.buildOptions
|
|
buildOpts.RecordHTTP = out.debugEnabled
|
|
buildOpts.StageModel = out.stageModel()
|
|
model, err := p.newModel(ctx, modelClientRequest{
|
|
Chat: spec.chat,
|
|
ModelName: modelName,
|
|
UserAgent: chatprovider.UserAgent(),
|
|
ExtraHeaders: chatprovider.CoderHeaders(spec.chat),
|
|
CallConfig: clientCallConfig,
|
|
}, out.route, buildOpts)
|
|
if err != nil {
|
|
return resolvedModelCall{}, xerrors.Errorf("create model: %w", err)
|
|
}
|
|
model = p.withThinkingDropBlock(model, out.route.Provider.ID, clientCallConfig, spec.chat.ID)
|
|
model = p.withQuickgenParamFallback(model, out.route.Provider.ID)
|
|
out.reasoningRejected = &atomic.Bool{}
|
|
model = p.withEncryptedReasoningFallback(model, spec.chat.ID, out.reasoningRejected)
|
|
|
|
if out.debugEnabled {
|
|
model = model.WithLanguageModel(chatdebug.WrapModel(model.LanguageModel(), debugSvc, chatdebug.RecorderOptions{
|
|
ChatID: spec.chat.ID,
|
|
OwnerID: spec.chat.OwnerID,
|
|
}))
|
|
}
|
|
out.model = model
|
|
|
|
// Only the derived provider options see the coerced config; callConfig
|
|
// stays as configured.
|
|
out.providerOptions = chatprovider.ProviderOptionsForCall(
|
|
out.model,
|
|
coerceBedrockReasoningSummary(out.route.Provider.Type, modelName, out.callConfig),
|
|
spec.requestedEffort,
|
|
)
|
|
|
|
p.logger.Debug(ctx, "resolved model call",
|
|
slog.F("purpose", spec.purpose),
|
|
slog.F("chat_id", spec.chat.ID),
|
|
slog.F("provider", out.resolvedProvider),
|
|
slog.F("model", out.resolvedModel),
|
|
slog.F("reasoning_effort", out.resolvedEffort),
|
|
slog.F("debug_enabled", out.debugEnabled),
|
|
)
|
|
return out, nil
|
|
}
|
|
|
|
// requireResponsesAPI returns an OpenAI config that selects the Responses API.
|
|
// It never mutates config.
|
|
func requireResponsesAPI(
|
|
config *codersdk.ChatModelOpenAIConfig,
|
|
modelName string,
|
|
) (*codersdk.ChatModelOpenAIConfig, error) {
|
|
var out codersdk.ChatModelOpenAIConfig
|
|
if config != nil {
|
|
if config.UseResponsesAPI != nil && !*config.UseResponsesAPI {
|
|
return nil, chaterror.WithClassification(
|
|
xerrors.Errorf("model %q disables the OpenAI Responses API, which computer use requires", modelName),
|
|
chaterror.ClassifiedError{
|
|
Kind: codersdk.ChatErrorKindConfig,
|
|
Retryable: false,
|
|
Detail: "Ask an administrator to enable the Responses API for this model or choose another computer use model.",
|
|
},
|
|
)
|
|
}
|
|
out = *config
|
|
}
|
|
out.UseResponsesAPI = ptr.Ref(true)
|
|
return &out, nil
|
|
}
|
|
|
|
func (r resolvedModelCall) newCall() fantasy.Call {
|
|
return fantasy.Call{
|
|
ProviderOptions: r.providerOptions,
|
|
MaxOutputTokens: r.callConfig.MaxOutputTokens,
|
|
Temperature: r.callConfig.Temperature,
|
|
TopP: r.callConfig.TopP,
|
|
TopK: r.callConfig.TopK,
|
|
PresencePenalty: r.callConfig.PresencePenalty,
|
|
FrequencyPenalty: r.callConfig.FrequencyPenalty,
|
|
}
|
|
}
|
|
|
|
// compactionSummaryCall follows the resolved call template, except summaries
|
|
// must not call tools. Streaming avoids the Anthropic SDK's non-streaming
|
|
// duration limit, while the explicit cap prevents adaptive thinking from
|
|
// exhausting fantasy's smaller provider default before producing summary text.
|
|
// The cap is not raised: the configured cap is the provider ceiling chatd
|
|
// knows, and providers reject a larger one. GenerateCompaction can only
|
|
// lower it, to the remaining context window.
|
|
func compactionSummaryCall(resolved resolvedModelCall) fantasy.Call {
|
|
call := resolved.newCall()
|
|
toolChoiceNone := fantasy.ToolChoiceNone
|
|
call.ToolChoice = &toolChoiceNone
|
|
return call
|
|
}
|
|
|
|
func (r resolvedModelCall) newObjectCall(schemaName, schemaDescription string, maxOutputTokens int64) fantasy.ObjectCall {
|
|
return fantasy.ObjectCall{
|
|
SchemaName: schemaName,
|
|
SchemaDescription: schemaDescription,
|
|
MaxOutputTokens: ptr.Ref(maxOutputTokens),
|
|
Temperature: r.callConfig.Temperature,
|
|
TopP: r.callConfig.TopP,
|
|
TopK: r.callConfig.TopK,
|
|
PresencePenalty: r.callConfig.PresencePenalty,
|
|
FrequencyPenalty: r.callConfig.FrequencyPenalty,
|
|
ProviderOptions: r.providerOptions,
|
|
}
|
|
}
|