fix(coderd/x/chatd): route quickgen side calls through the title generation model

The end-of-turn status label and the whole-chat summary ran structured
(forced tool call) generations on the chat's own model. Models that reject
forced tool_choice, or answer it with text, failed every call, so those
chats fell back to the generic "Finished latest turn" label.

Introduce resolveQuickgenModel as the single resolver for title, status
label, and summary calls: organization title_generation override first,
then an enabled preferred small model, then the chat model as a last
resort. The automatic title path now shares it too (it previously skipped
the preferred small models), and maybeGenerateChatTitle takes the resolved
call instead of resolving the override itself. Quickgen candidate failures
log at Warn with provider and model.

Refs CODAGT-1077
This commit is contained in:
Michael Suchacz committed 2026-09-20 17:16:00 +00:00
1 parent 854bd610f0
commit b4cc6fc336
8 files changed
+319 -383

No files matched your search

+13 -102
View File
@@ -2644,7 +2644,7 @@ func (p *Server) ProposeChatTitle(
) (string, error) {
//nolint:gocritic // Non-admin users need chatd-scoped config reads here.
chatdCtx := dbauthz.AsChatd(ctx)
return p.generateManualTitleCandidate(chatdCtx, p.db, chat)
return p.generateManualTitleCandidate(chatdCtx, chat)
}
// generateManualTitleCandidate generates a title candidate from the chat's
@@ -2652,10 +2652,9 @@ func (p *Server) ProposeChatTitle(
// Endpoint-specific commit paths decide whether to persist the title.
func (p *Server) generateManualTitleCandidate(
ctx context.Context,
store database.Store,
chat database.Chat,
) (string, error) {
headMessages, err := store.GetChatMessagesByChatIDAscPaginated(
headMessages, err := p.db.GetChatMessagesByChatIDAscPaginated(
ctx,
database.GetChatMessagesByChatIDAscPaginatedParams{
ChatID: chat.ID,
@@ -2666,7 +2665,7 @@ func (p *Server) generateManualTitleCandidate(
if err != nil {
return "", xerrors.Errorf("get head chat messages: %w", err)
}
tailMessages, err := store.GetChatMessagesByChatIDDescPaginated(
tailMessages, err := p.db.GetChatMessagesByChatIDDescPaginated(
ctx,
database.GetChatMessagesByChatIDDescPaginatedParams{
ChatID: chat.ID,
@@ -2681,7 +2680,7 @@ func (p *Server) generateManualTitleCandidate(
if len(messages) == 0 {
return "", nil
}
pasteText, err := titlePasteText(ctx, store, messages)
pasteText, err := titlePasteText(ctx, p.db, messages)
if err != nil {
return "", xerrors.Errorf("get pasted-text attachments for manual title: %w", err)
}
@@ -2691,7 +2690,7 @@ func (p *Server) generateManualTitleCandidate(
}
modelOpts := modelBuildOptions{ActiveAPIKeyID: apiKeyID}
resolved, err := p.resolveManualTitleModel(ctx, store, chat, modelOpts)
resolved, err := p.resolveQuickgenModel(ctx, "title", chat, modelOpts)
if err != nil {
return "", err
}
@@ -2850,94 +2849,6 @@ func deriveChatDebugSeed(messages []database.ChatMessage) (
return triggerMessageID, historyTipMessageID, triggerLabel
}
func (p *Server) resolveManualTitleModel(
ctx context.Context,
store database.Store,
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, error) {
overrideResolved, overrideSet, overrideErr := p.resolveTitleGenerationModelOverride(
ctx,
chat,
modelOpts,
)
if overrideErr != nil {
if overrideSet {
return resolvedModelCall{}, xerrors.Errorf(
"resolve manual title generation model override: %w",
overrideErr,
)
}
p.logger.Debug(ctx, "failed to resolve title generation model override for manual title",
slog.F("chat_id", chat.ID),
slog.Error(overrideErr),
)
} else if overrideSet {
return overrideResolved, nil
}
modelCtx, err := p.callerModelConfigContext(ctx, chat.OwnerID)
if err != nil {
return resolvedModelCall{}, err
}
configs, err := enabledChatModelConfigsForOrganization(modelCtx, store, chat.OrganizationID)
if err != nil {
p.logger.Debug(ctx, "failed to list manual title model configs",
slog.F("chat_id", chat.ID),
slog.Error(err),
)
return p.resolveFallbackManualTitleModel(ctx, chat, modelOpts)
}
config, ok := selectPreferredConfiguredShortTextModelConfig(configs)
if !ok {
return p.resolveFallbackManualTitleModel(ctx, chat, modelOpts)
}
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: "title",
chat: chat,
explicitConfig: &config,
buildOptions: modelOpts,
})
if err != nil {
p.logger.Debug(ctx, "manual title preferred model unavailable",
slog.F("chat_id", chat.ID),
slog.F("model", config.Model),
slog.Error(err),
)
return p.resolveFallbackManualTitleModel(ctx, chat, modelOpts)
}
return resolved, nil
}
func (p *Server) resolveFallbackManualTitleModel(
ctx context.Context,
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, error) {
config, err := p.resolveModelConfig(ctx, chat)
if err != nil {
return resolvedModelCall{}, xerrors.Errorf(
"resolve fallback manual title model config: %w",
err,
)
}
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: "title",
chat: chat,
explicitConfig: &config,
buildOptions: modelOpts,
})
if err != nil {
return resolvedModelCall{}, xerrors.Errorf(
"create fallback manual title model: %w",
err,
)
}
return resolved, nil
}
func mergeManualTitleMessages(
headMessages []database.ChatMessage,
tailMessagesDesc []database.ChatMessage,
@@ -4781,8 +4692,12 @@ func (p *Server) generateAndStoreChatSummary(
summary, _, genErr := generateChatSummary(summaryCtx, resolved.model.LanguageModel(), summaryObjectCall(resolved), transcript)
if genErr != nil {
logger.Debug(ctx, "failed to generate chat summary",
slog.F("chat_id", chat.ID), slog.Error(genErr))
logger.Warn(ctx, "failed to generate chat summary",
slog.F("chat_id", chat.ID),
slog.F("provider", resolved.resolvedProvider),
slog.F("model", resolved.resolvedModel),
slog.Error(genErr),
)
return
}
@@ -4795,13 +4710,9 @@ func (p *Server) resolveChatSummaryModel(
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, bool) {
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: "chat_summary",
chat: chat,
buildOptions: modelOpts,
})
resolved, err := p.resolveQuickgenModel(ctx, "chat_summary", chat, modelOpts)
if err != nil {
logger.Debug(ctx, "failed to resolve chat model for summary",
logger.Warn(ctx, "failed to resolve model for chat summary",
slog.F("chat_id", chat.ID), slog.Error(err))
return resolvedModelCall{}, false
}
+92
View File
@@ -31,6 +31,8 @@ import (
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
skillspkg "github.com/coder/coder/v2/coderd/x/skills"
"github.com/coder/coder/v2/codersdk"
@@ -327,6 +329,96 @@ func TestMaybeGenerateChatSummaryAsync_CloseCancelsInflight(t *testing.T) {
testutil.TryReceive(ctx, t, closed)
}
// The whole-chat summary is a structured side call, so it must run on the
// title generation model rather than the chat's own model.
func TestGenerateAndStoreChatSummary_UsesTitleGenerationModel(t *testing.T) {
t.Parallel()
db, ps := dbtestutil.NewDB(t)
ctx := chatdTestContext(t)
summaryModels := make(chan string, 1)
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
if strings.Contains(string(req.RawBody), "chat_summary") {
summaryModels <- req.Model
}
return chattest.OpenAINonStreamingResponse(`{"headline":"Explains the answer to the question","bullets":[]}`)
})
user := dbgen.User(t, db, database.User{})
org := dbgen.Organization(t, db, database.Organization{})
dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: user.ID,
OrganizationID: org.ID,
})
dbgen.ChatProvider(t, db, database.ChatProvider{
Provider: "openai",
DisplayName: "OpenAI",
BaseUrl: openAIURL,
})
insertModel := func(model string, isDefault bool) database.ChatModelConfig {
return dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
Model: model,
DisplayName: model,
Options: json.RawMessage(`{"openai_config":{"use_responses_api":false}}`),
OrganizationID: org.ID,
}, func(p *database.InsertChatModelConfigParams) {
p.Enabled = true
p.IsDefault = isDefault
})
}
chatModel := insertModel("gpt-4.1", true)
insertModel("gpt-4o-mini", false)
created, err := chatstate.CreateChat(ctx, db, ps, chatstate.CreateChatInput{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: chatModel.ID,
Title: "summary-model-chat",
ClientType: database.ChatClientTypeUi,
InitialMessages: []chatstate.Message{
{
Role: database.ChatMessageRoleUser,
Content: mustMarshalText(t, "what is the answer to life, the universe, and everything?"),
Visibility: database.ChatMessageVisibilityBoth,
ContentVersion: chatprompt.CurrentContentVersion,
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
ModelConfigID: uuid.NullUUID{UUID: chatModel.ID, Valid: true},
},
},
})
require.NoError(t, err)
chat := created.Chat
machine := chatstate.NewChatMachine(db, ps, chat.ID)
require.NoError(t, machine.Update(ctx, func(tx *chatstate.Tx, _ database.Store) error {
_, err := tx.CommitStep(chatstate.CommitStepInput{
Messages: []chatstate.Message{
{
Role: database.ChatMessageRoleAssistant,
Content: mustMarshalText(t, strings.Repeat("The answer is 42. ", summaryMinTranscriptRunes/10)),
Visibility: database.ChatMessageVisibilityBoth,
ContentVersion: chatprompt.CurrentContentVersion,
ModelConfigID: uuid.NullUUID{UUID: chatModel.ID, Valid: true},
},
},
})
return err
}))
server := newInternalTestServer(
t, db, ps, chatprovider.ProviderAPIKeys{},
withInternalTestServerTransportFactory(chattest.NewMockAIBridgeTransport(t, openAIURL)),
)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server.generateAndStoreChatSummary(ctx, logger, chat)
require.Equal(t, "gpt-4o-mini", testutil.RequireReceive(ctx, t, summaryModels))
fetched, err := db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.True(t, fetched.Summary.Valid)
}
func TestComputerUseProviderAndModelFromConfig(t *testing.T) {
t.Parallel()
+1 -5
View File
@@ -901,11 +901,7 @@ func (server *Server) deriveFinalTurnRunResult(
return runChatResult{FinalAssistantText: finalAssistantText, TriggerMessageID: triggerMessageID, HistoryTipMessageID: historyTipMessageID}
}
modelOpts := modelBuildOptions{ActiveAPIKeyID: apiKeyID}
resolved, err := server.resolveModelCall(ctx, modelCallSpec{
purpose: "turn_status_label",
chat: chat,
buildOptions: modelOpts,
})
resolved, err := server.resolveQuickgenModel(ctx, "turn_status_label", chat, modelOpts)
if err != nil {
// Preserve the text and IDs for the generic-label fallback.
logger.Warn(ctx, "derive final turn status label: resolve model", slog.Error(err))
@@ -483,7 +483,10 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
setup := func(t *testing.T) (*Server, database.Chat) {
// setup seeds an OpenAI provider with one enabled config per model. The
// first model is the chat's default model; the rest are extra enabled
// configs, returned by model name.
setup := func(t *testing.T, chatModel string, extraModels ...string) (*Server, database.Chat, map[string]database.ChatModelConfig) {
t.Helper()
db, ps := dbtestutil.NewDB(t)
ctx := chatdTestContext(t)
@@ -501,15 +504,22 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
Enabled: true,
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
})
modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
Model: "gpt-4o-mini",
DisplayName: "gpt-4o-mini",
Options: json.RawMessage(`{"openai_config":{"use_responses_api":false}}`),
OrganizationID: org.ID,
}, func(p *database.InsertChatModelConfigParams) {
p.Enabled = true
p.IsDefault = true
})
insertModel := func(model string, isDefault bool) database.ChatModelConfig {
return dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
Model: model,
DisplayName: model,
Options: json.RawMessage(`{"openai_config":{"use_responses_api":false}}`),
OrganizationID: org.ID,
}, func(p *database.InsertChatModelConfigParams) {
p.Enabled = true
p.IsDefault = isDefault
})
}
modelCfg := insertModel(chatModel, true)
extra := make(map[string]database.ChatModelConfig, len(extraModels))
for _, model := range extraModels {
extra[model] = insertModel(model, false)
}
created, err := chatstate.CreateChat(ctx, db, ps, chatstate.CreateChatInput{
OrganizationID: org.ID,
@@ -534,7 +544,7 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
t, db, ps, chatprovider.ProviderAPIKeys{},
withInternalTestServerTransportFactory(&aibridgeTestFactory{}),
)
return server, created.Chat
return server, created.Chat, extra
}
commitAssistant := func(t *testing.T, server *Server, chat database.Chat, text string) {
@@ -559,7 +569,7 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
t.Run("WaitingDerivesFromHistory", func(t *testing.T) {
t.Parallel()
server, chat := setup(t)
server, chat, _ := setup(t, "gpt-4o-mini")
ctx := chatdTestContext(t)
commitAssistant(t, server, chat, "the answer is 42")
@@ -587,9 +597,44 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
require.JSONEq(t, `{"openai_config":{"use_responses_api":false}}`, string(result.StatusLabelCall.dbConfig.Options))
})
// The status label is a structured side call, so it must prefer the
// title generation model over a chat model that may reject forced tools.
t.Run("WaitingPrefersSmallModelOverChatModel", func(t *testing.T) {
t.Parallel()
server, chat, extra := setup(t, "gpt-4.1", "gpt-4o-mini")
ctx := chatdTestContext(t)
commitAssistant(t, server, chat, "the answer is 42")
chat.Status = database.ChatStatusWaiting
result := server.deriveFinalTurnRunResult(ctx, chat, logger)
require.NotNil(t, result.StatusLabelCall)
require.Equal(t, extra["gpt-4o-mini"].ID, result.StatusLabelCall.dbConfig.ID)
require.Equal(t, "gpt-4o-mini", result.StatusLabelCall.resolvedModel)
})
t.Run("WaitingPrefersTitleGenerationOverride", func(t *testing.T) {
t.Parallel()
server, chat, extra := setup(t, "gpt-4.1", "gpt-4o-mini", "gpt-4.1-nano")
ctx := chatdTestContext(t)
commitAssistant(t, server, chat, "the answer is 42")
upsertInternalChatOrganizationModelOverride(
t, server.db, chat.OrganizationID,
codersdk.ChatModelOverrideContextTitleGeneration,
extra["gpt-4.1-nano"].ID.String(),
)
chat.Status = database.ChatStatusWaiting
result := server.deriveFinalTurnRunResult(ctx, chat, logger)
require.NotNil(t, result.StatusLabelCall)
require.Equal(t, extra["gpt-4.1-nano"].ID, result.StatusLabelCall.dbConfig.ID)
require.Equal(t, "gpt-4.1-nano", result.StatusLabelCall.resolvedModel)
})
t.Run("NonWaitingReturnsEmpty", func(t *testing.T) {
t.Parallel()
server, chat := setup(t)
server, chat, _ := setup(t, "gpt-4o-mini")
ctx := chatdTestContext(t)
commitAssistant(t, server, chat, "the answer is 42")
@@ -600,7 +645,7 @@ func TestDeriveFinalTurnRunResult(t *testing.T) {
t.Run("WaitingWithoutAssistantReturnsEmpty", func(t *testing.T) {
t.Parallel()
server, chat := setup(t)
server, chat, _ := setup(t, "gpt-4o-mini")
ctx := chatdTestContext(t)
// No assistant message was committed, so there is nothing to label.
+119 -64
View File
@@ -160,6 +160,101 @@ func selectPreferredConfiguredShortTextModelConfig(
return database.ChatModelConfig{}, false
}
// resolveQuickgenModel resolves the model for a quickgen side call (title,
// turn status label, chat summary). It prefers the organization's title
// generation model override, then an enabled preferred small model, and uses
// the chat's own model only as a last resort, so side calls avoid chat models
// that cannot serve structured generations. A configured but unusable
// override is a hard error. purpose labels resolver logs only.
func (p *Server) resolveQuickgenModel(
ctx context.Context,
purpose string,
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, error) {
overrideResolved, overrideSet, overrideErr := p.resolveTitleGenerationModelOverride(
ctx,
purpose,
chat,
modelOpts,
)
if overrideErr != nil {
if overrideSet {
return resolvedModelCall{}, xerrors.Errorf(
"resolve title generation model override for %s: %w",
purpose,
overrideErr,
)
}
p.logger.Debug(ctx, "failed to resolve title generation model override",
slog.F("purpose", purpose),
slog.F("chat_id", chat.ID),
slog.Error(overrideErr),
)
} else if overrideSet {
return overrideResolved, nil
}
modelCtx, err := p.callerModelConfigContext(ctx, chat.OwnerID)
if err != nil {
return resolvedModelCall{}, err
}
configs, err := enabledChatModelConfigsForOrganization(modelCtx, p.db, chat.OrganizationID)
if err != nil {
p.logger.Debug(ctx, "failed to list quickgen model configs",
slog.F("purpose", purpose),
slog.F("chat_id", chat.ID),
slog.Error(err),
)
return p.resolveQuickgenChatModelFallback(ctx, purpose, chat, modelOpts)
}
config, ok := selectPreferredConfiguredShortTextModelConfig(configs)
if !ok {
return p.resolveQuickgenChatModelFallback(ctx, purpose, chat, modelOpts)
}
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: purpose,
chat: chat,
explicitConfig: &config,
buildOptions: modelOpts,
})
if err != nil {
p.logger.Debug(ctx, "preferred quickgen model unavailable",
slog.F("purpose", purpose),
slog.F("chat_id", chat.ID),
slog.F("model", config.Model),
slog.Error(err),
)
return p.resolveQuickgenChatModelFallback(ctx, purpose, chat, modelOpts)
}
return resolved, nil
}
// resolveQuickgenChatModelFallback resolves the chat's own model as the last
// resort for a quickgen side call.
func (p *Server) resolveQuickgenChatModelFallback(
ctx context.Context,
purpose string,
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, error) {
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: purpose,
chat: chat,
buildOptions: modelOpts,
})
if err != nil {
return resolvedModelCall{}, xerrors.Errorf(
"resolve fallback chat model for %s: %w",
purpose,
err,
)
}
return resolved, nil
}
func normalizeShortTextOutput(text string) string {
text = strings.TrimSpace(text)
if text == "" {
@@ -187,10 +282,10 @@ type generatedTurnStatusLabel struct {
// request but bound to the server: it neither blocks the HTTP response
// nor is canceled when the request completes, and Close cancels it
// instead of blocking on the title timeout while a provider is
// unreachable. It resolves the chat's model and provider keys, then
// delegates to maybeGenerateChatTitle, which only acts on the first user
// turn (see titleInput) and is otherwise a no-op. Errors are logged and
// swallowed.
// unreachable. It resolves the title generation model (see
// resolveQuickgenModel), then delegates to maybeGenerateChatTitle, which
// only acts on the first user turn (see titleInput) and is otherwise a
// no-op. Errors are logged and swallowed.
func (p *Server) GenerateChatTitleAsync(ctx context.Context, chat database.Chat) {
logger := p.logger.With(
slog.F("chat_id", chat.ID),
@@ -227,25 +322,19 @@ func (p *Server) GenerateChatTitleAsync(ctx context.Context, chat database.Chat)
return
}
modelOpts := modelBuildOptions{ActiveAPIKeyID: apiKeyID}
turnCtx := titleCtx
fallback, err := p.resolveModelCall(turnCtx, modelCallSpec{
purpose: "title",
chat: chat,
buildOptions: modelOpts,
})
resolved, err := p.resolveQuickgenModel(titleCtx, "title", chat, modelOpts)
if err != nil {
logger.Debug(titleCtx, "failed to resolve model for automatic title generation",
logger.Warn(titleCtx, "failed to resolve model for automatic title generation",
slog.Error(err),
)
return
}
p.maybeGenerateChatTitle(
turnCtx,
titleCtx,
chat,
messages,
pasteText,
fallback,
modelOpts,
resolved,
&generatedChatTitle{},
logger,
p.existingDebugService(),
@@ -262,18 +351,15 @@ func (p *Server) GenerateChatTitleAsync(ctx context.Context, chat database.Chat)
// maybeGenerateChatTitle generates an AI title for the chat when
// appropriate (first user message, no assistant reply yet, and the
// current title is either empty or still the fallback truncation).
// It uses the configured title generation model override when set.
// Otherwise, it tries cheap, fast models first and falls back to the
// user's chat model. It is a best-effort operation that logs and
// swallows errors.
// current title is either empty or still the fallback truncation) using
// the resolved title generation model (see resolveQuickgenModel). It is a
// best-effort operation that logs and swallows errors.
func (p *Server) maybeGenerateChatTitle(
ctx context.Context,
chat database.Chat,
messages []database.ChatMessage,
pasteText map[uuid.UUID]string,
fallback resolvedModelCall,
modelOpts modelBuildOptions,
resolved resolvedModelCall,
generatedTitle *generatedChatTitle,
logger slog.Logger,
debugSvc *chatdebug.Service,
@@ -287,35 +373,10 @@ func (p *Server) maybeGenerateChatTitle(
titleCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
overrideResolved, overrideSet, overrideErr := p.resolveTitleGenerationModelOverride(
titleCtx,
chat,
modelOpts,
)
if overrideErr != nil {
if overrideSet {
logger.Warn(ctx, "title generation model override unavailable, skipping title generation",
slog.F("chat_id", chat.ID),
slog.F("override_context", titleGenerationOverrideContext),
slog.Error(overrideErr),
)
return
}
logger.Debug(ctx, "failed to resolve title generation model override",
slog.F("chat_id", chat.ID),
slog.F("override_context", titleGenerationOverrideContext),
slog.Error(overrideErr),
)
}
selected := fallback
if overrideSet {
selected = overrideResolved
}
candidate := shortTextCandidate{
provider: string(selected.route.Provider.Type),
model: selected.dbConfig.Model,
resolved: selected,
provider: string(resolved.route.Provider.Type),
model: resolved.dbConfig.Model,
resolved: resolved,
}
var historyTipMessageID int64
@@ -357,20 +418,12 @@ func (p *Server) maybeGenerateChatTitle(
title, err := generateTitle(candidateCtx, candidate.resolved.model.LanguageModel(), titleObjectCall(candidate.resolved), input)
finishDebugRun(err)
if err != nil {
if overrideSet {
logger.Warn(ctx, "title model candidate failed",
slog.F("chat_id", chat.ID),
slog.F("override_context", titleGenerationOverrideContext),
slog.F("provider", candidate.provider),
slog.F("model", candidate.model),
slog.Error(err),
)
} else {
logger.Debug(ctx, "title model candidate failed",
slog.F("chat_id", chat.ID),
slog.Error(err),
)
}
logger.Warn(ctx, "title model candidate failed",
slog.F("chat_id", chat.ID),
slog.F("provider", resolved.resolvedProvider),
slog.F("model", resolved.resolvedModel),
slog.Error(err),
)
return
}
if title == "" || title == chat.Title {
@@ -1388,7 +1441,9 @@ func generateTurnStatusLabel(
)
finishDebugRun(err)
if err != nil {
logger.Debug(ctx, "turn status label model candidate failed",
logger.Warn(ctx, "turn status label model candidate failed",
slog.F("provider", resolved.resolvedProvider),
slog.F("model", resolved.resolvedModel),
slog.Error(err),
)
return ""
-3
View File
@@ -591,7 +591,6 @@ func TestMaybeGenerateChatTitlePreservesUpdatedAt(t *testing.T) {
model: chatprovider.NewModel(model, nil),
dbConfig: database.ChatModelConfig{Model: "test-model"},
},
modelBuildOptions{},
generated,
logger,
nil,
@@ -643,7 +642,6 @@ func TestMaybeGenerateChatTitleAppliesModelConfigReasoningEffort(t *testing.T) {
}
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(database.ChatOrganizationModelOverride{}, sql.ErrNoRows)
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: "Reasoning title",
@@ -665,7 +663,6 @@ func TestMaybeGenerateChatTitleAppliesModelConfigReasoningEffort(t *testing.T) {
dbConfig: fallbackConfig,
providerOptions: chatprovider.ProviderOptionsForCall(fallbackModel, callConfig, nil),
},
modelBuildOptions{},
&generatedChatTitle{},
logger,
nil,
+3 -1
View File
@@ -37,8 +37,10 @@ func readTitleGenerationModelOverride(
// resolveTitleGenerationModelOverride resolves the chat organization's title
// generation model override. A configured but unusable override is a hard
// failure. When no row is configured, callers may use the default title model.
// purpose labels resolver logs only.
func (p *Server) resolveTitleGenerationModelOverride(
ctx context.Context,
purpose string,
chat database.Chat,
modelOpts modelBuildOptions,
) (resolvedModelCall, bool, error) {
@@ -73,7 +75,7 @@ func (p *Server) resolveTitleGenerationModelOverride(
return resolvedModelCall{}, false, nil
}
resolved, err := p.resolveModelCall(ctx, modelCallSpec{
purpose: "title",
purpose: purpose,
chat: chat,
explicitConfig: &modelConfig,
requestedEffort: overrideEffort,
+32 -194
View File
@@ -12,12 +12,10 @@ import (
"testing"
"time"
"charm.land/fantasy"
fantasyopenai "charm.land/fantasy/providers/openai"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"cdr.dev/slog/v3/sloggers/slogtest"
@@ -26,7 +24,6 @@ import (
"github.com/coder/coder/v2/coderd/database/dbmock"
dbpubsub "github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
@@ -40,108 +37,6 @@ func titleGenerationOverrideParams(chat database.Chat) database.GetChatOrganizat
}
}
func TestMaybeGenerateChatTitle_TitleGenerationOverrideUnset(t *testing.T) {
t.Parallel()
t.Run("falls back to chat model when preferred models are unavailable", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
chat, messages := titleOverrideTestChatAndMessages(t)
wantTitle := "Fallback title"
var fallbackCalls atomic.Int32
fallbackModel := &chattest.FakeModel{
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
fallbackCalls.Add(1)
return &fantasy.ObjectResponse{
Object: map[string]any{"title": wantTitle},
}, nil
},
}
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(database.ChatOrganizationModelOverride{}, sql.ErrNoRows)
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
generated := &generatedChatTitle{}
server := titleOverrideTestServer(db, logger)
server.maybeGenerateChatTitle(
ctx,
chat,
messages,
nil,
resolvedModelCall{
model: chatprovider.NewModel(fallbackModel, nil),
dbConfig: database.ChatModelConfig{Model: "fallback-chat-model"},
},
modelBuildOptions{},
generated,
logger,
nil,
)
require.Equal(t, int32(1), fallbackCalls.Load())
gotTitle, ok := generated.Load()
require.True(t, ok)
require.Equal(t, wantTitle, gotTitle)
})
}
func TestMaybeGenerateChatTitle_TitleGenerationOverrideReadDBError(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
chat, messages := titleOverrideTestChatAndMessages(t)
wantTitle := "Fallback title"
var fallbackCalls atomic.Int32
fallbackModel := &chattest.FakeModel{
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
fallbackCalls.Add(1)
return &fantasy.ObjectResponse{
Object: map[string]any{"title": wantTitle},
}, nil
},
}
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(database.ChatOrganizationModelOverride{}, sql.ErrConnDone)
db.EXPECT().UpdateChatTitleByID(gomock.Any(), database.UpdateChatTitleByIDParams{
ID: chat.ID,
Title: wantTitle,
}).Return(chatWithTitle(chat, wantTitle), nil)
generated := &generatedChatTitle{}
server := titleOverrideTestServer(db, logger)
server.maybeGenerateChatTitle(
ctx,
chat,
messages,
nil,
resolvedModelCall{
model: chatprovider.NewModel(fallbackModel, nil),
dbConfig: database.ChatModelConfig{Model: "fallback-chat-model"},
},
modelBuildOptions{},
generated,
logger,
nil,
)
require.Equal(t, int32(1), fallbackCalls.Load())
gotTitle, ok := generated.Load()
require.True(t, ok)
require.Equal(t, wantTitle, gotTitle)
}
func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUsable(t *testing.T) {
t.Parallel()
@@ -186,12 +81,6 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUsable(t *testing.T) {
Type: database.AIProviderTypeOpenai,
Enabled: true,
}
fallbackModel := &chattest.FakeModel{
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
t.Fatal("fallback model should not be called when override is usable")
return nil, xerrors.New("unexpected fallback model call")
},
}
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(orgModelOverride(chat, titleGenerationOverrideContext, overrideConfig.ID, "xhigh"), nil)
db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil)
@@ -208,16 +97,15 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUsable(t *testing.T) {
generated := &generatedChatTitle{}
server := titleOverrideTestServer(db, logger)
server.aibridgeTransportFactory = aibridgeTestFactoryPointer(factory)
resolved, err := server.resolveQuickgenModel(ctx, "title", chat, modelBuildOptions{ActiveAPIKeyID: uuid.NewString()})
require.NoError(t, err)
require.Equal(t, overrideConfig, resolved.dbConfig)
server.maybeGenerateChatTitle(
ctx,
chat,
messages,
nil,
resolvedModelCall{
model: chatprovider.NewModel(fallbackModel, nil),
dbConfig: database.ChatModelConfig{Model: "fallback-chat-model"},
},
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
resolved,
generated,
logger,
nil,
@@ -229,49 +117,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUsable(t *testing.T) {
require.Equal(t, wantTitle, gotTitle)
}
func TestMaybeGenerateChatTitle_TitleGenerationOverrideSetUnusableHardFails(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
chat, messages := titleOverrideTestChatAndMessages(t)
overrideConfig := titleOverrideModelConfig("gpt-4.1", false)
wantTitle := "Fallback title"
fallbackModel := &chattest.FakeModel{
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
return &fantasy.ObjectResponse{
Object: map[string]any{"title": wantTitle},
}, nil
},
}
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(orgModelOverride(chat, titleGenerationOverrideContext, overrideConfig.ID, ""), nil)
db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil)
generated := &generatedChatTitle{}
server := titleOverrideTestServer(db, logger)
server.maybeGenerateChatTitle(
ctx,
chat,
messages,
nil,
resolvedModelCall{
model: chatprovider.NewModel(fallbackModel, nil),
dbConfig: database.ChatModelConfig{Model: "fallback-chat-model"},
},
modelBuildOptions{},
generated,
logger,
nil,
)
_, ok := generated.Load()
require.False(t, ok, "a configured but unusable title override must not fall back")
}
func TestMaybeGenerateChatTitle_TitleGenerationOverrideCallFailureSkipsFallback(t *testing.T) {
func TestMaybeGenerateChatTitle_TitleGenerationOverrideCallFailure(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -288,12 +134,6 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideCallFailureSkipsFallback(
requestCount.Add(1)
return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil
})}
fallbackModel := &chattest.FakeModel{
GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
t.Fatal("fallback model should not be called after override call failure")
return nil, xerrors.New("unexpected fallback model call")
},
}
db.EXPECT().GetChatOrganizationModelOverride(gomock.Any(), titleGenerationOverrideParams(chat)).Return(orgModelOverride(chat, titleGenerationOverrideContext, overrideConfig.ID, ""), nil)
db.EXPECT().GetChatModelConfigByID(gomock.Any(), overrideConfig.ID).Return(overrideConfig, nil)
@@ -306,16 +146,14 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideCallFailureSkipsFallback(
generated := &generatedChatTitle{}
server := titleOverrideTestServer(db, logger)
server.aibridgeTransportFactory = aibridgeTestFactoryPointer(factory)
resolved, err := server.resolveQuickgenModel(ctx, "title", chat, modelBuildOptions{ActiveAPIKeyID: uuid.NewString()})
require.NoError(t, err)
server.maybeGenerateChatTitle(
ctx,
chat,
messages,
nil,
resolvedModelCall{
model: chatprovider.NewModel(fallbackModel, nil),
dbConfig: database.ChatModelConfig{Model: "fallback-chat-model"},
},
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
resolved,
generated,
logger,
nil,
@@ -326,7 +164,7 @@ func TestMaybeGenerateChatTitle_TitleGenerationOverrideCallFailureSkipsFallback(
require.False(t, ok)
}
func TestResolveManualTitleModel_TitleGenerationOverrideUnset(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideUnset(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -350,9 +188,9 @@ func TestResolveManualTitleModel_TitleGenerationOverrideUnset(t *testing.T) {
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary-openai", database.AIProviderTypeOpenai), nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
)
@@ -361,7 +199,7 @@ func TestResolveManualTitleModel_TitleGenerationOverrideUnset(t *testing.T) {
require.Equal(t, preferredConfig, resolved.dbConfig)
}
func TestResolveManualTitleModel_NonDefaultOrgDoesNotUseDefaultOrgConfigs(t *testing.T) {
func TestResolveQuickgenModel_NonDefaultOrgDoesNotUseDefaultOrgConfigs(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -375,9 +213,9 @@ func TestResolveManualTitleModel_NonDefaultOrgDoesNotUseDefaultOrgConfigs(t *tes
db.EXPECT().GetEnabledChatModelConfigsByOrganization(gomock.Any(), chat.OrganizationID).Return(nil, nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
)
@@ -385,7 +223,7 @@ func TestResolveManualTitleModel_NonDefaultOrgDoesNotUseDefaultOrgConfigs(t *tes
require.Equal(t, resolvedModelCall{}, resolved)
}
func TestResolveManualTitleModel_TitleGenerationOverrideUnsetAIProvider(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideUnsetAIProvider(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -423,9 +261,9 @@ func TestResolveManualTitleModel_TitleGenerationOverrideUnsetAIProvider(t *testi
}}, nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
)
@@ -434,7 +272,7 @@ func TestResolveManualTitleModel_TitleGenerationOverrideUnsetAIProvider(t *testi
require.Equal(t, preferredConfig, resolved.dbConfig)
}
func TestResolveManualTitleModel_TitleGenerationOverrideReadDBError(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideReadDBError(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -458,9 +296,9 @@ func TestResolveManualTitleModel_TitleGenerationOverrideReadDBError(t *testing.T
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary-openai", database.AIProviderTypeOpenai), nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
)
@@ -469,7 +307,7 @@ func TestResolveManualTitleModel_TitleGenerationOverrideReadDBError(t *testing.T
require.Equal(t, preferredConfig, resolved.dbConfig)
}
func TestResolveManualTitleModel_TitleGenerationOverrideSetUsable(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideSetUsable(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -490,9 +328,9 @@ func TestResolveManualTitleModel_TitleGenerationOverrideSetUsable(t *testing.T)
}}, nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
)
@@ -501,7 +339,7 @@ func TestResolveManualTitleModel_TitleGenerationOverrideSetUsable(t *testing.T)
require.Equal(t, overrideConfig, resolved.dbConfig)
}
func TestResolveManualTitleModel_TitleGenerationOverrideMissingCredentials(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideMissingCredentials(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -525,13 +363,13 @@ func TestResolveManualTitleModel_TitleGenerationOverrideMissingCredentials(t *te
// Missing override credentials hard-fail the configured override.
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{},
)
require.ErrorContains(t, err, "resolve manual title generation model override")
require.ErrorContains(t, err, "resolve title generation model override for title")
require.ErrorContains(t, err, "credentials are unavailable")
require.Equal(t, resolvedModelCall{}, resolved)
}
@@ -605,7 +443,7 @@ func TestGenerateManualTitleCandidate_UsesSyntheticAPIKey(t *testing.T) {
server := titleOverrideTestServer(db, logger)
server.clock = quartz.NewReal()
server.aibridgeTransportFactory = aibridgeTestFactoryPointer(factory)
title, err := server.generateManualTitleCandidate(ctx, db, chat)
title, err := server.generateManualTitleCandidate(ctx, chat)
require.NoError(t, err)
require.Equal(t, wantTitle, title)
require.Equal(t, apiKeyID, testutil.RequireReceive(ctx, t, seenAPIKeyID))
@@ -615,7 +453,7 @@ func TestGenerateManualTitleCandidate_UsesSyntheticAPIKey(t *testing.T) {
require.Equal(t, "title-options-sentinel", raw["user"])
}
func TestResolveManualTitleModel_TitleGenerationOverrideSetUnusable(t *testing.T) {
func TestResolveQuickgenModel_TitleGenerationOverrideSetUnusable(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
@@ -630,13 +468,13 @@ func TestResolveManualTitleModel_TitleGenerationOverrideSetUnusable(t *testing.T
// A disabled configured override is a hard failure.
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveManualTitleModel(
resolved, err := server.resolveQuickgenModel(
ctx,
db,
"title",
chat,
modelBuildOptions{},
)
require.ErrorContains(t, err, "resolve manual title generation model override")
require.ErrorContains(t, err, "resolve title generation model override for title")
require.ErrorContains(t, err, "model override is unavailable")
require.Equal(t, resolvedModelCall{}, resolved)
}