Files
coder/coderd/x/chatd/quickgen_debug_internal_test.go
T
Michael Suchacz aef1917171 refactor(coderd/x/chatd): delete the write-only generatedChatTitle (#30926)
Deletes `generatedChatTitle`, a mutex-guarded cell in which
`maybeGenerateChatTitle` stored each generated title so that tests could
read it. Production passed a fresh one and never read it. The PR removes
the type, its `Store` and `Load` methods, the parameter, and the six
`Load` checks in the title tests, including the one that #30933 added to
`quickgen_debug_internal_test.go`.

## Why it is safe

`git grep generatedChatTitle` finds nothing. Each removed `Load` check
has a remaining check on the stored chat row or the published event:

| Removed check | Remaining check |
|---|---|
| `WritesGeneratedTitle`: title stored | stored title and source,
`title_change` event |
| `KeepsCurrentTitleWhenNoTitleIsWritten`, both cases: nothing stored |
stored title unchanged, no publish |
| `TestQuickgenFollowsConfiguredRetries/Title`: nothing stored | `model
call failed` and `TitleGenerationOverrideCallFailure` cover the same
return |
| `TitleGenerationOverrideSetUsable`: title stored | gomock expects the
exact `UpdateChatTitleByID` |
| `TitleGenerationOverrideCallFailure`: nothing stored | gomock fails on
any `UpdateChatTitleByID` |
| `TestQuickgenDebugRun/TitleGeneration`: whether a title was stored |
whether the owner's channel got one event |

One gap needed a test change. Without the `return` after a refused title
write, the code publishes the zero `updatedChat` on the nil owner's
channel, which only the observer caught on `main`
(`quickgen_internal_test.go:688`). The two no-write cases now use a
`psmock` pubsub with no expectations, so any publish fails them.

Mutants of `maybeGenerateChatTitle` (`go test -overlay`) and the tests
they fail:

| Mutant | Fails |
|---|---|
| no `return` after a refused write | `write refused by a rename during
the call` |
| no `return` after a failed model call | `model call failed`,
`TitleGenerationOverrideCallFailure`, `TitleGeneration/ModelFails` |
| writes `title + "!"` | `WritesGeneratedTitle`,
`TitleGenerationOverrideSetUsable`,
`TestMaybeGenerateChatTitleAppliesModelConfigReasoningEffort` |
| no `publishChatPubsubEvent` | `WritesGeneratedTitle`,
`TitleGeneration/Completed` and `/CreateFails` |

## Review focus

- The mock in the two no-write cases is stricter than the old owner
subscription: a publish on any channel fails.
- `TitleGeneration` checks the event because a recorded
`UpdateChatTitleByID` call stays true when the write fails.
`MemoryPubsub.Publish` waits for its subscribers, so `len(events)` is
final when the call returns.

## Validation

- `go test ./coderd/x/chatd/ -count=1 -race -run 'Title|Quickgen'`:
pass.
- Mutants: each fails as listed, and the first passes with only the
observer removed. A copy with a syntax error fails to build, so the
overlay applies.
- `TitleGeneration` fails when `generate` always returns true
(`ModelFails`) or when the title write fails (`Completed`,
`CreateFails`).
- The pre-commit hook (`make pre-commit`) passed.

> Xum, an AI agent, opened this PR for @ibetitsmike and wrote this
description.
2026-10-27 10:14:11 +01:00

206 lines
7.9 KiB
Go

package chatd
import (
"context"
"database/sql"
"encoding/json"
"testing"
"charm.land/fantasy"
"github.com/google/uuid"
"github.com/sqlc-dev/pqtype"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func TestQuickgenDebugRun(t *testing.T) {
t.Parallel()
const prompt = "fix the flaky heartbeat test"
chat := database.Chat{
ID: uuid.New(),
OwnerID: uuid.New(),
Title: "Fix the flaky heartbeat test",
TitleSource: database.ChatTitleSourceFallback,
}
hidden := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityModel, codersdk.ChatMessageText("model context"))
hidden.ID = 3
user := mustChatMessage(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText(prompt))
user.ID = 5
tip := hidden
tip.ID = 8
messages := []database.ChatMessage{hidden, user, tip}
generators := []struct {
name string
kind chatdebug.RunKind
trigger int64
historyTip int64
provider string
model string
label string
object map[string]any
generate func(ctx context.Context, t *testing.T, db *dbmock.MockStore, svc *chatdebug.Service, resolved resolvedModelCall) bool
}{
{
name: "TitleGeneration",
kind: chatdebug.KindTitleGeneration,
trigger: user.ID,
historyTip: tip.ID,
provider: string(database.AIProviderTypeOpenaiCompat),
model: "configured-model",
label: prompt,
object: map[string]any{"title": "Flaky heartbeat test"},
generate: func(ctx context.Context, t *testing.T, db *dbmock.MockStore, svc *chatdebug.Service, resolved resolvedModelCall) bool {
db.EXPECT().UpdateChatTitleByID(gomock.Any(), gomock.Any()).
Return(chatWithGeneratedTitle(chat, "Flaky heartbeat test"), nil).AnyTimes()
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
server := titleOverrideTestServer(db, logger)
events := subscribeChatWatchEvents(t, server.pubsub, chat.OwnerID)
server.maybeGenerateChatTitle(ctx, chat, messages, nil, resolved, logger, svc)
return len(events) == 1
},
},
{
name: "TurnStatusLabel",
kind: chatdebug.KindQuickgen,
trigger: 21,
historyTip: 34,
provider: "resolved-provider",
model: "resolved-model",
label: "Turn status label",
object: map[string]any{"label": "Fixed flaky test"},
generate: func(ctx context.Context, t *testing.T, _ *dbmock.MockStore, svc *chatdebug.Service, resolved resolvedModelCall) bool {
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
return generateTurnStatusLabel(ctx, chat, database.ChatStatusWaiting, "All tests pass.", resolved, codersdk.DefaultChatMaxGenerationRetries, logger, svc, 21, 34) != ""
},
},
}
outcomes := []struct {
name string
createErr error
modelErr error
wantStatus chatdebug.Status
}{
{name: "Completed", wantStatus: chatdebug.StatusCompleted},
{name: "ModelFails", modelErr: xerrors.New("model failed"), wantStatus: chatdebug.StatusError},
{name: "CreateFails", createErr: xerrors.New("insert failed")},
}
for _, gen := range generators {
for _, outcome := range outcomes {
t.Run(gen.name+"/"+outcome.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
db := dbmock.NewMockStore(gomock.NewController(t))
svc := chatdebug.NewService(db, testutil.Logger(t), nil, chatdebug.WithAlwaysEnable(true))
runID := uuid.New()
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.InsertChatDebugRunParams) (database.ChatDebugRun, error) {
params.StartedAt, params.UpdatedAt = sql.NullTime{}, sql.NullTime{}
require.Equal(t, database.InsertChatDebugRunParams{
ChatID: chat.ID,
TriggerMessageID: sql.NullInt64{Int64: gen.trigger, Valid: true},
HistoryTipMessageID: sql.NullInt64{Int64: gen.historyTip, Valid: true},
Kind: string(gen.kind),
Status: string(chatdebug.StatusInProgress),
Provider: sql.NullString{String: gen.provider, Valid: true},
Model: sql.NullString{String: gen.model, Valid: true},
Summary: pqtype.NullRawMessage{RawMessage: json.RawMessage(`{"first_message":"` + gen.label + `"}`), Valid: true},
}, params)
if outcome.createErr != nil {
return database.ChatDebugRun{}, outcome.createErr
}
return database.ChatDebugRun{
ID: runID,
ChatID: params.ChatID,
TriggerMessageID: params.TriggerMessageID,
HistoryTipMessageID: params.HistoryTipMessageID,
Kind: params.Kind,
Status: params.Status,
Provider: params.Provider,
Model: params.Model,
}, nil
}).Times(1)
if outcome.createErr == nil {
db.EXPECT().GetChatDebugStepsByRunID(gomock.Any(), runID).Return(nil, nil).Times(1)
db.EXPECT().UpdateChatDebugRun(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, params database.UpdateChatDebugRunParams) (database.ChatDebugRun, error) {
require.Equal(t, runID, params.ID)
require.Equal(t, chat.ID, params.ChatID)
require.Equal(t, sql.NullString{String: string(outcome.wantStatus), Valid: true}, params.Status)
var summary map[string]any
require.NoError(t, json.Unmarshal(params.Summary.RawMessage, &summary))
require.Equal(t, gen.label, summary["first_message"])
return database.ChatDebugRun{ID: runID, ChatID: chat.ID}, nil
}).Times(1)
}
var runs []*chatdebug.RunContext
model := &chattest.FakeModel{
GenerateObjectFn: func(ctx context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) {
if run, ok := chatdebug.RunFromContext(ctx); ok {
runs = append(runs, run)
}
if outcome.modelErr != nil {
return nil, outcome.modelErr
}
return &fantasy.ObjectResponse{Object: gen.object}, nil
},
}
resolved := resolvedModelCall{
model: chatprovider.NewModel(model, nil),
dbConfig: database.ChatModelConfig{ID: uuid.New(), Model: "configured-model"},
resolvedProvider: "resolved-provider",
resolvedModel: "resolved-model",
route: aiGatewayModelRoute{Provider: database.AIProvider{Type: database.AIProviderTypeOpenaiCompat}},
}
require.Equal(t, outcome.modelErr == nil, gen.generate(ctx, t, db, svc, resolved))
if outcome.createErr != nil {
require.Empty(t, runs)
return
}
require.NotEmpty(t, runs)
for _, run := range runs {
require.Equal(t, chatdebug.RunContext{
RunID: runID,
ChatID: chat.ID,
TriggerMessageID: gen.trigger,
HistoryTipMessageID: gen.historyTip,
Provider: gen.provider,
Model: gen.model,
}, *run)
}
})
}
}
}
func TestPrepareManualTitleDebugRunCreateFailure(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().InsertChatDebugRun(gomock.Any(), gomock.Any()).
Return(database.ChatDebugRun{}, xerrors.New("insert failed")).Times(1)
ctx, finish := prepareManualTitleDebugRun(testutil.Context(t, testutil.WaitLong),
chatdebug.NewService(db, testutil.Logger(t), nil), database.Chat{ID: uuid.New()}, resolvedModelCall{}, nil)
_, ok := chatdebug.RunFromContext(ctx)
require.False(t, ok)
// The mock expects no finalize calls.
finish(nil)
}