Files
coder/coderd/x/chatd/chatdebug/bench_internal_test.go
Michael Suchacz eea839ccc4 refactor(coderd): number chat debug steps in the database (#31007)
`InsertChatDebugStep` now numbers a step after the run's highest step
when the caller passes zero, so chatdebug stops numbering steps in
process memory. `CreateStepParams` loses `StepNumber`, `beginStep` no
longer calls `nextStepNumber` or `syncStepCounter`, and `CreateStep`
replaces its collision retry, which listed the run's steps and bumped
the number, with a per-run lock. #31008 deletes the counters, which no
production code reads after this PR.

**Why it is safe:** Postgres takes the insert's snapshot before the
statement waits on the run row, so creates that waited on the same run
read the same highest step. `CreateStep` therefore takes
`pg_advisory_xact_lock(LockIDChatDebugRunSteps(run))` in its own
statement and inserts in the same transaction, so each insert starts
after the previous holder committed. The new
`TestService_CreateStep_NumbersConcurrentSteps` holds the run row,
starts 20 creates, waits until all of them wait on a lock, then commits
and checks the numbers 1 to 20. It passes 10 of 10 runs, and fails 10 of
10 with a 10-attempt collision retry in place of the lock.
`TestService_CreateStep_RetriesDuplicateStepNumbers` becomes
`TestService_CreateStep_NumbersStepsPerRun`, and
`TestService_CreateStep_ListRetryErrorWins` goes with the list call it
covered. Mutants that fail the chatdebug tests: no lock, the lock
outside the transaction, the lock keyed by chat, a constant number,
storing the zero, numbering across runs, a nonzero number from
`CreateStep`, and an unclassified missing run. One mutant survives and
is equivalent: running the insert on `s.db` instead of `tx` inside the
same `InTx` callback, because the insert still commits before the
transaction releases the lock. Running the insert after the transaction
commits fails the new test with a duplicate key. A copy of `service.go`
with garbage in it fails to build.

**Review focus:** `@step_number` stays as an override. Production passes
zero, and the fixtures in `querier_test.go`, `dbauthz_test.go`,
`dbpurge_test.go` and `exp_chats_test.go` keep their explicit numbers.
No test depends on the override (a mutant that ignores it survives),
because every fixture numbers its steps in insert order. Dropping the
parameter would touch about 20 fixtures outside chatdebug.

The PR removes more than it adds: +179/-233 hand-written lines. The
generated `querier.go` and `queries.sql.go` change 15 more.

Validation:
- `make gen/db` changes only `querier.go` and `queries.sql.go`.
- `go test -race -count=1 -p 4 -parallel 4
./coderd/x/chatd/chatdebug/...` passes, and `go test -count=10 -run
TestService_CreateStep ./coderd/x/chatd/chatdebug/` passes all 6 tests
10 of 10 times.
- `go test -count=1 -run ChatDebug ./coderd/database/
./coderd/database/dbpurge/` and `go test -count=1 -run Debug
./coderd/x/chatd/` (19 tests) pass, and the 25 chatd debug-run tests
pass under `-race`.
- `golangci-lint run --fix=false ./coderd/x/chatd/chatdebug/...
./coderd/database/` reports 0 issues, and the pre-commit hook passed.

## Stack

This stack deletes the stale debug-run sweep, then simplifies the
chatdebug debug-run, stream wrapper and step numbering code, drops the
indexes the sweep used, and last deletes a runner debug field that lost
its only reader with the heartbeats. The debug-run and stream wrapper
refactors each sit on a test-only layer that pins the behavior they
keep.

1. #30988 Delete the chat debug run and step heartbeats
2. #30989 Delete `FinalizeStale` and the stale threshold
3. #30995 Delete the `FinalizeStaleChatDebugRows` query
4. #30933 Pin the debug runs that turns, titles and status labels record
5. #30934 Create and finalize debug runs through one helper
6. #30980 Table the stream wrapper tests and cover their status rules
7. #30981 Share one finalize-once lifecycle between the stream wrappers
8. This PR: Number chat debug steps in the database
9. #31008 Delete the process-global step counters
10. #31009 Drop the chat debug stale-sweep indexes
11. #31010 Drop the unused `runnerCtx` from the runner debug turn

> Xum, an AI agent, opened this PR for @ibetitsmike and wrote this
description.
2026-10-21 08:23:28 +00:00

346 lines
9.5 KiB
Go

package chatdebug
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"slices"
"strings"
"testing"
"charm.land/fantasy"
"github.com/google/uuid"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/pubsub"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
)
// noopStore implements database.Store by embedding it (nil) and
// overriding only the methods chatdebug's hot paths call. Every
// override returns immediately with a canned value and does no I/O,
// so any time spent inside Service methods that use this store is
// chatdebug's own CPU/allocation cost, not Postgres's.
type noopStore struct {
database.Store
}
func (noopStore) GetChatDebugLoggingAllowUsers(context.Context) (bool, error) {
return true, nil
}
func (noopStore) GetUserChatDebugLoggingEnabled(context.Context, uuid.UUID) (bool, error) {
return true, nil
}
func (noopStore) InsertChatDebugStep(
_ context.Context,
arg database.InsertChatDebugStepParams,
) (database.ChatDebugStep, error) {
return database.ChatDebugStep{
ID: uuid.New(),
RunID: arg.RunID,
ChatID: arg.ChatID,
StepNumber: arg.StepNumber,
Operation: arg.Operation,
Status: arg.Status,
}, nil
}
func (noopStore) UpdateChatDebugStep(
_ context.Context,
arg database.UpdateChatDebugStepParams,
) (database.ChatDebugStep, error) {
return database.ChatDebugStep{
ID: arg.ID,
ChatID: arg.ChatID,
}, nil
}
func (s noopStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error {
return fn(s)
}
func (noopStore) AcquireLock(context.Context, int64) error {
return nil
}
var _ database.Store = noopStore{}
// benchService builds a Service backed by noopStore and an in-memory
// pubsub so publishEvent still marshals and dispatches DebugEvent
// payloads, mirroring a real deployment's wiring, without any network
// or disk I/O.
func benchService(b *testing.B) *Service {
b.Helper()
ps := pubsub.NewInMemory()
b.Cleanup(func() { _ = ps.Close() })
return NewService(noopStore{}, slog.Make(), ps)
}
// benchCall builds a realistic multi-turn prompt with tool
// definitions so normalizeCall walks a representative message/tool
// shape.
func benchCall(nMessages, nTools int) fantasy.Call {
prompt := make(fantasy.Prompt, 0, nMessages)
for i := range nMessages {
if i%2 == 0 {
prompt = append(prompt, fantasy.NewUserMessage(
fmt.Sprintf("Please investigate issue #%d and summarize the relevant log lines around the failure.", i)))
continue
}
prompt = append(prompt, fantasy.Message{
Role: fantasy.MessageRoleAssistant,
Content: []fantasy.MessagePart{
fantasy.TextPart{Text: strings.Repeat("analysis details ", 40)},
},
})
}
tools := make([]fantasy.Tool, 0, nTools)
for i := range nTools {
tools = append(tools, fantasy.FunctionTool{
Name: fmt.Sprintf("tool_%d", i),
Description: "Runs a workspace command and returns its output",
InputSchema: map[string]any{
"type": "object",
"properties": map[string]any{
"command": map[string]any{"type": "string"},
"timeout": map[string]any{"type": "integer"},
},
"required": []string{"command"},
},
})
}
return fantasy.Call{
Prompt: prompt,
Tools: tools,
}
}
func benchStreamParts(nDeltas int) []fantasy.StreamPart {
parts := make([]fantasy.StreamPart, 0, nDeltas+2)
for i := range nDeltas {
parts = append(parts, fantasy.StreamPart{
Type: fantasy.StreamPartTypeTextDelta,
ID: "text-1",
Delta: fmt.Sprintf("token-%d ", i),
})
}
parts = append(parts,
fantasy.StreamPart{Type: fantasy.StreamPartTypeToolCall, ID: "tool-1", ToolCallName: "tool_0"},
fantasy.StreamPart{
Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop,
Usage: fantasy.Usage{InputTokens: 512, OutputTokens: int64(nDeltas), TotalTokens: 512 + int64(nDeltas)},
},
)
return parts
}
// runContext returns a fresh RunContext/context pair for one
// benchmark iteration, mirroring a new chat turn.
func runContext(chatID uuid.UUID) context.Context {
return ContextWithRun(context.Background(), &RunContext{RunID: uuid.New(), ChatID: chatID})
}
func BenchmarkWrapModel_Generate(b *testing.B) {
svc := benchService(b)
chatID, ownerID := uuid.New(), uuid.New()
call := benchCall(6, 4)
resp := &fantasy.Response{
Content: fantasy.ResponseContent{
fantasy.TextContent{Text: strings.Repeat("response text ", 100)},
fantasy.ToolCallContent{ToolCallID: "tool-1", ToolName: "tool_0", Input: `{"command":"ls"}`},
},
FinishReason: fantasy.FinishReasonStop,
Usage: fantasy.Usage{InputTokens: 512, OutputTokens: 128, TotalTokens: 640},
}
inner := &chattest.FakeModel{GenerateFn: func(context.Context, fantasy.Call) (*fantasy.Response, error) {
return resp, nil
}}
model := WrapModel(inner, svc, RecorderOptions{ChatID: chatID, OwnerID: ownerID})
b.ReportAllocs()
for b.Loop() {
ctx := runContext(chatID)
if _, err := model.Generate(ctx, call); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkWrapModel_Stream(b *testing.B) {
svc := benchService(b)
chatID, ownerID := uuid.New(), uuid.New()
call := benchCall(6, 4)
parts := benchStreamParts(200)
inner := &chattest.FakeModel{StreamFn: func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) {
return slices.Values(parts), nil
}}
model := WrapModel(inner, svc, RecorderOptions{ChatID: chatID, OwnerID: ownerID})
b.ReportAllocs()
for b.Loop() {
ctx := runContext(chatID)
seq, err := model.Stream(ctx, call)
if err != nil {
b.Fatal(err)
}
for range seq { //nolint:revive // draining the stream is the point of the benchmark.
}
}
}
// cannedRoundTripper replays a fixed response body for every request,
// isolating RecordingTransport's own redaction/buffering cost from any
// real network or provider latency.
type cannedRoundTripper struct {
status int
header http.Header
body []byte
chunkSize int // 0 means return the whole body in one Read.
}
func (rt *cannedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
var body io.ReadCloser
if rt.chunkSize > 0 {
body = io.NopCloser(&chunkedReader{data: rt.body, chunkSize: rt.chunkSize})
} else {
body = io.NopCloser(bytes.NewReader(rt.body))
}
header := rt.header.Clone()
return &http.Response{
StatusCode: rt.status,
Header: header,
Body: body,
ContentLength: -1,
Request: req,
}, nil
}
// chunkedReader splits data into fixed-size Read() calls so the
// benchmark exercises recordingBody's incremental accumulation path
// the way a real SSE stream would, instead of handing back the whole
// body in a single Read.
type chunkedReader struct {
data []byte
chunkSize int
}
func (c *chunkedReader) Read(p []byte) (int, error) {
if len(c.data) == 0 {
return 0, io.EOF
}
n := min(c.chunkSize, len(c.data), len(p))
copy(p, c.data[:n])
c.data = c.data[n:]
return n, nil
}
func benchJSONResponseBody(n int) []byte {
type choice struct {
Index int `json:"index"`
Text string `json:"text"`
}
payload := struct {
ID string `json:"id"`
Object string `json:"object"`
Model string `json:"model"`
Choices []choice `json:"choices"`
Usage struct {
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
} `json:"usage"`
}{
ID: "resp-bench",
Object: "chat.completion",
Model: "bench-model",
}
for i := range n {
payload.Choices = append(payload.Choices, choice{
Index: i,
Text: strings.Repeat("word ", 50),
})
}
data, err := json.Marshal(payload)
if err != nil {
panic(err)
}
return data
}
func benchSSEResponseBody(nEvents int) []byte {
var buf bytes.Buffer
for i := range nEvents {
// bytes.Buffer.Write* never returns an error.
_, _ = fmt.Fprintf(&buf, "data: {\"delta\":\"token-%d \",\"index\":%d}\n\n", i, i)
}
_, _ = buf.WriteString("data: [DONE]\n\n")
return buf.Bytes()
}
func newBenchRequest(b *testing.B, sink *attemptSink) *http.Request {
b.Helper()
req, err := http.NewRequestWithContext(
withAttemptSink(context.Background(), sink),
http.MethodPost,
"https://api.example.com/v1/chat/completions",
bytes.NewReader([]byte(`{"model":"bench-model","messages":[{"role":"user","content":"hi"}]}`)),
)
if err != nil {
b.Fatal(err)
}
req.Header.Set("Authorization", "Bearer sk-should-be-redacted")
req.Header.Set("Content-Type", "application/json")
return req
}
func BenchmarkRecordingTransport_RoundTrip_JSON(b *testing.B) {
header := http.Header{"Content-Type": []string{"application/json"}}
body := benchJSONResponseBody(20)
transport := &RecordingTransport{Base: &cannedRoundTripper{status: 200, header: header, body: body}}
b.ReportAllocs()
for b.Loop() {
sink := &attemptSink{}
req := newBenchRequest(b, sink)
resp, err := transport.RoundTrip(req)
if err != nil {
b.Fatal(err)
}
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
b.Fatal(err)
}
if err := resp.Body.Close(); err != nil {
b.Fatal(err)
}
}
}
func BenchmarkRecordingTransport_RoundTrip_SSE(b *testing.B) {
header := http.Header{"Content-Type": []string{"text/event-stream"}}
body := benchSSEResponseBody(200)
transport := &RecordingTransport{Base: &cannedRoundTripper{status: 200, header: header, body: body, chunkSize: 256}}
b.ReportAllocs()
for b.Loop() {
sink := &attemptSink{}
req := newBenchRequest(b, sink)
resp, err := transport.RoundTrip(req)
if err != nil {
b.Fatal(err)
}
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
b.Fatal(err)
}
if err := resp.Body.Close(); err != nil {
b.Fatal(err)
}
}
}