`InsertChatDebugStep` now numbers a step after the run's highest step when the caller passes zero, so chatdebug stops numbering steps in process memory. `CreateStepParams` loses `StepNumber`, `beginStep` no longer calls `nextStepNumber` or `syncStepCounter`, and `CreateStep` replaces its collision retry, which listed the run's steps and bumped the number, with a per-run lock. #31008 deletes the counters, which no production code reads after this PR. **Why it is safe:** Postgres takes the insert's snapshot before the statement waits on the run row, so creates that waited on the same run read the same highest step. `CreateStep` therefore takes `pg_advisory_xact_lock(LockIDChatDebugRunSteps(run))` in its own statement and inserts in the same transaction, so each insert starts after the previous holder committed. The new `TestService_CreateStep_NumbersConcurrentSteps` holds the run row, starts 20 creates, waits until all of them wait on a lock, then commits and checks the numbers 1 to 20. It passes 10 of 10 runs, and fails 10 of 10 with a 10-attempt collision retry in place of the lock. `TestService_CreateStep_RetriesDuplicateStepNumbers` becomes `TestService_CreateStep_NumbersStepsPerRun`, and `TestService_CreateStep_ListRetryErrorWins` goes with the list call it covered. Mutants that fail the chatdebug tests: no lock, the lock outside the transaction, the lock keyed by chat, a constant number, storing the zero, numbering across runs, a nonzero number from `CreateStep`, and an unclassified missing run. One mutant survives and is equivalent: running the insert on `s.db` instead of `tx` inside the same `InTx` callback, because the insert still commits before the transaction releases the lock. Running the insert after the transaction commits fails the new test with a duplicate key. A copy of `service.go` with garbage in it fails to build. **Review focus:** `@step_number` stays as an override. Production passes zero, and the fixtures in `querier_test.go`, `dbauthz_test.go`, `dbpurge_test.go` and `exp_chats_test.go` keep their explicit numbers. No test depends on the override (a mutant that ignores it survives), because every fixture numbers its steps in insert order. Dropping the parameter would touch about 20 fixtures outside chatdebug. The PR removes more than it adds: +179/-233 hand-written lines. The generated `querier.go` and `queries.sql.go` change 15 more. Validation: - `make gen/db` changes only `querier.go` and `queries.sql.go`. - `go test -race -count=1 -p 4 -parallel 4 ./coderd/x/chatd/chatdebug/...` passes, and `go test -count=10 -run TestService_CreateStep ./coderd/x/chatd/chatdebug/` passes all 6 tests 10 of 10 times. - `go test -count=1 -run ChatDebug ./coderd/database/ ./coderd/database/dbpurge/` and `go test -count=1 -run Debug ./coderd/x/chatd/` (19 tests) pass, and the 25 chatd debug-run tests pass under `-race`. - `golangci-lint run --fix=false ./coderd/x/chatd/chatdebug/... ./coderd/database/` reports 0 issues, and the pre-commit hook passed. ## Stack This stack deletes the stale debug-run sweep, then simplifies the chatdebug debug-run, stream wrapper and step numbering code, drops the indexes the sweep used, and last deletes a runner debug field that lost its only reader with the heartbeats. The debug-run and stream wrapper refactors each sit on a test-only layer that pins the behavior they keep. 1. #30988 Delete the chat debug run and step heartbeats 2. #30989 Delete `FinalizeStale` and the stale threshold 3. #30995 Delete the `FinalizeStaleChatDebugRows` query 4. #30933 Pin the debug runs that turns, titles and status labels record 5. #30934 Create and finalize debug runs through one helper 6. #30980 Table the stream wrapper tests and cover their status rules 7. #30981 Share one finalize-once lifecycle between the stream wrappers 8. This PR: Number chat debug steps in the database 9. #31008 Delete the process-global step counters 10. #31009 Drop the chat debug stale-sweep indexes 11. #31010 Drop the unused `runnerCtx` from the runner debug turn > Xum, an AI agent, opened this PR for @ibetitsmike and wrote this description.
346 lines
9.5 KiB
Go
346 lines
9.5 KiB
Go
package chatdebug
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"testing"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/google/uuid"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/pubsub"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
)
|
|
|
|
// noopStore implements database.Store by embedding it (nil) and
|
|
// overriding only the methods chatdebug's hot paths call. Every
|
|
// override returns immediately with a canned value and does no I/O,
|
|
// so any time spent inside Service methods that use this store is
|
|
// chatdebug's own CPU/allocation cost, not Postgres's.
|
|
type noopStore struct {
|
|
database.Store
|
|
}
|
|
|
|
func (noopStore) GetChatDebugLoggingAllowUsers(context.Context) (bool, error) {
|
|
return true, nil
|
|
}
|
|
|
|
func (noopStore) GetUserChatDebugLoggingEnabled(context.Context, uuid.UUID) (bool, error) {
|
|
return true, nil
|
|
}
|
|
|
|
func (noopStore) InsertChatDebugStep(
|
|
_ context.Context,
|
|
arg database.InsertChatDebugStepParams,
|
|
) (database.ChatDebugStep, error) {
|
|
return database.ChatDebugStep{
|
|
ID: uuid.New(),
|
|
RunID: arg.RunID,
|
|
ChatID: arg.ChatID,
|
|
StepNumber: arg.StepNumber,
|
|
Operation: arg.Operation,
|
|
Status: arg.Status,
|
|
}, nil
|
|
}
|
|
|
|
func (noopStore) UpdateChatDebugStep(
|
|
_ context.Context,
|
|
arg database.UpdateChatDebugStepParams,
|
|
) (database.ChatDebugStep, error) {
|
|
return database.ChatDebugStep{
|
|
ID: arg.ID,
|
|
ChatID: arg.ChatID,
|
|
}, nil
|
|
}
|
|
|
|
func (s noopStore) InTx(fn func(database.Store) error, _ *database.TxOptions) error {
|
|
return fn(s)
|
|
}
|
|
|
|
func (noopStore) AcquireLock(context.Context, int64) error {
|
|
return nil
|
|
}
|
|
|
|
var _ database.Store = noopStore{}
|
|
|
|
// benchService builds a Service backed by noopStore and an in-memory
|
|
// pubsub so publishEvent still marshals and dispatches DebugEvent
|
|
// payloads, mirroring a real deployment's wiring, without any network
|
|
// or disk I/O.
|
|
func benchService(b *testing.B) *Service {
|
|
b.Helper()
|
|
ps := pubsub.NewInMemory()
|
|
b.Cleanup(func() { _ = ps.Close() })
|
|
return NewService(noopStore{}, slog.Make(), ps)
|
|
}
|
|
|
|
// benchCall builds a realistic multi-turn prompt with tool
|
|
// definitions so normalizeCall walks a representative message/tool
|
|
// shape.
|
|
func benchCall(nMessages, nTools int) fantasy.Call {
|
|
prompt := make(fantasy.Prompt, 0, nMessages)
|
|
for i := range nMessages {
|
|
if i%2 == 0 {
|
|
prompt = append(prompt, fantasy.NewUserMessage(
|
|
fmt.Sprintf("Please investigate issue #%d and summarize the relevant log lines around the failure.", i)))
|
|
continue
|
|
}
|
|
prompt = append(prompt, fantasy.Message{
|
|
Role: fantasy.MessageRoleAssistant,
|
|
Content: []fantasy.MessagePart{
|
|
fantasy.TextPart{Text: strings.Repeat("analysis details ", 40)},
|
|
},
|
|
})
|
|
}
|
|
|
|
tools := make([]fantasy.Tool, 0, nTools)
|
|
for i := range nTools {
|
|
tools = append(tools, fantasy.FunctionTool{
|
|
Name: fmt.Sprintf("tool_%d", i),
|
|
Description: "Runs a workspace command and returns its output",
|
|
InputSchema: map[string]any{
|
|
"type": "object",
|
|
"properties": map[string]any{
|
|
"command": map[string]any{"type": "string"},
|
|
"timeout": map[string]any{"type": "integer"},
|
|
},
|
|
"required": []string{"command"},
|
|
},
|
|
})
|
|
}
|
|
|
|
return fantasy.Call{
|
|
Prompt: prompt,
|
|
Tools: tools,
|
|
}
|
|
}
|
|
|
|
func benchStreamParts(nDeltas int) []fantasy.StreamPart {
|
|
parts := make([]fantasy.StreamPart, 0, nDeltas+2)
|
|
for i := range nDeltas {
|
|
parts = append(parts, fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeTextDelta,
|
|
ID: "text-1",
|
|
Delta: fmt.Sprintf("token-%d ", i),
|
|
})
|
|
}
|
|
parts = append(parts,
|
|
fantasy.StreamPart{Type: fantasy.StreamPartTypeToolCall, ID: "tool-1", ToolCallName: "tool_0"},
|
|
fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop,
|
|
Usage: fantasy.Usage{InputTokens: 512, OutputTokens: int64(nDeltas), TotalTokens: 512 + int64(nDeltas)},
|
|
},
|
|
)
|
|
return parts
|
|
}
|
|
|
|
// runContext returns a fresh RunContext/context pair for one
|
|
// benchmark iteration, mirroring a new chat turn.
|
|
func runContext(chatID uuid.UUID) context.Context {
|
|
return ContextWithRun(context.Background(), &RunContext{RunID: uuid.New(), ChatID: chatID})
|
|
}
|
|
|
|
func BenchmarkWrapModel_Generate(b *testing.B) {
|
|
svc := benchService(b)
|
|
chatID, ownerID := uuid.New(), uuid.New()
|
|
call := benchCall(6, 4)
|
|
resp := &fantasy.Response{
|
|
Content: fantasy.ResponseContent{
|
|
fantasy.TextContent{Text: strings.Repeat("response text ", 100)},
|
|
fantasy.ToolCallContent{ToolCallID: "tool-1", ToolName: "tool_0", Input: `{"command":"ls"}`},
|
|
},
|
|
FinishReason: fantasy.FinishReasonStop,
|
|
Usage: fantasy.Usage{InputTokens: 512, OutputTokens: 128, TotalTokens: 640},
|
|
}
|
|
inner := &chattest.FakeModel{GenerateFn: func(context.Context, fantasy.Call) (*fantasy.Response, error) {
|
|
return resp, nil
|
|
}}
|
|
model := WrapModel(inner, svc, RecorderOptions{ChatID: chatID, OwnerID: ownerID})
|
|
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
ctx := runContext(chatID)
|
|
if _, err := model.Generate(ctx, call); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkWrapModel_Stream(b *testing.B) {
|
|
svc := benchService(b)
|
|
chatID, ownerID := uuid.New(), uuid.New()
|
|
call := benchCall(6, 4)
|
|
parts := benchStreamParts(200)
|
|
inner := &chattest.FakeModel{StreamFn: func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return slices.Values(parts), nil
|
|
}}
|
|
model := WrapModel(inner, svc, RecorderOptions{ChatID: chatID, OwnerID: ownerID})
|
|
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
ctx := runContext(chatID)
|
|
seq, err := model.Stream(ctx, call)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
for range seq { //nolint:revive // draining the stream is the point of the benchmark.
|
|
}
|
|
}
|
|
}
|
|
|
|
// cannedRoundTripper replays a fixed response body for every request,
|
|
// isolating RecordingTransport's own redaction/buffering cost from any
|
|
// real network or provider latency.
|
|
type cannedRoundTripper struct {
|
|
status int
|
|
header http.Header
|
|
body []byte
|
|
chunkSize int // 0 means return the whole body in one Read.
|
|
}
|
|
|
|
func (rt *cannedRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
var body io.ReadCloser
|
|
if rt.chunkSize > 0 {
|
|
body = io.NopCloser(&chunkedReader{data: rt.body, chunkSize: rt.chunkSize})
|
|
} else {
|
|
body = io.NopCloser(bytes.NewReader(rt.body))
|
|
}
|
|
header := rt.header.Clone()
|
|
return &http.Response{
|
|
StatusCode: rt.status,
|
|
Header: header,
|
|
Body: body,
|
|
ContentLength: -1,
|
|
Request: req,
|
|
}, nil
|
|
}
|
|
|
|
// chunkedReader splits data into fixed-size Read() calls so the
|
|
// benchmark exercises recordingBody's incremental accumulation path
|
|
// the way a real SSE stream would, instead of handing back the whole
|
|
// body in a single Read.
|
|
type chunkedReader struct {
|
|
data []byte
|
|
chunkSize int
|
|
}
|
|
|
|
func (c *chunkedReader) Read(p []byte) (int, error) {
|
|
if len(c.data) == 0 {
|
|
return 0, io.EOF
|
|
}
|
|
n := min(c.chunkSize, len(c.data), len(p))
|
|
copy(p, c.data[:n])
|
|
c.data = c.data[n:]
|
|
return n, nil
|
|
}
|
|
|
|
func benchJSONResponseBody(n int) []byte {
|
|
type choice struct {
|
|
Index int `json:"index"`
|
|
Text string `json:"text"`
|
|
}
|
|
payload := struct {
|
|
ID string `json:"id"`
|
|
Object string `json:"object"`
|
|
Model string `json:"model"`
|
|
Choices []choice `json:"choices"`
|
|
Usage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
} `json:"usage"`
|
|
}{
|
|
ID: "resp-bench",
|
|
Object: "chat.completion",
|
|
Model: "bench-model",
|
|
}
|
|
for i := range n {
|
|
payload.Choices = append(payload.Choices, choice{
|
|
Index: i,
|
|
Text: strings.Repeat("word ", 50),
|
|
})
|
|
}
|
|
data, err := json.Marshal(payload)
|
|
if err != nil {
|
|
panic(err)
|
|
}
|
|
return data
|
|
}
|
|
|
|
func benchSSEResponseBody(nEvents int) []byte {
|
|
var buf bytes.Buffer
|
|
for i := range nEvents {
|
|
// bytes.Buffer.Write* never returns an error.
|
|
_, _ = fmt.Fprintf(&buf, "data: {\"delta\":\"token-%d \",\"index\":%d}\n\n", i, i)
|
|
}
|
|
_, _ = buf.WriteString("data: [DONE]\n\n")
|
|
return buf.Bytes()
|
|
}
|
|
|
|
func newBenchRequest(b *testing.B, sink *attemptSink) *http.Request {
|
|
b.Helper()
|
|
req, err := http.NewRequestWithContext(
|
|
withAttemptSink(context.Background(), sink),
|
|
http.MethodPost,
|
|
"https://api.example.com/v1/chat/completions",
|
|
bytes.NewReader([]byte(`{"model":"bench-model","messages":[{"role":"user","content":"hi"}]}`)),
|
|
)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
req.Header.Set("Authorization", "Bearer sk-should-be-redacted")
|
|
req.Header.Set("Content-Type", "application/json")
|
|
return req
|
|
}
|
|
|
|
func BenchmarkRecordingTransport_RoundTrip_JSON(b *testing.B) {
|
|
header := http.Header{"Content-Type": []string{"application/json"}}
|
|
body := benchJSONResponseBody(20)
|
|
transport := &RecordingTransport{Base: &cannedRoundTripper{status: 200, header: header, body: body}}
|
|
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
sink := &attemptSink{}
|
|
req := newBenchRequest(b, sink)
|
|
resp, err := transport.RoundTrip(req)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
if err := resp.Body.Close(); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkRecordingTransport_RoundTrip_SSE(b *testing.B) {
|
|
header := http.Header{"Content-Type": []string{"text/event-stream"}}
|
|
body := benchSSEResponseBody(200)
|
|
transport := &RecordingTransport{Base: &cannedRoundTripper{status: 200, header: header, body: body, chunkSize: 256}}
|
|
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
sink := &attemptSink{}
|
|
req := newBenchRequest(b, sink)
|
|
resp, err := transport.RoundTrip(req)
|
|
if err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
if _, err := io.Copy(io.Discard, resp.Body); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
if err := resp.Body.Close(); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
}
|