Files
coder/coderd/x/chatd/chatdebug/summary_internal_test.go
T
Michael Suchacz 5c9815b474 refactor(coderd/x/chatd/chatdebug): replace run-summary Postgres subtests with literal rows (#30972)
Moves the post-query reduction in
`chatdebug.(*Service).AggregateRunSummary` into a private
`summarizeRunSteps` method and tests it with table cases over literal
`database.ChatDebugStep` rows. This deletes the 11 Postgres-backed
subtests of `TestService_AggregateRunSummary` whose assertions the table
covers; each one built its own database.

**Why it is safe:**
- The moved body is unchanged apart from `return result`.
`AggregateRunSummary` still owns the nil run ID check, the
chatd-authorized read and the error return.
- Every assertion of the removed subtests maps to a table row that
compares the whole output map, so absent keys are checked too. Each row
also checks that the base map is not mutated.
- `NilRunID`, `RecomputesHasErrorAndCompletedEndpointLabel` and
`MultipleStepsSumTokens` stay on Postgres here; #30973 replaces them.
- I ran 42 mutants of `summary.go` with `go test -overlay`: `main`'s
tests kill 22, this PR's tests kill 31, including all 22. The new kills
cover JSON null and empty error payloads, error payloads on completed
steps, malformed usage and attempts, empty paths, first-match selection
and caller map mutation. Of the 11 survivors, 3 are query-boundary
mutants that `main` also misses (#30973 kills them) and 8 are equivalent
or only change logging.

**Review focus:** `summarizeRunSteps` in `summary.go`. The moved block
should match `main` apart from the return statement.

Validation:
- `go test -race -count=1 -p 4 -parallel 4
./coderd/x/chatd/chatdebug/...` passes.
- `golangci-lint run --fix=false ./coderd/x/chatd/chatdebug/...` reports
0 issues, and the pre-commit hook passed.

## Stack

This stack makes the debug run-summary reduction database-free and
replaces its Postgres fixtures with literal rows.

1. #30972 Replace run-summary Postgres subtests with literal rows
2. #30973 Check the run-summary query boundary

> Xum, an AI agent, opened this PR for @ibetitsmike and wrote this
description.
2026-10-21 10:13:45 +00:00

153 lines
5.5 KiB
Go

package chatdebug
import (
"encoding/json"
"maps"
"testing"
"github.com/google/uuid"
"github.com/sqlc-dev/pqtype"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/testutil"
)
func TestSummarizeRunSteps(t *testing.T) {
t.Parallel()
raw := func(s string) pqtype.NullRawMessage {
return pqtype.NullRawMessage{RawMessage: json.RawMessage(s), Valid: true}
}
withUsage := func(s string) database.ChatDebugStep {
return database.ChatDebugStep{Status: string(StatusCompleted), Usage: raw(s)}
}
withAttempts := func(s string) database.ChatDebugStep {
return database.ChatDebugStep{Status: string(StatusCompleted), Attempts: json.RawMessage(s)}
}
tests := []struct {
name string
base map[string]any
steps []database.ChatDebugStep
want map[string]any
}{
{
name: "NoStepsKeepsSeededValues",
base: map[string]any{"first_message": "hello world"},
want: map[string]any{
"first_message": "hello world",
"step_count": 0,
"total_input_tokens": int64(0),
"total_output_tokens": int64(0),
},
},
{
name: "NilBaseOmitsZeroOptionalTotals",
steps: []database.ChatDebugStep{withUsage(`{"input_tokens":10,"output_tokens":5,"reasoning_tokens":0,"cache_creation_tokens":0,"cache_read_tokens":0}`)},
want: map[string]any{
"step_count": 1,
"total_input_tokens": int64(10),
"total_output_tokens": int64(5),
},
},
{
name: "ClearsStaleDerivedKeys",
base: map[string]any{
"first_message": "hello world",
"step_count": 9,
"total_input_tokens": 999,
"total_output_tokens": 888,
"total_reasoning_tokens": 777,
"total_cache_creation_tokens": 100,
"total_cache_read_tokens": 200,
"has_error": true,
"endpoint_label": "POST /stale",
},
steps: []database.ChatDebugStep{withUsage(`{"input_tokens":20,"output_tokens":10}`)},
want: map[string]any{
"first_message": "hello world",
"step_count": 1,
"total_input_tokens": int64(20),
"total_output_tokens": int64(10),
},
},
{
name: "SumsUsageAcrossSteps",
steps: []database.ChatDebugStep{
withUsage(`{"input_tokens":10,"output_tokens":5,"reasoning_tokens":20,"cache_creation_tokens":2,"cache_read_tokens":3}`),
withUsage(`{"input_tokens":15,"output_tokens":7,"reasoning_tokens":30,"cache_creation_tokens":1,"cache_read_tokens":4}`),
},
want: map[string]any{
"step_count": 2,
"total_input_tokens": int64(25),
"total_output_tokens": int64(12),
"total_reasoning_tokens": int64(50),
"total_cache_creation_tokens": int64(3),
"total_cache_read_tokens": int64(7),
},
},
{
name: "AbsentAndMalformedUsageAddNothing",
steps: []database.ChatDebugStep{
withUsage(`{"input_tokens":10,"output_tokens":5}`),
{Status: string(StatusCompleted)},
{Status: string(StatusCompleted), Usage: pqtype.NullRawMessage{Valid: true}},
withUsage(`{"input_tokens":"not-a-number","output_tokens":7}`),
},
want: map[string]any{
"step_count": 4,
"total_input_tokens": int64(10),
"total_output_tokens": int64(5),
},
},
{
name: "ErrorStatusSetsHasError",
steps: []database.ChatDebugStep{{Status: string(StatusError)}},
want: map[string]any{"step_count": 1, "total_input_tokens": int64(0), "total_output_tokens": int64(0), "has_error": true},
},
{
name: "ErrorPayloadSetsHasError",
steps: []database.ChatDebugStep{{Status: string(StatusCompleted), Error: raw(`{"message":"boom"}`)}},
want: map[string]any{"step_count": 1, "total_input_tokens": int64(0), "total_output_tokens": int64(0), "has_error": true},
},
{
name: "InterruptedAndClearedErrorsIgnored",
steps: []database.ChatDebugStep{
{Status: string(StatusInterrupted), Error: raw(`{"message":"user canceled"}`)},
{Status: string(StatusCompleted), Error: raw(`null`)},
{Status: string(StatusCompleted), Error: pqtype.NullRawMessage{Valid: true}},
},
want: map[string]any{"step_count": 3, "total_input_tokens": int64(0), "total_output_tokens": int64(0)},
},
{
name: "EndpointFromFirstCompletedAttemptWithPath",
steps: []database.ChatDebugStep{
{Status: string(StatusCompleted)},
withAttempts(`{"status":"completed"`),
withAttempts(`[{"status":"failed","method":"POST","path":"/failed"},{"status":"completed","method":"POST"},{"status":"completed","method":"POST","path":"/v1/messages"},{"status":"completed","method":"GET","path":"/later"}]`),
withAttempts(`[{"status":"completed","method":"POST","path":"/next-step"}]`),
},
want: map[string]any{"step_count": 4, "total_input_tokens": int64(0), "total_output_tokens": int64(0), "endpoint_label": "POST /v1/messages"},
},
{
name: "EndpointPathOnlyWithoutMethod",
steps: []database.ChatDebugStep{withAttempts(`[{"status":"completed","path":"/v1/messages"}]`)},
want: map[string]any{"step_count": 1, "total_input_tokens": int64(0), "total_output_tokens": int64(0), "endpoint_label": "/v1/messages"},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
svc := &Service{log: testutil.Logger(t)}
base := maps.Clone(tc.base)
got := svc.summarizeRunSteps(ctx, uuid.New(), tc.steps, tc.base)
require.Equal(t, tc.want, got)
require.Equal(t, base, tc.base, "base summary must not be mutated")
})
}
}