Files
coder/coderd/x/chatd/chatloop/runtime_test.go
T
Michael Suchacz e35cbe2a7f fix(coderd/x/chatd): stream compaction summaries (#28886)
Chats could get permanently stuck once their context crossed the
compaction threshold on adaptive-thinking Claude models: every
compaction attempt failed with `compaction produced an empty summary`,
and both automatic and manual compaction take the same path, so the chat
could never recover.

## Problem

`compactionSummaryCall` deliberately set `MaxOutputTokens = nil` because
the summary request was non-streaming and the Anthropic SDK rejects
non-streaming requests whose `max_tokens` implies a completion longer
than ten minutes. But a nil cap makes fantasy's Anthropic provider
silently default to 4096 output tokens, and adaptive-thinking Claude
models can spend that entire budget on a thinking block, leaving zero
summary text.

## Fix

- Stream the compaction summary request. Streaming lifts the Anthropic
SDK's non-streaming duration restriction, which was the only reason the
cap was nil'd.
- Keep the resolved output cap (32k default) on the summary call instead
of nil'ing it.
- When the summary comes back empty with truncation evidence (length
finish reason, or reasoning with no text), fail with `compaction summary
was truncated at the output token cap` instead of the generic
empty-summary error, so the next occurrence is attributable at a glance.

Unit tests cover text-block joining and the truncation classification;
the compaction e2e fixtures now serve streamed summaries and assert the
summary request streams with `max_tokens: 32000`. Dogfood UAT on a
deployment built from this branch verified manual compaction with Claude
Opus 5 (adaptive thinking, twice, with reload persistence) and an OpenAI
regression chat.

> 🤖 Opened by [Xum](https://mux.coder.com) on behalf of @ibetitsmike.
2026-09-21 21:28:45 +02:00

642 lines
20 KiB
Go

package chatloop_test
import (
"context"
"testing"
"time"
"charm.land/fantasy"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func TestGenerateAssistant_RecordsModelInvocationRuntime(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
model := &chattest.FakeModel{
ProviderName: "test-provider",
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return func(yield func(fantasy.StreamPart) bool) {
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
return
}
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "t", Delta: "hello"}) {
return
}
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "t"}) {
return
}
clock.Advance(1500 * time.Millisecond)
yield(fantasy.StreamPart{
Type: fantasy.StreamPartTypeFinish,
FinishReason: fantasy.FinishReasonStop,
})
}, nil
},
}
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
Model: model,
Clock: clock,
})
require.NoError(t, err)
require.Equal(t, 1500*time.Millisecond, outcome.Step.Runtime)
}
// The interrupt path bills the window OnModelStreamStart opens, so that
// hook must fire at the instant PersistedStep.Runtime starts measuring.
func TestGenerateAssistant_ModelStreamStartMatchesRuntimeWindow(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
model := &chattest.FakeModel{
ProviderName: "test-provider",
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return func(yield func(fantasy.StreamPart) bool) {
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
return
}
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "t", Delta: "hello"}) {
return
}
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "t"}) {
return
}
clock.Advance(1500 * time.Millisecond)
yield(fantasy.StreamPart{
Type: fantasy.StreamPartTypeFinish,
FinishReason: fantasy.FinishReasonStop,
})
}, nil
},
}
var startedAt []time.Time
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
Model: model,
Clock: clock,
OnModelStreamStart: func() {
startedAt = append(startedAt, clock.Now())
},
})
require.NoError(t, err)
require.Len(t, startedAt, 1)
require.Equal(t, outcome.Step.Runtime, clock.Since(startedAt[0]))
}
func TestGenerateAssistant_ErroredStreamReturnsNoStep(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
model := &chattest.FakeModel{
ProviderName: "test-provider",
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return func(yield func(fantasy.StreamPart) bool) {
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
return
}
clock.Advance(1500 * time.Millisecond)
yield(fantasy.StreamPart{
Type: fantasy.StreamPartTypeError,
Error: xerrors.New("stream blew up"),
})
}, nil
},
}
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
Model: model,
Clock: clock,
})
require.Error(t, err)
require.Zero(t, outcome.Step.Runtime)
require.Empty(t, outcome.Step.Content)
}
func TestGenerateCompaction_RecordsRuntime(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
model := &chattest.FakeModel{
ProviderName: "test-provider",
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
clock.Advance(1500 * time.Millisecond)
return func(yield func(fantasy.StreamPart) bool) {
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "text"})
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "text", Delta: "summary"})
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "text"})
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
}, nil
},
}
var startedAt []time.Time
result, err := chatloop.GenerateCompaction(context.Background(), chatloop.GenerateCompactionOptions{
Model: model,
Messages: []fantasy.Message{{
Role: fantasy.MessageRoleUser,
Content: []fantasy.MessagePart{fantasy.TextPart{Text: "hello"}},
}},
ThresholdPercent: 70,
ContextLimit: 100,
StepUsage: fantasy.Usage{InputTokens: 90},
Clock: clock,
OnModelStreamStart: func() {
startedAt = append(startedAt, clock.Now())
},
})
require.NoError(t, err)
require.Equal(t, "summary", result.SummaryReport)
require.Equal(t, 1500*time.Millisecond, result.Runtime)
require.Len(t, startedAt, 1)
require.Equal(t, result.Runtime, clock.Since(startedAt[0]))
}
// executeToolBatch lets tests release trapped clock events in order, so
// goroutines cannot race clock advances.
func executeToolBatch(
t *testing.T,
clock *quartz.Mock,
opts chatloop.ExecuteLocalToolsOptions,
) <-chan chatloop.PersistedStep {
t.Helper()
opts.Clock = clock
resultCh := make(chan chatloop.PersistedStep, 1)
go func() {
outcome, err := chatloop.ExecuteLocalTools(context.Background(), opts)
assert.NoError(t, err)
resultCh <- outcome
}()
return resultCh
}
func blockingTool(name string, release <-chan struct{}, response fantasy.ToolResponse) fantasy.AgentTool {
return fantasy.NewAgentTool(
name,
"test tool that completes when released",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
<-release
return response, nil
},
)
}
type serialTool struct {
fantasy.AgentTool
}
func (serialTool) SerialToolCalls() bool { return true }
// Each case scripts one tool batch through the trapped clock: the batch
// start timestamp is released first, then every step advances the clock,
// lets one tool's calls finish, and releases the Now() calls they trap.
func TestExecuteLocalTools_BatchRuntime(t *testing.T) {
t.Parallel()
type tool struct {
name string
response fantasy.ToolResponse
serial bool
}
type step struct {
advance time.Duration
finish string
// releases counts the trapped Now() calls that follow finish: one
// per completing call, plus one for the next serial call's start.
releases int
}
cases := []struct {
name string
tools []tool
unbilled map[string]bool
aliases map[string]string
calls []fantasy.ToolCallContent
steps []step
wantRuntime time.Duration
wantBilledCalls int
}{
{
name: "BatchWindowIsMaxNotSum",
tools: []tool{
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
{name: "slow_tool", response: fantasy.NewTextErrorResponse("blew up")},
},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
{ToolCallID: "call-slow", ToolName: "slow_tool", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
{advance: 50 * time.Second, finish: "slow_tool", releases: 1},
},
wantRuntime: 60 * time.Second,
wantBilledCalls: 2,
},
{
name: "SimultaneousCompletionsBillOnce",
tools: []tool{
{name: "read_tool", response: fantasy.NewTextResponse("done")},
},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-1", ToolName: "read_tool", Input: "{}"},
{ToolCallID: "call-2", ToolName: "read_tool", Input: "{}"},
{ToolCallID: "call-3", ToolName: "read_tool", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "read_tool", releases: 3},
},
wantRuntime: 10 * time.Second,
wantBilledCalls: 3,
},
{
name: "UnbilledToolNeverExtendsWindow",
tools: []tool{
{name: "execute", response: fantasy.NewTextResponse("done")},
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
},
unbilled: map[string]bool{"wait_agent": true},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-execute", ToolName: "execute", Input: "{}"},
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "execute", releases: 1},
{advance: 50 * time.Second, finish: "wait_agent", releases: 1},
},
wantRuntime: 10 * time.Second,
wantBilledCalls: 1,
},
{
name: "UnbilledOnlyBatchBillsNothing",
tools: []tool{
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
},
unbilled: map[string]bool{"wait_agent": true},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
},
steps: []step{
{advance: 60 * time.Second, finish: "wait_agent", releases: 1},
},
wantRuntime: 0,
wantBilledCalls: 0,
},
{
name: "AliasNamesClassifyAsCalled",
tools: []tool{
{name: "execute", response: fantasy.NewTextResponse("done")},
{name: "interrupt_agent", response: fantasy.NewTextResponse("stopped")},
},
aliases: map[string]string{"close_agent": "interrupt_agent"},
unbilled: map[string]bool{
"interrupt_agent": true,
"close_agent": true,
},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-execute", ToolName: "execute", Input: "{}"},
{ToolCallID: "call-legacy", ToolName: "close_agent", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "execute", releases: 1},
{advance: 50 * time.Second, finish: "interrupt_agent", releases: 1},
},
wantRuntime: 10 * time.Second,
wantBilledCalls: 1,
},
{
name: "DuplicateToolCallIDsKeepOccurrenceCompletions",
tools: []tool{
{name: "slow_tool", response: fantasy.NewTextResponse("done")},
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-dup", ToolName: "slow_tool", Input: "{}"},
{ToolCallID: "call-dup", ToolName: "fast_tool", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
{advance: 50 * time.Second, finish: "slow_tool", releases: 1},
},
wantRuntime: 60 * time.Second,
wantBilledCalls: 2,
},
{
name: "EmptyToolCallIDStillBillsWindow",
tools: []tool{
{name: "idless_tool", response: fantasy.NewTextResponse("done")},
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
},
calls: []fantasy.ToolCallContent{
{ToolCallID: "", ToolName: "idless_tool", Input: "{}"},
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
{advance: 50 * time.Second, finish: "idless_tool", releases: 1},
},
wantRuntime: 60 * time.Second,
wantBilledCalls: 2,
},
{
// A serial call bills its own execution, not the unbilled wait
// that delayed its launch.
name: "SerialCallBillsFromItsOwnStart",
tools: []tool{
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
{name: "serial_tool", response: fantasy.NewTextResponse("done"), serial: true},
},
unbilled: map[string]bool{"wait_agent": true},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
{ToolCallID: "call-serial", ToolName: "serial_tool", Input: "{}"},
},
steps: []step{
{advance: 10 * time.Minute, finish: "wait_agent", releases: 2},
{advance: 2 * time.Second, finish: "serial_tool", releases: 1},
},
wantRuntime: 2 * time.Second,
wantBilledCalls: 1,
},
{
// The 3s concurrent window and the 2s serial window bill; the 7s
// span where only wait_agent ran does not.
name: "SerialAfterBilledSiblingBillsUnion",
tools: []tool{
{name: "execute", response: fantasy.NewTextResponse("done")},
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
{name: "serial_tool", response: fantasy.NewTextResponse("done"), serial: true},
},
unbilled: map[string]bool{"wait_agent": true},
calls: []fantasy.ToolCallContent{
{ToolCallID: "call-execute", ToolName: "execute", Input: "{}"},
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
{ToolCallID: "call-serial", ToolName: "serial_tool", Input: "{}"},
},
steps: []step{
{advance: 3 * time.Second, finish: "execute", releases: 1},
{advance: 7 * time.Second, finish: "wait_agent", releases: 2},
{advance: 2 * time.Second, finish: "serial_tool", releases: 1},
},
wantRuntime: 5 * time.Second,
wantBilledCalls: 2,
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
clock := quartz.NewMock(t)
trap := clock.Trap().Now()
defer trap.Close()
releases := make(map[string]chan struct{}, len(tc.tools))
tools := make([]fantasy.AgentTool, 0, len(tc.tools))
activeTools := make([]string, 0, len(tc.tools))
for _, tl := range tc.tools {
release := make(chan struct{})
releases[tl.name] = release
agentTool := blockingTool(tl.name, release, tl.response)
if tl.serial {
agentTool = serialTool{agentTool}
}
tools = append(tools, agentTool)
activeTools = append(activeTools, tl.name)
}
resultCh := executeToolBatch(t, clock, chatloop.ExecuteLocalToolsOptions{
Tools: tools,
ActiveTools: activeTools,
ToolNameAliases: tc.aliases,
UnbilledToolNames: tc.unbilled,
ToolCalls: tc.calls,
})
trap.MustWait(ctx).MustRelease(ctx)
for _, st := range tc.steps {
clock.Advance(st.advance)
close(releases[st.finish])
for range st.releases {
trap.MustWait(ctx).MustRelease(ctx)
}
}
outcome := testutil.RequireReceive(ctx, t, resultCh)
require.Equal(t, tc.wantRuntime, outcome.BatchRuntime)
require.Equal(t, tc.wantBilledCalls, outcome.BatchBilledCalls)
})
}
}
// recordingToolBillingRecorder is a test ToolBillingRecorder that
// counts start/complete calls and can publish live completions.
type recordingToolBillingRecorder struct {
starts int
completions int
completeCh chan recordedToolCompletion
}
type recordedToolCompletion struct {
dispatchIndex int
completedAt time.Time
}
func (r *recordingToolBillingRecorder) RecordStart(int, time.Time) {
r.starts++
}
func (r *recordingToolBillingRecorder) RecordComplete(dispatchIndex int, completedAt time.Time) {
r.completions++
if r.completeCh != nil {
r.completeCh <- recordedToolCompletion{
dispatchIndex: dispatchIndex,
completedAt: completedAt,
}
}
}
func TestExecuteLocalTools_BillingRecorderRecordsOnlyRuns(t *testing.T) {
t.Parallel()
t.Run("started call records a paired lifecycle", func(t *testing.T) {
t.Parallel()
recorder := &recordingToolBillingRecorder{}
startedWhenToolRan := false
tool := fantasy.NewAgentTool(
"fast_tool",
"test tool",
func(context.Context, struct{}, fantasy.ToolCall) (fantasy.ToolResponse, error) {
startedWhenToolRan = recorder.starts > 0
return fantasy.NewTextResponse("done"), nil
},
)
outcome, err := chatloop.ExecuteLocalTools(context.Background(), chatloop.ExecuteLocalToolsOptions{
Clock: quartz.NewMock(t),
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"fast_tool"},
BillingRecorder: recorder,
ToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "call-1", ToolName: "fast_tool", Input: "{}"},
},
})
require.NoError(t, err)
require.Len(t, outcome.Content, 1)
require.Equal(t, 1, recorder.starts)
require.Equal(t, 1, recorder.completions)
require.True(t, startedWhenToolRan, "RecordStart must run before the tool runs")
})
t.Run("canceled context records no lifecycle", func(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(context.Background())
cancel()
recorder := &recordingToolBillingRecorder{}
_, err := chatloop.ExecuteLocalTools(ctx, chatloop.ExecuteLocalToolsOptions{
Clock: quartz.NewMock(t),
BillingRecorder: recorder,
ToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "call-1", ToolName: "fast_tool", Input: "{}"},
},
})
require.ErrorIs(t, err, context.Canceled)
require.Zero(t, recorder.starts)
require.Zero(t, recorder.completions)
})
t.Run("exclusive violation records no lifecycle", func(t *testing.T) {
t.Parallel()
recorder := &recordingToolBillingRecorder{}
outcome, err := chatloop.ExecuteLocalTools(context.Background(), chatloop.ExecuteLocalToolsOptions{
Clock: quartz.NewMock(t),
ExclusiveToolNames: map[string]bool{"exclusive_tool": true},
BillingRecorder: recorder,
ToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "call-1", ToolName: "exclusive_tool", Input: "{}"},
{ToolCallID: "call-2", ToolName: "fast_tool", Input: "{}"},
},
})
require.NoError(t, err)
require.Len(t, outcome.Content, 2, "the whole batch resolves to synthesized policy errors")
require.Zero(t, recorder.starts)
require.Zero(t, recorder.completions)
})
}
func TestExecuteLocalTools_BillingRecorderReportsLiveCompletions(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
clock := quartz.NewMock(t)
trap := clock.Trap().Now()
defer trap.Close()
recorder := &recordingToolBillingRecorder{
completeCh: make(chan recordedToolCompletion, 2),
}
fastGo := make(chan struct{})
slowGo := make(chan struct{})
resultCh := executeToolBatch(t, clock, chatloop.ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{
blockingTool("fast_tool", fastGo, fantasy.NewTextResponse("done")),
blockingTool("slow_tool", slowGo, fantasy.NewTextResponse("done")),
},
ActiveTools: []string{"fast_tool", "slow_tool"},
BillingRecorder: recorder,
ToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
{ToolCallID: "call-slow", ToolName: "slow_tool", Input: "{}"},
},
})
trap.MustWait(ctx).MustRelease(ctx)
clock.Advance(10 * time.Second)
close(fastGo)
trap.MustWait(ctx).MustRelease(ctx)
fast := testutil.RequireReceive(ctx, t, recorder.completeCh)
require.Equal(t, 0, fast.dispatchIndex)
clock.Advance(50 * time.Second)
close(slowGo)
trap.MustWait(ctx).MustRelease(ctx)
slow := testutil.RequireReceive(ctx, t, recorder.completeCh)
require.Equal(t, 1, slow.dispatchIndex)
require.Equal(t, 50*time.Second, slow.completedAt.Sub(fast.completedAt))
outcome := testutil.RequireReceive(ctx, t, resultCh)
require.Equal(t, map[string]time.Time{
"call-fast": fast.completedAt,
"call-slow": slow.completedAt,
}, outcome.ToolResultCreatedAt)
}
func TestBilledIntervalsDuration(t *testing.T) {
t.Parallel()
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
at := base.Add
for _, tc := range []struct {
name string
intervals []chatloop.BilledInterval
want time.Duration
}{
{name: "empty", want: 0},
{
name: "overlapping intervals bill once",
intervals: []chatloop.BilledInterval{
{Start: at(0), End: at(10 * time.Second)},
{Start: at(0), End: at(4 * time.Second)},
},
want: 10 * time.Second,
},
{
name: "gap between intervals is not billed",
intervals: []chatloop.BilledInterval{
{Start: at(0), End: at(3 * time.Second)},
{Start: at(10 * time.Second), End: at(12 * time.Second)},
},
want: 5 * time.Second,
},
{
name: "unsorted contained interval adds nothing",
intervals: []chatloop.BilledInterval{
{Start: at(2 * time.Second), End: at(4 * time.Second)},
{Start: at(0), End: at(10 * time.Second)},
},
want: 10 * time.Second,
},
{
name: "touching intervals merge without a gap",
intervals: []chatloop.BilledInterval{
{Start: at(0), End: at(3 * time.Second)},
{Start: at(3 * time.Second), End: at(5 * time.Second)},
},
want: 5 * time.Second,
},
{
name: "inverted interval is ignored",
intervals: []chatloop.BilledInterval{
{Start: at(5 * time.Second), End: at(0)},
{Start: at(0), End: at(2 * time.Second)},
},
want: 2 * time.Second,
},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
require.Equal(t, tc.want, chatloop.BilledIntervalsDuration(tc.intervals))
})
}
}