This PR breaks compatibility with chat histories that contain `close_agent` tool calls, which chats from v2.34 and earlier have. #26673 (v2.35) renamed `close_agent` to `interrupt_agent` and kept a hidden alias, so a model that copied the old name from such a history still interrupted the child. This PR deletes the alias: `subagentToolNameAliases`, `ExecuteLocalToolsOptions.ToolNameAliases` with `(*toolBatch).resolve` and its five call sites in `chatloop`, the alias step in builtin input validation, and the `close_agent` entries in the unbilled tool set, the plan policy, and the chatprompt lifecycle list. Coder Agents is experimental (`coderd/x`), and our rule for experimental persisted data is to drop the legacy shape outright and delete the code that existed only to support it, instead of carrying it forward. What happens to an old chat: - Replay does not change, because the replay path does not look up tool names: the model still sees the historical `close_agent` call and its stored result, and the UI renders them as before. The write path reads a tool name only to parse `execute` commands and to keep structured error results from subagent lifecycle tools. Dropping `close_agent` from that lifecycle list changes nothing, because its calls are now rejected with a plain-text error. - If the model calls `close_agent` again, nothing runs and the child keeps running. The tool result is the error `Tool not active in this turn: close_agent`, and the model can retry with `interrupt_agent`. **Why it is safe** - Deleted alias-only tests: `TestExecuteSingleTool_ResolvesToolNameAlias`, the `AliasNamesClassifyAsCalled` billing row, the alias subtest of `TestPartitionAmbiguousToolCallsGatesOnBuiltins`, the alias loop in `TestUnbilledSubagentToolNamesMatchCatalog`, and the aliased calls that #30939 added to `TestExecuteToolsSerialToolCallOrder` and `TestExecuteLocalTools_ProviderToolRunner`; both tests are back to their #30938 form. - `TestExecuteSingleTool_UnknownAliasFallsThrough` is now `TestExecuteSingleTool_RejectsUnknownTool` and asserts the exact error above. The observer tests call the tool by its real name. - Mutants of the active tool check, step observer notification, serial dispatch, the unbilled set, the plan policy and builtin input validation each fail a remaining test. Only the deleted alias test caught an empty name passed to the tool handler; no chatd tool reads that name (MCP tools use their original name). - `git grep` finds no use of the removed identifiers and no backend code that resolves `close_agent`. The rejection test still uses that name as its input. The frontend keeps its `close_agent` display mapping, which old histories still use. **Review focus** - A rejected `close_agent` call is now billed like any rejected tool call; the rejection is immediate. - `ARCHITECTURE.md`, "Generation goroutine": this PR deletes the sub-bullet that called `close_agent` a deprecated alias of `interrupt_agent`. - The `resolved_tool_name` debug log field is gone. It differed from `tool_name` only for aliased calls, and the alias is gone. **Validation** - `go test -count=1 ./coderd/x/chatd/ ./coderd/x/chatd/chatloop/ ./coderd/x/chatd/chatprompt/`: ok - `go vet` and `golangci-lint run --fix=false ./coderd/x/chatd/...`: clean - `paralleltestctx -custom-funcs="testutil.Context,chatdTestContext" ./coderd/x/chatd/...`: clean - pre-commit hook: passed **Stack** This stack retires chatd compatibility paths for chat data that older Coder versions stored. This PR replaces #31003 and carries its change, adapted to the `toolBatch` code that #30938 and #30939 merged. Mike chose to keep the fallbacks that #31001 and #31002 removed, so those two layers and #31003 left the stack. 1. #30996 stamp chat message fixtures with the current content version 2. #30997 drop tests of the legacy content codec 3. #30998 retire the V0 chat message content decoder 4. #31000 inline the model config provider identity 5. #31005 retire the `close_agent` tool alias (this PR) > Xum, an AI agent, opened this PR for @ibetitsmike and wrote this description.
618 lines
19 KiB
Go
618 lines
19 KiB
Go
package chatloop_test
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func TestGenerateAssistant_RecordsModelInvocationRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
|
|
return
|
|
}
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "t", Delta: "hello"}) {
|
|
return
|
|
}
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "t"}) {
|
|
return
|
|
}
|
|
clock.Advance(1500 * time.Millisecond)
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeFinish,
|
|
FinishReason: fantasy.FinishReasonStop,
|
|
})
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1500*time.Millisecond, outcome.Step.Runtime)
|
|
}
|
|
|
|
// The interrupt path bills the window OnModelStreamStart opens, so that
|
|
// hook must fire at the instant PersistedStep.Runtime starts measuring.
|
|
func TestGenerateAssistant_ModelStreamStartMatchesRuntimeWindow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
|
|
return
|
|
}
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "t", Delta: "hello"}) {
|
|
return
|
|
}
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "t"}) {
|
|
return
|
|
}
|
|
clock.Advance(1500 * time.Millisecond)
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeFinish,
|
|
FinishReason: fantasy.FinishReasonStop,
|
|
})
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
var startedAt []time.Time
|
|
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
OnModelStreamStart: func() {
|
|
startedAt = append(startedAt, clock.Now())
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, startedAt, 1)
|
|
require.Equal(t, outcome.Step.Runtime, clock.Since(startedAt[0]))
|
|
}
|
|
|
|
func TestGenerateAssistant_ErroredStreamReturnsNoStep(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "t"}) {
|
|
return
|
|
}
|
|
clock.Advance(1500 * time.Millisecond)
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeError,
|
|
Error: xerrors.New("stream blew up"),
|
|
})
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
outcome, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
})
|
|
require.Error(t, err)
|
|
require.Zero(t, outcome.Step.Runtime)
|
|
require.Empty(t, outcome.Step.Content)
|
|
}
|
|
|
|
func TestGenerateCompaction_RecordsRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "test-provider",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
clock.Advance(1500 * time.Millisecond)
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "text"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "text", Delta: "summary"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "text"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
}, nil
|
|
},
|
|
}
|
|
|
|
var startedAt []time.Time
|
|
result, err := chatloop.GenerateCompaction(context.Background(), chatloop.GenerateCompactionOptions{
|
|
Model: model,
|
|
Messages: []fantasy.Message{{
|
|
Role: fantasy.MessageRoleUser,
|
|
Content: []fantasy.MessagePart{fantasy.TextPart{Text: "hello"}},
|
|
}},
|
|
ThresholdPercent: 70,
|
|
ContextLimit: 100,
|
|
StepUsage: fantasy.Usage{InputTokens: 90},
|
|
Clock: clock,
|
|
OnModelStreamStart: func() {
|
|
startedAt = append(startedAt, clock.Now())
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, "summary", result.SummaryReport)
|
|
require.Equal(t, 1500*time.Millisecond, result.Runtime)
|
|
require.Len(t, startedAt, 1)
|
|
require.Equal(t, result.Runtime, clock.Since(startedAt[0]))
|
|
}
|
|
|
|
// executeToolBatch lets tests release trapped clock events in order, so
|
|
// goroutines cannot race clock advances.
|
|
func executeToolBatch(
|
|
t *testing.T,
|
|
clock *quartz.Mock,
|
|
opts chatloop.ExecuteLocalToolsOptions,
|
|
) <-chan chatloop.PersistedStep {
|
|
t.Helper()
|
|
opts.Clock = clock
|
|
resultCh := make(chan chatloop.PersistedStep, 1)
|
|
go func() {
|
|
outcome, err := chatloop.ExecuteLocalTools(context.Background(), opts)
|
|
assert.NoError(t, err)
|
|
resultCh <- outcome
|
|
}()
|
|
return resultCh
|
|
}
|
|
|
|
func blockingTool(name string, release <-chan struct{}, response fantasy.ToolResponse) fantasy.AgentTool {
|
|
return fantasy.NewAgentTool(
|
|
name,
|
|
"test tool that completes when released",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
<-release
|
|
return response, nil
|
|
},
|
|
)
|
|
}
|
|
|
|
type serialTool struct {
|
|
fantasy.AgentTool
|
|
}
|
|
|
|
func (serialTool) SerialToolCalls() bool { return true }
|
|
|
|
// Each case scripts one tool batch through the trapped clock: the batch
|
|
// start timestamp is released first, then every step advances the clock,
|
|
// lets one tool's calls finish, and releases the Now() calls they trap.
|
|
func TestExecuteLocalTools_BatchRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
type tool struct {
|
|
name string
|
|
response fantasy.ToolResponse
|
|
serial bool
|
|
}
|
|
type step struct {
|
|
advance time.Duration
|
|
finish string
|
|
// releases counts the trapped Now() calls that follow finish: one
|
|
// per completing call, plus one for the next serial call's start.
|
|
releases int
|
|
}
|
|
cases := []struct {
|
|
name string
|
|
tools []tool
|
|
unbilled map[string]bool
|
|
calls []fantasy.ToolCallContent
|
|
steps []step
|
|
wantRuntime time.Duration
|
|
wantBilledCalls int
|
|
}{
|
|
{
|
|
name: "BatchWindowIsMaxNotSum",
|
|
tools: []tool{
|
|
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
|
|
{name: "slow_tool", response: fantasy.NewTextErrorResponse("blew up")},
|
|
},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
|
|
{ToolCallID: "call-slow", ToolName: "slow_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
|
|
{advance: 50 * time.Second, finish: "slow_tool", releases: 1},
|
|
},
|
|
wantRuntime: 60 * time.Second,
|
|
wantBilledCalls: 2,
|
|
},
|
|
{
|
|
name: "SimultaneousCompletionsBillOnce",
|
|
tools: []tool{
|
|
{name: "read_tool", response: fantasy.NewTextResponse("done")},
|
|
},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-1", ToolName: "read_tool", Input: "{}"},
|
|
{ToolCallID: "call-2", ToolName: "read_tool", Input: "{}"},
|
|
{ToolCallID: "call-3", ToolName: "read_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Second, finish: "read_tool", releases: 3},
|
|
},
|
|
wantRuntime: 10 * time.Second,
|
|
wantBilledCalls: 3,
|
|
},
|
|
{
|
|
name: "UnbilledToolNeverExtendsWindow",
|
|
tools: []tool{
|
|
{name: "execute", response: fantasy.NewTextResponse("done")},
|
|
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
|
|
},
|
|
unbilled: map[string]bool{"wait_agent": true},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-execute", ToolName: "execute", Input: "{}"},
|
|
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Second, finish: "execute", releases: 1},
|
|
{advance: 50 * time.Second, finish: "wait_agent", releases: 1},
|
|
},
|
|
wantRuntime: 10 * time.Second,
|
|
wantBilledCalls: 1,
|
|
},
|
|
{
|
|
name: "UnbilledOnlyBatchBillsNothing",
|
|
tools: []tool{
|
|
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
|
|
},
|
|
unbilled: map[string]bool{"wait_agent": true},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 60 * time.Second, finish: "wait_agent", releases: 1},
|
|
},
|
|
wantRuntime: 0,
|
|
wantBilledCalls: 0,
|
|
},
|
|
{
|
|
name: "DuplicateToolCallIDsKeepOccurrenceCompletions",
|
|
tools: []tool{
|
|
{name: "slow_tool", response: fantasy.NewTextResponse("done")},
|
|
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
|
|
},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-dup", ToolName: "slow_tool", Input: "{}"},
|
|
{ToolCallID: "call-dup", ToolName: "fast_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
|
|
{advance: 50 * time.Second, finish: "slow_tool", releases: 1},
|
|
},
|
|
wantRuntime: 60 * time.Second,
|
|
wantBilledCalls: 2,
|
|
},
|
|
{
|
|
name: "EmptyToolCallIDStillBillsWindow",
|
|
tools: []tool{
|
|
{name: "idless_tool", response: fantasy.NewTextResponse("done")},
|
|
{name: "fast_tool", response: fantasy.NewTextResponse("done")},
|
|
},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "", ToolName: "idless_tool", Input: "{}"},
|
|
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Second, finish: "fast_tool", releases: 1},
|
|
{advance: 50 * time.Second, finish: "idless_tool", releases: 1},
|
|
},
|
|
wantRuntime: 60 * time.Second,
|
|
wantBilledCalls: 2,
|
|
},
|
|
{
|
|
// A serial call bills its own execution, not the unbilled wait
|
|
// that delayed its launch.
|
|
name: "SerialCallBillsFromItsOwnStart",
|
|
tools: []tool{
|
|
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
|
|
{name: "serial_tool", response: fantasy.NewTextResponse("done"), serial: true},
|
|
},
|
|
unbilled: map[string]bool{"wait_agent": true},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
|
|
{ToolCallID: "call-serial", ToolName: "serial_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 10 * time.Minute, finish: "wait_agent", releases: 2},
|
|
{advance: 2 * time.Second, finish: "serial_tool", releases: 1},
|
|
},
|
|
wantRuntime: 2 * time.Second,
|
|
wantBilledCalls: 1,
|
|
},
|
|
{
|
|
// The 3s concurrent window and the 2s serial window bill; the 7s
|
|
// span where only wait_agent ran does not.
|
|
name: "SerialAfterBilledSiblingBillsUnion",
|
|
tools: []tool{
|
|
{name: "execute", response: fantasy.NewTextResponse("done")},
|
|
{name: "wait_agent", response: fantasy.NewTextResponse("child report")},
|
|
{name: "serial_tool", response: fantasy.NewTextResponse("done"), serial: true},
|
|
},
|
|
unbilled: map[string]bool{"wait_agent": true},
|
|
calls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-execute", ToolName: "execute", Input: "{}"},
|
|
{ToolCallID: "call-wait", ToolName: "wait_agent", Input: "{}"},
|
|
{ToolCallID: "call-serial", ToolName: "serial_tool", Input: "{}"},
|
|
},
|
|
steps: []step{
|
|
{advance: 3 * time.Second, finish: "execute", releases: 1},
|
|
{advance: 7 * time.Second, finish: "wait_agent", releases: 2},
|
|
{advance: 2 * time.Second, finish: "serial_tool", releases: 1},
|
|
},
|
|
wantRuntime: 5 * time.Second,
|
|
wantBilledCalls: 2,
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().Now()
|
|
defer trap.Close()
|
|
|
|
releases := make(map[string]chan struct{}, len(tc.tools))
|
|
tools := make([]fantasy.AgentTool, 0, len(tc.tools))
|
|
activeTools := make([]string, 0, len(tc.tools))
|
|
for _, tl := range tc.tools {
|
|
release := make(chan struct{})
|
|
releases[tl.name] = release
|
|
agentTool := blockingTool(tl.name, release, tl.response)
|
|
if tl.serial {
|
|
agentTool = serialTool{agentTool}
|
|
}
|
|
tools = append(tools, agentTool)
|
|
activeTools = append(activeTools, tl.name)
|
|
}
|
|
resultCh := executeToolBatch(t, clock, chatloop.ExecuteLocalToolsOptions{
|
|
Tools: tools,
|
|
ActiveTools: activeTools,
|
|
UnbilledToolNames: tc.unbilled,
|
|
ToolCalls: tc.calls,
|
|
})
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
for _, st := range tc.steps {
|
|
clock.Advance(st.advance)
|
|
close(releases[st.finish])
|
|
for range st.releases {
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
}
|
|
}
|
|
|
|
outcome := testutil.RequireReceive(ctx, t, resultCh)
|
|
require.Equal(t, tc.wantRuntime, outcome.BatchRuntime)
|
|
require.Equal(t, tc.wantBilledCalls, outcome.BatchBilledCalls)
|
|
})
|
|
}
|
|
}
|
|
|
|
// recordingToolBillingRecorder is a test ToolBillingRecorder that
|
|
// counts start/complete calls and can publish live completions.
|
|
type recordingToolBillingRecorder struct {
|
|
starts int
|
|
completions int
|
|
completeCh chan recordedToolCompletion
|
|
}
|
|
|
|
type recordedToolCompletion struct {
|
|
dispatchIndex int
|
|
completedAt time.Time
|
|
}
|
|
|
|
func (r *recordingToolBillingRecorder) RecordStart(int, time.Time) {
|
|
r.starts++
|
|
}
|
|
|
|
func (r *recordingToolBillingRecorder) RecordComplete(dispatchIndex int, completedAt time.Time) {
|
|
r.completions++
|
|
if r.completeCh != nil {
|
|
r.completeCh <- recordedToolCompletion{
|
|
dispatchIndex: dispatchIndex,
|
|
completedAt: completedAt,
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestExecuteLocalTools_BillingRecorderRecordsOnlyRuns(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("started call records a paired lifecycle", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
recorder := &recordingToolBillingRecorder{}
|
|
startedWhenToolRan := false
|
|
tool := fantasy.NewAgentTool(
|
|
"fast_tool",
|
|
"test tool",
|
|
func(context.Context, struct{}, fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
startedWhenToolRan = recorder.starts > 0
|
|
return fantasy.NewTextResponse("done"), nil
|
|
},
|
|
)
|
|
outcome, err := chatloop.ExecuteLocalTools(context.Background(), chatloop.ExecuteLocalToolsOptions{
|
|
Clock: quartz.NewMock(t),
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"fast_tool"},
|
|
BillingRecorder: recorder,
|
|
ToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-1", ToolName: "fast_tool", Input: "{}"},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, outcome.Content, 1)
|
|
require.Equal(t, 1, recorder.starts)
|
|
require.Equal(t, 1, recorder.completions)
|
|
require.True(t, startedWhenToolRan, "RecordStart must run before the tool runs")
|
|
})
|
|
|
|
t.Run("canceled context records no lifecycle", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
recorder := &recordingToolBillingRecorder{}
|
|
_, err := chatloop.ExecuteLocalTools(ctx, chatloop.ExecuteLocalToolsOptions{
|
|
Clock: quartz.NewMock(t),
|
|
BillingRecorder: recorder,
|
|
ToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-1", ToolName: "fast_tool", Input: "{}"},
|
|
},
|
|
})
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
require.Zero(t, recorder.starts)
|
|
require.Zero(t, recorder.completions)
|
|
})
|
|
|
|
t.Run("exclusive violation records no lifecycle", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
recorder := &recordingToolBillingRecorder{}
|
|
outcome, err := chatloop.ExecuteLocalTools(context.Background(), chatloop.ExecuteLocalToolsOptions{
|
|
Clock: quartz.NewMock(t),
|
|
ExclusiveToolNames: map[string]bool{"exclusive_tool": true},
|
|
BillingRecorder: recorder,
|
|
ToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-1", ToolName: "exclusive_tool", Input: "{}"},
|
|
{ToolCallID: "call-2", ToolName: "fast_tool", Input: "{}"},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, outcome.Content, 2, "the whole batch resolves to synthesized policy errors")
|
|
require.Zero(t, recorder.starts)
|
|
require.Zero(t, recorder.completions)
|
|
})
|
|
}
|
|
|
|
func TestExecuteLocalTools_BillingRecorderReportsLiveCompletions(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().Now()
|
|
defer trap.Close()
|
|
|
|
recorder := &recordingToolBillingRecorder{
|
|
completeCh: make(chan recordedToolCompletion, 2),
|
|
}
|
|
fastGo := make(chan struct{})
|
|
slowGo := make(chan struct{})
|
|
resultCh := executeToolBatch(t, clock, chatloop.ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{
|
|
blockingTool("fast_tool", fastGo, fantasy.NewTextResponse("done")),
|
|
blockingTool("slow_tool", slowGo, fantasy.NewTextResponse("done")),
|
|
},
|
|
ActiveTools: []string{"fast_tool", "slow_tool"},
|
|
BillingRecorder: recorder,
|
|
ToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-fast", ToolName: "fast_tool", Input: "{}"},
|
|
{ToolCallID: "call-slow", ToolName: "slow_tool", Input: "{}"},
|
|
},
|
|
})
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
clock.Advance(10 * time.Second)
|
|
close(fastGo)
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
fast := testutil.RequireReceive(ctx, t, recorder.completeCh)
|
|
require.Equal(t, 0, fast.dispatchIndex)
|
|
clock.Advance(50 * time.Second)
|
|
close(slowGo)
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
slow := testutil.RequireReceive(ctx, t, recorder.completeCh)
|
|
require.Equal(t, 1, slow.dispatchIndex)
|
|
require.Equal(t, 50*time.Second, slow.completedAt.Sub(fast.completedAt))
|
|
|
|
outcome := testutil.RequireReceive(ctx, t, resultCh)
|
|
require.Equal(t, map[string]time.Time{
|
|
"call-fast": fast.completedAt,
|
|
"call-slow": slow.completedAt,
|
|
}, outcome.ToolResultCreatedAt)
|
|
}
|
|
|
|
func TestBilledIntervalsDuration(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
base := time.Date(2024, 1, 1, 0, 0, 0, 0, time.UTC)
|
|
at := base.Add
|
|
for _, tc := range []struct {
|
|
name string
|
|
intervals []chatloop.BilledInterval
|
|
want time.Duration
|
|
}{
|
|
{name: "empty", want: 0},
|
|
{
|
|
name: "overlapping intervals bill once",
|
|
intervals: []chatloop.BilledInterval{
|
|
{Start: at(0), End: at(10 * time.Second)},
|
|
{Start: at(0), End: at(4 * time.Second)},
|
|
},
|
|
want: 10 * time.Second,
|
|
},
|
|
{
|
|
name: "gap between intervals is not billed",
|
|
intervals: []chatloop.BilledInterval{
|
|
{Start: at(0), End: at(3 * time.Second)},
|
|
{Start: at(10 * time.Second), End: at(12 * time.Second)},
|
|
},
|
|
want: 5 * time.Second,
|
|
},
|
|
{
|
|
name: "unsorted contained interval adds nothing",
|
|
intervals: []chatloop.BilledInterval{
|
|
{Start: at(2 * time.Second), End: at(4 * time.Second)},
|
|
{Start: at(0), End: at(10 * time.Second)},
|
|
},
|
|
want: 10 * time.Second,
|
|
},
|
|
{
|
|
name: "touching intervals merge without a gap",
|
|
intervals: []chatloop.BilledInterval{
|
|
{Start: at(0), End: at(3 * time.Second)},
|
|
{Start: at(3 * time.Second), End: at(5 * time.Second)},
|
|
},
|
|
want: 5 * time.Second,
|
|
},
|
|
{
|
|
name: "inverted interval is ignored",
|
|
intervals: []chatloop.BilledInterval{
|
|
{Start: at(5 * time.Second), End: at(0)},
|
|
{Start: at(0), End: at(2 * time.Second)},
|
|
},
|
|
want: 2 * time.Second,
|
|
},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
require.Equal(t, tc.want, chatloop.BilledIntervalsDuration(tc.intervals))
|
|
})
|
|
}
|
|
}
|