Files
coder/coderd/x/chatd/chatloop/chatloop_run_internal_test.go
Ethan 5cbf086517 feat: send tool result media to providers like user files (#31383)
Part of [CODAGT-1430](https://linear.app/codercom/issue/CODAGT-1430).
This is the first of four stacked PRs.

Today a PDF in an MCP tool result reaches no provider. On Anthropic we
strip it in chatd. On OpenAI Chat Completions fantasy swaps it for a
"cannot be displayed" note, and on Responses it sends a placeholder that
points at a user message it never sends.

coder/fantasy#73 and coder/fantasy#74 make fantasy convert tool result
media with the converter it already uses for user files: an Anthropic
document inside `tool_result`, an `input_file` inside
`function_call_output` on Responses, and a user file part straight after
the tool message on Chat Completions. Both have merged, and this PR pins
`charm.land/fantasy` to the coder_2_33 tip that has them. Chat
Completions file parts now refuse text files, as Responses already did,
so `AcceptsFilePartMediaType` follows.

The converters take a filename, so an MCP embedded resource blob is now
named after the last element of its URI path, and we store the name with
the tool result so later turns send the same one. OpenAI picks the file
type from the extension, so we append the right one when it's missing or
wrong. We drop names with control characters, as Postgres `jsonb`
rejects NUL, and we redact the name like the rest of the tool response,
since a URI like `https://host/sse/<token>` would otherwise put the
token in the chat history.

There's also a test proving aibridge's Chat Completions re-encode keeps
a user file part, as that's where these PDFs end up.

Anthropic still strips tool result PDFs until the next PR in the stack.
2026-11-11 00:06:40 +11:00

1630 lines
50 KiB
Go

package chatloop
import (
"bytes"
"context"
"encoding/base64"
"errors"
"image"
"image/png"
"iter"
"runtime"
"slices"
"sync"
"sync/atomic"
"testing"
"time"
"unicode/utf8"
"charm.land/fantasy"
fantasyanthropic "charm.land/fantasy/providers/anthropic"
fantasyopenai "charm.land/fantasy/providers/openai"
"github.com/prometheus/client_golang/prometheus"
promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
"github.com/coder/coder/v2/coderd/x/chatd/chatsanitize"
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func validWebSearchProviderMetadataForTest() fantasy.ProviderMetadata {
return fantasy.ProviderMetadata{
fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{
Results: []fantasyanthropic.WebSearchResultItem{
{
URL: "https://example.com",
Title: "Example",
EncryptedContent: "encrypted",
},
},
},
}
}
func safeToolCallContent(block fantasy.Content) (fantasy.ToolCallContent, bool) {
var zero fantasy.ToolCallContent
switch value := block.(type) {
case fantasy.ToolCallContent:
return value, true
case *fantasy.ToolCallContent:
if value == nil {
return zero, false
}
return *value, true
default:
return zero, false
}
}
func safeToolResultContent(block fantasy.Content) (fantasy.ToolResultContent, bool) {
var zero fantasy.ToolResultContent
switch value := block.(type) {
case fantasy.ToolResultContent:
return value, true
case *fantasy.ToolResultContent:
if value == nil {
return zero, false
}
return *value, true
default:
return zero, false
}
}
func toolCallContentToPart(toolCall fantasy.ToolCallContent) fantasy.ToolCallPart {
return fantasy.ToolCallPart{
ToolCallID: toolCall.ToolCallID,
ToolName: toolCall.ToolName,
Input: toolCall.Input,
ProviderExecuted: toolCall.ProviderExecuted,
ProviderOptions: fantasy.ProviderOptions(toolCall.ProviderMetadata),
}
}
func toolResultContentToPart(toolResult fantasy.ToolResultContent) fantasy.ToolResultPart {
return fantasy.ToolResultPart{
ToolCallID: toolResult.ToolCallID,
Output: toolResult.Result,
ProviderExecuted: toolResult.ProviderExecuted,
ProviderOptions: fantasy.ProviderOptions(toolResult.ProviderMetadata),
}
}
func TestStreamSilenceGuard_DisarmAndFireRace(t *testing.T) {
t.Parallel()
for range 128 {
var cancels atomic.Int32
guard := newStreamSilenceGuard(quartz.NewReal(), time.Hour, func(err error) {
if errors.Is(err, errStreamSilenceTimeout) {
cancels.Add(1)
}
})
start := make(chan struct{})
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
<-start
guard.onTimeout()
}()
go func() {
defer wg.Done()
<-start
guard.Disarm()
}()
close(start)
wg.Wait()
guard.onTimeout()
guard.Disarm()
require.LessOrEqual(t, cancels.Load(), int32(1))
}
}
func TestStreamSilenceGuard_DisarmPreservesPermanentError(t *testing.T) {
t.Parallel()
attemptCtx, cancelAttempt := context.WithCancelCause(context.Background())
defer cancelAttempt(nil)
guard := newStreamSilenceGuard(quartz.NewReal(), time.Hour, cancelAttempt)
guard.Disarm()
guard.onTimeout()
classified := chaterror.Classify(classifyStreamSilenceTimeout(
attemptCtx,
"openai",
xerrors.New("invalid model"),
))
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
require.False(t, classified.Retryable)
require.Nil(t, context.Cause(attemptCtx))
}
func TestGenerateAssistant_ProviderContextSurvivesStreamError(t *testing.T) {
t.Parallel()
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return nil, xerrors.New("upstream returned status 400")
},
}
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Messages: []fantasy.Message{
textMessage(fantasy.MessageRoleUser, "hello"),
},
})
require.Error(t, err)
classified := chaterror.Classify(err)
require.Equal(t, "openai", classified.Provider)
require.Equal(t, "OpenAI returned an unexpected error.", classified.Message)
}
func TestGenerateAssistant_ProviderResponseID(t *testing.T) {
t.Parallel()
cases := []struct {
name string
finish fantasy.StreamPart
want string
}{
{
name: "anthropic message ID",
finish: fantasy.StreamPart{ID: "msg_anthropic"},
want: "msg_anthropic",
},
{
name: "openai responses metadata",
finish: fantasy.StreamPart{ProviderMetadata: fantasy.ProviderMetadata{
fantasyopenai.Name: &fantasyopenai.ResponsesProviderMetadata{ResponseID: "resp_openai"},
}},
want: "resp_openai",
},
{
name: "not reported",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
finish := tc.finish
finish.Type = fantasy.StreamPartTypeFinish
finish.FinishReason = fantasy.FinishReasonStop
model := &chattest.FakeModel{
ProviderName: "fake",
ModelName: "fake-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return streamFromParts([]fantasy.StreamPart{
{Type: fantasy.StreamPartTypeTextStart, ID: "text"},
{Type: fantasy.StreamPartTypeTextDelta, ID: "text", Delta: "hi"},
{Type: fantasy.StreamPartTypeTextEnd, ID: "text"},
finish,
}), nil
},
}
outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Messages: []fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
})
require.NoError(t, err)
require.Equal(t, tc.want, outcome.Step.ProviderResponseID)
})
}
}
func TestGenerateAssistant_ErrorProviderOverridesTransportLabel(t *testing.T) {
t.Parallel()
// Bedrock routed through aibridge uses the Anthropic transport, so
// Model.Provider() reports "anthropic". The user-facing error must use
// the configured provider from ErrorProvider instead.
model := &chattest.FakeModel{
ProviderName: "anthropic",
ModelName: "qwen3-coder-next",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return nil, xerrors.New("upstream returned status 400")
},
}
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
ErrorProvider: "bedrock",
Messages: []fantasy.Message{
textMessage(fantasy.MessageRoleUser, "hello"),
},
})
require.Error(t, err)
classified := chaterror.Classify(err)
require.Equal(t, "bedrock", classified.Provider)
require.Equal(t, "AWS Bedrock returned an unexpected error.", classified.Message)
}
func TestGenerateAssistant_HTTP2TransportErrorClassifiedAsRetryableTimeout(t *testing.T) {
t.Parallel()
for _, provider := range []string{"anthropic", "openai"} {
provider := provider
t.Run(provider, func(t *testing.T) {
t.Parallel()
model := &chattest.FakeModel{
ProviderName: provider,
ModelName: "test-model",
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
return nil, xerrors.New("http2: client connection force closed via ClientConn.Close")
},
}
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
})
require.Error(t, err)
classified := chaterror.Classify(err)
require.Equal(t, codersdk.ChatErrorKindTimeout, classified.Kind)
require.Equal(t, provider, classified.Provider)
require.True(t, classified.Retryable)
})
}
}
func TestGenerateAssistant_StreamSilenceTimeoutRetryClassification(t *testing.T) {
t.Parallel()
t.Run("timeout while opening stream", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
const silenceTimeout = 5 * time.Millisecond
clock := quartz.NewMock(t)
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
defer trap.Close()
var calls atomic.Int32
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
if calls.Add(1) == 1 {
<-ctx.Done()
return nil, ctx.Err()
}
return streamFromParts([]fantasy.StreamPart{{
Type: fantasy.StreamPartTypeFinish,
FinishReason: fantasy.FinishReasonStop,
}}), nil
},
}
done := make(chan error, 1)
go func() {
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Clock: clock,
StreamSilenceTimeout: silenceTimeout,
})
done <- err
}()
trap.MustWait(ctx).MustRelease(ctx)
_, waiter := clock.AdvanceNext()
waiter.MustWait(ctx)
require.Error(t, <-done)
require.Equal(t, int32(1), calls.Load())
})
t.Run("timeout before first part", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
const silenceTimeout = 5 * time.Millisecond
clock := quartz.NewMock(t)
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
defer trap.Close()
var calls atomic.Int32
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
calls.Add(1)
return func(yield func(fantasy.StreamPart) bool) {
<-ctx.Done()
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: ctx.Err()})
}, nil
},
}
done := make(chan error, 1)
go func() {
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Clock: clock,
StreamSilenceTimeout: silenceTimeout,
})
done <- err
}()
trap.MustWait(ctx).MustRelease(ctx)
_, waiter := clock.AdvanceNext()
waiter.MustWait(ctx)
err := <-done
require.Error(t, err)
classified := chaterror.Classify(err)
require.Equal(t, codersdk.ChatErrorKindStreamSilenceTimeout, classified.Kind)
require.Equal(t, "openai", classified.Provider)
require.True(t, classified.Retryable)
require.Equal(t, int32(1), calls.Load())
})
t.Run("first part disarms timeout", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
const silenceTimeout = 5 * time.Millisecond
clock := quartz.NewMock(t)
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
defer trap.Close()
var calls atomic.Int32
continueStream := make(chan struct{})
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
calls.Add(1)
return func(yield func(fantasy.StreamPart) bool) {
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}) {
return
}
select {
case <-continueStream:
case <-ctx.Done():
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: ctx.Err()})
return
}
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"})
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"})
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
}, nil
},
}
done := make(chan error, 1)
go func() {
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Clock: clock,
StreamSilenceTimeout: silenceTimeout,
})
done <- err
}()
trap.MustWait(ctx).MustRelease(ctx)
close(continueStream)
require.NoError(t, <-done)
require.Equal(t, int32(1), calls.Load())
})
t.Run("silent stream close after timeout", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
const silenceTimeout = 5 * time.Millisecond
clock := quartz.NewMock(t)
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
defer trap.Close()
var calls atomic.Int32
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
calls.Add(1)
return func(func(fantasy.StreamPart) bool) {
<-ctx.Done()
}, nil
},
}
done := make(chan error, 1)
go func() {
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
Clock: clock,
StreamSilenceTimeout: silenceTimeout,
})
done <- err
}()
trap.MustWait(ctx).MustRelease(ctx)
_, waiter := clock.AdvanceNext()
waiter.MustWait(ctx)
err := <-done
require.Error(t, err)
classified := chaterror.Classify(err)
require.Equal(t, codersdk.ChatErrorKindStreamSilenceTimeout, classified.Kind)
require.Equal(t, int32(1), calls.Load())
})
}
func TestGenerateAssistant_PanicInPublishMessagePartReleasesAttempt(t *testing.T) {
t.Parallel()
attemptReleased := make(chan struct{})
model := &chattest.FakeModel{
ProviderName: "openai",
ModelName: "test-model",
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
go func() {
<-ctx.Done()
close(attemptReleased)
}()
return streamFromParts([]fantasy.StreamPart{
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "boom"},
}), nil
},
}
defer func() {
r := recover()
require.NotNil(t, r)
select {
case <-attemptReleased:
case <-time.After(time.Second):
t.Fatal("attempt context was not released after panic")
}
}()
_, _ = GenerateAssistant(context.Background(), GenerateAssistantOptions{
Model: model,
PublishMessagePart: func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {
panic("publish panic")
},
})
t.Fatal("expected GenerateAssistant to panic")
}
func requireToolResultErrorMessage(
t *testing.T,
result fantasy.ToolResultContent,
expected string,
) {
t.Helper()
output, ok := result.Result.(fantasy.ToolResultOutputContentError)
require.Truef(t, ok, "expected error tool result, got %T", result.Result)
require.Error(t, output.Error)
require.Equal(t, expected, output.Error.Error())
}
func streamFromParts(parts []fantasy.StreamPart) fantasy.StreamResponse {
return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
for _, part := range parts {
if !yield(part) {
return
}
}
})
}
func textMessage(role fantasy.MessageRole, text string) fantasy.Message {
return fantasy.Message{
Role: role,
Content: []fantasy.MessagePart{
fantasy.TextPart{Text: text},
},
}
}
func requireTextContent(t *testing.T, content []fantasy.Content, text string) fantasy.TextContent {
t.Helper()
for _, block := range content {
textContent, ok := fantasy.AsContentType[fantasy.TextContent](block)
if ok && textContent.Text == text {
return textContent
}
}
t.Fatalf("missing text content %q", text)
return fantasy.TextContent{}
}
func TestExclusiveToolPolicy_MixedBatchErrors(t *testing.T) {
t.Parallel()
results, violated := applyExclusiveToolPolicy(
[]fantasy.ToolCallContent{
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
{ToolCallID: "read-1", ToolName: "read_file", Input: `{"path":"main.go"}`},
},
map[string]bool{"advisor": true},
NopMetrics(),
"fake",
"",
)
require.True(t, violated)
require.Len(t, results, 2)
require.Equal(t, "advisor-1", results[0].ToolCallID)
require.Equal(t, "read-1", results[1].ToolCallID)
requireToolResultErrorMessage(
t,
results[0],
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
)
requireToolResultErrorMessage(
t,
results[1],
"this tool was skipped because advisor must run alone in its batch. Retry your tool calls without advisor, or call advisor separately first.",
)
}
func TestApplyExclusiveToolPolicy_RecordsErrorMetrics(t *testing.T) {
t.Parallel()
reg := prometheus.NewPedanticRegistry()
m := NewMetrics(reg)
_, violated := applyExclusiveToolPolicy(
[]fantasy.ToolCallContent{
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
{ToolCallID: "read-1", ToolName: "read_file", Input: `{"path":"main.go"}`},
},
map[string]bool{"advisor": true},
m,
"fake",
"claude-test",
)
require.True(t, violated)
require.Equal(t, 1.0, promtestutil.ToFloat64(
m.ToolErrorsTotal.WithLabelValues("fake", "claude-test", "advisor"),
))
require.Equal(t, 1.0, promtestutil.ToFloat64(
m.ToolErrorsTotal.WithLabelValues("fake", "claude-test", "read_file"),
))
}
func TestExclusiveToolPolicy_MultipleExclusive(t *testing.T) {
t.Parallel()
results, violated := applyExclusiveToolPolicy(
[]fantasy.ToolCallContent{
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
{ToolCallID: "advisor-2", ToolName: "advisor", Input: `{"mode":"second-opinion"}`},
},
map[string]bool{"advisor": true},
NopMetrics(),
"fake",
"",
)
require.True(t, violated)
require.Len(t, results, 2)
requireToolResultErrorMessage(
t,
results[0],
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
)
requireToolResultErrorMessage(
t,
results[1],
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
)
}
func TestSanitizeAnthropicProviderToolContent(t *testing.T) {
t.Parallel()
providerCall := func(id, name, input string) fantasy.ToolCallContent {
return fantasy.ToolCallContent{
ToolCallID: id,
ToolName: name,
Input: input,
ProviderExecuted: true,
}
}
providerResult := func(id, name string) fantasy.ToolResultContent {
return fantasy.ToolResultContent{
ToolCallID: id,
ToolName: name,
ProviderExecuted: true,
ProviderMetadata: validWebSearchProviderMetadataForTest(),
Result: fantasy.ToolResultOutputContentText{Text: "ok"},
}
}
localCall := func(id, name string) fantasy.ToolCallContent {
return fantasy.ToolCallContent{
ToolCallID: id,
ToolName: name,
Input: `{}`,
}
}
localResult := func(id, name string) fantasy.ToolResultContent {
return fantasy.ToolResultContent{
ToolCallID: id,
ToolName: name,
Result: fantasy.ToolResultOutputContentText{Text: "ok"},
}
}
type contentSummary struct {
providerCalls []string
providerResults []string
localCalls []string
localResults []string
}
summarizeContent := func(content []fantasy.Content) contentSummary {
var summary contentSummary
for _, block := range content {
if toolCall, ok := safeToolCallContent(block); ok {
if toolCall.ProviderExecuted {
summary.providerCalls = append(summary.providerCalls, toolCall.ToolCallID)
} else {
summary.localCalls = append(summary.localCalls, toolCall.ToolCallID)
}
continue
}
if toolResult, ok := safeToolResultContent(block); ok {
if toolResult.ProviderExecuted {
summary.providerResults = append(summary.providerResults, toolResult.ToolCallID)
} else {
summary.localResults = append(summary.localResults, toolResult.ToolCallID)
}
}
}
return summary
}
assertProviderHistoryValid := func(t *testing.T, content []fantasy.Content) {
t.Helper()
parts := make([]fantasy.MessagePart, 0)
for _, block := range content {
if toolCall, ok := safeToolCallContent(block); ok && toolCall.ProviderExecuted {
parts = append(parts, toolCallContentToPart(toolCall))
continue
}
if toolResult, ok := safeToolResultContent(block); ok && toolResult.ProviderExecuted {
parts = append(parts, toolResultContentToPart(toolResult))
}
}
if len(parts) == 0 {
return
}
require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory([]fantasy.Message{
{
Role: fantasy.MessageRoleAssistant,
Content: parts,
},
}))
}
metadataCall := providerCall("ws-meta", "web_search", `{"query":"coder"}`)
metadataCall.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil}
metadataResult := providerResult("ws-meta", "web_search")
metadataResult.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil}
pointerCall := providerCall("ws-pointer", "web_search", `{"query":"coder"}`)
var nilToolCall *fantasy.ToolCallContent
testCases := []struct {
name string
provider string
content []fantasy.Content
wantSummary contentSummary
wantRemovedCalls int
wantRemovedResults int
wantTexts []string
validateAnthropic bool
}{
{
name: "orphan provider result textified",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
fantasy.TextContent{Text: "keep"},
providerResult("ws-1", "web_search"),
},
wantRemovedResults: 1,
wantTexts: []string{"keep", "ok"},
validateAnthropic: true,
},
{
name: "result before call removes both provider blocks",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerResult("ws-1", "web_search"),
providerCall("ws-1", "web_search", `{"query":"coder"}`),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "valid web search pair preserved",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("ws-1", "web_search", `{"query":"coder"}`),
providerResult("ws-1", "web_search"),
fantasy.TextContent{Text: "search done"},
},
wantSummary: contentSummary{
providerCalls: []string{"ws-1"},
providerResults: []string{"ws-1"},
},
wantTexts: []string{"search done"},
validateAnthropic: true,
},
{
name: "invalid JSON provider call drops pair",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("ws-1", "web_search", `{`),
providerResult("ws-1", "web_search"),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "empty ID provider call drops pair",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("", "web_search", `{"query":"coder"}`),
providerResult("", "web_search"),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "empty tool name provider call drops pair",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("ws-empty", "", `{"query":"coder"}`),
providerResult("ws-empty", ""),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "non web search provider pair drops through serializable helper",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("code-1", "code_execution", `{"code":"print(1)"}`),
providerResult("code-1", "code_execution"),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "mismatched provider result tool name drops pair",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("ws-mismatch", "web_search", `{"query":"coder"}`),
providerResult("ws-mismatch", "code_execution"),
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "duplicate provider IDs drop all provider content for ID",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("dup-1", "web_search", `{"query":"coder"}`),
providerResult("dup-1", "web_search"),
providerCall("dup-1", "web_search", `{"query":"coder"}`),
},
wantRemovedCalls: 2,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "mismatched provider flags remove only provider side",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
providerCall("mix-1", "web_search", `{"query":"coder"}`),
localResult("mix-1", "web_search"),
localCall("mix-2", "read_file"),
providerResult("mix-2", "web_search"),
},
wantSummary: contentSummary{
localCalls: []string{"mix-2"},
localResults: []string{"mix-1"},
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "malformed provider metadata textifies result",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
metadataCall,
metadataResult,
},
wantRemovedCalls: 1,
wantRemovedResults: 1,
wantTexts: []string{"ok"},
validateAnthropic: true,
},
{
name: "pointer and nil pointer variants are handled safely",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
nilToolCall,
&pointerCall,
providerResult("ws-pointer", "web_search"),
},
wantSummary: contentSummary{
providerCalls: []string{"ws-pointer"},
providerResults: []string{"ws-pointer"},
},
validateAnthropic: true,
},
{
name: "local tool content is unchanged",
provider: fantasyanthropic.Name,
content: []fantasy.Content{
localCall("tc-1", "read_file"),
localResult("tc-1", "read_file"),
},
wantSummary: contentSummary{
localCalls: []string{"tc-1"},
localResults: []string{"tc-1"},
},
validateAnthropic: true,
},
{
name: "non Anthropic provider content is unchanged",
provider: "fake",
content: []fantasy.Content{
providerCall("ws-1", "web_search", `{"query":"coder"}`),
},
wantSummary: contentSummary{
providerCalls: []string{"ws-1"},
},
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
sanitized, stats := chatsanitize.SanitizeAnthropicProviderToolContent(tc.provider, tc.content)
require.Equal(t, tc.wantRemovedCalls, stats.RemovedToolCalls)
require.Equal(t, tc.wantRemovedResults, stats.RemovedToolResults)
require.Zero(t, stats.DroppedMessages)
summary := summarizeContent(sanitized)
assert.ElementsMatch(t, tc.wantSummary.providerCalls, summary.providerCalls)
assert.ElementsMatch(t, tc.wantSummary.providerResults, summary.providerResults)
assert.ElementsMatch(t, tc.wantSummary.localCalls, summary.localCalls)
assert.ElementsMatch(t, tc.wantSummary.localResults, summary.localResults)
for _, text := range tc.wantTexts {
requireTextContent(t, sanitized, text)
}
if tc.validateAnthropic {
assertProviderHistoryValid(t, sanitized)
}
})
}
}
type serialMarkerTool struct{ fantasy.AgentTool }
func (serialMarkerTool) SerialToolCalls() bool { return true }
type observerMarkerTool struct {
fantasy.AgentTool
observed func(names []string)
}
func (t observerMarkerTool) ObserveStepToolCalls(names []string) { t.observed(names) }
func TestExecuteToolsNotifiesStepToolCallObservers(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var observedNames []string
observedBeforeRun := false
observer := observerMarkerTool{
AgentTool: fantasy.NewAgentTool(
"observer_tool",
"records sibling calls",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
mu.Lock()
observedBeforeRun = observedNames != nil
mu.Unlock()
return fantasy.NewTextResponse("ok"), nil
},
),
observed: func(names []string) {
mu.Lock()
defer mu.Unlock()
observedNames = append([]string{}, names...)
},
}
var uncalledObserved atomic.Bool
uncalledObserver := observerMarkerTool{
AgentTool: fantasy.NewAgentTool(
"uncalled_observer",
"never called this step",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("ok"), nil
},
),
observed: func([]string) { uncalledObserved.Store(true) },
}
other := fantasy.NewAgentTool(
"other_tool",
"plain tool",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("ok"), nil
},
)
newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{observer, uncalledObserver, other},
ObservedToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
{ToolCallID: "3", ToolName: "denied_tool", Input: "{}"},
},
Clock: quartz.NewReal(),
}, "fake", "fake-model").run(context.Background(), []fantasy.ToolCallContent{
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
}, time.Time{})
require.Equal(t, []string{"observer_tool", "other_tool", "denied_tool"}, observedNames,
"a called observer sees every observed tool-call name, including calls denied before execution")
require.True(t, observedBeforeRun, "observers are notified before any tool call executes")
require.False(t, uncalledObserved.Load(), "tools not called this step are not notified")
}
type resultObserverMarkerTool struct {
fantasy.AgentTool
observedResults func(names []string, errored []bool)
}
func (t resultObserverMarkerTool) ObserveStepToolResults(names []string, errored []bool) {
t.observedResults(names, errored)
}
func TestExecuteToolsNotifiesStepToolResultObservers(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var gotNames []string
var gotErrored []bool
notifications := 0
observer := resultObserverMarkerTool{
AgentTool: fantasy.NewAgentTool(
"observer_tool",
"records sibling outcomes",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("ok"), nil
},
),
observedResults: func(names []string, errored []bool) {
mu.Lock()
defer mu.Unlock()
notifications++
gotNames = append([]string{}, names...)
gotErrored = append([]bool{}, errored...)
},
}
failing := fantasy.NewAgentTool(
"failing_tool",
"returns an error result",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextErrorResponse("remote error"), nil
},
)
executed := []fantasy.ToolCallContent{
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
{ToolCallID: "3", ToolName: "missing_tool", Input: "{}"},
}
newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{observer, failing},
ObservedToolCalls: append(slices.Clone(executed), fantasy.ToolCallContent{
ToolCallID: "4", ToolName: "rejected_tool", Input: "{not json",
}),
Clock: quartz.NewReal(),
}, "fake", "fake-model").run(context.Background(), executed, time.Time{})
require.Equal(t, 1, notifications, "each called observer is notified once per step")
require.Equal(t, []string{"observer_tool", "failing_tool", "missing_tool", "rejected_tool"}, gotNames,
"outcomes are reported per call in observed order")
require.Equal(t, []bool{false, true, true, true}, gotErrored,
"error results, unresolvable tools, and observed calls rejected before execution all settle as errored outcomes")
}
type serialResultObserverTool struct {
fantasy.AgentTool
observedResults func(names []string, errored []bool)
}
func (serialResultObserverTool) SerialToolCalls() bool { return true }
func (t serialResultObserverTool) ObserveStepToolResults(names []string, errored []bool) {
t.observedResults(names, errored)
}
func TestExecuteToolsReconcilesResultsBeforeSerialCalls(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var erroredAtNotify []string
var erroredAtRun []string
notified := false
serial := serialResultObserverTool{
AgentTool: fantasy.NewAgentTool(
"serial_observer",
"observes sibling outcomes before running",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
mu.Lock()
erroredAtRun = append([]string{}, erroredAtNotify...)
mu.Unlock()
return fantasy.NewTextResponse("ok"), nil
},
),
observedResults: func(names []string, errored []bool) {
mu.Lock()
defer mu.Unlock()
notified = true
erroredAtNotify = nil
for i, name := range names {
if errored[i] {
erroredAtNotify = append(erroredAtNotify, name)
}
}
},
}
failing := fantasy.NewAgentTool(
"failing_tool",
"returns an error result",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextErrorResponse("remote error"), nil
},
)
results := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{serial, failing},
Clock: quartz.NewReal(),
}, "fake", "fake-model").run(context.Background(), []fantasy.ToolCallContent{
{ToolCallID: "1", ToolName: "serial_observer", Input: "{}"},
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
}, time.Time{})
require.True(t, notified)
require.Equal(t, []string{"failing_tool"}, erroredAtRun,
"a serial tool must see settled sibling outcomes before it executes")
require.Len(t, results, 2)
require.Equal(t, "1", results[0].content.ToolCallID, "results keep original call order")
}
type haltingMarkerTool struct{ fantasy.AgentTool }
func (haltingMarkerTool) SerialToolCalls() bool { return true }
func (haltingMarkerTool) HaltedToolCallError() (string, bool) { return "halted", true }
func TestExecuteToolsHaltsToolAfterFailedCall(t *testing.T) {
t.Parallel()
const (
succeed = `{}`
fail = `{"fail":true}`
)
halting := func(id, input string) fantasy.ToolCallContent {
return fantasy.ToolCallContent{ToolCallID: id, ToolName: "halting_tool", Input: input}
}
other := fantasy.ToolCallContent{ToolCallID: "3", ToolName: "other_tool", Input: succeed}
tests := []struct {
name string
observed []fantasy.ToolCallContent
executed []fantasy.ToolCallContent
wantRan []string
// wantResults maps each executed call ID to its result text.
wantResults map[string]string
}{
{
name: "FailedCall",
executed: []fantasy.ToolCallContent{halting("1", succeed), halting("2", fail), other, halting("4", succeed)},
wantRan: []string{"1", "2", "3"},
wantResults: map[string]string{"1": "ok", "2": "error: failed", "3": "ok", "4": "error: halted"},
},
{
name: "RejectedCall",
observed: []fantasy.ToolCallContent{halting("1", succeed), halting("2", succeed), other, halting("4", succeed)},
executed: []fantasy.ToolCallContent{halting("1", succeed), other, halting("4", succeed)},
wantRan: []string{"1", "3"},
wantResults: map[string]string{"1": "ok", "3": "ok", "4": "error: halted"},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var ran []string
record := func(id string) {
mu.Lock()
defer mu.Unlock()
ran = append(ran, id)
}
haltingTool := haltingMarkerTool{AgentTool: fantasy.NewAgentTool(
"halting_tool",
"fails when asked",
func(_ context.Context, input struct {
Fail bool `json:"fail"`
}, call fantasy.ToolCall,
) (fantasy.ToolResponse, error) {
record(call.ID)
if input.Fail {
return fantasy.NewTextErrorResponse("failed"), nil
}
return fantasy.NewTextResponse("ok"), nil
},
)}
otherTool := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
"other_tool",
"runs between halting calls",
func(_ context.Context, _ struct{}, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
record(call.ID)
return fantasy.NewTextResponse("ok"), nil
},
)}
outcome, err := ExecuteLocalTools(context.Background(), ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{haltingTool, otherTool},
ToolCalls: tt.executed,
ObservedToolCalls: tt.observed,
Clock: quartz.NewReal(),
})
require.NoError(t, err)
require.Equal(t, tt.wantRan, ran)
results := make(map[string]string)
for _, content := range outcome.Content {
result, ok := content.(fantasy.ToolResultContent)
require.True(t, ok, "%T", content)
switch output := result.Result.(type) {
case fantasy.ToolResultOutputContentText:
results[result.ToolCallID] = output.Text
case fantasy.ToolResultOutputContentError:
results[result.ToolCallID] = "error: " + output.Error.Error()
default:
t.Fatalf("unexpected result %T", result.Result)
}
}
require.Equal(t, tt.wantResults, results)
require.Equal(t, len(tt.wantRan), outcome.BatchBilledCalls, "halted calls are not billed")
})
}
}
func TestExecuteToolsSerialToolCallOrder(t *testing.T) {
t.Parallel()
var mu sync.Mutex
var events []string
inFlight := 0
maxInFlight := 0
record := func(event string, delta int) {
mu.Lock()
defer mu.Unlock()
inFlight += delta
if inFlight > maxInFlight {
maxInFlight = inFlight
}
events = append(events, event)
}
serial := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
"serial_tool",
"records call order",
func(_ context.Context, _ struct{}, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
record(call.ID+":start", 1)
runtime.Gosched()
record(call.ID+":end", -1)
return fantasy.NewTextResponse("ok"), nil
},
)}
parallelRan := make(chan struct{})
parallel := fantasy.NewAgentTool(
"parallel_tool",
"plain tool",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
close(parallelRan)
return fantasy.NewTextResponse("ok"), nil
},
)
calls := []fantasy.ToolCallContent{
{ToolCallID: "a", ToolName: "serial_tool", Input: "{}"},
{ToolCallID: "p", ToolName: "parallel_tool", Input: "{}"},
{ToolCallID: "b", ToolName: "serial_tool", Input: "{}"},
{ToolCallID: "c", ToolName: "serial_tool", Input: "{}"},
}
results := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{serial, parallel},
Clock: quartz.NewReal(),
}, "fake", "fake-model").run(context.Background(), calls, time.Time{})
require.Equal(t, []string{"a:start", "a:end", "b:start", "b:end", "c:start", "c:end"}, events,
"serial tool calls must run one at a time in tool-call order")
require.Equal(t, 1, maxInFlight)
select {
case <-parallelRan:
default:
t.Fatal("parallel tool call did not run")
}
require.Len(t, results, len(calls))
for i, tc := range calls {
require.Equal(t, tc.ToolCallID, results[i].content.ToolCallID, "results keep original call order")
}
}
type timingEvent struct {
dispatchIndex int
at time.Time
}
// liveToolBillingRecorder is a test ToolBillingRecorder that publishes
// start and completion timestamps as they happen.
type liveToolBillingRecorder struct {
started chan timingEvent
completed chan timingEvent
}
func (r liveToolBillingRecorder) RecordStart(dispatchIndex int, startedAt time.Time) {
r.started <- timingEvent{dispatchIndex: dispatchIndex, at: startedAt}
}
func (r liveToolBillingRecorder) RecordComplete(dispatchIndex int, completedAt time.Time) {
r.completed <- timingEvent{dispatchIndex: dispatchIndex, at: completedAt}
}
func TestExecuteToolsReturnsExecutionIntervals(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
started := make(chan timingEvent, 3)
completed := make(chan timingEvent, 3)
slowGo := make(chan struct{})
fastGo := make(chan struct{})
blocking := func(name string, release <-chan struct{}) fantasy.AgentTool {
return fantasy.NewAgentTool(
name,
"waits for the test to release it",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
<-release
return fantasy.NewTextResponse("ok"), nil
},
)
}
slow := blocking("slow_tool", slowGo)
fast := blocking("fast_tool", fastGo)
serial := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
"serial_tool",
"runs after concurrent tools",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("ok"), nil
},
)}
calls := []fantasy.ToolCallContent{
{ToolCallID: "slow", ToolName: "slow_tool", Input: "{}"},
{ToolCallID: "fast", ToolName: "fast_tool", Input: "{}"},
{ToolCallID: "serial", ToolName: "serial_tool", Input: "{}"},
}
batchStart := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC)
resultsCh := make(chan []toolExecutionResult, 1)
go func() {
resultsCh <- newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{slow, fast, serial},
BillingRecorder: liveToolBillingRecorder{started: started, completed: completed},
Clock: quartz.NewReal(),
}, "fake", "fake-model").run(ctx, calls, batchStart)
}()
startByIndex := make([]time.Time, len(calls))
for range 2 {
event := testutil.RequireReceive(ctx, t, started)
startByIndex[event.dispatchIndex] = event.at
}
close(fastGo)
fastCompleted := testutil.RequireReceive(ctx, t, completed)
require.Equal(t, 1, fastCompleted.dispatchIndex)
close(slowGo)
slowCompleted := testutil.RequireReceive(ctx, t, completed)
require.Equal(t, 0, slowCompleted.dispatchIndex)
serialStarted := testutil.RequireReceive(ctx, t, started)
require.Equal(t, 2, serialStarted.dispatchIndex)
startByIndex[serialStarted.dispatchIndex] = serialStarted.at
serialCompleted := testutil.RequireReceive(ctx, t, completed)
require.Equal(t, 2, serialCompleted.dispatchIndex)
endByIndex := []time.Time{slowCompleted.at, fastCompleted.at, serialCompleted.at}
results := testutil.RequireReceive(ctx, t, resultsCh)
require.Len(t, results, len(calls))
for i, call := range calls {
require.Equal(t, call.ToolCallID, results[i].content.ToolCallID,
"results keep dispatch order when calls complete out of order")
require.Equal(t, startByIndex[i], results[i].interval.Start)
require.Equal(t, endByIndex[i], results[i].interval.End)
}
require.Equal(t, batchStart, results[0].interval.Start)
require.Equal(t, batchStart, results[1].interval.Start)
require.NotEqual(t, batchStart, results[2].interval.Start)
require.False(t, results[2].interval.Start.Before(slowCompleted.at),
"serial execution starts only after concurrent calls settle")
}
func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
t.Parallel()
originalBytes := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10}
t.Run("EncodesRawBytesToBase64", func(t *testing.T) {
t.Parallel()
tool := fantasy.NewAgentTool(
"screenshot",
"takes a screenshot",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.ToolResponse{
Type: "image",
Data: originalBytes,
MediaType: "image/jpeg",
Filename: "photo.jpg",
}, nil
},
)
tc := fantasy.ToolCallContent{
ToolCallID: "call-1",
ToolName: "screenshot",
Input: "{}",
}
result, _ := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"screenshot"},
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
media, ok := result.Result.(fantasy.ToolResultOutputContentMedia)
require.True(t, ok, "expected ToolResultOutputContentMedia")
require.Equal(t, "image/jpeg", media.MediaType)
require.Equal(t, "photo.jpg", media.Filename)
decoded, err := base64.StdEncoding.DecodeString(media.Data)
require.NoError(t, err, "Data should be valid base64")
require.Equal(t, originalBytes, decoded)
})
t.Run("SanitizesInvalidUTF8InContent", func(t *testing.T) {
t.Parallel()
tool := fantasy.NewAgentTool(
"screenshot",
"takes a screenshot",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.ToolResponse{
Type: "image",
Data: originalBytes,
MediaType: "image/png",
Content: "hello\xffworld",
}, nil
},
)
tc := fantasy.ToolCallContent{
ToolCallID: "call-2",
ToolName: "screenshot",
Input: "{}",
}
result, _ := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"screenshot"},
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
media, ok := result.Result.(fantasy.ToolResultOutputContentMedia)
require.True(t, ok, "expected ToolResultOutputContentMedia")
require.True(t, utf8.ValidString(media.Text), "Text should be valid UTF-8")
require.Contains(t, media.Text, "hello")
require.Contains(t, media.Text, "world")
})
t.Run("SanitizesInvalidUTF8InTextResult", func(t *testing.T) {
t.Parallel()
tool := fantasy.NewAgentTool(
"echo",
"echoes input",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.ToolResponse{
Content: "hello\xffworld",
}, nil
},
)
tc := fantasy.ToolCallContent{
ToolCallID: "call-3",
ToolName: "echo",
Input: "{}",
}
result, _ := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"echo"},
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
textOutput, ok := result.Result.(fantasy.ToolResultOutputContentText)
require.True(t, ok, "expected ToolResultOutputContentText, got %T", result.Result)
require.True(t, utf8.ValidString(textOutput.Text), "Text should be valid UTF-8")
require.Contains(t, textOutput.Text, "hello")
require.Contains(t, textOutput.Text, "world")
})
}
func TestExecuteSingleTool_NormalizesMedia(t *testing.T) {
t.Parallel()
var pngData bytes.Buffer
require.NoError(t, png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 1, 1))))
for _, tc := range []struct {
name string
mediaType string
data []byte
// wantMediaType is empty when the media is rejected and only text survives.
wantMediaType string
wantText string
}{
{name: "OversizedMediaKeepsText", mediaType: "image/png", data: make([]byte, codersdk.MaxChatFileSizeBytes+1), wantText: "Ran Playwright code\n[image/png content omitted"},
{name: "NonImageBytesDeclaredAsImageKeepsText", mediaType: "image/png", data: []byte("<html>not an image</html>"), wantText: "Ran Playwright code\n[image omitted: payload declared as image/png is text/html"},
{name: "ImageTypeFollowsBytes", mediaType: "image/jpeg", data: pngData.Bytes(), wantMediaType: "image/png", wantText: "Ran Playwright code"},
{name: "NonImageMediaPassesThrough", mediaType: "audio/mpeg", data: []byte{1, 2, 3}, wantMediaType: "audio/mpeg", wantText: "Ran Playwright code"},
} {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
tool := fantasy.NewAgentTool("screenshot", "takes a screenshot",
func(context.Context, struct{}, fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.ToolResponse{Type: "media", Data: tc.data, MediaType: tc.mediaType, Content: "Ran Playwright code"}, nil
},
)
result, _ := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"screenshot"},
}, "openai", "model").runOne(
context.Background(),
fantasy.ToolCallContent{ToolCallID: "call-1", ToolName: "screenshot", Input: "{}"},
nil,
)
switch out := result.Result.(type) {
case fantasy.ToolResultOutputContentMedia:
require.Equal(t, tc.wantMediaType, out.MediaType)
require.Equal(t, tc.wantText, out.Text)
case fantasy.ToolResultOutputContentText:
require.Empty(t, tc.wantMediaType, "media rejected: %s", out.Text)
require.Contains(t, out.Text, tc.wantText)
default:
t.Fatalf("unexpected result %T", out)
}
})
}
}
func TestExecuteSingleTool_RejectsUnknownTool(t *testing.T) {
t.Parallel()
tc := fantasy.ToolCallContent{
ToolCallID: "call-missing",
ToolName: "close_agent",
Input: "{}",
}
result, _ := newToolBatch(ExecuteLocalToolsOptions{
ActiveTools: []string{"interrupt_agent"},
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
errOutput, ok := result.Result.(fantasy.ToolResultOutputContentError)
require.True(t, ok, "expected error output, got %T", result.Result)
require.EqualError(t, errOutput.Error, "Tool not active in this turn: close_agent")
}
func TestExecuteSingleTool_AllowsDeferredDirectCall(t *testing.T) {
t.Parallel()
tool := fantasy.NewAgentTool(
"server__direct",
"direct",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("ok"), nil
},
)
result, _ := newToolBatch(ExecuteLocalToolsOptions{
Tools: []fantasy.AgentTool{tool},
ActiveTools: []string{"find_tools"},
AllowInactiveTools: map[string]bool{"server__direct": true},
}, "fake", "fake-model").runOne(
context.Background(),
fantasy.ToolCallContent{ToolCallID: "call-direct", ToolName: "server__direct", Input: "{}"},
nil,
)
text, ok := result.Result.(fantasy.ToolResultOutputContentText)
require.True(t, ok)
require.Equal(t, "ok", text.Text)
}
func TestExecuteLocalTools_ProviderToolRunner(t *testing.T) {
t.Parallel()
runner := fantasy.NewAgentTool(
"computer",
"runs a provider-defined tool locally",
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
return fantasy.NewTextResponse("screenshot taken"), nil
},
)
metadata := fantasy.ProviderMetadata{
fantasyopenai.Name: &fantasyopenai.ComputerCallOutputOptions{Detail: "original"},
}
step, err := ExecuteLocalTools(context.Background(), ExecuteLocalToolsOptions{
// The runner is neither a regular tool nor an active one.
ActiveTools: []string{"read_file"},
ProviderTools: []ProviderTool{
{Definition: fantasy.ProviderDefinedTool{ID: "web_search", Name: "web_search"}},
{
Runner: runner,
ResultProviderMetadata: func(fantasy.ToolResponse) fantasy.ProviderMetadata {
return metadata
},
},
},
ToolCalls: []fantasy.ToolCallContent{
{ToolCallID: "call-search", ToolName: "web_search", Input: "{}", ProviderExecuted: true},
{ToolCallID: "call-computer", ToolName: "computer", Input: "{}"},
},
Clock: quartz.NewReal(),
})
require.NoError(t, err)
require.Len(t, step.Content, 1, "provider-executed calls are skipped")
result, ok := step.Content[0].(fantasy.ToolResultContent)
require.True(t, ok)
require.Equal(t, "call-computer", result.ToolCallID)
require.Equal(t, fantasy.ToolResultOutputContentText{Text: "screenshot taken"}, result.Result)
require.Equal(t, metadata, result.ProviderMetadata)
}