Part of [CODAGT-1430](https://linear.app/codercom/issue/CODAGT-1430). This is the first of four stacked PRs. Today a PDF in an MCP tool result reaches no provider. On Anthropic we strip it in chatd. On OpenAI Chat Completions fantasy swaps it for a "cannot be displayed" note, and on Responses it sends a placeholder that points at a user message it never sends. coder/fantasy#73 and coder/fantasy#74 make fantasy convert tool result media with the converter it already uses for user files: an Anthropic document inside `tool_result`, an `input_file` inside `function_call_output` on Responses, and a user file part straight after the tool message on Chat Completions. Both have merged, and this PR pins `charm.land/fantasy` to the coder_2_33 tip that has them. Chat Completions file parts now refuse text files, as Responses already did, so `AcceptsFilePartMediaType` follows. The converters take a filename, so an MCP embedded resource blob is now named after the last element of its URI path, and we store the name with the tool result so later turns send the same one. OpenAI picks the file type from the extension, so we append the right one when it's missing or wrong. We drop names with control characters, as Postgres `jsonb` rejects NUL, and we redact the name like the rest of the tool response, since a URI like `https://host/sse/<token>` would otherwise put the token in the chat history. There's also a test proving aibridge's Chat Completions re-encode keeps a user file part, as that's where these PDFs end up. Anthropic still strips tool result PDFs until the next PR in the stack.
1630 lines
50 KiB
Go
1630 lines
50 KiB
Go
package chatloop
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/base64"
|
|
"errors"
|
|
"image"
|
|
"image/png"
|
|
"iter"
|
|
"runtime"
|
|
"slices"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
"unicode/utf8"
|
|
|
|
"charm.land/fantasy"
|
|
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
|
fantasyopenai "charm.land/fantasy/providers/openai"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatsanitize"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func validWebSearchProviderMetadataForTest() fantasy.ProviderMetadata {
|
|
return fantasy.ProviderMetadata{
|
|
fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{
|
|
Results: []fantasyanthropic.WebSearchResultItem{
|
|
{
|
|
URL: "https://example.com",
|
|
Title: "Example",
|
|
EncryptedContent: "encrypted",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func safeToolCallContent(block fantasy.Content) (fantasy.ToolCallContent, bool) {
|
|
var zero fantasy.ToolCallContent
|
|
switch value := block.(type) {
|
|
case fantasy.ToolCallContent:
|
|
return value, true
|
|
case *fantasy.ToolCallContent:
|
|
if value == nil {
|
|
return zero, false
|
|
}
|
|
return *value, true
|
|
default:
|
|
return zero, false
|
|
}
|
|
}
|
|
|
|
func safeToolResultContent(block fantasy.Content) (fantasy.ToolResultContent, bool) {
|
|
var zero fantasy.ToolResultContent
|
|
switch value := block.(type) {
|
|
case fantasy.ToolResultContent:
|
|
return value, true
|
|
case *fantasy.ToolResultContent:
|
|
if value == nil {
|
|
return zero, false
|
|
}
|
|
return *value, true
|
|
default:
|
|
return zero, false
|
|
}
|
|
}
|
|
|
|
func toolCallContentToPart(toolCall fantasy.ToolCallContent) fantasy.ToolCallPart {
|
|
return fantasy.ToolCallPart{
|
|
ToolCallID: toolCall.ToolCallID,
|
|
ToolName: toolCall.ToolName,
|
|
Input: toolCall.Input,
|
|
ProviderExecuted: toolCall.ProviderExecuted,
|
|
ProviderOptions: fantasy.ProviderOptions(toolCall.ProviderMetadata),
|
|
}
|
|
}
|
|
|
|
func toolResultContentToPart(toolResult fantasy.ToolResultContent) fantasy.ToolResultPart {
|
|
return fantasy.ToolResultPart{
|
|
ToolCallID: toolResult.ToolCallID,
|
|
Output: toolResult.Result,
|
|
ProviderExecuted: toolResult.ProviderExecuted,
|
|
ProviderOptions: fantasy.ProviderOptions(toolResult.ProviderMetadata),
|
|
}
|
|
}
|
|
|
|
func TestStreamSilenceGuard_DisarmAndFireRace(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for range 128 {
|
|
var cancels atomic.Int32
|
|
guard := newStreamSilenceGuard(quartz.NewReal(), time.Hour, func(err error) {
|
|
if errors.Is(err, errStreamSilenceTimeout) {
|
|
cancels.Add(1)
|
|
}
|
|
})
|
|
|
|
start := make(chan struct{})
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
guard.onTimeout()
|
|
}()
|
|
|
|
go func() {
|
|
defer wg.Done()
|
|
<-start
|
|
guard.Disarm()
|
|
}()
|
|
|
|
close(start)
|
|
wg.Wait()
|
|
|
|
guard.onTimeout()
|
|
guard.Disarm()
|
|
|
|
require.LessOrEqual(t, cancels.Load(), int32(1))
|
|
}
|
|
}
|
|
|
|
func TestStreamSilenceGuard_DisarmPreservesPermanentError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
attemptCtx, cancelAttempt := context.WithCancelCause(context.Background())
|
|
defer cancelAttempt(nil)
|
|
|
|
guard := newStreamSilenceGuard(quartz.NewReal(), time.Hour, cancelAttempt)
|
|
guard.Disarm()
|
|
guard.onTimeout()
|
|
|
|
classified := chaterror.Classify(classifyStreamSilenceTimeout(
|
|
attemptCtx,
|
|
"openai",
|
|
xerrors.New("invalid model"),
|
|
))
|
|
require.Equal(t, codersdk.ChatErrorKindConfig, classified.Kind)
|
|
require.False(t, classified.Retryable)
|
|
require.Nil(t, context.Cause(attemptCtx))
|
|
}
|
|
|
|
func TestGenerateAssistant_ProviderContextSurvivesStreamError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return nil, xerrors.New("upstream returned status 400")
|
|
},
|
|
}
|
|
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Messages: []fantasy.Message{
|
|
textMessage(fantasy.MessageRoleUser, "hello"),
|
|
},
|
|
})
|
|
require.Error(t, err)
|
|
classified := chaterror.Classify(err)
|
|
require.Equal(t, "openai", classified.Provider)
|
|
require.Equal(t, "OpenAI returned an unexpected error.", classified.Message)
|
|
}
|
|
|
|
func TestGenerateAssistant_ProviderResponseID(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
cases := []struct {
|
|
name string
|
|
finish fantasy.StreamPart
|
|
want string
|
|
}{
|
|
{
|
|
name: "anthropic message ID",
|
|
finish: fantasy.StreamPart{ID: "msg_anthropic"},
|
|
want: "msg_anthropic",
|
|
},
|
|
{
|
|
name: "openai responses metadata",
|
|
finish: fantasy.StreamPart{ProviderMetadata: fantasy.ProviderMetadata{
|
|
fantasyopenai.Name: &fantasyopenai.ResponsesProviderMetadata{ResponseID: "resp_openai"},
|
|
}},
|
|
want: "resp_openai",
|
|
},
|
|
{
|
|
name: "not reported",
|
|
},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
finish := tc.finish
|
|
finish.Type = fantasy.StreamPartTypeFinish
|
|
finish.FinishReason = fantasy.FinishReasonStop
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "fake",
|
|
ModelName: "fake-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text", Delta: "hi"},
|
|
{Type: fantasy.StreamPartTypeTextEnd, ID: "text"},
|
|
finish,
|
|
}), nil
|
|
},
|
|
}
|
|
|
|
outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Messages: []fantasy.Message{textMessage(fantasy.MessageRoleUser, "hello")},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, tc.want, outcome.Step.ProviderResponseID)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateAssistant_ErrorProviderOverridesTransportLabel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Bedrock routed through aibridge uses the Anthropic transport, so
|
|
// Model.Provider() reports "anthropic". The user-facing error must use
|
|
// the configured provider from ErrorProvider instead.
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "anthropic",
|
|
ModelName: "qwen3-coder-next",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return nil, xerrors.New("upstream returned status 400")
|
|
},
|
|
}
|
|
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
ErrorProvider: "bedrock",
|
|
Messages: []fantasy.Message{
|
|
textMessage(fantasy.MessageRoleUser, "hello"),
|
|
},
|
|
})
|
|
require.Error(t, err)
|
|
classified := chaterror.Classify(err)
|
|
require.Equal(t, "bedrock", classified.Provider)
|
|
require.Equal(t, "AWS Bedrock returned an unexpected error.", classified.Message)
|
|
}
|
|
|
|
func TestGenerateAssistant_HTTP2TransportErrorClassifiedAsRetryableTimeout(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, provider := range []string{"anthropic", "openai"} {
|
|
provider := provider
|
|
t.Run(provider, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
model := &chattest.FakeModel{
|
|
ProviderName: provider,
|
|
ModelName: "test-model",
|
|
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
return nil, xerrors.New("http2: client connection force closed via ClientConn.Close")
|
|
},
|
|
}
|
|
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
})
|
|
require.Error(t, err)
|
|
classified := chaterror.Classify(err)
|
|
require.Equal(t, codersdk.ChatErrorKindTimeout, classified.Kind)
|
|
require.Equal(t, provider, classified.Provider)
|
|
require.True(t, classified.Retryable)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerateAssistant_StreamSilenceTimeoutRetryClassification(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("timeout while opening stream", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
const silenceTimeout = 5 * time.Millisecond
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
|
|
defer trap.Close()
|
|
var calls atomic.Int32
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
if calls.Add(1) == 1 {
|
|
<-ctx.Done()
|
|
return nil, ctx.Err()
|
|
}
|
|
return streamFromParts([]fantasy.StreamPart{{
|
|
Type: fantasy.StreamPartTypeFinish,
|
|
FinishReason: fantasy.FinishReasonStop,
|
|
}}), nil
|
|
},
|
|
}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
StreamSilenceTimeout: silenceTimeout,
|
|
})
|
|
done <- err
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
_, waiter := clock.AdvanceNext()
|
|
waiter.MustWait(ctx)
|
|
require.Error(t, <-done)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
})
|
|
|
|
t.Run("timeout before first part", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
const silenceTimeout = 5 * time.Millisecond
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
|
|
defer trap.Close()
|
|
var calls atomic.Int32
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
calls.Add(1)
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
<-ctx.Done()
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: ctx.Err()})
|
|
}, nil
|
|
},
|
|
}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
StreamSilenceTimeout: silenceTimeout,
|
|
})
|
|
done <- err
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
_, waiter := clock.AdvanceNext()
|
|
waiter.MustWait(ctx)
|
|
err := <-done
|
|
require.Error(t, err)
|
|
classified := chaterror.Classify(err)
|
|
require.Equal(t, codersdk.ChatErrorKindStreamSilenceTimeout, classified.Kind)
|
|
require.Equal(t, "openai", classified.Provider)
|
|
require.True(t, classified.Retryable)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
})
|
|
|
|
t.Run("first part disarms timeout", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
const silenceTimeout = 5 * time.Millisecond
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
|
|
defer trap.Close()
|
|
var calls atomic.Int32
|
|
continueStream := make(chan struct{})
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
calls.Add(1)
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}) {
|
|
return
|
|
}
|
|
select {
|
|
case <-continueStream:
|
|
case <-ctx.Done():
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeError, Error: ctx.Err()})
|
|
return
|
|
}
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
}, nil
|
|
},
|
|
}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
StreamSilenceTimeout: silenceTimeout,
|
|
})
|
|
done <- err
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
close(continueStream)
|
|
require.NoError(t, <-done)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
})
|
|
|
|
t.Run("silent stream close after timeout", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
const silenceTimeout = 5 * time.Millisecond
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().AfterFunc(streamSilenceGuardTimerTag)
|
|
defer trap.Close()
|
|
var calls atomic.Int32
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
calls.Add(1)
|
|
return func(func(fantasy.StreamPart) bool) {
|
|
<-ctx.Done()
|
|
}, nil
|
|
},
|
|
}
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
_, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
Clock: clock,
|
|
StreamSilenceTimeout: silenceTimeout,
|
|
})
|
|
done <- err
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
_, waiter := clock.AdvanceNext()
|
|
waiter.MustWait(ctx)
|
|
err := <-done
|
|
require.Error(t, err)
|
|
classified := chaterror.Classify(err)
|
|
require.Equal(t, codersdk.ChatErrorKindStreamSilenceTimeout, classified.Kind)
|
|
require.Equal(t, int32(1), calls.Load())
|
|
})
|
|
}
|
|
|
|
func TestGenerateAssistant_PanicInPublishMessagePartReleasesAttempt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
attemptReleased := make(chan struct{})
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "openai",
|
|
ModelName: "test-model",
|
|
StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
|
go func() {
|
|
<-ctx.Done()
|
|
close(attemptReleased)
|
|
}()
|
|
return streamFromParts([]fantasy.StreamPart{
|
|
{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
|
|
{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "boom"},
|
|
}), nil
|
|
},
|
|
}
|
|
|
|
defer func() {
|
|
r := recover()
|
|
require.NotNil(t, r)
|
|
select {
|
|
case <-attemptReleased:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("attempt context was not released after panic")
|
|
}
|
|
}()
|
|
|
|
_, _ = GenerateAssistant(context.Background(), GenerateAssistantOptions{
|
|
Model: model,
|
|
PublishMessagePart: func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {
|
|
panic("publish panic")
|
|
},
|
|
})
|
|
|
|
t.Fatal("expected GenerateAssistant to panic")
|
|
}
|
|
|
|
func requireToolResultErrorMessage(
|
|
t *testing.T,
|
|
result fantasy.ToolResultContent,
|
|
expected string,
|
|
) {
|
|
t.Helper()
|
|
|
|
output, ok := result.Result.(fantasy.ToolResultOutputContentError)
|
|
require.Truef(t, ok, "expected error tool result, got %T", result.Result)
|
|
require.Error(t, output.Error)
|
|
require.Equal(t, expected, output.Error.Error())
|
|
}
|
|
|
|
func streamFromParts(parts []fantasy.StreamPart) fantasy.StreamResponse {
|
|
return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
for _, part := range parts {
|
|
if !yield(part) {
|
|
return
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
func textMessage(role fantasy.MessageRole, text string) fantasy.Message {
|
|
return fantasy.Message{
|
|
Role: role,
|
|
Content: []fantasy.MessagePart{
|
|
fantasy.TextPart{Text: text},
|
|
},
|
|
}
|
|
}
|
|
|
|
func requireTextContent(t *testing.T, content []fantasy.Content, text string) fantasy.TextContent {
|
|
t.Helper()
|
|
|
|
for _, block := range content {
|
|
textContent, ok := fantasy.AsContentType[fantasy.TextContent](block)
|
|
if ok && textContent.Text == text {
|
|
return textContent
|
|
}
|
|
}
|
|
t.Fatalf("missing text content %q", text)
|
|
return fantasy.TextContent{}
|
|
}
|
|
|
|
func TestExclusiveToolPolicy_MixedBatchErrors(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
results, violated := applyExclusiveToolPolicy(
|
|
[]fantasy.ToolCallContent{
|
|
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
|
|
{ToolCallID: "read-1", ToolName: "read_file", Input: `{"path":"main.go"}`},
|
|
},
|
|
map[string]bool{"advisor": true},
|
|
NopMetrics(),
|
|
"fake",
|
|
"",
|
|
)
|
|
|
|
require.True(t, violated)
|
|
require.Len(t, results, 2)
|
|
require.Equal(t, "advisor-1", results[0].ToolCallID)
|
|
require.Equal(t, "read-1", results[1].ToolCallID)
|
|
requireToolResultErrorMessage(
|
|
t,
|
|
results[0],
|
|
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
|
|
)
|
|
requireToolResultErrorMessage(
|
|
t,
|
|
results[1],
|
|
"this tool was skipped because advisor must run alone in its batch. Retry your tool calls without advisor, or call advisor separately first.",
|
|
)
|
|
}
|
|
|
|
func TestApplyExclusiveToolPolicy_RecordsErrorMetrics(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
reg := prometheus.NewPedanticRegistry()
|
|
m := NewMetrics(reg)
|
|
|
|
_, violated := applyExclusiveToolPolicy(
|
|
[]fantasy.ToolCallContent{
|
|
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
|
|
{ToolCallID: "read-1", ToolName: "read_file", Input: `{"path":"main.go"}`},
|
|
},
|
|
map[string]bool{"advisor": true},
|
|
m,
|
|
"fake",
|
|
"claude-test",
|
|
)
|
|
require.True(t, violated)
|
|
|
|
require.Equal(t, 1.0, promtestutil.ToFloat64(
|
|
m.ToolErrorsTotal.WithLabelValues("fake", "claude-test", "advisor"),
|
|
))
|
|
require.Equal(t, 1.0, promtestutil.ToFloat64(
|
|
m.ToolErrorsTotal.WithLabelValues("fake", "claude-test", "read_file"),
|
|
))
|
|
}
|
|
|
|
func TestExclusiveToolPolicy_MultipleExclusive(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
results, violated := applyExclusiveToolPolicy(
|
|
[]fantasy.ToolCallContent{
|
|
{ToolCallID: "advisor-1", ToolName: "advisor", Input: `{}`},
|
|
{ToolCallID: "advisor-2", ToolName: "advisor", Input: `{"mode":"second-opinion"}`},
|
|
},
|
|
map[string]bool{"advisor": true},
|
|
NopMetrics(),
|
|
"fake",
|
|
"",
|
|
)
|
|
|
|
require.True(t, violated)
|
|
require.Len(t, results, 2)
|
|
requireToolResultErrorMessage(
|
|
t,
|
|
results[0],
|
|
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
|
|
)
|
|
requireToolResultErrorMessage(
|
|
t,
|
|
results[1],
|
|
"advisor must be called alone, without other tools in the same batch. If you need more information to feed into the advisor call, execute the other tools first, then retry with only the advisor call.",
|
|
)
|
|
}
|
|
|
|
func TestSanitizeAnthropicProviderToolContent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
providerCall := func(id, name, input string) fantasy.ToolCallContent {
|
|
return fantasy.ToolCallContent{
|
|
ToolCallID: id,
|
|
ToolName: name,
|
|
Input: input,
|
|
ProviderExecuted: true,
|
|
}
|
|
}
|
|
providerResult := func(id, name string) fantasy.ToolResultContent {
|
|
return fantasy.ToolResultContent{
|
|
ToolCallID: id,
|
|
ToolName: name,
|
|
ProviderExecuted: true,
|
|
ProviderMetadata: validWebSearchProviderMetadataForTest(),
|
|
Result: fantasy.ToolResultOutputContentText{Text: "ok"},
|
|
}
|
|
}
|
|
localCall := func(id, name string) fantasy.ToolCallContent {
|
|
return fantasy.ToolCallContent{
|
|
ToolCallID: id,
|
|
ToolName: name,
|
|
Input: `{}`,
|
|
}
|
|
}
|
|
localResult := func(id, name string) fantasy.ToolResultContent {
|
|
return fantasy.ToolResultContent{
|
|
ToolCallID: id,
|
|
ToolName: name,
|
|
Result: fantasy.ToolResultOutputContentText{Text: "ok"},
|
|
}
|
|
}
|
|
type contentSummary struct {
|
|
providerCalls []string
|
|
providerResults []string
|
|
localCalls []string
|
|
localResults []string
|
|
}
|
|
summarizeContent := func(content []fantasy.Content) contentSummary {
|
|
var summary contentSummary
|
|
for _, block := range content {
|
|
if toolCall, ok := safeToolCallContent(block); ok {
|
|
if toolCall.ProviderExecuted {
|
|
summary.providerCalls = append(summary.providerCalls, toolCall.ToolCallID)
|
|
} else {
|
|
summary.localCalls = append(summary.localCalls, toolCall.ToolCallID)
|
|
}
|
|
continue
|
|
}
|
|
if toolResult, ok := safeToolResultContent(block); ok {
|
|
if toolResult.ProviderExecuted {
|
|
summary.providerResults = append(summary.providerResults, toolResult.ToolCallID)
|
|
} else {
|
|
summary.localResults = append(summary.localResults, toolResult.ToolCallID)
|
|
}
|
|
}
|
|
}
|
|
return summary
|
|
}
|
|
assertProviderHistoryValid := func(t *testing.T, content []fantasy.Content) {
|
|
t.Helper()
|
|
|
|
parts := make([]fantasy.MessagePart, 0)
|
|
for _, block := range content {
|
|
if toolCall, ok := safeToolCallContent(block); ok && toolCall.ProviderExecuted {
|
|
parts = append(parts, toolCallContentToPart(toolCall))
|
|
continue
|
|
}
|
|
if toolResult, ok := safeToolResultContent(block); ok && toolResult.ProviderExecuted {
|
|
parts = append(parts, toolResultContentToPart(toolResult))
|
|
}
|
|
}
|
|
if len(parts) == 0 {
|
|
return
|
|
}
|
|
require.Empty(t, chatsanitize.ValidateAnthropicProviderToolHistory([]fantasy.Message{
|
|
{
|
|
Role: fantasy.MessageRoleAssistant,
|
|
Content: parts,
|
|
},
|
|
}))
|
|
}
|
|
|
|
metadataCall := providerCall("ws-meta", "web_search", `{"query":"coder"}`)
|
|
metadataCall.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil}
|
|
metadataResult := providerResult("ws-meta", "web_search")
|
|
metadataResult.ProviderMetadata = fantasy.ProviderMetadata{fantasyanthropic.Name: nil}
|
|
pointerCall := providerCall("ws-pointer", "web_search", `{"query":"coder"}`)
|
|
var nilToolCall *fantasy.ToolCallContent
|
|
|
|
testCases := []struct {
|
|
name string
|
|
provider string
|
|
content []fantasy.Content
|
|
wantSummary contentSummary
|
|
wantRemovedCalls int
|
|
wantRemovedResults int
|
|
wantTexts []string
|
|
validateAnthropic bool
|
|
}{
|
|
{
|
|
name: "orphan provider result textified",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
fantasy.TextContent{Text: "keep"},
|
|
providerResult("ws-1", "web_search"),
|
|
},
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"keep", "ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "result before call removes both provider blocks",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerResult("ws-1", "web_search"),
|
|
providerCall("ws-1", "web_search", `{"query":"coder"}`),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "valid web search pair preserved",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("ws-1", "web_search", `{"query":"coder"}`),
|
|
providerResult("ws-1", "web_search"),
|
|
fantasy.TextContent{Text: "search done"},
|
|
},
|
|
wantSummary: contentSummary{
|
|
providerCalls: []string{"ws-1"},
|
|
providerResults: []string{"ws-1"},
|
|
},
|
|
wantTexts: []string{"search done"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "invalid JSON provider call drops pair",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("ws-1", "web_search", `{`),
|
|
providerResult("ws-1", "web_search"),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "empty ID provider call drops pair",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("", "web_search", `{"query":"coder"}`),
|
|
providerResult("", "web_search"),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "empty tool name provider call drops pair",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("ws-empty", "", `{"query":"coder"}`),
|
|
providerResult("ws-empty", ""),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "non web search provider pair drops through serializable helper",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("code-1", "code_execution", `{"code":"print(1)"}`),
|
|
providerResult("code-1", "code_execution"),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "mismatched provider result tool name drops pair",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("ws-mismatch", "web_search", `{"query":"coder"}`),
|
|
providerResult("ws-mismatch", "code_execution"),
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "duplicate provider IDs drop all provider content for ID",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("dup-1", "web_search", `{"query":"coder"}`),
|
|
providerResult("dup-1", "web_search"),
|
|
providerCall("dup-1", "web_search", `{"query":"coder"}`),
|
|
},
|
|
wantRemovedCalls: 2,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "mismatched provider flags remove only provider side",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
providerCall("mix-1", "web_search", `{"query":"coder"}`),
|
|
localResult("mix-1", "web_search"),
|
|
localCall("mix-2", "read_file"),
|
|
providerResult("mix-2", "web_search"),
|
|
},
|
|
wantSummary: contentSummary{
|
|
localCalls: []string{"mix-2"},
|
|
localResults: []string{"mix-1"},
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "malformed provider metadata textifies result",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
metadataCall,
|
|
metadataResult,
|
|
},
|
|
wantRemovedCalls: 1,
|
|
wantRemovedResults: 1,
|
|
wantTexts: []string{"ok"},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "pointer and nil pointer variants are handled safely",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
nilToolCall,
|
|
&pointerCall,
|
|
providerResult("ws-pointer", "web_search"),
|
|
},
|
|
wantSummary: contentSummary{
|
|
providerCalls: []string{"ws-pointer"},
|
|
providerResults: []string{"ws-pointer"},
|
|
},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "local tool content is unchanged",
|
|
provider: fantasyanthropic.Name,
|
|
content: []fantasy.Content{
|
|
localCall("tc-1", "read_file"),
|
|
localResult("tc-1", "read_file"),
|
|
},
|
|
wantSummary: contentSummary{
|
|
localCalls: []string{"tc-1"},
|
|
localResults: []string{"tc-1"},
|
|
},
|
|
validateAnthropic: true,
|
|
},
|
|
{
|
|
name: "non Anthropic provider content is unchanged",
|
|
provider: "fake",
|
|
content: []fantasy.Content{
|
|
providerCall("ws-1", "web_search", `{"query":"coder"}`),
|
|
},
|
|
wantSummary: contentSummary{
|
|
providerCalls: []string{"ws-1"},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range testCases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sanitized, stats := chatsanitize.SanitizeAnthropicProviderToolContent(tc.provider, tc.content)
|
|
require.Equal(t, tc.wantRemovedCalls, stats.RemovedToolCalls)
|
|
require.Equal(t, tc.wantRemovedResults, stats.RemovedToolResults)
|
|
require.Zero(t, stats.DroppedMessages)
|
|
|
|
summary := summarizeContent(sanitized)
|
|
assert.ElementsMatch(t, tc.wantSummary.providerCalls, summary.providerCalls)
|
|
assert.ElementsMatch(t, tc.wantSummary.providerResults, summary.providerResults)
|
|
assert.ElementsMatch(t, tc.wantSummary.localCalls, summary.localCalls)
|
|
assert.ElementsMatch(t, tc.wantSummary.localResults, summary.localResults)
|
|
for _, text := range tc.wantTexts {
|
|
requireTextContent(t, sanitized, text)
|
|
}
|
|
if tc.validateAnthropic {
|
|
assertProviderHistoryValid(t, sanitized)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type serialMarkerTool struct{ fantasy.AgentTool }
|
|
|
|
func (serialMarkerTool) SerialToolCalls() bool { return true }
|
|
|
|
type observerMarkerTool struct {
|
|
fantasy.AgentTool
|
|
observed func(names []string)
|
|
}
|
|
|
|
func (t observerMarkerTool) ObserveStepToolCalls(names []string) { t.observed(names) }
|
|
|
|
func TestExecuteToolsNotifiesStepToolCallObservers(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var mu sync.Mutex
|
|
var observedNames []string
|
|
observedBeforeRun := false
|
|
observer := observerMarkerTool{
|
|
AgentTool: fantasy.NewAgentTool(
|
|
"observer_tool",
|
|
"records sibling calls",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
mu.Lock()
|
|
observedBeforeRun = observedNames != nil
|
|
mu.Unlock()
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
),
|
|
observed: func(names []string) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
observedNames = append([]string{}, names...)
|
|
},
|
|
}
|
|
var uncalledObserved atomic.Bool
|
|
uncalledObserver := observerMarkerTool{
|
|
AgentTool: fantasy.NewAgentTool(
|
|
"uncalled_observer",
|
|
"never called this step",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
),
|
|
observed: func([]string) { uncalledObserved.Store(true) },
|
|
}
|
|
other := fantasy.NewAgentTool(
|
|
"other_tool",
|
|
"plain tool",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)
|
|
|
|
newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{observer, uncalledObserver, other},
|
|
ObservedToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
|
|
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
|
|
{ToolCallID: "3", ToolName: "denied_tool", Input: "{}"},
|
|
},
|
|
Clock: quartz.NewReal(),
|
|
}, "fake", "fake-model").run(context.Background(), []fantasy.ToolCallContent{
|
|
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
|
|
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
|
|
}, time.Time{})
|
|
|
|
require.Equal(t, []string{"observer_tool", "other_tool", "denied_tool"}, observedNames,
|
|
"a called observer sees every observed tool-call name, including calls denied before execution")
|
|
require.True(t, observedBeforeRun, "observers are notified before any tool call executes")
|
|
require.False(t, uncalledObserved.Load(), "tools not called this step are not notified")
|
|
}
|
|
|
|
type resultObserverMarkerTool struct {
|
|
fantasy.AgentTool
|
|
observedResults func(names []string, errored []bool)
|
|
}
|
|
|
|
func (t resultObserverMarkerTool) ObserveStepToolResults(names []string, errored []bool) {
|
|
t.observedResults(names, errored)
|
|
}
|
|
|
|
func TestExecuteToolsNotifiesStepToolResultObservers(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var mu sync.Mutex
|
|
var gotNames []string
|
|
var gotErrored []bool
|
|
notifications := 0
|
|
observer := resultObserverMarkerTool{
|
|
AgentTool: fantasy.NewAgentTool(
|
|
"observer_tool",
|
|
"records sibling outcomes",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
),
|
|
observedResults: func(names []string, errored []bool) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
notifications++
|
|
gotNames = append([]string{}, names...)
|
|
gotErrored = append([]bool{}, errored...)
|
|
},
|
|
}
|
|
failing := fantasy.NewAgentTool(
|
|
"failing_tool",
|
|
"returns an error result",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextErrorResponse("remote error"), nil
|
|
},
|
|
)
|
|
|
|
executed := []fantasy.ToolCallContent{
|
|
{ToolCallID: "1", ToolName: "observer_tool", Input: "{}"},
|
|
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
|
|
{ToolCallID: "3", ToolName: "missing_tool", Input: "{}"},
|
|
}
|
|
newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{observer, failing},
|
|
ObservedToolCalls: append(slices.Clone(executed), fantasy.ToolCallContent{
|
|
ToolCallID: "4", ToolName: "rejected_tool", Input: "{not json",
|
|
}),
|
|
Clock: quartz.NewReal(),
|
|
}, "fake", "fake-model").run(context.Background(), executed, time.Time{})
|
|
|
|
require.Equal(t, 1, notifications, "each called observer is notified once per step")
|
|
require.Equal(t, []string{"observer_tool", "failing_tool", "missing_tool", "rejected_tool"}, gotNames,
|
|
"outcomes are reported per call in observed order")
|
|
require.Equal(t, []bool{false, true, true, true}, gotErrored,
|
|
"error results, unresolvable tools, and observed calls rejected before execution all settle as errored outcomes")
|
|
}
|
|
|
|
type serialResultObserverTool struct {
|
|
fantasy.AgentTool
|
|
observedResults func(names []string, errored []bool)
|
|
}
|
|
|
|
func (serialResultObserverTool) SerialToolCalls() bool { return true }
|
|
|
|
func (t serialResultObserverTool) ObserveStepToolResults(names []string, errored []bool) {
|
|
t.observedResults(names, errored)
|
|
}
|
|
|
|
func TestExecuteToolsReconcilesResultsBeforeSerialCalls(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var mu sync.Mutex
|
|
var erroredAtNotify []string
|
|
var erroredAtRun []string
|
|
notified := false
|
|
serial := serialResultObserverTool{
|
|
AgentTool: fantasy.NewAgentTool(
|
|
"serial_observer",
|
|
"observes sibling outcomes before running",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
mu.Lock()
|
|
erroredAtRun = append([]string{}, erroredAtNotify...)
|
|
mu.Unlock()
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
),
|
|
observedResults: func(names []string, errored []bool) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
notified = true
|
|
erroredAtNotify = nil
|
|
for i, name := range names {
|
|
if errored[i] {
|
|
erroredAtNotify = append(erroredAtNotify, name)
|
|
}
|
|
}
|
|
},
|
|
}
|
|
failing := fantasy.NewAgentTool(
|
|
"failing_tool",
|
|
"returns an error result",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextErrorResponse("remote error"), nil
|
|
},
|
|
)
|
|
|
|
results := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{serial, failing},
|
|
Clock: quartz.NewReal(),
|
|
}, "fake", "fake-model").run(context.Background(), []fantasy.ToolCallContent{
|
|
{ToolCallID: "1", ToolName: "serial_observer", Input: "{}"},
|
|
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
|
|
}, time.Time{})
|
|
|
|
require.True(t, notified)
|
|
require.Equal(t, []string{"failing_tool"}, erroredAtRun,
|
|
"a serial tool must see settled sibling outcomes before it executes")
|
|
require.Len(t, results, 2)
|
|
require.Equal(t, "1", results[0].content.ToolCallID, "results keep original call order")
|
|
}
|
|
|
|
type haltingMarkerTool struct{ fantasy.AgentTool }
|
|
|
|
func (haltingMarkerTool) SerialToolCalls() bool { return true }
|
|
|
|
func (haltingMarkerTool) HaltedToolCallError() (string, bool) { return "halted", true }
|
|
|
|
func TestExecuteToolsHaltsToolAfterFailedCall(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const (
|
|
succeed = `{}`
|
|
fail = `{"fail":true}`
|
|
)
|
|
halting := func(id, input string) fantasy.ToolCallContent {
|
|
return fantasy.ToolCallContent{ToolCallID: id, ToolName: "halting_tool", Input: input}
|
|
}
|
|
other := fantasy.ToolCallContent{ToolCallID: "3", ToolName: "other_tool", Input: succeed}
|
|
|
|
tests := []struct {
|
|
name string
|
|
observed []fantasy.ToolCallContent
|
|
executed []fantasy.ToolCallContent
|
|
wantRan []string
|
|
// wantResults maps each executed call ID to its result text.
|
|
wantResults map[string]string
|
|
}{
|
|
{
|
|
name: "FailedCall",
|
|
executed: []fantasy.ToolCallContent{halting("1", succeed), halting("2", fail), other, halting("4", succeed)},
|
|
wantRan: []string{"1", "2", "3"},
|
|
wantResults: map[string]string{"1": "ok", "2": "error: failed", "3": "ok", "4": "error: halted"},
|
|
},
|
|
{
|
|
name: "RejectedCall",
|
|
observed: []fantasy.ToolCallContent{halting("1", succeed), halting("2", succeed), other, halting("4", succeed)},
|
|
executed: []fantasy.ToolCallContent{halting("1", succeed), other, halting("4", succeed)},
|
|
wantRan: []string{"1", "3"},
|
|
wantResults: map[string]string{"1": "ok", "3": "ok", "4": "error: halted"},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var mu sync.Mutex
|
|
var ran []string
|
|
record := func(id string) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
ran = append(ran, id)
|
|
}
|
|
haltingTool := haltingMarkerTool{AgentTool: fantasy.NewAgentTool(
|
|
"halting_tool",
|
|
"fails when asked",
|
|
func(_ context.Context, input struct {
|
|
Fail bool `json:"fail"`
|
|
}, call fantasy.ToolCall,
|
|
) (fantasy.ToolResponse, error) {
|
|
record(call.ID)
|
|
if input.Fail {
|
|
return fantasy.NewTextErrorResponse("failed"), nil
|
|
}
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)}
|
|
otherTool := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
|
|
"other_tool",
|
|
"runs between halting calls",
|
|
func(_ context.Context, _ struct{}, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
record(call.ID)
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)}
|
|
|
|
outcome, err := ExecuteLocalTools(context.Background(), ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{haltingTool, otherTool},
|
|
ToolCalls: tt.executed,
|
|
ObservedToolCalls: tt.observed,
|
|
Clock: quartz.NewReal(),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, tt.wantRan, ran)
|
|
results := make(map[string]string)
|
|
for _, content := range outcome.Content {
|
|
result, ok := content.(fantasy.ToolResultContent)
|
|
require.True(t, ok, "%T", content)
|
|
switch output := result.Result.(type) {
|
|
case fantasy.ToolResultOutputContentText:
|
|
results[result.ToolCallID] = output.Text
|
|
case fantasy.ToolResultOutputContentError:
|
|
results[result.ToolCallID] = "error: " + output.Error.Error()
|
|
default:
|
|
t.Fatalf("unexpected result %T", result.Result)
|
|
}
|
|
}
|
|
require.Equal(t, tt.wantResults, results)
|
|
require.Equal(t, len(tt.wantRan), outcome.BatchBilledCalls, "halted calls are not billed")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExecuteToolsSerialToolCallOrder(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var mu sync.Mutex
|
|
var events []string
|
|
inFlight := 0
|
|
maxInFlight := 0
|
|
record := func(event string, delta int) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
inFlight += delta
|
|
if inFlight > maxInFlight {
|
|
maxInFlight = inFlight
|
|
}
|
|
events = append(events, event)
|
|
}
|
|
serial := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
|
|
"serial_tool",
|
|
"records call order",
|
|
func(_ context.Context, _ struct{}, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
record(call.ID+":start", 1)
|
|
runtime.Gosched()
|
|
record(call.ID+":end", -1)
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)}
|
|
parallelRan := make(chan struct{})
|
|
parallel := fantasy.NewAgentTool(
|
|
"parallel_tool",
|
|
"plain tool",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
close(parallelRan)
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)
|
|
|
|
calls := []fantasy.ToolCallContent{
|
|
{ToolCallID: "a", ToolName: "serial_tool", Input: "{}"},
|
|
{ToolCallID: "p", ToolName: "parallel_tool", Input: "{}"},
|
|
{ToolCallID: "b", ToolName: "serial_tool", Input: "{}"},
|
|
{ToolCallID: "c", ToolName: "serial_tool", Input: "{}"},
|
|
}
|
|
results := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{serial, parallel},
|
|
Clock: quartz.NewReal(),
|
|
}, "fake", "fake-model").run(context.Background(), calls, time.Time{})
|
|
|
|
require.Equal(t, []string{"a:start", "a:end", "b:start", "b:end", "c:start", "c:end"}, events,
|
|
"serial tool calls must run one at a time in tool-call order")
|
|
require.Equal(t, 1, maxInFlight)
|
|
select {
|
|
case <-parallelRan:
|
|
default:
|
|
t.Fatal("parallel tool call did not run")
|
|
}
|
|
require.Len(t, results, len(calls))
|
|
for i, tc := range calls {
|
|
require.Equal(t, tc.ToolCallID, results[i].content.ToolCallID, "results keep original call order")
|
|
}
|
|
}
|
|
|
|
type timingEvent struct {
|
|
dispatchIndex int
|
|
at time.Time
|
|
}
|
|
|
|
// liveToolBillingRecorder is a test ToolBillingRecorder that publishes
|
|
// start and completion timestamps as they happen.
|
|
type liveToolBillingRecorder struct {
|
|
started chan timingEvent
|
|
completed chan timingEvent
|
|
}
|
|
|
|
func (r liveToolBillingRecorder) RecordStart(dispatchIndex int, startedAt time.Time) {
|
|
r.started <- timingEvent{dispatchIndex: dispatchIndex, at: startedAt}
|
|
}
|
|
|
|
func (r liveToolBillingRecorder) RecordComplete(dispatchIndex int, completedAt time.Time) {
|
|
r.completed <- timingEvent{dispatchIndex: dispatchIndex, at: completedAt}
|
|
}
|
|
|
|
func TestExecuteToolsReturnsExecutionIntervals(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
started := make(chan timingEvent, 3)
|
|
completed := make(chan timingEvent, 3)
|
|
|
|
slowGo := make(chan struct{})
|
|
fastGo := make(chan struct{})
|
|
blocking := func(name string, release <-chan struct{}) fantasy.AgentTool {
|
|
return fantasy.NewAgentTool(
|
|
name,
|
|
"waits for the test to release it",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
<-release
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)
|
|
}
|
|
slow := blocking("slow_tool", slowGo)
|
|
fast := blocking("fast_tool", fastGo)
|
|
serial := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
|
|
"serial_tool",
|
|
"runs after concurrent tools",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)}
|
|
calls := []fantasy.ToolCallContent{
|
|
{ToolCallID: "slow", ToolName: "slow_tool", Input: "{}"},
|
|
{ToolCallID: "fast", ToolName: "fast_tool", Input: "{}"},
|
|
{ToolCallID: "serial", ToolName: "serial_tool", Input: "{}"},
|
|
}
|
|
batchStart := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
resultsCh := make(chan []toolExecutionResult, 1)
|
|
go func() {
|
|
resultsCh <- newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{slow, fast, serial},
|
|
BillingRecorder: liveToolBillingRecorder{started: started, completed: completed},
|
|
Clock: quartz.NewReal(),
|
|
}, "fake", "fake-model").run(ctx, calls, batchStart)
|
|
}()
|
|
|
|
startByIndex := make([]time.Time, len(calls))
|
|
for range 2 {
|
|
event := testutil.RequireReceive(ctx, t, started)
|
|
startByIndex[event.dispatchIndex] = event.at
|
|
}
|
|
|
|
close(fastGo)
|
|
fastCompleted := testutil.RequireReceive(ctx, t, completed)
|
|
require.Equal(t, 1, fastCompleted.dispatchIndex)
|
|
|
|
close(slowGo)
|
|
slowCompleted := testutil.RequireReceive(ctx, t, completed)
|
|
require.Equal(t, 0, slowCompleted.dispatchIndex)
|
|
|
|
serialStarted := testutil.RequireReceive(ctx, t, started)
|
|
require.Equal(t, 2, serialStarted.dispatchIndex)
|
|
startByIndex[serialStarted.dispatchIndex] = serialStarted.at
|
|
serialCompleted := testutil.RequireReceive(ctx, t, completed)
|
|
require.Equal(t, 2, serialCompleted.dispatchIndex)
|
|
|
|
endByIndex := []time.Time{slowCompleted.at, fastCompleted.at, serialCompleted.at}
|
|
results := testutil.RequireReceive(ctx, t, resultsCh)
|
|
require.Len(t, results, len(calls))
|
|
for i, call := range calls {
|
|
require.Equal(t, call.ToolCallID, results[i].content.ToolCallID,
|
|
"results keep dispatch order when calls complete out of order")
|
|
require.Equal(t, startByIndex[i], results[i].interval.Start)
|
|
require.Equal(t, endByIndex[i], results[i].interval.End)
|
|
}
|
|
require.Equal(t, batchStart, results[0].interval.Start)
|
|
require.Equal(t, batchStart, results[1].interval.Start)
|
|
require.NotEqual(t, batchStart, results[2].interval.Start)
|
|
require.False(t, results[2].interval.Start.Before(slowCompleted.at),
|
|
"serial execution starts only after concurrent calls settle")
|
|
}
|
|
|
|
func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
originalBytes := []byte{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10}
|
|
|
|
t.Run("EncodesRawBytesToBase64", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := fantasy.NewAgentTool(
|
|
"screenshot",
|
|
"takes a screenshot",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.ToolResponse{
|
|
Type: "image",
|
|
Data: originalBytes,
|
|
MediaType: "image/jpeg",
|
|
Filename: "photo.jpg",
|
|
}, nil
|
|
},
|
|
)
|
|
|
|
tc := fantasy.ToolCallContent{
|
|
ToolCallID: "call-1",
|
|
ToolName: "screenshot",
|
|
Input: "{}",
|
|
}
|
|
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"screenshot"},
|
|
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
|
|
|
|
media, ok := result.Result.(fantasy.ToolResultOutputContentMedia)
|
|
require.True(t, ok, "expected ToolResultOutputContentMedia")
|
|
require.Equal(t, "image/jpeg", media.MediaType)
|
|
require.Equal(t, "photo.jpg", media.Filename)
|
|
|
|
decoded, err := base64.StdEncoding.DecodeString(media.Data)
|
|
require.NoError(t, err, "Data should be valid base64")
|
|
require.Equal(t, originalBytes, decoded)
|
|
})
|
|
|
|
t.Run("SanitizesInvalidUTF8InContent", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := fantasy.NewAgentTool(
|
|
"screenshot",
|
|
"takes a screenshot",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.ToolResponse{
|
|
Type: "image",
|
|
Data: originalBytes,
|
|
MediaType: "image/png",
|
|
Content: "hello\xffworld",
|
|
}, nil
|
|
},
|
|
)
|
|
|
|
tc := fantasy.ToolCallContent{
|
|
ToolCallID: "call-2",
|
|
ToolName: "screenshot",
|
|
Input: "{}",
|
|
}
|
|
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"screenshot"},
|
|
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
|
|
|
|
media, ok := result.Result.(fantasy.ToolResultOutputContentMedia)
|
|
require.True(t, ok, "expected ToolResultOutputContentMedia")
|
|
require.True(t, utf8.ValidString(media.Text), "Text should be valid UTF-8")
|
|
require.Contains(t, media.Text, "hello")
|
|
require.Contains(t, media.Text, "world")
|
|
})
|
|
|
|
t.Run("SanitizesInvalidUTF8InTextResult", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := fantasy.NewAgentTool(
|
|
"echo",
|
|
"echoes input",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.ToolResponse{
|
|
Content: "hello\xffworld",
|
|
}, nil
|
|
},
|
|
)
|
|
|
|
tc := fantasy.ToolCallContent{
|
|
ToolCallID: "call-3",
|
|
ToolName: "echo",
|
|
Input: "{}",
|
|
}
|
|
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"echo"},
|
|
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
|
|
|
|
textOutput, ok := result.Result.(fantasy.ToolResultOutputContentText)
|
|
require.True(t, ok, "expected ToolResultOutputContentText, got %T", result.Result)
|
|
require.True(t, utf8.ValidString(textOutput.Text), "Text should be valid UTF-8")
|
|
require.Contains(t, textOutput.Text, "hello")
|
|
require.Contains(t, textOutput.Text, "world")
|
|
})
|
|
}
|
|
|
|
func TestExecuteSingleTool_NormalizesMedia(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var pngData bytes.Buffer
|
|
require.NoError(t, png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 1, 1))))
|
|
|
|
for _, tc := range []struct {
|
|
name string
|
|
mediaType string
|
|
data []byte
|
|
// wantMediaType is empty when the media is rejected and only text survives.
|
|
wantMediaType string
|
|
wantText string
|
|
}{
|
|
{name: "OversizedMediaKeepsText", mediaType: "image/png", data: make([]byte, codersdk.MaxChatFileSizeBytes+1), wantText: "Ran Playwright code\n[image/png content omitted"},
|
|
{name: "NonImageBytesDeclaredAsImageKeepsText", mediaType: "image/png", data: []byte("<html>not an image</html>"), wantText: "Ran Playwright code\n[image omitted: payload declared as image/png is text/html"},
|
|
{name: "ImageTypeFollowsBytes", mediaType: "image/jpeg", data: pngData.Bytes(), wantMediaType: "image/png", wantText: "Ran Playwright code"},
|
|
{name: "NonImageMediaPassesThrough", mediaType: "audio/mpeg", data: []byte{1, 2, 3}, wantMediaType: "audio/mpeg", wantText: "Ran Playwright code"},
|
|
} {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tool := fantasy.NewAgentTool("screenshot", "takes a screenshot",
|
|
func(context.Context, struct{}, fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.ToolResponse{Type: "media", Data: tc.data, MediaType: tc.mediaType, Content: "Ran Playwright code"}, nil
|
|
},
|
|
)
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"screenshot"},
|
|
}, "openai", "model").runOne(
|
|
context.Background(),
|
|
fantasy.ToolCallContent{ToolCallID: "call-1", ToolName: "screenshot", Input: "{}"},
|
|
nil,
|
|
)
|
|
|
|
switch out := result.Result.(type) {
|
|
case fantasy.ToolResultOutputContentMedia:
|
|
require.Equal(t, tc.wantMediaType, out.MediaType)
|
|
require.Equal(t, tc.wantText, out.Text)
|
|
case fantasy.ToolResultOutputContentText:
|
|
require.Empty(t, tc.wantMediaType, "media rejected: %s", out.Text)
|
|
require.Contains(t, out.Text, tc.wantText)
|
|
default:
|
|
t.Fatalf("unexpected result %T", out)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestExecuteSingleTool_RejectsUnknownTool(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tc := fantasy.ToolCallContent{
|
|
ToolCallID: "call-missing",
|
|
ToolName: "close_agent",
|
|
Input: "{}",
|
|
}
|
|
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
ActiveTools: []string{"interrupt_agent"},
|
|
}, "fake", "fake-model").runOne(context.Background(), tc, nil)
|
|
|
|
errOutput, ok := result.Result.(fantasy.ToolResultOutputContentError)
|
|
require.True(t, ok, "expected error output, got %T", result.Result)
|
|
require.EqualError(t, errOutput.Error, "Tool not active in this turn: close_agent")
|
|
}
|
|
|
|
func TestExecuteSingleTool_AllowsDeferredDirectCall(t *testing.T) {
|
|
t.Parallel()
|
|
tool := fantasy.NewAgentTool(
|
|
"server__direct",
|
|
"direct",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
},
|
|
)
|
|
result, _ := newToolBatch(ExecuteLocalToolsOptions{
|
|
Tools: []fantasy.AgentTool{tool},
|
|
ActiveTools: []string{"find_tools"},
|
|
AllowInactiveTools: map[string]bool{"server__direct": true},
|
|
}, "fake", "fake-model").runOne(
|
|
context.Background(),
|
|
fantasy.ToolCallContent{ToolCallID: "call-direct", ToolName: "server__direct", Input: "{}"},
|
|
nil,
|
|
)
|
|
text, ok := result.Result.(fantasy.ToolResultOutputContentText)
|
|
require.True(t, ok)
|
|
require.Equal(t, "ok", text.Text)
|
|
}
|
|
|
|
func TestExecuteLocalTools_ProviderToolRunner(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
runner := fantasy.NewAgentTool(
|
|
"computer",
|
|
"runs a provider-defined tool locally",
|
|
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("screenshot taken"), nil
|
|
},
|
|
)
|
|
metadata := fantasy.ProviderMetadata{
|
|
fantasyopenai.Name: &fantasyopenai.ComputerCallOutputOptions{Detail: "original"},
|
|
}
|
|
step, err := ExecuteLocalTools(context.Background(), ExecuteLocalToolsOptions{
|
|
// The runner is neither a regular tool nor an active one.
|
|
ActiveTools: []string{"read_file"},
|
|
ProviderTools: []ProviderTool{
|
|
{Definition: fantasy.ProviderDefinedTool{ID: "web_search", Name: "web_search"}},
|
|
{
|
|
Runner: runner,
|
|
ResultProviderMetadata: func(fantasy.ToolResponse) fantasy.ProviderMetadata {
|
|
return metadata
|
|
},
|
|
},
|
|
},
|
|
ToolCalls: []fantasy.ToolCallContent{
|
|
{ToolCallID: "call-search", ToolName: "web_search", Input: "{}", ProviderExecuted: true},
|
|
{ToolCallID: "call-computer", ToolName: "computer", Input: "{}"},
|
|
},
|
|
Clock: quartz.NewReal(),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, step.Content, 1, "provider-executed calls are skipped")
|
|
result, ok := step.Content[0].(fantasy.ToolResultContent)
|
|
require.True(t, ok)
|
|
require.Equal(t, "call-computer", result.ToolCallID)
|
|
require.Equal(t, fantasy.ToolResultOutputContentText{Text: "screenshot taken"}, result.Result)
|
|
require.Equal(t, metadata, result.ProviderMetadata)
|
|
}
|