2026-02-27 11:50:56 -05:00
package chatloop
import (
2026-06-16 11:14:37 +01:00
"cmp"
2026-02-27 11:50:56 -05:00
"context"
"database/sql"
2026-04-23 19:58:38 +01:00
"encoding/base64"
2026-02-27 11:50:56 -05:00
"encoding/json"
"errors"
2026-09-17 15:09:19 +02:00
"fmt"
2026-10-04 22:55:37 +02:00
"maps"
2026-03-02 18:51:57 -05:00
"slices"
2026-02-27 11:50:56 -05:00
"strconv"
"strings"
2026-03-11 11:00:08 -07:00
"sync"
2026-02-27 18:34:33 -05:00
"time"
2026-04-02 10:07:01 -07:00
"unicode"
2026-02-27 11:50:56 -05:00
"charm.land/fantasy"
fantasyanthropic "charm.land/fantasy/providers/anthropic"
2026-09-23 19:01:24 +02:00
fantasyopenai "charm.land/fantasy/providers/openai"
2026-10-04 22:55:37 +02:00
fantasyopenaicompat "charm.land/fantasy/providers/openaicompat"
fantasyopenrouter "charm.land/fantasy/providers/openrouter"
fantasyvercel "charm.land/fantasy/providers/vercel"
2026-06-12 13:33:12 +02:00
"github.com/google/uuid"
2026-09-30 11:53:58 -05:00
"go.opentelemetry.io/otel/attribute"
2026-02-27 11:50:56 -05:00
"golang.org/x/xerrors"
2026-04-22 20:11:53 +10:00
"cdr.dev/slog/v3"
2026-04-08 12:42:03 -04:00
"github.com/coder/coder/v2/coderd/database/dbtime"
2026-04-20 12:27:16 +02:00
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
2026-03-25 13:47:54 +11:00
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
2026-03-23 17:34:43 +00:00
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
2026-10-04 22:55:37 +02:00
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
2026-04-28 12:45:23 +02:00
"github.com/coder/coder/v2/coderd/x/chatd/chatsanitize"
2026-04-22 20:11:53 +10:00
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
2026-09-17 15:09:19 +02:00
"github.com/coder/coder/v2/coderd/x/chatfiles"
2026-02-27 11:50:56 -05:00
"github.com/coder/coder/v2/codersdk"
2026-04-10 00:40:09 +10:00
"github.com/coder/quartz"
2026-02-27 11:50:56 -05:00
)
const (
2026-09-18 15:24:15 +10:00
// DefaultStreamSilenceTimeout bounds how long an individual
2026-05-28 21:02:40 +10:00
// model attempt may go without receiving a stream part before
2026-03-25 13:47:54 +11:00
// the attempt is canceled and retried.
2026-09-18 15:24:15 +10:00
DefaultStreamSilenceTimeout = 10 * time . Minute
// StreamSilenceTimeoutDisabled turns the silence guard off.
StreamSilenceTimeoutDisabled time . Duration = - 1
streamSilenceGuardTimerTag = "streamSilenceGuard"
2026-02-27 11:50:56 -05:00
)
2026-03-25 13:47:54 +11:00
var (
2026-07-15 16:36:31 +10:00
// ErrContentFiltered is returned when the provider's safety
// classifiers blocked the response and the model produced no
// content, e.g. Anthropic's stop_reason "refusal".
ErrContentFiltered = xerrors . New ( "response blocked by provider content filter" )
2026-08-24 17:09:00 +02:00
// ErrNoModelOutput is returned when a stream ends without visible content or
// tool calls and its finish reason indicates an incomplete response.
ErrNoModelOutput = xerrors . New ( "model stream finished without output" )
2026-03-25 13:47:54 +11:00
2026-05-28 21:02:40 +10:00
errStreamSilenceTimeout = xerrors . New (
"chat stream was silent for longer than the configured timeout" ,
2026-03-25 13:47:54 +11:00
)
)
2026-02-27 11:50:56 -05:00
2026-07-06 11:57:12 +01:00
// PersistedStep is the unit the persistence layer splits into role-separated
// database messages. Content mixes assistant blocks (text, reasoning, tool
// calls) and tool result blocks from one completed or interrupted agent step.
2026-02-27 11:50:56 -05:00
type PersistedStep struct {
2026-07-06 11:57:12 +01:00
Content [] fantasy . Content
Usage fantasy . Usage
ContextLimit sql . NullInt64
2026-08-25 16:40:41 +07:00
// Runtime is the wall-clock duration from opening to consuming the
// model stream.
2026-03-18 10:57:35 -04:00
Runtime time . Duration
2026-09-23 19:01:24 +02:00
// ProviderResponseID is the response ID the model endpoint reported, if
// any. Through the AI Gateway, Anthropic Messages responses carry the
// gateway's interception ID instead of the upstream message ID.
ProviderResponseID string
2026-08-25 16:40:41 +07:00
// BatchRuntime is the union of billed local-tool execution intervals.
// Parallel calls count once and serial calls count from their own start.
BatchRuntime time . Duration
// BatchBilledCalls counts the executed calls whose intervals produced
// BatchRuntime. Audit metadata for the batch usage record.
BatchBilledCalls int
2026-04-08 12:42:03 -04:00
// ToolCallCreatedAt maps tool-call IDs to the time
// the model emitted each tool call. Applied by the
// persistence layer to set CreatedAt on persisted
// tool-call ChatMessageParts.
ToolCallCreatedAt map [ string ] time . Time
// ToolResultCreatedAt maps tool-call IDs to the time
// each tool result was produced (or interrupted).
// Applied by the persistence layer to set CreatedAt
// on persisted tool-result ChatMessageParts.
ToolResultCreatedAt map [ string ] time . Time
2026-05-18 12:30:30 -04:00
// ReasoningStartedAt and ReasoningCompletedAt are parallel
// slices indexed by the occurrence order of reasoning
// content in Content. The persistence layer walks reasoning
// parts in order and applies these timestamps to the
// corresponding ChatMessageParts so the frontend can render
// reasoning duration. Reasoning parts have no provider-side
// stable ID, so order is the only correlation we have.
ReasoningStartedAt [] time . Time
ReasoningCompletedAt [] time . Time
2026-02-27 11:50:56 -05:00
}
2026-06-12 13:33:12 +02:00
// GenerateAssistantOptions configures one assistant model call.
type GenerateAssistantOptions struct {
2026-06-16 11:14:37 +01:00
Model fantasy . LanguageModel
// ErrorProvider labels user-facing errors with the configured provider
// identity (e.g. "bedrock"). It differs from Model.Provider(), which
// reflects the fantasy transport client and is "anthropic" for Bedrock
// routed through aibridge. Metrics and prompt preparation keep using
// Model.Provider(). When empty, Model.Provider() is used.
ErrorProvider string
2026-06-12 13:33:12 +02:00
Messages [] fantasy . Message
Tools [] fantasy . AgentTool
ActiveTools [] string
ProviderTools [] ProviderTool
StreamSilenceTimeout time . Duration
Clock quartz . Clock
ContextLimitFallback int64
2026-08-20 22:54:52 +02:00
// CallTemplate is copied before GenerateAssistant attaches the prompt and
// tools.
CallTemplate fantasy . Call
2026-06-12 13:33:12 +02:00
PublishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart )
2026-08-06 16:09:38 +07:00
// OnModelStreamStart runs immediately before the provider stream is
// opened, at the instant PersistedStep.Runtime starts measuring. It
// lets callers record the billable window's start out of band, so an
// interrupted attempt bills the same window a completed step reports.
OnModelStreamStart func ()
2026-06-12 13:33:12 +02:00
Logger slog . Logger
Metrics * Metrics
2026-09-30 11:53:58 -05:00
Stages * StageTracer
// StageModel should match the model identity on provider_attempt stages.
StageModel StageModel
2026-06-12 13:33:12 +02:00
}
// AssistantOutcome is the durable assistant-side result from one model call.
type AssistantOutcome struct {
Step PersistedStep
ToolCalls [] fantasy . ToolCallContent
FinishReason fantasy . FinishReason
}
// ExecuteLocalToolsOptions configures one local tool execution batch.
type ExecuteLocalToolsOptions struct {
2026-08-18 19:12:47 +02:00
Tools [] fantasy . AgentTool
ActiveTools [] string
AllowInactiveTools map [ string ] bool
ProviderTools [] ProviderTool
ToolCalls [] fantasy . ToolCallContent
// ObservedToolCalls optionally carries the step's full assistant
// tool-call batch, including calls denied before execution, so
// step observers account for denied siblings that derivation will
2026-11-02 11:28:22 +01:00
// still count, and halting tools skip the calls after a denied
// call. Calls missing from ToolCalls count as denied. Defaults to
// ToolCalls.
2026-08-18 19:12:47 +02:00
ObservedToolCalls [] fantasy . ToolCallContent
2026-06-12 13:33:12 +02:00
ExclusiveToolNames map [ string ] bool
BuiltinToolNames map [ string ] bool
ModelProvider string
ModelName string
2026-06-24 09:16:38 -06:00
// ContextLimit is the model's context window in tokens. It is used
// to derive a per-result byte budget so a single oversized tool
// result cannot overflow the prompt. Zero means unknown, in which
// case a default budget applies.
ContextLimit int64
2026-08-25 16:40:41 +07:00
// UnbilledToolNames lists called tool names excluded from the batch
2026-10-30 03:00:59 +01:00
// window.
2026-08-25 16:40:41 +07:00
UnbilledToolNames map [ string ] bool
// BillingRecorder observes each local call's start and completion
// for interrupt billing. Serial calls may start after concurrent
// siblings settle, so interrupts bill actual starts and skip calls
// that never run. Optional.
BillingRecorder ToolBillingRecorder
2026-09-30 14:31:52 +03:00
// ToolCallContext returns the context to run tc with. Optional.
ToolCallContext func ( ctx context . Context , tc fantasy . ToolCallContent ) context . Context
2026-08-25 16:40:41 +07:00
2026-09-30 11:53:58 -05:00
Stages * StageTracer
StageModel StageModel
2026-06-12 13:33:12 +02:00
PublishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart )
Logger slog . Logger
Metrics * Metrics
Clock quartz . Clock
}
2026-08-25 16:40:41 +07:00
// ToolBillingRecorder records live start and completion timestamps for
// local tool calls. Interrupt billing uses these so a cancel can bill
// work that already started and skip calls that never ran.
// dispatchIndex identifies the dispatch-order occurrence.
// RecordComplete may run from multiple tool goroutines; implementations
// must be concurrency-safe.
type ToolBillingRecorder interface {
RecordStart ( dispatchIndex int , startedAt time . Time )
RecordComplete ( dispatchIndex int , completedAt time . Time )
}
2026-06-12 13:33:12 +02:00
// GenerateCompactionOptions configures one context compaction call.
type GenerateCompactionOptions struct {
Model fantasy . LanguageModel
Messages [] fantasy . Message
ThresholdPercent int32
ContextLimit int64
ContextLimitFallback int64
SummaryPrompt string
2026-07-29 13:39:12 +02:00
SummaryHint string
2026-06-12 13:33:12 +02:00
SystemSummaryPrefix string
StepUsage fantasy . Usage
2026-07-21 10:58:08 +02:00
// Force skips the threshold gate (including the threshold=100
// disable and the zero-usage early return). Set for manual,
// user-requested compactions.
Force bool
// Source labels what triggered the compaction. Defaults to
// CompactionSourceAutomatic when empty.
Source CompactionSource
2026-06-12 13:33:12 +02:00
DebugSvc * chatdebug . Service
ChatID uuid . UUID
HistoryTipMessageID int64
2026-10-20 11:29:57 +02:00
// ToolCallID and ToolName identify the synthetic tool call
// used to represent compaction in the message stream.
ToolCallID string
ToolName string
2026-06-12 13:33:12 +02:00
2026-07-14 16:57:04 +02:00
// ResolvedProvider, ResolvedModel, and ModelConfigID identify the
// summary model, which can differ from the chat model when a
// compaction override is configured. Debug runs record these.
ResolvedProvider string
ResolvedModel string
ModelConfigID uuid . UUID
2026-10-11 12:10:59 +02:00
// SummaryContextLimit is the summary model's window. When positive, it
// bounds the summary cap instead of the trigger's limit.
SummaryContextLimit int64
// ChatContextLimit is the chat model's window, recorded for chat usage
// display.
ChatContextLimit int64
2026-07-14 16:57:04 +02:00
2026-08-20 22:54:52 +02:00
// SummaryCall is copied before GenerateCompaction attaches the summary
2026-09-18 23:19:00 +02:00
// prompt and prepared tool definitions.
2026-08-20 22:54:52 +02:00
SummaryCall fantasy . Call
2026-09-18 23:19:00 +02:00
// ToolDefinitions is copied from the parent generation request so the
// summary call uses the exact same ordered definitions.
ToolDefinitions [] fantasy . Tool
2026-07-14 16:57:04 +02:00
2026-10-20 11:29:57 +02:00
// PublishMessagePart publishes streaming parts to connected
// clients so they see "Summarizing..." / "Summarized" UI
// transitions during compaction.
2026-06-12 13:33:12 +02:00
PublishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart )
2026-08-06 16:09:38 +07:00
// Clock measures the summary call duration. Required.
Clock quartz . Clock
2026-09-21 21:28:45 +02:00
// StreamSilenceTimeout bounds how long the summary stream may go
// without yielding a part. Defaults to defaultStreamSilenceTimeout.
StreamSilenceTimeout time . Duration
2026-08-06 16:09:38 +07:00
// OnModelStreamStart runs immediately before the summary model call,
// at the instant CompactionResult.Runtime starts measuring.
OnModelStreamStart func ()
2026-10-20 00:30:05 +02:00
// Logger records summary requests the model rejects for their size.
Logger slog . Logger
2026-06-12 13:33:12 +02:00
}
2026-11-09 07:26:10 +01:00
// ProviderTool pairs a tool definition that chatd builds for one
// provider, either provider-native or a plain function tool, with an
2026-03-13 19:49:34 +01:00
// optional local executor. When Runner is nil the tool is fully
// provider-executed (e.g. web search). When Runner is non-nil
// the definition is sent to the API but execution is handled
// locally (e.g. computer use).
type ProviderTool struct {
Definition fantasy . Tool
Runner fantasy . AgentTool
2026-05-04 20:30:50 +02:00
// ResultProviderMetadata extracts provider-specific metadata from successful
// local runner responses. The chat loop attaches returned metadata to the tool
// result sent back to the model. OpenAI computer-use uses this to request
// original screenshot detail for image results.
ResultProviderMetadata func ( response fantasy . ToolResponse ) fantasy . ProviderMetadata
2026-03-13 19:49:34 +01:00
}
2026-03-02 18:51:57 -05:00
// stepResult holds the accumulated output of a single streaming
// step. Since we own the stream consumer, all content is tracked
2026-04-24 23:57:30 +02:00
// directly here, no shadow draft state needed.
2026-03-02 18:51:57 -05:00
type stepResult struct {
2026-05-18 12:30:30 -04:00
content [] fantasy . Content
usage fantasy . Usage
2026-10-28 18:35:20 -07:00
responseBytes int
2026-05-18 12:30:30 -04:00
providerMetadata fantasy . ProviderMetadata
2026-09-23 19:01:24 +02:00
providerResponseID string
2026-05-18 12:30:30 -04:00
finishReason fantasy . FinishReason
toolCalls [] fantasy . ToolCallContent
toolCallCreatedAt map [ string ] time . Time
toolResultCreatedAt map [ string ] time . Time
reasoningStartedAt [] time . Time
reasoningCompletedAt [] time . Time
2026-09-30 12:20:03 +02:00
// unfinishedToolCalls holds local calls whose input started streaming
// but never completed into a ToolCall.
unfinishedToolCalls [] fantasy . ToolCallContent
2026-03-02 18:51:57 -05:00
}
// reasoningState accumulates reasoning content and provider
// metadata while the stream is in flight.
type reasoningState struct {
2026-05-18 12:30:30 -04:00
text string
options fantasy . ProviderMetadata
startedAt time . Time
2026-03-02 18:51:57 -05:00
}
2026-10-12 22:53:07 +02:00
type textState struct {
text string
options fantasy . ProviderMetadata
narration bool
}
2026-06-12 13:33:12 +02:00
// GenerateAssistant performs one assistant model stream and returns the
// durable assistant-side content. It does not execute tools, retry, or persist.
2026-09-30 11:53:58 -05:00
func GenerateAssistant ( ctx context . Context , opts GenerateAssistantOptions ) ( _ AssistantOutcome , retErr error ) {
2026-02-27 11:50:56 -05:00
if opts . Model == nil {
2026-06-12 13:33:12 +02:00
return AssistantOutcome {}, xerrors . New ( "chat model is required" )
2026-02-27 11:50:56 -05:00
}
2026-09-18 15:24:15 +10:00
if opts . StreamSilenceTimeout == 0 {
opts . StreamSilenceTimeout = DefaultStreamSilenceTimeout
2026-03-25 13:47:54 +11:00
}
2026-04-10 00:40:09 +10:00
if opts . Clock == nil {
opts . Clock = quartz . NewReal ()
}
2026-04-15 19:53:10 +01:00
if opts . Metrics == nil {
opts . Metrics = NopMetrics ()
}
2026-02-27 11:50:56 -05:00
2026-03-13 17:53:26 +02:00
publishMessagePart := func ( role codersdk . ChatMessageRole , part codersdk . ChatMessagePart ) {
2026-06-12 13:33:12 +02:00
if opts . PublishMessagePart != nil {
opts . PublishMessagePart ( role , part )
2026-02-27 11:50:56 -05:00
}
}
2026-06-12 13:33:12 +02:00
provider := opts . Model . Provider ()
modelName := opts . Model . Model ()
2026-06-16 11:14:37 +01:00
// errorProvider labels user-facing errors with the configured provider;
// see GenerateAssistantOptions.ErrorProvider. The transport provider is
// kept for prompt preparation, Anthropic history sanitization, and the
// metric labels below.
errorProvider := cmp . Or ( opts . ErrorProvider , provider )
2026-09-05 11:39:10 +02:00
prepared , err := prepareMessagesForRequest ( ctx , opts . Model , opts . Logger , opts . Messages , provider , modelName , 0 , 1 )
2026-06-12 13:33:12 +02:00
if err != nil {
return AssistantOutcome {}, xerrors . Errorf ( "prepare prompt: %w" , err )
}
opts . Metrics . MessageCount . WithLabelValues ( provider , modelName ). Observe ( float64 ( len ( prepared )))
opts . Metrics . PromptSizeBytes . WithLabelValues ( provider , modelName ). Observe ( float64 ( EstimatePromptSize ( prepared )))
opts . Metrics . StepsTotal . WithLabelValues ( provider , modelName ). Inc ()
2026-02-27 11:50:56 -05:00
2026-08-20 22:54:52 +02:00
call := opts . CallTemplate
call . Prompt = prepared
2026-09-18 23:19:00 +02:00
call . Tools = BuildToolDefinitions ( opts . Tools , opts . ActiveTools , opts . ProviderTools )
2026-06-12 13:33:12 +02:00
stepStart := opts . Clock . Now ()
2026-08-06 16:09:38 +07:00
if opts . OnModelStreamStart != nil {
opts . OnModelStreamStart ()
}
2026-06-12 13:33:12 +02:00
stepCtx := chatdebug . ReuseStep ( ctx )
2026-09-30 11:53:58 -05:00
streamCtx , streamSpan := opts . Stages . Start ( stepCtx , StageStream )
streamSpan . SetModel ( opts . StageModel )
2026-06-12 13:33:12 +02:00
attempt , streamErr := guardedStream (
2026-09-30 11:53:58 -05:00
streamCtx ,
2026-06-12 13:33:12 +02:00
provider ,
modelName ,
opts . Clock ,
opts . StreamSilenceTimeout ,
func ( attemptCtx context . Context ) ( fantasy . StreamResponse , error ) {
return opts . Model . Stream ( attemptCtx , call )
},
opts . Metrics ,
2026-09-30 11:53:58 -05:00
opts . Stages ,
opts . StageModel ,
2026-06-12 13:33:12 +02:00
)
if streamErr != nil {
2026-06-16 11:14:37 +01:00
wrappedErr := wrapProviderStreamError ( errorProvider , streamErr )
2026-09-30 11:53:58 -05:00
streamSpan . End ( wrappedErr )
2026-06-16 11:14:37 +01:00
classified := chaterror . Classify ( wrappedErr ). WithProvider ( errorProvider )
2026-06-12 13:33:12 +02:00
if classified . Retryable {
opts . Metrics . RecordStreamRetry ( provider , modelName , classified )
2026-03-24 14:57:40 +01:00
}
2026-06-12 13:33:12 +02:00
return AssistantOutcome {}, wrappedErr
}
2026-09-30 11:53:58 -05:00
// release closes the time_to_first_token window, so it must run
// before the stream stage ends.
defer func () {
attempt . release ()
streamSpan . End ( retErr )
}()
2026-06-12 13:33:12 +02:00
2026-09-14 10:51:22 +02:00
result , processErr := processStepStream ( attempt . stream , opts . Clock , publishMessagePart )
2026-10-28 18:35:20 -07:00
// Observe before error handling so failed or rejected responses are still counted.
opts . Metrics . ResponseSizeBytes . WithLabelValues ( provider , modelName ). Observe ( float64 ( result . responseBytes ))
2026-06-12 13:33:12 +02:00
if err := attempt . finish ( processErr ); err != nil {
2026-06-16 11:14:37 +01:00
wrappedErr := wrapProviderStreamError ( errorProvider , err )
classified := chaterror . Classify ( wrappedErr ). WithProvider ( errorProvider )
2026-06-12 13:33:12 +02:00
if classified . Retryable {
opts . Metrics . RecordStreamRetry ( provider , modelName , classified )
}
return AssistantOutcome {}, wrappedErr
}
contextLimit := extractContextLimitWithFallback ( result . providerMetadata , opts . ContextLimitFallback )
result . content = chatsanitize . SanitizeAnthropicProviderToolStepContent (
ctx , opts . Logger , provider , modelName ,
"assistant_helper" , 0 , result . finishReason , result . content ,
)
2026-09-30 12:20:03 +02:00
if result . finishReason == fantasy . FinishReasonLength {
resolveOutputLimitToolCalls ( & result , call . MaxOutputTokens , opts . Clock , publishMessagePart )
}
2026-07-24 19:40:53 +02:00
// A content-filter finish without user-visible output means the
// provider's safety classifiers blocked the whole response (e.g.
// Anthropic stop_reason "refusal"). The refusal can arrive after
// reasoning has already streamed, so reasoning alone must not
// count as output.
if result . finishReason == fantasy . FinishReasonContentFilter && ! hasUserVisibleContent ( result . content ) {
2026-07-15 16:36:31 +10:00
return AssistantOutcome {}, contentFilterError ( errorProvider , result . providerMetadata )
}
2026-08-24 17:09:00 +02:00
// Treat discarded responses as retryable so an empty turn is not persisted.
if silentNoOutputFinish ( result . finishReason ) && ! hasUserVisibleContent ( result . content ) && len ( result . toolCalls ) == 0 {
noOutputErr := noModelOutputError ( errorProvider , result . finishReason )
opts . Metrics . RecordStreamRetry ( provider , modelName , chaterror . Classify ( noOutputErr ))
return AssistantOutcome {}, noOutputErr
}
2026-06-12 13:33:12 +02:00
step := PersistedStep {
Content : result . content ,
Usage : result . usage ,
ContextLimit : contextLimit ,
Runtime : opts . Clock . Since ( stepStart ),
2026-09-23 19:01:24 +02:00
ProviderResponseID : result . providerResponseID ,
2026-06-12 13:33:12 +02:00
ToolCallCreatedAt : result . toolCallCreatedAt ,
ToolResultCreatedAt : result . toolResultCreatedAt ,
ReasoningStartedAt : result . reasoningStartedAt ,
ReasoningCompletedAt : result . reasoningCompletedAt ,
}
return AssistantOutcome {
Step : step ,
ToolCalls : append ([] fantasy . ToolCallContent ( nil ), result . toolCalls ... ),
FinishReason : result . finishReason ,
}, nil
}
2026-09-30 12:20:03 +02:00
// resolveOutputLimitToolCalls gives every call whose input the output token
// limit cut off an error result, so the model learns why the call failed
// instead of retrying the same oversized input or silently losing the call.
func resolveOutputLimitToolCalls (
result * stepResult ,
maxOutputTokens * int64 ,
clock quartz . Clock ,
publishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart ),
) {
limit := "the output token limit"
if maxOutputTokens != nil && * maxOutputTokens > 0 {
limit = fmt . Sprintf ( "the output token limit (%d tokens)" , * maxOutputTokens )
}
message := "This tool call was not executed because the response reached " + limit +
" while writing its input. Split the content into smaller tool calls."
appendResult := func ( toolCall fantasy . ToolCallContent ) {
toolResult := fantasy . ToolResultContent {
ToolCallID : toolCall . ToolCallID ,
ToolName : toolCall . ToolName ,
Result : fantasy . ToolResultOutputContentError { Error : xerrors . New ( message )},
}
result . content = append ( result . content , toolResult )
now := clockNow ( clock )
if result . toolResultCreatedAt == nil {
result . toolResultCreatedAt = make ( map [ string ] time . Time )
}
result . toolResultCreatedAt [ toolCall . ToolCallID ] = now
part := chatprompt . PartFromContent ( toolResult )
part . CreatedAt = & now
publishMessagePart ( codersdk . ChatMessageRoleTool , part )
}
var truncated [] fantasy . ToolCallContent
for _ , block := range result . content {
toolCall , ok := fantasy . AsContentType [ fantasy . ToolCallContent ]( block )
if ok && ! toolCall . ProviderExecuted && ! json . Valid ([] byte ( toolCall . Input )) {
truncated = append ( truncated , toolCall )
}
}
for _ , toolCall := range truncated {
appendResult ( toolCall )
}
for _ , toolCall := range result . unfinishedToolCalls {
result . content = append ( result . content , toolCall )
now := clockNow ( clock )
if result . toolCallCreatedAt == nil {
result . toolCallCreatedAt = make ( map [ string ] time . Time )
}
result . toolCallCreatedAt [ toolCall . ToolCallID ] = now
part := chatprompt . PartFromContent ( toolCall )
part . CreatedAt = & now
publishMessagePart ( codersdk . ChatMessageRoleAssistant , part )
appendResult ( toolCall )
}
}
2026-06-12 13:33:12 +02:00
func wrapProviderStreamError ( provider string , err error ) error {
if err == nil {
2026-03-24 14:57:40 +01:00
return nil
}
2026-06-12 13:33:12 +02:00
classified := chaterror . Classify ( err ). WithProvider ( provider )
if ! classified . Retryable && classified . StatusCode == 0 && errors . Is ( err , context . Canceled ) {
wrapped := errors . Join ( chaterror . ErrProviderTransportReset , err )
reclassified := chaterror . Classify ( wrapped ). WithProvider ( provider )
if reclassified . Retryable {
classified = reclassified
err = wrapped
2026-02-27 11:50:56 -05:00
}
2026-06-12 13:33:12 +02:00
}
return xerrors . Errorf ( "stream response: %w" , chaterror . WithClassification ( err , classified ))
}
2026-02-27 11:50:56 -05:00
2026-08-24 17:09:00 +02:00
// hasUserVisibleContent ignores reasoning and blank text because neither can
// complete a user-facing response.
2026-07-24 19:40:53 +02:00
func hasUserVisibleContent ( content [] fantasy . Content ) bool {
for _ , part := range content {
2026-08-24 17:09:00 +02:00
switch value := part .( type ) {
2026-07-24 19:40:53 +02:00
case fantasy . ReasoningContent , * fantasy . ReasoningContent :
2026-08-24 17:09:00 +02:00
case fantasy . TextContent :
if strings . TrimSpace ( value . Text ) != "" {
return true
}
case * fantasy . TextContent :
if value != nil && strings . TrimSpace ( value . Text ) != "" {
return true
}
2026-07-24 19:40:53 +02:00
default :
return true
}
}
return false
}
2026-08-24 17:09:00 +02:00
func silentNoOutputFinish ( reason fantasy . FinishReason ) bool {
switch reason {
case fantasy . FinishReasonUnknown , fantasy . FinishReasonError ,
fantasy . FinishReasonOther , fantasy . FinishReasonToolCalls :
return true
default :
return false
}
}
func noModelOutputError ( provider string , reason fantasy . FinishReason ) error {
classified := chaterror . ClassifiedError {
Message : "The model ended its response without producing any output." ,
Detail : "finish reason: " + string ( reason ),
Kind : codersdk . ChatErrorKindGeneric ,
Provider : provider ,
Retryable : true ,
}
return chaterror . WithClassification ( ErrNoModelOutput , classified )
}
2026-07-15 16:36:31 +10:00
func contentFilterError ( provider string , metadata fantasy . ProviderMetadata ) error {
classified := chaterror . ClassifiedError {
Kind : codersdk . ChatErrorKindContentFilter ,
Provider : provider ,
Retryable : false ,
}
if refusal := fantasyanthropic . GetRefusalMetadata ( metadata ); refusal != nil {
classified . Message = chaterror . ContentFilterMessage ( provider , refusal . Category )
classified . Detail = strings . TrimSpace ( refusal . Explanation )
}
return chaterror . WithClassification ( ErrContentFiltered , classified )
}
2026-06-12 13:33:12 +02:00
// ExecuteLocalTools runs local tool calls and returns durable tool results. It
// does not retry or persist.
2026-08-25 16:31:06 +07:00
func ExecuteLocalTools ( ctx context . Context , opts ExecuteLocalToolsOptions ) ( PersistedStep , error ) {
2026-06-12 13:33:12 +02:00
if opts . Metrics == nil {
opts . Metrics = NopMetrics ()
}
provider := opts . ModelProvider
if provider == "" {
provider = "unknown"
}
modelName := opts . ModelName
if modelName == "" {
modelName = "unknown"
}
publishMessagePart := func ( role codersdk . ChatMessageRole , part codersdk . ChatMessagePart ) {
if opts . PublishMessagePart != nil {
opts . PublishMessagePart ( role , part )
2026-03-24 14:57:40 +01:00
}
2026-06-12 13:33:12 +02:00
}
// Expose the publisher on the execution context so tools that stream
// intermediate output (e.g. the advisor tool) can publish parts
// without capturing the publisher at construction time.
ctx = WithMessagePartPublisher ( ctx , opts . PublishMessagePart )
if ctx . Err () != nil {
2026-08-25 16:31:06 +07:00
return PersistedStep {}, ctx . Err ()
2026-02-27 11:50:56 -05:00
}
2026-06-12 13:33:12 +02:00
localCalls := make ([] fantasy . ToolCallContent , 0 , len ( opts . ToolCalls ))
for _ , tc := range opts . ToolCalls {
if ! tc . ProviderExecuted {
localCalls = append ( localCalls , tc )
}
}
if len ( localCalls ) == 0 {
2026-08-25 16:31:06 +07:00
return PersistedStep {}, nil
2026-06-12 13:33:12 +02:00
}
var result stepResult
policyResults , exclusiveViolation := applyExclusiveToolPolicy (
localCalls ,
opts . ExclusiveToolNames ,
opts . Metrics ,
provider ,
modelName ,
)
if exclusiveViolation {
now := clockNow ( opts . Clock )
for _ , tr := range policyResults {
recordToolResultTimestamp ( & result , tr . ToolCallID , now )
publishToolAttachments ( ctx , opts . Logger , tr , now , publishMessagePart )
ssePart := chatprompt . PartFromContentWithLogger ( ctx , opts . Logger , tr )
ssePart . CreatedAt = & now
publishMessagePart ( codersdk . ChatMessageRoleTool , ssePart )
result . content = append ( result . content , tr )
}
if ctx . Err () != nil {
2026-08-25 16:31:06 +07:00
return PersistedStep {}, ctx . Err ()
2026-06-12 13:33:12 +02:00
}
2026-08-25 16:31:06 +07:00
return PersistedStep {
2026-06-12 13:33:12 +02:00
Content : result . content ,
ToolResultCreatedAt : result . toolResultCreatedAt ,
2026-08-25 16:31:06 +07:00
}, nil
2026-06-12 13:33:12 +02:00
}
2026-08-25 16:40:41 +07:00
batchStart := clockNow ( opts . Clock )
2026-10-20 11:26:42 +02:00
toolExecutions := newToolBatch ( opts , provider , modelName ). run ( ctx , localCalls , batchStart )
2026-08-25 16:40:41 +07:00
for _ , execution := range toolExecutions {
tr := execution . content
completedAt := execution . interval . End
recordToolResultTimestamp ( & result , tr . ToolCallID , completedAt )
publishToolAttachments ( ctx , opts . Logger , tr , completedAt , publishMessagePart )
ssePart := chatprompt . PartFromContentWithLogger ( ctx , opts . Logger , tr )
ssePart . CreatedAt = & completedAt
2026-10-26 17:55:52 +01:00
userResult , projectionErr := chattool . UserResultFromMetadata ( tr . ClientMetadata )
if projectionErr != nil {
return PersistedStep {}, projectionErr
}
if len ( userResult ) > 0 {
ssePart . Result = userResult
}
2026-08-25 16:40:41 +07:00
publishMessagePart ( codersdk . ChatMessageRoleTool , ssePart )
result . content = append ( result . content , tr )
}
2026-06-12 13:33:12 +02:00
if ctx . Err () != nil {
2026-08-25 16:31:06 +07:00
return PersistedStep {}, ctx . Err ()
2026-06-12 13:33:12 +02:00
}
2026-08-25 16:40:41 +07:00
billedIntervals := billableBatchIntervals ( toolExecutions , opts . UnbilledToolNames )
2026-08-25 16:31:06 +07:00
return PersistedStep {
2026-06-12 13:33:12 +02:00
Content : result . content ,
ToolResultCreatedAt : result . toolResultCreatedAt ,
2026-08-25 16:40:41 +07:00
BatchRuntime : BilledIntervalsDuration ( billedIntervals ),
BatchBilledCalls : len ( billedIntervals ),
2026-08-25 16:31:06 +07:00
}, nil
2026-03-02 18:51:57 -05:00
}
2026-08-25 16:40:41 +07:00
// billableBatchIntervals returns the billed execution intervals.
// Unbilled tools and calls without both stamps do not count.
func billableBatchIntervals (
executions [] toolExecutionResult ,
unbilledToolNames map [ string ] bool ,
) [] BilledInterval {
intervals := make ([] BilledInterval , 0 , len ( executions ))
for _ , execution := range executions {
if unbilledToolNames [ execution . content . ToolName ] ||
execution . interval . Start . IsZero () ||
execution . interval . End . IsZero () {
continue
}
intervals = append ( intervals , execution . interval )
}
return intervals
}
// BilledInterval is one billed tool call's execution window.
type BilledInterval struct {
Start time . Time
End time . Time
}
// BilledIntervalsDuration returns the union duration of valid intervals.
// Overlaps count once, gaps do not, and inverted intervals are ignored.
// Committed and interrupted batches share this helper.
func BilledIntervalsDuration ( intervals [] BilledInterval ) time . Duration {
valid := slices . DeleteFunc ( slices . Clone ( intervals ), func ( iv BilledInterval ) bool {
return iv . End . Before ( iv . Start )
})
if len ( valid ) == 0 {
return 0
}
slices . SortFunc ( valid , func ( a , b BilledInterval ) int {
return a . Start . Compare ( b . Start )
})
curStart , curEnd := valid [ 0 ]. Start , valid [ 0 ]. End
var total time . Duration
for _ , iv := range valid [ 1 :] {
if iv . Start . After ( curEnd ) {
total += curEnd . Sub ( curStart )
curStart , curEnd = iv . Start , iv . End
continue
}
if iv . End . After ( curEnd ) {
curEnd = iv . End
}
}
return total + curEnd . Sub ( curStart )
}
2026-05-11 17:43:40 +01:00
// prepareMessagesForRequest applies the prompt preparation pipeline used
// immediately before sending messages to a provider. It returns the
// possibly updated canonical messages and an independent provider-ready
2026-05-18 15:20:33 +10:00
// prompt. When preparation fails, the prompt result is nil and err is the
// terminal prompt-preparation failure.
2026-05-11 17:43:40 +01:00
func prepareMessagesForRequest (
ctx context . Context ,
2026-09-05 11:39:10 +02:00
model fantasy . LanguageModel ,
logger slog . Logger ,
2026-05-11 17:43:40 +01:00
messages [] fantasy . Message ,
provider string ,
modelName string ,
step int ,
totalSteps int ,
2026-09-05 11:39:10 +02:00
) ([] fantasy . Message , error ) {
2026-05-11 17:43:40 +01:00
// Copy messages so provider-specific caching mutations don't leak
// back to the canonical message slice.
2026-09-05 11:39:10 +02:00
prompt := slices . Clone ( messages )
2026-05-11 17:43:40 +01:00
prompt , sanitizeStats := chatsanitize . SanitizeAnthropicProviderToolHistory ( provider , prompt )
chatsanitize . LogAnthropicProviderToolSanitization (
2026-09-05 11:39:10 +02:00
ctx , logger , "pre_request" , provider , modelName , sanitizeStats ,
2026-05-11 17:43:40 +01:00
slog . F ( "step_index" , step ),
slog . F ( "total_steps" , totalSteps ),
)
2026-09-05 11:39:10 +02:00
prompt , err := chatsanitize . ApplyAnthropicProviderToolGuard (
ctx , logger , provider , modelName , prompt ,
2026-05-11 17:43:40 +01:00
)
2026-05-18 15:20:33 +10:00
if err != nil {
err = chaterror . WithClassification (
xerrors . Errorf ( "apply anthropic provider tool guard: %w" , err ),
chaterror . ClassifiedError {
Message : "The chat continuation failed due to an internal state mismatch. This is not a configuration or billing issue. Start a new chat to continue." ,
Detail : "Anthropic replay diagnostic: match=provider_tool_guard_postcondition_failed." ,
Kind : codersdk . ChatErrorKindGeneric ,
Provider : provider ,
Retryable : false ,
},
)
2026-09-05 11:39:10 +02:00
return nil , err
2026-05-18 15:20:33 +10:00
}
2026-10-04 22:55:37 +02:00
if applyPromptCaching := promptCachingStrategyFor ( model , modelName ); applyPromptCaching != nil {
applyPromptCaching ( prompt )
2026-05-11 17:43:40 +01:00
}
2026-09-05 11:39:10 +02:00
return prompt , nil
2026-05-11 17:43:40 +01:00
}
2026-05-28 21:02:40 +10:00
// guardedAttempt owns an attempt-scoped context and silence guard
2026-03-25 13:47:54 +11:00
// around a provider stream. release is idempotent and frees the
2026-05-28 21:02:40 +10:00
// attempt-scoped timer/context. finish canonicalizes silence timeout
2026-03-25 13:47:54 +11:00
// errors before the retry loop classifies them.
type guardedAttempt struct {
ctx context . Context
stream fantasy . StreamResponse
release func ()
finish func ( error ) error
}
2026-05-28 21:02:40 +10:00
// streamSilenceGuard arbitrates whether an attempt times out while
// waiting for the next stream part. Exactly one outcome wins: the
// timer cancels the attempt, or release disarms the timer.
type streamSilenceGuard struct {
mu sync . Mutex
timer * quartz . Timer
cancel context . CancelCauseFunc
timeout time . Duration
settled bool
2026-03-25 13:47:54 +11:00
}
2026-05-28 21:02:40 +10:00
func newStreamSilenceGuard (
2026-04-10 00:40:09 +10:00
clock quartz . Clock ,
2026-03-25 13:47:54 +11:00
timeout time . Duration ,
cancel context . CancelCauseFunc ,
2026-05-28 21:02:40 +10:00
) * streamSilenceGuard {
guard := & streamSilenceGuard {
cancel : cancel ,
timeout : timeout ,
}
2026-09-18 15:24:15 +10:00
if timeout < 0 {
return guard
}
2026-05-28 21:02:40 +10:00
guard . timer = clock . AfterFunc (
timeout ,
guard . onTimeout ,
streamSilenceGuardTimerTag ,
)
2026-03-25 13:47:54 +11:00
return guard
}
2026-05-28 21:02:40 +10:00
func ( g * streamSilenceGuard ) settle () bool {
g . mu . Lock ()
defer g . mu . Unlock ()
if g . settled {
return false
}
g . settled = true
return true
2026-03-25 13:47:54 +11:00
}
2026-05-28 21:02:40 +10:00
func ( g * streamSilenceGuard ) onTimeout () {
if ! g . settle () {
return
}
g . cancel ( errStreamSilenceTimeout )
2026-03-25 13:47:54 +11:00
}
2026-05-28 21:02:40 +10:00
func ( g * streamSilenceGuard ) Reset () {
g . mu . Lock ()
defer g . mu . Unlock ()
2026-09-18 15:24:15 +10:00
if g . settled || g . timer == nil {
2026-05-28 21:02:40 +10:00
return
}
g . timer . Reset ( g . timeout , streamSilenceGuardTimerTag )
}
func ( g * streamSilenceGuard ) Disarm () {
2026-09-18 15:24:15 +10:00
if ! g . settle () || g . timer == nil {
2026-05-28 21:02:40 +10:00
return
}
g . timer . Stop ()
}
func classifyStreamSilenceTimeout (
2026-03-25 13:47:54 +11:00
attemptCtx context . Context ,
provider string ,
err error ,
) error {
2026-05-28 21:02:40 +10:00
if ! errors . Is ( context . Cause ( attemptCtx ), errStreamSilenceTimeout ) {
2026-03-25 13:47:54 +11:00
return err
}
if err == nil {
2026-05-28 21:02:40 +10:00
err = errStreamSilenceTimeout
2026-03-25 13:47:54 +11:00
}
return chaterror . WithClassification ( err , chaterror . ClassifiedError {
2026-06-04 18:36:02 +10:00
Kind : codersdk . ChatErrorKindStreamSilenceTimeout ,
2026-03-25 13:47:54 +11:00
Provider : provider ,
Retryable : true ,
})
}
2026-09-18 15:24:15 +10:00
type streamWatchdogKey struct {}
// WithStreamWatchdog returns a context whose guarded streams call kick
// with the silence timeout whenever the guard arms or resets.
func WithStreamWatchdog ( ctx context . Context , kick func ( silence time . Duration )) context . Context {
return context . WithValue ( ctx , streamWatchdogKey {}, kick )
}
2026-09-30 11:53:58 -05:00
var errNoFirstToken = xerrors . New ( "stream ended before the first token" )
2026-03-25 13:47:54 +11:00
func guardedStream (
parent context . Context ,
2026-04-17 16:16:30 +01:00
provider , model string ,
2026-04-10 00:40:09 +10:00
clock quartz . Clock ,
2026-03-25 13:47:54 +11:00
timeout time . Duration ,
openStream func ( context . Context ) ( fantasy . StreamResponse , error ),
2026-04-15 19:53:10 +01:00
metrics * Metrics ,
2026-09-30 11:53:58 -05:00
stages * StageTracer ,
stageModel StageModel ,
2026-03-25 13:47:54 +11:00
) ( guardedAttempt , error ) {
attemptCtx , cancelAttempt := context . WithCancelCause ( parent )
2026-09-18 15:24:15 +10:00
kick , _ := parent . Value ( streamWatchdogKey {}).( func ( time . Duration ))
if kick == nil {
kick = func ( time . Duration ) {}
}
2026-05-28 21:02:40 +10:00
guard := newStreamSilenceGuard ( clock , timeout , cancelAttempt )
2026-09-18 15:24:15 +10:00
kick ( timeout )
2026-09-30 11:53:58 -05:00
// A nil tracer still times the window for TTFTSeconds.
if stages == nil {
stages = NewStageTracer ( nil , nil , WithClock ( clock ))
}
_ , ttftSpan := stages . Start ( parent , StageTimeToFirstToken )
ttftSpan . SetModel ( stageModel )
var ttftOnce sync . Once
// A silence guard cancellation surfaces as a context error, so it is
// replaced with the classified timeout.
finishTTFT := func ( err error ) {
ttftOnce . Do ( func () {
if err != nil {
if errors . Is ( context . Cause ( attemptCtx ), errStreamSilenceTimeout ) {
err = classifyStreamSilenceTimeout ( attemptCtx , provider , nil )
}
ttftSpan . EndWithoutObservation ( err )
return
}
elapsed := ttftSpan . End ( nil )
metrics . TTFTSeconds . WithLabelValues ( provider , model ). Observe ( elapsed . Seconds ())
})
}
2026-03-25 13:47:54 +11:00
var releaseOnce sync . Once
release := func () {
releaseOnce . Do ( func () {
guard . Disarm ()
cancelAttempt ( nil )
2026-09-30 11:53:58 -05:00
finishTTFT ( errNoFirstToken )
2026-03-25 13:47:54 +11:00
})
}
stream , err := openStream ( attemptCtx )
if err != nil {
2026-05-28 21:02:40 +10:00
err = classifyStreamSilenceTimeout ( attemptCtx , provider , err )
2026-09-30 11:53:58 -05:00
finishTTFT ( err )
2026-03-25 13:47:54 +11:00
release ()
return guardedAttempt {}, err
}
return guardedAttempt {
ctx : attemptCtx ,
stream : fantasy . StreamResponse ( func ( yield func ( fantasy . StreamPart ) bool ) {
for part := range stream {
2026-05-28 21:02:40 +10:00
guard . Reset ()
2026-09-18 15:24:15 +10:00
kick ( timeout )
2026-09-30 11:53:58 -05:00
switch part . Type {
case fantasy . StreamPartTypeError :
finishTTFT ( part . Error )
2026-10-27 02:29:10 +02:00
case fantasy . StreamPartTypeWarnings , fantasy . StreamPartTypeKeepalive , fantasy . StreamPartTypeFinish :
// None is model output, so the window stays open.
2026-09-30 11:53:58 -05:00
default :
// Start markers such as text_start count as output.
finishTTFT ( nil )
}
2026-03-25 13:47:54 +11:00
if ! yield ( part ) {
return
}
}
}),
release : release ,
finish : func ( err error ) error {
2026-05-28 21:02:40 +10:00
return classifyStreamSilenceTimeout ( attemptCtx , provider , err )
2026-03-25 13:47:54 +11:00
},
}, nil
}
2026-06-12 13:33:12 +02:00
// clockNow returns the clock's current time normalized the same
// way as dbtime.Now so persisted timestamps are Postgres-safe.
func clockNow ( clock quartz . Clock ) time . Time {
return dbtime . Time ( clock . Now (). UTC ())
}
2026-03-02 18:51:57 -05:00
// processStepStream consumes a fantasy StreamResponse and
// accumulates all content into a stepResult. Callbacks fire
// inline and their errors propagate directly.
func processStepStream (
stream fantasy . StreamResponse ,
2026-06-12 13:33:12 +02:00
clock quartz . Clock ,
2026-03-13 17:53:26 +02:00
publishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart ),
2026-03-02 18:51:57 -05:00
) ( stepResult , error ) {
var result stepResult
2026-09-14 10:51:22 +02:00
providerExecutedCalls := make ( map [ string ] bool )
2026-10-12 22:53:07 +02:00
activeTextContent := make ( map [ string ] textState )
2026-03-02 18:51:57 -05:00
activeReasoningContent := make ( map [ string ] reasoningState )
// Track tool names by ID for input delta publishing.
toolNames := make ( map [ string ] string )
2026-09-30 12:20:03 +02:00
var startedToolInputIDs [] string
2026-03-02 18:51:57 -05:00
for part := range stream {
2026-10-28 18:35:20 -07:00
// Counts Delta on every part type. This relies on providers keeping
// summary data, such as ToolCall input, out of Delta. Only
// TestGenerateAssistant_ResponseSizeBytesByPartType checks this, so
// update it when adding new part types.
result . responseBytes += len ( part . Delta )
2026-03-02 18:51:57 -05:00
switch part . Type {
case fantasy . StreamPartTypeTextStart :
2026-10-12 22:53:07 +02:00
// OpenAI labels narration when the text starts, so every delta
// can carry the label before the text ends.
activeTextContent [ part . ID ] = textState {
options : part . ProviderMetadata ,
narration : chatprompt . IsNarration ( part . ProviderMetadata ),
}
2026-03-02 18:51:57 -05:00
case fantasy . StreamPartTypeTextDelta :
2026-10-12 22:53:07 +02:00
textPart := codersdk . ChatMessageText ( part . Delta )
if active , exists := activeTextContent [ part . ID ]; exists {
active . text += part . Delta
activeTextContent [ part . ID ] = active
textPart . Narration = active . narration
2026-03-02 18:51:57 -05:00
}
2026-10-12 22:53:07 +02:00
publishMessagePart ( codersdk . ChatMessageRoleAssistant , textPart )
2026-03-02 18:51:57 -05:00
case fantasy . StreamPartTypeTextEnd :
2026-10-12 22:53:07 +02:00
if active , exists := activeTextContent [ part . ID ]; exists {
if len ( part . ProviderMetadata ) > 0 {
active . options = part . ProviderMetadata
}
2026-03-02 18:51:57 -05:00
result . content = append ( result . content , fantasy . TextContent {
2026-10-12 22:53:07 +02:00
Text : active . text ,
ProviderMetadata : active . options ,
2026-03-02 18:51:57 -05:00
})
delete ( activeTextContent , part . ID )
}
case fantasy . StreamPartTypeReasoningStart :
activeReasoningContent [ part . ID ] = reasoningState {
2026-05-18 12:30:30 -04:00
text : part . Delta ,
options : part . ProviderMetadata ,
2026-06-12 13:33:12 +02:00
startedAt : clockNow ( clock ),
2026-03-02 18:51:57 -05:00
}
case fantasy . StreamPartTypeReasoningDelta :
2026-06-12 13:33:12 +02:00
reasoningPart := codersdk . ChatMessageReasoning ( part . Delta )
2026-03-02 18:51:57 -05:00
if active , exists := activeReasoningContent [ part . ID ]; exists {
active . text += part . Delta
2026-05-18 15:20:33 +10:00
if len ( part . ProviderMetadata ) > 0 {
active . options = part . ProviderMetadata
}
2026-03-02 18:51:57 -05:00
activeReasoningContent [ part . ID ] = active
2026-06-12 13:33:12 +02:00
if ! active . startedAt . IsZero () {
startedAt := active . startedAt
reasoningPart . CreatedAt = & startedAt
}
2026-03-02 18:51:57 -05:00
}
2026-06-12 13:33:12 +02:00
publishMessagePart ( codersdk . ChatMessageRoleAssistant , reasoningPart )
2026-03-02 18:51:57 -05:00
case fantasy . StreamPartTypeReasoningEnd :
if active , exists := activeReasoningContent [ part . ID ]; exists {
2026-05-18 15:20:33 +10:00
if len ( part . ProviderMetadata ) > 0 {
2026-03-02 18:51:57 -05:00
active . options = part . ProviderMetadata
}
content := fantasy . ReasoningContent {
Text : active . text ,
ProviderMetadata : active . options ,
}
result . content = append ( result . content , content )
2026-05-18 12:30:30 -04:00
result . reasoningStartedAt = append ( result . reasoningStartedAt , active . startedAt )
2026-06-12 13:33:12 +02:00
result . reasoningCompletedAt = append ( result . reasoningCompletedAt , clockNow ( clock ))
2026-03-02 18:51:57 -05:00
delete ( activeReasoningContent , part . ID )
}
case fantasy . StreamPartTypeToolInputStart :
2026-09-30 12:20:03 +02:00
if _ , exists := providerExecutedCalls [ part . ID ]; ! exists {
startedToolInputIDs = append ( startedToolInputIDs , part . ID )
}
2026-09-14 10:51:22 +02:00
providerExecutedCalls [ part . ID ] = part . ProviderExecuted
2026-03-02 18:51:57 -05:00
if strings . TrimSpace ( part . ToolCallName ) != "" {
toolNames [ part . ID ] = part . ToolCallName
}
case fantasy . StreamPartTypeToolInputDelta :
2026-09-14 10:51:22 +02:00
providerExecuted := providerExecutedCalls [ part . ID ]
2026-03-02 18:51:57 -05:00
toolName := toolNames [ part . ID ]
2026-03-13 17:53:26 +02:00
publishMessagePart ( codersdk . ChatMessageRoleAssistant , codersdk . ChatMessagePart {
2026-03-11 14:33:15 -07:00
Type : codersdk . ChatMessagePartTypeToolCall ,
ToolCallID : part . ID ,
ToolName : toolName ,
ArgsDelta : part . Delta ,
ProviderExecuted : providerExecuted ,
2026-03-02 18:51:57 -05:00
})
case fantasy . StreamPartTypeToolInputEnd :
// No callback needed; the full tool call arrives in
// StreamPartTypeToolCall.
case fantasy . StreamPartTypeToolCall :
tc := fantasy . ToolCallContent {
ToolCallID : part . ID ,
ToolName : part . ToolCallName ,
Input : part . ToolCallInput ,
ProviderExecuted : part . ProviderExecuted ,
ProviderMetadata : part . ProviderMetadata ,
}
result . toolCalls = append ( result . toolCalls , tc )
result . content = append ( result . content , tc )
if strings . TrimSpace ( part . ToolCallName ) != "" {
toolNames [ part . ID ] = part . ToolCallName
}
// Clean up active tool call tracking.
2026-09-14 10:51:22 +02:00
delete ( providerExecutedCalls , part . ID )
2026-03-02 18:51:57 -05:00
2026-04-08 12:42:03 -04:00
// Record when the model emitted this tool call
// so the persisted part carries an accurate
// timestamp for duration computation.
2026-06-12 13:33:12 +02:00
now := clockNow ( clock )
2026-04-08 12:42:03 -04:00
if result . toolCallCreatedAt == nil {
result . toolCallCreatedAt = make ( map [ string ] time . Time )
}
result . toolCallCreatedAt [ part . ID ] = now
ssePart := chatprompt . PartFromContent ( tc )
ssePart . CreatedAt = & now
2026-03-02 18:51:57 -05:00
publishMessagePart (
2026-03-13 17:53:26 +02:00
codersdk . ChatMessageRoleAssistant ,
2026-04-08 12:42:03 -04:00
ssePart ,
2026-03-02 18:51:57 -05:00
)
case fantasy . StreamPartTypeSource :
sourceContent := fantasy . SourceContent {
SourceType : part . SourceType ,
ID : part . ID ,
URL : part . URL ,
Title : part . Title ,
2026-10-27 12:46:27 +02:00
ToolCallID : part . SourceToolCallID ,
2026-03-02 18:51:57 -05:00
ProviderMetadata : part . ProviderMetadata ,
}
result . content = append ( result . content , sourceContent )
publishMessagePart (
2026-03-13 17:53:26 +02:00
codersdk . ChatMessageRoleAssistant ,
2026-03-02 18:51:57 -05:00
chatprompt . PartFromContent ( sourceContent ),
)
2026-03-11 14:33:15 -07:00
case fantasy . StreamPartTypeToolResult :
// Provider-executed tool results (e.g. web search)
// are emitted by the provider and added directly
// to the step content for multi-turn round-tripping.
// This mirrors fantasy's agent.go accumulation logic.
if part . ProviderExecuted {
tr := fantasy . ToolResultContent {
ToolCallID : part . ID ,
ToolName : part . ToolCallName ,
ProviderExecuted : part . ProviderExecuted ,
ProviderMetadata : part . ProviderMetadata ,
}
2026-10-27 12:46:27 +02:00
if err := providerToolResultError ( part . ProviderMetadata ); err != nil {
tr . Result = fantasy . ToolResultOutputContentError { Error : err }
}
2026-03-11 14:33:15 -07:00
result . content = append ( result . content , tr )
2026-04-08 12:42:03 -04:00
2026-06-12 13:33:12 +02:00
now := clockNow ( clock )
2026-04-08 12:42:03 -04:00
if result . toolResultCreatedAt == nil {
result . toolResultCreatedAt = make ( map [ string ] time . Time )
}
result . toolResultCreatedAt [ part . ID ] = now
ssePart := chatprompt . PartFromContent ( tr )
ssePart . CreatedAt = & now
2026-03-11 14:33:15 -07:00
publishMessagePart (
2026-03-13 17:53:26 +02:00
codersdk . ChatMessageRoleTool ,
2026-04-08 12:42:03 -04:00
ssePart ,
2026-03-11 14:33:15 -07:00
)
}
2026-03-02 18:51:57 -05:00
case fantasy . StreamPartTypeFinish :
result . usage = part . Usage
result . finishReason = part . FinishReason
result . providerMetadata = part . ProviderMetadata
2026-09-23 19:01:24 +02:00
result . providerResponseID = providerResponseID ( part )
2026-03-02 18:51:57 -05:00
case fantasy . StreamPartTypeError :
return result , part . Error
}
}
2026-09-30 12:20:03 +02:00
for _ , id := range startedToolInputIDs {
providerExecuted , unfinished := providerExecutedCalls [ id ]
if ! unfinished || providerExecuted {
continue
}
result . unfinishedToolCalls = append ( result . unfinishedToolCalls , fantasy . ToolCallContent {
ToolCallID : id ,
ToolName : toolNames [ id ],
})
}
2026-02-27 11:50:56 -05:00
return result , nil
}
2026-09-23 19:01:24 +02:00
// providerResponseID returns the response ID from a finish part.
// Anthropic and Bedrock report the message ID as the part ID, and the OpenAI
// Responses API reports it in provider metadata. Chat Completions providers
// report neither.
func providerResponseID ( part fantasy . StreamPart ) string {
if part . ID != "" {
return part . ID
}
for _ , metadata := range part . ProviderMetadata {
if responses , ok := metadata .( * fantasyopenai . ResponsesProviderMetadata ); ok && responses != nil {
return responses . ResponseID
}
}
return ""
}
2026-08-25 16:40:41 +07:00
type toolExecutionResult struct {
content fantasy . ToolResultContent
interval BilledInterval
}
2026-10-20 11:26:42 +02:00
type toolBatch struct {
opts ExecuteLocalToolsOptions
provider , model string
maxResultBytes int
tools map [ string ] fantasy . AgentTool
providerRunners map [ string ] struct {}
resultMetadata map [ string ] func ( fantasy . ToolResponse ) fantasy . ProviderMetadata
}
2026-02-27 11:50:56 -05:00
2026-10-20 11:26:42 +02:00
func newToolBatch ( opts ExecuteLocalToolsOptions , provider , model string ) * toolBatch {
toolMap := make ( map [ string ] fantasy . AgentTool , len ( opts . Tools ))
for _ , t := range opts . Tools {
2026-03-02 18:51:57 -05:00
toolMap [ t . Info (). Name ] = t
2026-02-27 11:50:56 -05:00
}
2026-10-20 11:26:42 +02:00
providerRunnerNames := make ( map [ string ] struct {}, len ( opts . ProviderTools ))
2026-05-04 20:30:50 +02:00
resultProviderMetadata := make (
map [ string ] func ( fantasy . ToolResponse ) fantasy . ProviderMetadata ,
2026-10-20 11:26:42 +02:00
len ( opts . ProviderTools ),
2026-05-04 20:30:50 +02:00
)
2026-03-13 19:49:34 +01:00
// Include runners from provider tools so locally-executed
// provider tools (e.g. computer use) can be dispatched.
2026-10-20 11:26:42 +02:00
for _ , pt := range opts . ProviderTools {
2026-05-04 20:30:50 +02:00
if pt . Runner == nil {
continue
}
name := pt . Runner . Info (). Name
toolMap [ name ] = pt . Runner
providerRunnerNames [ name ] = struct {}{}
if pt . ResultProviderMetadata != nil {
resultProviderMetadata [ name ] = pt . ResultProviderMetadata
2026-03-13 19:49:34 +01:00
}
}
2026-10-20 11:26:42 +02:00
return & toolBatch {
opts : opts ,
provider : provider ,
model : model ,
maxResultBytes : ToolResultByteBudget ( opts . ContextLimit ),
tools : toolMap ,
providerRunners : providerRunnerNames ,
resultMetadata : resultProviderMetadata ,
}
}
2026-03-02 18:51:57 -05:00
2026-10-20 11:26:42 +02:00
func ( b * toolBatch ) run ( ctx context . Context , localToolCalls [] fantasy . ToolCallContent , batchStart time . Time ) [] toolExecutionResult {
observed := b . opts . ObservedToolCalls
2026-08-18 19:12:47 +02:00
if observed == nil {
observed = localToolCalls
}
2026-10-20 11:39:54 +02:00
b . notifyStepToolCallObservers ( observed )
2026-08-18 19:12:47 +02:00
2026-10-20 11:26:42 +02:00
recorder := b . opts . BillingRecorder
2026-08-25 16:40:41 +07:00
executions := make ([] toolExecutionResult , len ( localToolCalls ))
2026-08-18 19:12:47 +02:00
runCall := func ( i int , tc fantasy . ToolCallContent ) {
2026-09-30 11:53:58 -05:00
// Started only if the tool runs, so rejected calls record no stage.
var toolSpan * StageSpan
startRun := func ( ctx context . Context ) context . Context {
var toolCtx context . Context
2026-10-20 11:26:42 +02:00
toolCtx , toolSpan = b . opts . Stages . Start ( ctx , StageToolCall ,
2026-09-30 11:53:58 -05:00
attribute . String ( AttrToolName , tc . ToolName ),
)
2026-10-20 11:26:42 +02:00
toolSpan . SetModel ( b . opts . StageModel )
2026-09-30 11:53:58 -05:00
return toolCtx
}
var execErr error
2026-08-18 19:12:47 +02:00
defer func () {
if r := recover (); r != nil {
2026-09-30 11:53:58 -05:00
execErr = xerrors . Errorf ( "tool panicked: %v" , r )
2026-08-25 16:40:41 +07:00
executions [ i ]. content = fantasy . ToolResultContent {
2026-08-18 19:12:47 +02:00
ToolCallID : tc . ToolCallID ,
ToolName : tc . ToolName ,
Result : fantasy . ToolResultOutputContentError {
2026-09-30 11:53:58 -05:00
Error : execErr ,
2026-08-18 19:12:47 +02:00
},
}
}
// Record when this tool completed (or panicked).
// Captured per call so parallel tools get
// accurate individual completion times.
2026-10-20 11:26:42 +02:00
completedAt := clockNow ( b . opts . Clock )
2026-08-25 16:40:41 +07:00
executions [ i ]. interval . End = completedAt
if recorder != nil {
recorder . RecordComplete ( i , completedAt )
}
2026-09-30 11:53:58 -05:00
toolSpan . End ( execErr )
2026-08-18 19:12:47 +02:00
}()
2026-09-30 14:31:52 +03:00
callCtx := ctx
2026-10-20 11:26:42 +02:00
if b . opts . ToolCallContext != nil {
callCtx = b . opts . ToolCallContext ( ctx , tc )
2026-09-30 14:31:52 +03:00
}
2026-10-20 11:26:42 +02:00
executions [ i ]. content , execErr = b . runOne ( callCtx , tc , startRun )
2026-08-18 19:12:47 +02:00
}
2026-08-25 16:40:41 +07:00
// SerialToolCalls run in call order after concurrent siblings settle, so
// order-sensitive state observes final sibling outcomes.
2026-08-18 19:12:47 +02:00
var serialIndexes [] int
2026-03-11 11:00:08 -07:00
var wg sync . WaitGroup
2026-03-11 14:33:15 -07:00
for i , tc := range localToolCalls {
2026-10-20 11:39:54 +02:00
if b . isSerialToolCall ( tc . ToolName ) {
2026-08-18 19:12:47 +02:00
serialIndexes = append ( serialIndexes , i )
continue
}
2026-08-25 16:40:41 +07:00
executions [ i ]. interval . Start = batchStart
if recorder != nil {
recorder . RecordStart ( i , batchStart )
}
2026-08-18 19:12:47 +02:00
wg . Add ( 1 )
2026-04-08 12:42:03 -04:00
go func () {
2026-03-11 11:00:08 -07:00
defer wg . Done ()
2026-08-18 19:12:47 +02:00
runCall ( i , tc )
2026-04-08 12:42:03 -04:00
}()
2026-03-11 11:00:08 -07:00
}
wg . Wait ()
2026-08-25 16:40:41 +07:00
// Reconcile concurrent results before serial tools inspect shared state.
settled := make ([] fantasy . ToolResultContent , 0 , len ( executions ))
for i := range executions {
2026-08-18 19:12:47 +02:00
if ! slices . Contains ( serialIndexes , i ) {
2026-08-25 16:40:41 +07:00
settled = append ( settled , executions [ i ]. content )
2026-08-18 19:12:47 +02:00
}
}
2026-10-20 11:39:54 +02:00
b . notifyStepToolResultObservers ( localToolCalls , observed , settled )
2026-08-18 19:12:47 +02:00
2026-11-02 11:28:22 +01:00
// failed holds the halting tools with a call that failed in this batch.
failed := make ( map [ string ] bool )
afterRejection := callsAfterRejection ( localToolCalls , observed )
2026-08-18 19:12:47 +02:00
for _ , i := range serialIndexes {
2026-11-02 11:28:22 +01:00
tc := localToolCalls [ i ]
name := tc . ToolName
haltedText , halts := b . haltedCallError ( name )
if halts && ( failed [ name ] || afterRejection [ tc . ToolCallID ]) {
// The call never runs, so it records no start and is not billed.
executions [ i ]. content = fantasy . ToolResultContent {
ToolCallID : tc . ToolCallID ,
ToolName : tc . ToolName ,
Result : fantasy . ToolResultOutputContentError { Error : xerrors . New ( haltedText )},
}
executions [ i ]. interval . End = clockNow ( b . opts . Clock )
recordToolResultMetrics ( b . opts . Metrics , b . provider , b . model , executions [ i ]. content )
continue
}
2026-08-25 16:40:41 +07:00
// Stamp serial calls at launch, not batch start.
2026-10-20 11:26:42 +02:00
startedAt := clockNow ( b . opts . Clock )
2026-08-25 16:40:41 +07:00
executions [ i ]. interval . Start = startedAt
if recorder != nil {
recorder . RecordStart ( i , startedAt )
}
2026-11-02 11:28:22 +01:00
runCall ( i , tc )
if _ , isErr := executions [ i ]. content . Result .( fantasy . ToolResultOutputContentError ); halts && isErr {
failed [ name ] = true
}
2026-08-18 19:12:47 +02:00
}
2026-08-25 16:40:41 +07:00
return executions
2026-03-02 18:51:57 -05:00
}
2026-04-30 14:08:59 +02:00
// applyExclusiveToolPolicy checks whether toolCalls violate the
// exclusive-tool policy declared by exclusiveToolNames. When a
// violation is detected it synthesizes deterministic policy-error
// results for every tool call and records size/error metrics so the
// exclusivity failure mode is visible to operators. Returns
// (results, true) on violation; (nil, false) otherwise.
func applyExclusiveToolPolicy (
toolCalls [] fantasy . ToolCallContent ,
exclusiveToolNames map [ string ] bool ,
metrics * Metrics ,
provider , model string ,
) ([] fantasy . ToolResultContent , bool ) {
blockingToolName , ok := firstExclusiveToolName ( toolCalls , exclusiveToolNames )
if ! ok {
return nil , false
}
results := exclusiveToolPolicyResults ( toolCalls , exclusiveToolNames , blockingToolName )
for _ , tr := range results {
recordToolResultMetrics ( metrics , provider , model , tr )
}
return results , true
}
func recordToolResultMetrics ( metrics * Metrics , provider , model string , tr fantasy . ToolResultContent ) {
if metrics == nil {
return
}
label := tr . ToolName
if label == "" {
label = "unknown"
}
metrics . ToolResultSizeBytes . WithLabelValues ( provider , model , label ). Observe (
float64 ( ToolResultSize ( tr )),
)
if _ , ok := tr . Result .( fantasy . ToolResultOutputContentError ); ok {
metrics . RecordToolError ( provider , model , label )
}
}
func firstExclusiveToolName (
toolCalls [] fantasy . ToolCallContent ,
exclusiveToolNames map [ string ] bool ,
) ( string , bool ) {
if len ( toolCalls ) <= 1 || len ( exclusiveToolNames ) == 0 {
return "" , false
}
for _ , tc := range toolCalls {
if exclusiveToolNames [ tc . ToolName ] {
return tc . ToolName , true
}
}
return "" , false
}
func exclusiveToolPolicyResults (
toolCalls [] fantasy . ToolCallContent ,
exclusiveToolNames map [ string ] bool ,
blockingToolName string ,
) [] fantasy . ToolResultContent {
results := make ([] fantasy . ToolResultContent , len ( toolCalls ))
for i , tc := range toolCalls {
message := exclusiveToolSkippedErrorMessage ( blockingToolName )
if exclusiveToolNames [ tc . ToolName ] {
message = exclusiveToolMustRunAloneErrorMessage ( tc . ToolName )
}
results [ i ] = fantasy . ToolResultContent {
ToolCallID : tc . ToolCallID ,
ToolName : tc . ToolName ,
Result : fantasy . ToolResultOutputContentError {
Error : xerrors . New ( message ),
},
}
}
return results
}
func exclusiveToolMustRunAloneErrorMessage ( toolName string ) string {
2026-09-15 17:02:48 +02:00
return toolName + " must be called alone, without other tools in the same batch. If you need more information to feed into the " + toolName + " call, execute the other tools first, then retry with only the " + toolName + " call."
2026-04-30 14:08:59 +02:00
}
func exclusiveToolSkippedErrorMessage ( toolName string ) string {
return "this tool was skipped because " + toolName + " must run alone in its batch. Retry your tool calls without " + toolName + ", or call " + toolName + " separately first."
}
2026-10-20 11:26:42 +02:00
// runOne returns an error only when tool.Run fails, so stage tracing
// can tell those failures from other model-facing error results.
func ( b * toolBatch ) runOne (
2026-03-02 18:51:57 -05:00
ctx context . Context ,
tc fantasy . ToolCallContent ,
2026-09-30 11:53:58 -05:00
startRun func ( context . Context ) context . Context ,
) ( fantasy . ToolResultContent , error ) {
2026-03-02 18:51:57 -05:00
result := fantasy . ToolResultContent {
ToolCallID : tc . ToolCallID ,
ToolName : tc . ToolName ,
ProviderExecuted : false ,
}
2026-04-15 19:53:10 +01:00
defer func () {
2026-10-20 11:26:42 +02:00
recordToolResultMetrics ( b . opts . Metrics , b . provider , b . model , result )
2026-04-15 19:53:10 +01:00
}()
2026-03-02 18:51:57 -05:00
2026-10-30 03:00:59 +01:00
_ , isProviderRunner := b . providerRunners [ tc . ToolName ]
if ! isProviderRunner && ! isToolActive ( tc . ToolName , b . opts . ActiveTools ) && ! b . opts . AllowInactiveTools [ tc . ToolName ] {
2026-04-16 11:12:01 +02:00
result . Result = fantasy . ToolResultOutputContentError {
2026-10-30 03:00:59 +01:00
Error : xerrors . New ( "Tool not active in this turn: " + tc . ToolName ),
2026-04-16 11:12:01 +02:00
}
2026-09-30 11:53:58 -05:00
return result , nil
2026-04-16 11:12:01 +02:00
}
2026-10-30 03:00:59 +01:00
tool , exists := b . tools [ tc . ToolName ]
2026-03-02 18:51:57 -05:00
if ! exists {
result . Result = fantasy . ToolResultOutputContentError {
2026-10-30 03:00:59 +01:00
Error : xerrors . New ( "Tool not found: " + tc . ToolName ),
2026-03-02 18:51:57 -05:00
}
2026-09-30 11:53:58 -05:00
return result , nil
2026-03-02 18:51:57 -05:00
}
2026-10-20 11:26:42 +02:00
b . opts . Logger . Debug ( ctx , "tool execution" ,
2026-04-22 17:19:56 +01:00
slog . F ( "tool_name" , tc . ToolName ),
slog . F ( "tool_call_id" , tc . ToolCallID ),
2026-10-30 03:00:59 +01:00
slog . F ( "builtin" , b . opts . BuiltinToolNames [ tc . ToolName ]),
2026-04-22 17:19:56 +01:00
slog . F ( "is_provider_runner" , isProviderRunner ),
)
2026-09-30 11:53:58 -05:00
if startRun != nil {
ctx = startRun ( ctx )
}
2026-03-02 18:51:57 -05:00
resp , err := tool . Run ( ctx , fantasy . ToolCall {
ID : tc . ToolCallID ,
2026-10-30 03:00:59 +01:00
Name : tc . ToolName ,
2026-03-02 18:51:57 -05:00
Input : tc . Input ,
})
if err != nil {
result . Result = fantasy . ToolResultOutputContentError {
Error : err ,
}
result . ClientMetadata = resp . Metadata
2026-10-20 11:26:42 +02:00
b . opts . Logger . Error ( ctx , "tool execution failed" ,
2026-04-22 17:19:56 +01:00
slog . F ( "tool_name" , tc . ToolName ),
slog . F ( "tool_call_id" , tc . ToolCallID ),
slog . Error ( err ),
)
2026-09-30 11:53:58 -05:00
return result , err
2026-03-02 18:51:57 -05:00
}
result . ClientMetadata = resp . Metadata
2026-06-24 09:16:38 -06:00
2026-09-17 15:09:19 +02:00
// Bound text so one tool result cannot overflow the model's context window.
// Media limits are applied separately by normalizeToolMedia.
2026-06-24 09:16:38 -06:00
content := resp . Content
2026-10-20 11:26:42 +02:00
if truncated , didTruncate := TruncateToolResultText ( content , b . maxResultBytes ); didTruncate {
b . opts . Metrics . RecordToolResultTruncated ( b . provider , b . model , tc . ToolName )
b . opts . Logger . Warn ( ctx , "tool result truncated to fit model context" ,
2026-06-24 09:16:38 -06:00
slog . F ( "tool_name" , tc . ToolName ),
slog . F ( "tool_call_id" , tc . ToolCallID ),
slog . F ( "original_bytes" , len ( content )),
2026-10-20 11:26:42 +02:00
slog . F ( "max_bytes" , b . maxResultBytes ),
2026-06-24 09:16:38 -06:00
)
content = truncated
}
2026-03-02 18:51:57 -05:00
switch {
case resp . IsError :
result . Result = fantasy . ToolResultOutputContentError {
2026-06-24 09:16:38 -06:00
Error : xerrors . New ( content ),
2026-03-02 18:51:57 -05:00
}
2026-10-20 11:26:42 +02:00
b . opts . Logger . Info ( ctx , "tool returned error result" ,
2026-04-22 17:19:56 +01:00
slog . F ( "tool_name" , tc . ToolName ),
slog . F ( "tool_call_id" , tc . ToolCallID ),
2026-06-24 09:16:38 -06:00
slog . F ( "tool_error" , content ),
2026-04-22 17:19:56 +01:00
)
2026-03-02 18:51:57 -05:00
case resp . Type == "image" || resp . Type == "media" :
2026-09-17 15:09:19 +02:00
text := strings . ToValidUTF8 ( content , "\uFFFD" )
if note , rejected := normalizeToolMedia ( & resp ); rejected {
2026-10-20 11:26:42 +02:00
b . opts . Logger . Warn ( ctx , "tool result media rejected, keeping text only" ,
2026-09-17 15:09:19 +02:00
slog . F ( "tool_name" , tc . ToolName ),
slog . F ( "tool_call_id" , tc . ToolCallID ),
slog . F ( "media_type" , resp . MediaType ),
slog . F ( "media_bytes" , len ( resp . Data )),
)
if text != "" {
text += "\n"
}
result . Result = fantasy . ToolResultOutputContentText { Text : text + note }
break
}
2026-03-02 18:51:57 -05:00
result . Result = fantasy . ToolResultOutputContentMedia {
2026-04-23 19:58:38 +01:00
Data : base64 . StdEncoding . EncodeToString ( resp . Data ),
2026-03-02 18:51:57 -05:00
MediaType : resp . MediaType ,
2026-09-17 15:09:19 +02:00
Text : text ,
2026-11-11 00:06:40 +11:00
Filename : resp . Filename ,
2026-03-02 18:51:57 -05:00
}
default :
result . Result = fantasy . ToolResultOutputContentText {
2026-06-24 09:16:38 -06:00
Text : strings . ToValidUTF8 ( content , "\uFFFD" ),
2026-03-02 18:51:57 -05:00
}
2026-02-27 11:50:56 -05:00
}
2026-05-04 20:30:50 +02:00
if _ , isError := result . Result .( fantasy . ToolResultOutputContentError ); isError {
2026-09-30 11:53:58 -05:00
return result , nil
2026-05-04 20:30:50 +02:00
}
if len ( result . ProviderMetadata ) == 0 {
2026-10-30 03:00:59 +01:00
if callback := b . resultMetadata [ tc . ToolName ]; callback != nil {
2026-05-04 20:30:50 +02:00
metadata := callback ( resp )
if len ( metadata ) > 0 {
result . ProviderMetadata = metadata
}
}
}
2026-09-30 11:53:58 -05:00
return result , nil
2026-02-27 11:50:56 -05:00
}
2026-09-17 15:09:19 +02:00
// normalizeToolMedia bounds persisted payloads and corrects image MIME types
// by signature, not full image decoding. Transport limits apply at prompt build.
func normalizeToolMedia ( resp * fantasy . ToolResponse ) ( string , bool ) {
if len ( resp . Data ) > codersdk . MaxChatFileSizeBytes {
return fmt . Sprintf (
"[%s content omitted: %d bytes exceeds the %d byte tool media limit]" ,
resp . MediaType , len ( resp . Data ), codersdk . MaxChatFileSizeBytes ,
), true
}
if ! strings . HasPrefix ( chatfiles . BaseMediaType ( resp . MediaType ), "image/" ) {
return "" , false
}
detected := chatfiles . DetectMediaType ( resp . Data )
if ! strings . HasPrefix ( detected , "image/" ) {
return fmt . Sprintf (
"[image omitted: payload declared as %s is %s]" ,
resp . MediaType , detected ,
), true
}
resp . MediaType = detected
return "" , false
}
2026-04-16 11:12:01 +02:00
func isToolActive ( name string , activeTools [] string ) bool {
return len ( activeTools ) == 0 || slices . Contains ( activeTools , name )
}
2026-08-18 19:12:47 +02:00
// serialToolCaller is implemented by tools whose calls within one step
// must execute in tool-call order because they claim from shared state.
type serialToolCaller interface { SerialToolCalls () bool }
2026-11-02 11:28:22 +01:00
// haltingToolCaller is implemented by serial tools whose calls in one step
// build on each other, for example computer actions that the model plans
// against one screen. After a call of the tool fails, or is rejected before
// it runs, no later call of the tool runs in the step. Each later call gets
// an error result with the returned text. ok is false if the tool does not
// halt.
type haltingToolCaller interface {
HaltedToolCallError () ( text string , ok bool )
}
// haltedCallError returns the error text for the calls of the tool name that
// follow a failed call, and false if the tool does not halt.
func ( b * toolBatch ) haltedCallError ( name string ) ( string , bool ) {
halting , ok := b . tools [ name ].( haltingToolCaller )
if ! ok {
return "" , false
}
return halting . HaltedToolCallError ()
}
// callsAfterRejection returns the IDs of the calls that follow, in observed
// order, a rejected call of the same tool. Observed calls missing from calls
// were rejected before execution.
func callsAfterRejection ( calls , observed [] fantasy . ToolCallContent ) map [ string ] bool {
executed := make ( map [ string ] bool , len ( calls ))
for _ , tc := range calls {
executed [ tc . ToolCallID ] = true
}
rejectedTools := make ( map [ string ] bool )
after := make ( map [ string ] bool )
for _ , tc := range observed {
name := tc . ToolName
switch {
case ! executed [ tc . ToolCallID ]:
rejectedTools [ name ] = true
case rejectedTools [ name ]:
after [ tc . ToolCallID ] = true
}
}
return after
}
2026-08-18 19:12:47 +02:00
// stepToolCallObserver is implemented by tools that need to see every
// tool-call name in the step before any call executes, for example so
// find_tools can charge same-step direct calls against its budget.
type stepToolCallObserver interface { ObserveStepToolCalls ( names [] string ) }
2026-10-30 03:00:59 +01:00
// notifyStepToolCallObservers passes the step's tool-call names to
// each distinct called tool that observes them.
2026-10-20 11:39:54 +02:00
func ( b * toolBatch ) notifyStepToolCallObservers ( calls [] fantasy . ToolCallContent ) {
2026-08-18 19:12:47 +02:00
names := make ([] string , 0 , len ( calls ))
for _ , tc := range calls {
2026-10-30 03:00:59 +01:00
names = append ( names , tc . ToolName )
2026-08-18 19:12:47 +02:00
}
notified := make ( map [ string ] struct {}, len ( names ))
for _ , name := range names {
if _ , dup := notified [ name ]; dup {
continue
}
notified [ name ] = struct {}{}
2026-10-20 11:39:54 +02:00
if observer , ok := b . tools [ name ].( stepToolCallObserver ); ok {
2026-08-18 19:12:47 +02:00
observer . ObserveStepToolCalls ( names )
}
}
}
// stepToolResultObserver is implemented by tools that need the step's
// per-call execution outcomes, for example so find_tools can refund
// budget it reserved for a direct call whose execution errored. names
// and errored are parallel slices in the observed tool-call order;
// outcomes are kept per call because one tool can be called several
// times in a step with different results.
type stepToolResultObserver interface {
ObserveStepToolResults ( names [] string , errored [] bool )
}
// notifyStepToolResultObservers passes the settled sibling outcomes to
// each distinct called tool that observes them, per call in observed
// order. Observed calls missing from the executed batch were rejected
// before execution (for example malformed JSON partitioned into
// synthetic denials) and settle as errored, since their persisted
// results always carry IsError. Serial calls have not run yet, so
// their own outcomes are reported as not errored; observers only need
// the concurrent siblings they share state with.
2026-10-20 11:39:54 +02:00
func ( b * toolBatch ) notifyStepToolResultObservers ( calls , observed [] fantasy . ToolCallContent , settled [] fantasy . ToolResultContent ) {
2026-08-18 19:12:47 +02:00
erroredByID := make ( map [ string ] bool , len ( settled ))
for _ , tr := range settled {
_ , isErr := tr . Result .( fantasy . ToolResultOutputContentError )
erroredByID [ tr . ToolCallID ] = isErr
}
executedIDs := make ( map [ string ] struct {}, len ( calls ))
for _ , tc := range calls {
executedIDs [ tc . ToolCallID ] = struct {}{}
}
names := make ([] string , 0 , len ( observed ))
errored := make ([] bool , 0 , len ( observed ))
for _ , tc := range observed {
2026-10-30 03:00:59 +01:00
names = append ( names , tc . ToolName )
2026-08-18 19:12:47 +02:00
if isErr , ok := erroredByID [ tc . ToolCallID ]; ok {
errored = append ( errored , isErr )
continue
}
_ , executed := executedIDs [ tc . ToolCallID ]
errored = append ( errored , ! executed )
}
notified := make ( map [ string ] struct {}, len ( calls ))
for _ , tc := range calls {
2026-10-30 03:00:59 +01:00
name := tc . ToolName
2026-08-18 19:12:47 +02:00
if _ , dup := notified [ name ]; dup {
continue
}
notified [ name ] = struct {}{}
2026-10-20 11:39:54 +02:00
if observer , ok := b . tools [ name ].( stepToolResultObserver ); ok {
2026-08-18 19:12:47 +02:00
observer . ObserveStepToolResults ( names , errored )
}
}
}
2026-10-20 11:39:54 +02:00
func ( b * toolBatch ) isSerialToolCall ( name string ) bool {
2026-10-30 03:00:59 +01:00
serial , ok := b . tools [ name ].( serialToolCaller )
2026-08-18 19:12:47 +02:00
return ok && serial . SerialToolCalls ()
}
2026-10-04 22:55:37 +02:00
// promptCachingStrategy marks prompt cache breakpoints on a prepared
// prompt in the shape the model's fantasy client serializes.
type promptCachingStrategy func ( messages [] fantasy . Message )
// promptCachingStrategyFor returns the breakpoint marker for Claude models
// on transports that forward Anthropic prompt caching, or nil.
func promptCachingStrategyFor ( model fantasy . LanguageModel , modelName string ) promptCachingStrategy {
2026-02-27 11:50:56 -05:00
if model == nil {
2026-10-04 22:55:37 +02:00
return nil
2026-02-27 11:50:56 -05:00
}
2026-10-04 22:55:37 +02:00
switch model . Provider () {
case fantasyanthropic . Name :
return addAnthropicPromptCaching
case fantasyopenrouter . Name , fantasyvercel . Name :
// Both clients translate Anthropic-keyed cache options into
// cache_control on the OpenAI-compatible wire.
if chatprovider . IsAnthropicFamilyModelID ( modelName ) {
return addAnthropicPromptCaching
}
case fantasyopenaicompat . Name :
// The client only knows ContentExtraFields, and the AI Gateway
// re-applies client-sent cache_control when it rebuilds the
// upstream body, so the marker reaches OpenRouter-style upstreams.
if chatprovider . IsAnthropicFamilyModelID ( modelName ) {
return addOpenAICompatPromptCaching
}
}
return nil
2026-02-27 11:50:56 -05:00
}
2026-03-02 18:51:57 -05:00
// addAnthropicPromptCaching mutates messages in-place, setting
// ProviderOptions for Anthropic prompt caching on the last system
// message and the final two messages.
func addAnthropicPromptCaching ( messages [] fantasy . Message ) {
2026-02-27 11:50:56 -05:00
for i := range messages {
messages [ i ]. ProviderOptions = nil
}
providerOption := fantasy . ProviderOptions {
fantasyanthropic . Name : & fantasyanthropic . ProviderCacheControlOptions {
CacheControl : fantasyanthropic . CacheControl { Type : "ephemeral" },
},
}
lastSystemRoleIdx := - 1
systemMessageUpdated := false
for i , msg := range messages {
if msg . Role == fantasy . MessageRoleSystem {
lastSystemRoleIdx = i
} else if ! systemMessageUpdated && lastSystemRoleIdx >= 0 {
messages [ lastSystemRoleIdx ]. ProviderOptions = providerOption
systemMessageUpdated = true
}
if i > len ( messages ) - 3 {
messages [ i ]. ProviderOptions = providerOption
}
}
}
2026-10-04 22:55:37 +02:00
// addOpenAICompatPromptCaching marks breakpoints on the last text part
// of the last system message and of the final two text-bearing user or
// assistant messages; tool messages have no text block on the
// OpenAI-compatible wire. Markers are part-level because message-level
// ContentExtraFields shares a key with openaicompat.ProviderOptions.
func addOpenAICompatPromptCaching ( messages [] fantasy . Message ) {
marker := & fantasyopenaicompat . ContentExtraFields {
Fields : map [ string ] any { "cache_control" : map [ string ] string { "type" : "ephemeral" }},
}
lastSystemIdx := - 1
for i := range messages {
// Content slices are shared with the canonical messages.
messages [ i ]. Content = slices . Clone ( messages [ i ]. Content )
if messages [ i ]. Role == fantasy . MessageRoleSystem {
lastSystemIdx = i
}
}
if lastSystemIdx >= 0 {
markLastTextPart ( messages [ lastSystemIdx ]. Content , marker )
}
remaining := 2
for i := len ( messages ) - 1 ; i >= 0 && remaining > 0 ; i -- {
if messages [ i ]. Role != fantasy . MessageRoleUser && messages [ i ]. Role != fantasy . MessageRoleAssistant {
continue
}
if markLastTextPart ( messages [ i ]. Content , marker ) {
remaining --
}
}
}
func markLastTextPart ( content [] fantasy . MessagePart , marker fantasy . ProviderOptionsData ) bool {
for j := len ( content ) - 1 ; j >= 0 ; j -- {
textPart , ok := content [ j ].( fantasy . TextPart )
if ! ok || strings . TrimSpace ( textPart . Text ) == "" {
continue
}
// The options map is shared with the canonical part; never mutate it.
options := maps . Clone ( textPart . ProviderOptions )
if options == nil {
options = fantasy . ProviderOptions {}
}
options [ fantasyopenaicompat . Name ] = marker
textPart . ProviderOptions = options
content [ j ] = textPart
return true
}
return false
}
2026-04-08 12:42:03 -04:00
// recordToolResultTimestamp lazily initializes the
// toolResultCreatedAt map on the stepResult and records
// the completion timestamp for the given tool-call ID.
func recordToolResultTimestamp ( result * stepResult , toolCallID string , ts time . Time ) {
if result . toolResultCreatedAt == nil {
result . toolResultCreatedAt = make ( map [ string ] time . Time )
}
result . toolResultCreatedAt [ toolCallID ] = ts
}
2026-04-22 20:11:53 +10:00
func publishToolAttachments (
ctx context . Context ,
logger slog . Logger ,
tr fantasy . ToolResultContent ,
createdAt time . Time ,
publishMessagePart func ( codersdk . ChatMessageRole , codersdk . ChatMessagePart ),
) {
attachments , err := chattool . AttachmentsFromMetadata ( tr . ClientMetadata )
if err != nil {
logger . Warn ( ctx , "skipping malformed tool attachment metadata" ,
slog . F ( "tool_name" , tr . ToolName ),
slog . F ( "tool_call_id" , tr . ToolCallID ),
slog . Error ( err ),
)
return
}
for _ , attachment := range attachments {
filePart := codersdk . ChatMessageFile (
attachment . FileID ,
attachment . MediaType ,
attachment . Name ,
)
filePart . CreatedAt = & createdAt
publishMessagePart ( codersdk . ChatMessageRoleAssistant , filePart )
}
}
2026-02-27 11:50:56 -05:00
func extractContextLimit ( metadata fantasy . ProviderMetadata ) sql . NullInt64 {
if len ( metadata ) == 0 {
return sql . NullInt64 {}
}
encoded , err := json . Marshal ( metadata )
if err != nil || len ( encoded ) == 0 {
return sql . NullInt64 {}
}
var payload any
if err := json . Unmarshal ( encoded , & payload ); err != nil {
return sql . NullInt64 {}
}
limit , ok := findContextLimitValue ( payload )
if ! ok {
return sql . NullInt64 {}
}
return sql . NullInt64 {
Int64 : limit ,
Valid : true ,
}
}
2026-04-24 23:57:30 +02:00
func extractContextLimitWithFallback ( metadata fantasy . ProviderMetadata , fallback int64 ) sql . NullInt64 {
contextLimit := extractContextLimit ( metadata )
if contextLimit . Valid || fallback <= 0 {
return contextLimit
}
return sql . NullInt64 {
Int64 : fallback ,
Valid : true ,
}
}
2026-02-27 11:50:56 -05:00
func findContextLimitValue ( value any ) ( int64 , bool ) {
var (
limit int64
found bool
)
collectContextLimitValues ( value , func ( candidate int64 ) {
if ! found || candidate > limit {
limit = candidate
found = true
}
})
return limit , found
}
func collectContextLimitValues ( value any , onValue func ( int64 )) {
switch typed := value .( type ) {
case map [ string ] any :
for key , child := range typed {
if isContextLimitKey ( key ) {
if numeric , ok := numericContextLimitValue ( child ); ok {
onValue ( numeric )
}
}
collectContextLimitValues ( child , onValue )
}
case [] any :
for _ , child := range typed {
collectContextLimitValues ( child , onValue )
}
}
}
func isContextLimitKey ( key string ) bool {
normalized := normalizeMetadataKey ( key )
if normalized == "" {
return false
}
switch normalized {
case
"contextlimit" ,
"contextwindow" ,
"contextlength" ,
"maxcontext" ,
"maxcontexttokens" ,
"maxinputtokens" ,
"maxinputtoken" ,
"inputtokenlimit" :
return true
}
2026-04-02 10:07:01 -07:00
words := metadataKeyWords ( key )
if ! slices . Contains ( words , "context" ) {
return false
}
if slices . Contains ( words , "limit" ) {
return true
}
if slices . Contains ( words , "window" ) {
return slices . Contains ( words , "size" ) || slices . Contains ( words , "max" )
}
if slices . Contains ( words , "length" ) {
return slices . Contains ( words , "max" )
}
return ( slices . Contains ( words , "token" ) || slices . Contains ( words , "tokens" )) &&
( slices . Contains ( words , "max" ) || slices . Contains ( words , "limit" ))
2026-02-27 11:50:56 -05:00
}
func normalizeMetadataKey ( key string ) string {
var b strings . Builder
b . Grow ( len ( key ))
for _ , r := range key {
switch {
case r >= 'a' && r <= 'z' :
_ , _ = b . WriteRune ( r )
case r >= 'A' && r <= 'Z' :
_ , _ = b . WriteRune ( r + ( 'a' - 'A' ))
case r >= '0' && r <= '9' :
_ , _ = b . WriteRune ( r )
}
}
return b . String ()
}
2026-04-02 10:07:01 -07:00
func metadataKeyWords ( key string ) [] string {
words := make ([] string , 0 , 4 )
var current strings . Builder
flush := func () {
if current . Len () == 0 {
return
}
words = append ( words , current . String ())
current . Reset ()
}
var prev rune
var hasPrev bool
for _ , r := range key {
if ! unicode . IsLetter ( r ) {
flush ()
hasPrev = false
continue
}
if hasPrev && unicode . IsUpper ( r ) && unicode . IsLower ( prev ) {
flush ()
}
_ , _ = current . WriteRune ( unicode . ToLower ( r ))
prev = r
hasPrev = true
}
flush ()
return words
}
2026-02-27 11:50:56 -05:00
func numericContextLimitValue ( value any ) ( int64 , bool ) {
switch typed := value .( type ) {
case float64 :
casted := int64 ( typed )
if typed > 0 && float64 ( casted ) == typed {
return casted , true
}
case string :
parsed , err := strconv . ParseInt ( strings . TrimSpace ( typed ), 10 , 64 )
if err == nil {
return positiveInt64 ( parsed )
}
}
return 0 , false
}
func positiveInt64 ( value int64 ) ( int64 , bool ) {
if value <= 0 {
return 0 , false
}
return value , true
}
2026-10-27 12:46:27 +02:00
// providerToolResultError returns the failure a provider reported for a
// provider-executed web search, or nil. Anthropic reports an error code and
// OpenAI a failed item status, both only in the result's metadata.
func providerToolResultError ( metadata fantasy . ProviderMetadata ) error {
if meta , ok := metadata [ fantasyanthropic . Name ].( * fantasyanthropic . WebSearchResultMetadata ); ok && meta != nil && meta . ErrorCode != "" {
return xerrors . Errorf ( "web search failed: %s" , meta . ErrorCode )
}
if meta , ok := metadata [ fantasyopenai . Name ].( * fantasyopenai . WebSearchCallMetadata ); ok && meta != nil && meta . Status == "failed" {
return xerrors . New ( "web search failed" )
}
return nil
}