Provider-executed web searches were invisible in chats. The UI skipped provider-executed parts, and the only trace was one source row per message that mixed OpenAI citations with Anthropic search results. The fantasy pin moves to coder/fantasy#62, which tags each found page with the search that found it and makes OpenAI request and report its found pages and search actions. chatd stores the tag and marks a search failed on an Anthropic error_code or an OpenAI failed status. Each search renders as one row with its queries or opened page, a running or unfinished state, and the pages it found. Cited pages stay in the answer's source row, which now follows the answer text. Closes CODAGT-795
234 lines
9.5 KiB
Go
234 lines
9.5 KiB
Go
package chatloop
|
|
|
|
import (
|
|
"iter"
|
|
"testing"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
fantasyanthropic "charm.land/fantasy/providers/anthropic"
|
|
fantasyopenai "charm.land/fantasy/providers/openai"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func TestProcessStepStreamPreservesReasoningMetadataAcrossNilDelta(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningStart, ID: "0"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", Delta: "thinking"})
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeReasoningDelta,
|
|
ID: "0",
|
|
ProviderMetadata: fantasy.ProviderMetadata{
|
|
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
|
Signature: "sig",
|
|
},
|
|
},
|
|
})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", ProviderMetadata: fantasy.ProviderMetadata{}})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningEnd, ID: "0", ProviderMetadata: fantasy.ProviderMetadata{}})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
})
|
|
|
|
result, err := processStepStream(stream, quartz.NewMock(t), func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {})
|
|
require.NoError(t, err)
|
|
require.Len(t, result.content, 1)
|
|
reasoning, ok := fantasy.AsContentType[fantasy.ReasoningContent](result.content[0])
|
|
require.True(t, ok)
|
|
require.Equal(t, "thinking", reasoning.Text)
|
|
metadata := fantasyanthropic.GetReasoningMetadata(fantasy.ProviderOptions(reasoning.ProviderMetadata))
|
|
require.NotNil(t, metadata)
|
|
require.Equal(t, "sig", metadata.Signature)
|
|
}
|
|
|
|
func TestProcessStepStreamPersistsRedactedThinkingOnEnd(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
reasoningMetadata := fantasy.ProviderMetadata{
|
|
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
|
RedactedData: "redacted-payload",
|
|
},
|
|
}
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeReasoningStart,
|
|
ID: "0",
|
|
ProviderMetadata: reasoningMetadata,
|
|
})
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeReasoningEnd,
|
|
ID: "0",
|
|
ProviderMetadata: reasoningMetadata,
|
|
})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "1"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "1", Delta: "done"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "1"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
})
|
|
|
|
result, err := processStepStream(stream, quartz.NewMock(t), func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {})
|
|
require.NoError(t, err)
|
|
require.Len(t, result.content, 2)
|
|
reasoning, ok := fantasy.AsContentType[fantasy.ReasoningContent](result.content[0])
|
|
require.True(t, ok)
|
|
require.Empty(t, reasoning.Text)
|
|
metadata := fantasyanthropic.GetReasoningMetadata(fantasy.ProviderOptions(reasoning.ProviderMetadata))
|
|
require.NotNil(t, metadata)
|
|
require.Equal(t, "redacted-payload", metadata.RedactedData)
|
|
}
|
|
|
|
func TestProcessStepStreamLabelsNarration(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
phase := func(itemID, phase string) fantasy.ProviderMetadata {
|
|
return fantasy.ProviderMetadata{
|
|
fantasyopenai.Name: &fantasyopenai.ResponsesTextMetadata{ItemID: itemID, Phase: phase},
|
|
}
|
|
}
|
|
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
// The narration's end carries no metadata, so the start's label
|
|
// must persist.
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "msg_1", ProviderMetadata: phase("msg_1", "commentary")})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "msg_1", Delta: "Reading"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "msg_1", Delta: " the file."})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "msg_1", ProviderMetadata: fantasy.ProviderMetadata{}})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "msg_2", ProviderMetadata: phase("msg_2", "final_answer")})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "msg_2", Delta: "It is flaky."})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "msg_2", ProviderMetadata: phase("msg_2", "final_answer")})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
})
|
|
|
|
var published []codersdk.ChatMessagePart
|
|
result, err := processStepStream(stream, quartz.NewMock(t), func(_ codersdk.ChatMessageRole, part codersdk.ChatMessagePart) {
|
|
published = append(published, part)
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
narration := codersdk.ChatMessageText("Reading")
|
|
narration.Narration = true
|
|
rest := codersdk.ChatMessageText(" the file.")
|
|
rest.Narration = true
|
|
require.Equal(t, []codersdk.ChatMessagePart{
|
|
narration,
|
|
rest,
|
|
codersdk.ChatMessageText("It is flaky."),
|
|
}, published)
|
|
|
|
require.Equal(t, []fantasy.Content{
|
|
fantasy.TextContent{Text: "Reading the file.", ProviderMetadata: phase("msg_1", "commentary")},
|
|
fantasy.TextContent{Text: "It is flaky.", ProviderMetadata: phase("msg_2", "final_answer")},
|
|
}, result.content)
|
|
}
|
|
|
|
func TestProcessStepStreamSetsReasoningDeltaCreatedAtToBlockStart(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
start := clock.Now()
|
|
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningStart, ID: "0"})
|
|
clock.Advance(time.Second)
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", Delta: "first "})
|
|
clock.Advance(time.Second)
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", Delta: "thought"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningEnd, ID: "0"})
|
|
clock.Advance(time.Second)
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningStart, ID: "1"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "1", Delta: "second"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningEnd, ID: "1"})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
})
|
|
|
|
var sinceStart []time.Duration
|
|
_, err := processStepStream(stream, clock, func(_ codersdk.ChatMessageRole, part codersdk.ChatMessagePart) {
|
|
if part.Type == codersdk.ChatMessagePartTypeReasoning {
|
|
require.NotNil(t, part.CreatedAt)
|
|
sinceStart = append(sinceStart, part.CreatedAt.Sub(start))
|
|
}
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, []time.Duration{0, 0, 3 * time.Second}, sinceStart,
|
|
"interrupted turns split reasoning blocks by CreatedAt, so each delta's CreatedAt must be its block's start time")
|
|
}
|
|
|
|
func TestProcessStepStreamProviderWebSearch(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
metadata fantasy.ProviderMetadata
|
|
wantError string
|
|
}{
|
|
{
|
|
name: "AnthropicResults",
|
|
metadata: fantasy.ProviderMetadata{
|
|
fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{
|
|
Results: []fantasyanthropic.WebSearchResultItem{{URL: "https://go.dev/doc/go1.27", Title: "Go 1.27"}},
|
|
},
|
|
},
|
|
},
|
|
{
|
|
name: "AnthropicErrorCode",
|
|
metadata: fantasy.ProviderMetadata{
|
|
fantasyanthropic.Name: &fantasyanthropic.WebSearchResultMetadata{ErrorCode: "max_uses_exceeded"},
|
|
},
|
|
wantError: "web search failed: max_uses_exceeded",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeSource,
|
|
ID: "https://go.dev/doc/go1.27",
|
|
SourceType: fantasy.SourceTypeURL,
|
|
URL: "https://go.dev/doc/go1.27",
|
|
Title: "Go 1.27",
|
|
SourceToolCallID: "srvtoolu_1",
|
|
})
|
|
yield(fantasy.StreamPart{
|
|
Type: fantasy.StreamPartTypeToolResult,
|
|
ID: "srvtoolu_1",
|
|
ToolCallName: "web_search",
|
|
ProviderExecuted: true,
|
|
ProviderMetadata: tt.metadata,
|
|
})
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
})
|
|
|
|
var published []codersdk.ChatMessagePart
|
|
result, err := processStepStream(stream, quartz.NewMock(t), func(_ codersdk.ChatMessageRole, part codersdk.ChatMessagePart) {
|
|
published = append(published, part)
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, result.content, 2)
|
|
|
|
source, ok := fantasy.AsContentType[fantasy.SourceContent](result.content[0])
|
|
require.True(t, ok)
|
|
require.Equal(t, "srvtoolu_1", source.ToolCallID)
|
|
toolResult, ok := fantasy.AsContentType[fantasy.ToolResultContent](result.content[1])
|
|
require.True(t, ok)
|
|
if tt.wantError == "" {
|
|
require.Nil(t, toolResult.Result)
|
|
} else {
|
|
errorOutput, ok := toolResult.Result.(fantasy.ToolResultOutputContentError)
|
|
require.True(t, ok, "result should be an error output, got %T", toolResult.Result)
|
|
require.EqualError(t, errorOutput.Error, tt.wantError)
|
|
}
|
|
|
|
require.Len(t, published, 2)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeSource, published[0].Type)
|
|
require.Equal(t, "srvtoolu_1", published[0].ToolCallID)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, published[1].Type)
|
|
require.Equal(t, tt.wantError != "", published[1].IsError)
|
|
})
|
|
}
|
|
}
|