> Xum acted for Mike: this PR was written and opened by an AI agent on
Mike's behalf.
Computer use subagents on Claude 5.5 models (`claude-opus-5-5`,
`claude-sonnet-5-5`) failed on their first request: these models reject
`computer_20251124` and accept only the GA toolset
`computer_toolset_20260801`.
chatd now picks the Anthropic computer use tool version from the chat's
model with `ComputerUseToolVersionForModel` from coder/fantasy: the
toolset for Claude 4.8 and later, `computer_20251124` for Opus 4.5 to
4.7, Sonnet 4.6 and model IDs it cannot parse, and `computer_20250124`
for Haiku and Sonnet 4.5. Claude Platform on AWS and Amazon Bedrock
offer computer use only through the earlier versions, so an Anthropic
provider configured for either declares `computer_20251124` where the
model would take the toolset. For the toolset, chatd answers like
Anthropic's SDK tool runner: actions answer with text and only
`screenshot` returns an image, `cursor_position` returns the position as
text, `key` honors `repeat` (1 to 100), and zoom is declared disabled
because the agent desktop has no zoom action. The earlier versions still
answer every action with a screenshot.
A step's computer calls run one at a time in call order. For every
Anthropic version, the calls after one that failed, or that was rejected
before it ran (malformed input or a pre_tool_use hook denial), return
`Not executed: an earlier computer action in this turn failed.` without
running, which is the SDK runner's text. The chat loop enforces this
halt, so the computer tool keeps no state between calls. `wait` and
`hold_key` durations are read as seconds (capped at 300).
The frontend marks Claude Opus 5.5 and Sonnet 5.5 as verified for
computer use, and the computer tool row now labels each action ("Left
click", "Type", "Press key") instead of "Screenshot", shows text output,
and shows the error of failed or halted calls inline. The error block is
shared with the write_file and edit_files rows as
`ToolCall.ErrorDetail`.
Uses coder/fantasy#80 and coder/fantasy#82, both merged. `go.mod` pins
the fork to `7882f906a89a`.
<details>
<summary>UAT</summary>
TL;DR: it works. A remote Coder Agents UAT on dogfood passed on
`00143d5b12`, with all model traffic logged through a proxy:
- `claude-opus-5-5`, `claude-sonnet-5-5` and `claude-opus-4-8` declared
`computer_toolset_20260801` with zoom disabled and no beta header.
`claude-opus-4-6` and an unknown model ID declared `computer_20251124`
with the `computer-use-2025-11-24` beta, `claude-haiku-4-5` declared
`computer_20250124`, and a Claude Platform on AWS provider declared
`computer_20251124`. Opus 5.5, Sonnet 5.5, Opus 4.6 and Haiku 4.5 each
completed a desktop task.
- Toolset actions answered with text and only `screenshot` returned an
image. `repeat` 100 ran, 150 failed, and `wait` 400 stopped at 300
seconds.
- A failed call (`repeat` 150 on the toolset, a desktop error on 4.6)
and a pre_tool_use hook denial each halted the rest of the step without
touching the desktop. The next step ran normally.
- The computer tool rows showed action labels, text output and the halt
error. Opus 5.5 and Sonnet 5.5 showed no warning in the computer use
model selector. Switching a chat between 4.6 and 5.5 worked in both
directions.
Not tested: the tool type on Bedrock, and a halt after malformed model
input. Known, unchanged behavior: the desktop reports success for
unknown key names and off-screen coordinates, the model can still emit
`zoom`, which fails, and `scroll` up fails in `agent/x/agentdesktop`
because the negative amount is read as a flag. Both failures halt the
rest of their batch.
</details>
---------
Co-authored-by: Ethan Dickson <ethanndickson@gmail.com>
1808 lines
80 KiB
Go
1808 lines
80 KiB
Go
package chatd //nolint:testpackage // Uses unexported chatworker helpers.
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/google/uuid"
|
|
"github.com/sqlc-dev/pqtype"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatadvisor"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
)
|
|
|
|
func TestBuildCommitStepMessages_AssistantTextAndReasoning(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
startedAt := time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC)
|
|
completedAt := startedAt.Add(2 * time.Second)
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: modelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
reasoningProvenance: reasoningProvenance{
|
|
ProviderIdentity: "provider-a",
|
|
Model: "model-a",
|
|
Credential: reasoningCredentialUser,
|
|
Epoch: 2,
|
|
},
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ReasoningContent{Text: "thinking"},
|
|
fantasy.TextContent{Text: "hello"},
|
|
},
|
|
ReasoningStartedAt: []time.Time{startedAt},
|
|
ReasoningCompletedAt: []time.Time{completedAt},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 1)
|
|
|
|
msg := got.Messages[0]
|
|
require.Equal(t, database.ChatMessageRoleAssistant, msg.Role)
|
|
require.Equal(t, database.ChatMessageVisibilityBoth, msg.Visibility)
|
|
require.Equal(t, uuid.NullUUID{UUID: modelConfigID, Valid: true}, msg.ModelConfigID)
|
|
require.Equal(t, chatprompt.CurrentContentVersion, msg.ContentVersion)
|
|
parts := parseMessageParts(t, msg.Role, msg.Content)
|
|
require.Len(t, parts, 2)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeReasoning, parts[0].Type)
|
|
require.Equal(t, "thinking", parts[0].Text)
|
|
require.Equal(t, startedAt, requireNotNilTime(t, parts[0].CreatedAt))
|
|
require.Equal(t, completedAt, requireNotNilTime(t, parts[0].CompletedAt))
|
|
require.Equal(t, "provider-a", parts[0].ProviderIdentity)
|
|
require.Equal(t, "model-a", parts[0].ProviderModel)
|
|
require.Equal(t, reasoningCredentialUser, parts[0].ProviderCredential)
|
|
require.Equal(t, 2, parts[0].ReasoningEpoch)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeText, parts[1].Type)
|
|
require.Equal(t, "hello", parts[1].Text)
|
|
require.Empty(t, parts[1].ProviderIdentity)
|
|
require.Empty(t, parts[1].ProviderModel)
|
|
require.Empty(t, parts[1].ProviderCredential)
|
|
require.Zero(t, parts[1].ReasoningEpoch)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_LocalToolResultsBecomeToolMessages(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: modelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{ToolCallID: "call-1", ToolName: "execute", Input: `{"cmd":"pwd"}`},
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "call-1",
|
|
ToolName: "execute",
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"stdout":"/tmp"}`},
|
|
},
|
|
},
|
|
Runtime: 1500 * time.Millisecond,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 2)
|
|
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, got.Messages[0].RuntimeMs)
|
|
require.False(t, got.Messages[1].RuntimeMs.Valid)
|
|
|
|
assistantParts := parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)
|
|
require.Len(t, assistantParts, 1)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, assistantParts[0].Type)
|
|
require.Equal(t, "call-1", assistantParts[0].ToolCallID)
|
|
require.Equal(t, "execute", assistantParts[0].ToolName)
|
|
|
|
toolParts := parseMessageParts(t, got.Messages[1].Role, got.Messages[1].Content)
|
|
require.Len(t, toolParts, 1)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, toolParts[0].Type)
|
|
require.Equal(t, "call-1", toolParts[0].ToolCallID)
|
|
require.Equal(t, "execute", toolParts[0].ToolName)
|
|
require.JSONEq(t, `{"stdout":"/tmp"}`, string(toolParts[0].Result))
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_BatchRuntimeBillsDedicatedUsageRow(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "call-1",
|
|
ToolName: "read_file",
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"data":"fast"}`},
|
|
},
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "call-2",
|
|
ToolName: "execute",
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"stdout":"/tmp"}`},
|
|
},
|
|
},
|
|
BatchRuntime: 10 * time.Second,
|
|
BatchBilledCalls: 2,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 3)
|
|
// Real tool results never carry batch-level runtime.
|
|
require.False(t, got.Messages[0].RuntimeMs.Valid)
|
|
require.False(t, got.Messages[1].RuntimeMs.Valid)
|
|
|
|
stamp := got.Messages[2]
|
|
require.Equal(t, database.ChatMessageRoleTool, stamp.Role)
|
|
// The usage record is model-only bookkeeping, never published to clients.
|
|
require.Equal(t, database.ChatMessageVisibilityModel, stamp.Visibility)
|
|
require.Equal(t, sql.NullInt64{Int64: 10000, Valid: true}, stamp.RuntimeMs)
|
|
stampParts, err := chatprompt.ParseContent(database.ChatMessage{
|
|
Role: stamp.Role,
|
|
Content: stamp.Content,
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, stampParts, 1)
|
|
require.Equal(t, toolBatchUsagePartType, stampParts[0].Type)
|
|
require.JSONEq(t, `{"billed_ms":10000,"billed_calls":2}`, string(stampParts[0].Result))
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_BatchAttachmentAssistantRowStaysNull(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "call-1",
|
|
ToolName: "attach_file",
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`},
|
|
ClientMetadata: `{"attachments":[{"file_id":"` + uuid.NewString() + `","media_type":"image/png","name":"shot.png"}]}`,
|
|
},
|
|
},
|
|
BatchRuntime: 3 * time.Second,
|
|
BatchBilledCalls: 1,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 3)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got.Messages[0].Role)
|
|
require.False(t, got.Messages[0].RuntimeMs.Valid)
|
|
require.Equal(t, database.ChatMessageRoleTool, got.Messages[1].Role)
|
|
require.False(t, got.Messages[1].RuntimeMs.Valid)
|
|
require.Equal(t, database.ChatMessageVisibilityModel, got.Messages[2].Visibility)
|
|
require.Equal(t, sql.NullInt64{Int64: 3000, Valid: true}, got.Messages[2].RuntimeMs)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_ZeroBatchRuntimeLeavesRuntimeNull(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{ToolCallID: "call-1", ToolName: "execute", Input: `{"cmd":"pwd"}`},
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "call-1",
|
|
ToolName: "execute",
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"stdout":"/tmp"}`},
|
|
},
|
|
},
|
|
Runtime: 0,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 2)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got.Messages[0].Role)
|
|
require.False(t, got.Messages[0].RuntimeMs.Valid)
|
|
require.False(t, got.Messages[1].RuntimeMs.Valid)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_ProviderExecutedResultsStayAssistantContent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{
|
|
ToolCallID: "web-1",
|
|
ToolName: "web_search",
|
|
ProviderExecuted: true,
|
|
},
|
|
fantasy.ToolResultContent{
|
|
ToolCallID: "web-1",
|
|
ToolName: "web_search",
|
|
ProviderExecuted: true,
|
|
Result: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`},
|
|
},
|
|
}},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 1)
|
|
parts := parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)
|
|
require.Len(t, parts, 2)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, parts[0].Type)
|
|
require.True(t, parts[0].ProviderExecuted)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, parts[1].Type)
|
|
require.True(t, parts[1].ProviderExecuted)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_UsageRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{fantasy.TextContent{Text: "usage"}},
|
|
Usage: fantasy.Usage{InputTokens: 100, OutputTokens: 20, TotalTokens: 120, ReasoningTokens: 3, CacheCreationTokens: 4, CacheReadTokens: 5},
|
|
ContextLimit: sql.NullInt64{Int64: 4096, Valid: true},
|
|
Runtime: 1500 * time.Millisecond,
|
|
ProviderResponseID: "msg_usage",
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 1)
|
|
msg := got.Messages[0]
|
|
require.Equal(t, sql.NullInt64{Int64: 100, Valid: true}, msg.InputTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 20, Valid: true}, msg.OutputTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 120, Valid: true}, msg.TotalTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 3, Valid: true}, msg.ReasoningTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 4, Valid: true}, msg.CacheCreationTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 5, Valid: true}, msg.CacheReadTokens)
|
|
require.Equal(t, sql.NullInt64{Int64: 4096, Valid: true}, msg.ContextLimit)
|
|
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, msg.RuntimeMs)
|
|
require.Equal(t, sql.NullString{String: "msg_usage", Valid: true}, msg.ProviderResponseID)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_ToolTimestampsAndMCPConfigIDs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
callAt := time.Date(2026, 2, 3, 4, 5, 6, 0, time.UTC)
|
|
resultAt := callAt.Add(3 * time.Second)
|
|
configID := uuid.New()
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
toolNameToConfigID: map[string]uuid.UUID{
|
|
"mcp_tool": configID,
|
|
},
|
|
step: stepData{Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{ToolCallID: "call-1", ToolName: "mcp_tool", Input: `{}`},
|
|
fantasy.ToolResultContent{ToolCallID: "call-1", ToolName: "mcp_tool", Result: fantasy.ToolResultOutputContentText{Text: `{"ok":true}`}},
|
|
}, ToolCallCreatedAt: map[string]time.Time{
|
|
"call-1": callAt,
|
|
}, ToolResultCreatedAt: map[string]time.Time{
|
|
"call-1": resultAt,
|
|
}},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 2)
|
|
callPart := parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)[0]
|
|
resultPart := parseMessageParts(t, got.Messages[1].Role, got.Messages[1].Content)[0]
|
|
require.Equal(t, uuid.NullUUID{UUID: configID, Valid: true}, callPart.MCPServerConfigID)
|
|
require.Equal(t, callAt, requireNotNilTime(t, callPart.CreatedAt))
|
|
require.Equal(t, uuid.NullUUID{UUID: configID, Valid: true}, resultPart.MCPServerConfigID)
|
|
require.Equal(t, resultAt, requireNotNilTime(t, resultPart.CreatedAt))
|
|
}
|
|
|
|
func TestBuildCompactionMessages_CompressedSummaryToolCallAndResult(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
got, err := buildCompactionMessages(buildCompactionMessagesInput{
|
|
modelConfigID: modelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
toolCallID: "summary-1",
|
|
toolName: "chat_summarized",
|
|
compaction: chatloop.CompactionResult{
|
|
SystemSummary: "system summary",
|
|
SummaryReport: "user report",
|
|
ThresholdPercent: 70,
|
|
UsagePercent: 81.5,
|
|
ContextTokens: 815,
|
|
ChatContextLimit: 1000,
|
|
TriggerContextLimit: 500,
|
|
Runtime: 1500 * time.Millisecond,
|
|
EstimatedContextTokens: 5,
|
|
ProviderResponseID: "resp_summary",
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 3)
|
|
|
|
require.Equal(t, database.ChatMessageRoleUser, got.Messages[0].Role)
|
|
require.Equal(t, database.ChatMessageVisibilityModel, got.Messages[0].Visibility)
|
|
require.True(t, got.Messages[0].Compressed)
|
|
require.Equal(t, uuid.NullUUID{UUID: modelConfigID, Valid: true}, got.Messages[0].ModelConfigID)
|
|
require.Equal(t, "system summary", parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)[0].Text)
|
|
require.False(t, got.Messages[0].RuntimeMs.Valid)
|
|
require.False(t, got.Messages[0].ProviderResponseID.Valid)
|
|
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got.Messages[1].Role)
|
|
require.Equal(t, database.ChatMessageVisibilityUser, got.Messages[1].Visibility)
|
|
require.True(t, got.Messages[1].Compressed)
|
|
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, got.Messages[1].RuntimeMs)
|
|
require.Equal(t, sql.NullString{String: "resp_summary", Valid: true}, got.Messages[1].ProviderResponseID)
|
|
callPart := parseMessageParts(t, got.Messages[1].Role, got.Messages[1].Content)[0]
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, callPart.Type)
|
|
require.Equal(t, "summary-1", callPart.ToolCallID)
|
|
require.JSONEq(t, `{"source":"automatic","threshold_percent":70}`, string(callPart.Args))
|
|
|
|
require.Equal(t, database.ChatMessageRoleTool, got.Messages[2].Role)
|
|
require.Equal(t, database.ChatMessageVisibilityBoth, got.Messages[2].Visibility)
|
|
require.True(t, got.Messages[2].Compressed)
|
|
require.False(t, got.Messages[2].RuntimeMs.Valid)
|
|
require.False(t, got.Messages[2].ProviderResponseID.Valid)
|
|
resultPart := parseMessageParts(t, got.Messages[2].Role, got.Messages[2].Content)[0]
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, resultPart.Type)
|
|
require.Equal(t, "summary-1", resultPart.ToolCallID)
|
|
require.JSONEq(t, `{"summary":"user report","source":"automatic","threshold_percent":70,"usage_percent":81.5,"context_tokens":815,"context_limit_tokens":1000,"estimated_context_tokens":5,"trigger_context_limit_tokens":500}`, string(resultPart.Result))
|
|
|
|
boundary := database.ChatMessage{Role: got.Messages[2].Role, Content: got.Messages[2].Content, ContentVersion: got.Messages[2].ContentVersion, Compressed: true}
|
|
require.False(t, summaryRecordedUnderOtherTrigger(boundary, 70, 500))
|
|
require.True(t, summaryRecordedUnderOtherTrigger(boundary, 40, 500))
|
|
require.True(t, summaryRecordedUnderOtherTrigger(boundary, 70, 1000))
|
|
}
|
|
|
|
// A compaction that never reached the summary model call carries no
|
|
// runtime, so its assistant row must persist runtime_ms NULL.
|
|
func TestBuildCompactionMessages_ZeroRuntimeLeavesRuntimeNull(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCompactionMessages(buildCompactionMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
toolCallID: "summary-1",
|
|
toolName: "chat_summarized",
|
|
compaction: chatloop.CompactionResult{
|
|
SystemSummary: "system summary",
|
|
SummaryReport: "user report",
|
|
ThresholdPercent: 70,
|
|
UsagePercent: 81.5,
|
|
ContextTokens: 815,
|
|
ChatContextLimit: 1000,
|
|
Runtime: 0,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 3)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got.Messages[1].Role)
|
|
for i := range got.Messages {
|
|
require.False(t, got.Messages[i].RuntimeMs.Valid)
|
|
}
|
|
}
|
|
|
|
func TestCurrentTurnStepCount_ExcludesCompressedCompactionMessages(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("first")),
|
|
dbMessage(t, 3, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("compressed summary")),
|
|
dbMessage(t, 4, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary", "chat_summarized", nil)),
|
|
dbMessage(t, 5, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 6, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("second")),
|
|
}
|
|
got := currentTurnStepCount(messages)
|
|
require.Equal(t, 2, got)
|
|
}
|
|
|
|
func TestCurrentTurnStepCount_CountsAssistantMessagesAfterLatestUser(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("old")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("old answer")),
|
|
dbMessage(t, 3, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("new")),
|
|
dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("one")),
|
|
dbMessage(t, 5, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("call", "tool", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 6, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("two")),
|
|
}
|
|
got := currentTurnStepCount(messages)
|
|
require.Equal(t, 2, got)
|
|
}
|
|
|
|
func TestCurrentTurnStepCount_IgnoresHookModelContext(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hookContext := dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("hook context"))
|
|
hookContext.Visibility = database.ChatMessageVisibilityModel
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("prompt")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("one")),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("call", "tool", json.RawMessage(`{}`), false, false)),
|
|
hookContext,
|
|
dbMessage(t, 5, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("two")),
|
|
}
|
|
got := currentTurnStepCount(messages)
|
|
require.Equal(t, 2, got)
|
|
}
|
|
|
|
func TestDecisionCompactsAgainAfterPostCompactionTurn(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("initial request")),
|
|
dbMessage(t, 2, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("compacted summary")),
|
|
dbMessage(t, 3, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 4, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 5, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("continued after compaction")),
|
|
dbMessage(t, 6, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("next request")),
|
|
}
|
|
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
messages: messages,
|
|
compactionEnabled: true,
|
|
compactionNeeded: true,
|
|
compactionThresholdPercent: 70,
|
|
compactionContextLimit: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionCompact, decision.kind)
|
|
}
|
|
|
|
func TestDecideGenerationActionMaxSteps(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// historyWithSteps returns a prompt followed by steps resolved tool
|
|
// calls, so the history is incomplete and only the step limit can
|
|
// finish the turn.
|
|
historyWithSteps := func(t *testing.T, steps int) []database.ChatMessage {
|
|
t.Helper()
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("prompt")),
|
|
}
|
|
for i := range steps {
|
|
callID := fmt.Sprintf("call-%d", i)
|
|
nextID := int64(len(messages) + 1)
|
|
messages = append(messages,
|
|
dbMessage(t, nextID, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall(callID, "tool", json.RawMessage(`{}`))),
|
|
dbMessage(t, nextID+1, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult(callID, "tool", json.RawMessage(`{}`), false, false)),
|
|
)
|
|
}
|
|
return messages
|
|
}
|
|
|
|
for _, maxSteps := range []int{1, 3, codersdk.DefaultChatMaxStepsPerTurn} {
|
|
t.Run(fmt.Sprintf("MaxSteps%d", maxSteps), func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
below, err := decideGenerationAction(generationDecisionInput{
|
|
messages: historyWithSteps(t, maxSteps-1),
|
|
maxSteps: maxSteps,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionGenerateAssistant, below.kind)
|
|
|
|
atLimit, err := decideGenerationAction(generationDecisionInput{
|
|
messages: historyWithSteps(t, maxSteps),
|
|
maxSteps: maxSteps,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionFinishTurn, atLimit.kind)
|
|
require.Equal(t, generationFinishReasonMaxSteps, atLimit.finishReason)
|
|
})
|
|
}
|
|
|
|
for name, maxSteps := range map[string]int{"RejectsZero": 0, "RejectsNegative": -1} {
|
|
t.Run(name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
_, err := decideGenerationAction(generationDecisionInput{
|
|
messages: historyWithSteps(t, 1),
|
|
maxSteps: maxSteps,
|
|
})
|
|
require.Error(t, err)
|
|
require.True(t, isTerminalGeneration(err), "a step limit below 1 cannot succeed on retry")
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBuildCompactionMessages_ManualSource(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCompactionMessages(buildCompactionMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
toolCallID: "summary-1",
|
|
toolName: "chat_summarized",
|
|
compaction: chatloop.CompactionResult{
|
|
SystemSummary: "system summary",
|
|
SummaryReport: "user report",
|
|
Source: chatloop.CompactionSourceManual,
|
|
ThresholdPercent: 70,
|
|
UsagePercent: 10,
|
|
EstimatedContextTokens: 5,
|
|
ContextTokens: 100,
|
|
ChatContextLimit: 1000,
|
|
TriggerContextLimit: 500,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 3)
|
|
|
|
callPart := parseMessageParts(t, got.Messages[1].Role, got.Messages[1].Content)[0]
|
|
require.JSONEq(t, `{"source":"manual","threshold_percent":70}`, string(callPart.Args))
|
|
resultPart := parseMessageParts(t, got.Messages[2].Role, got.Messages[2].Content)[0]
|
|
require.JSONEq(t, `{"summary":"user report","source":"manual","threshold_percent":70,"usage_percent":10,"context_tokens":100,"context_limit_tokens":1000,"estimated_context_tokens":5,"trigger_context_limit_tokens":500}`, string(resultPart.Result))
|
|
}
|
|
|
|
// TestDecisionForcedCompaction verifies the manual compaction request
|
|
// ordering contract: a pending request beats the history-complete
|
|
// FinishTurn decision on idle chats, loses to unresolved tool calls,
|
|
// and is skipped when nothing after the latest boundary is
|
|
// compactable.
|
|
func TestDecisionForcedCompaction(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
requestedChat := database.Chat{
|
|
CompactionRequestedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
|
}
|
|
|
|
t.Run("beats history complete", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
}
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: requestedChat,
|
|
messages: messages,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionCompact, decision.kind)
|
|
require.True(t, decision.forced)
|
|
})
|
|
|
|
t.Run("loses to unresolved tool calls", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))),
|
|
}
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: requestedChat,
|
|
messages: messages,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionExecuteLocalTools, decision.kind)
|
|
require.False(t, decision.forced)
|
|
})
|
|
|
|
t.Run("skipped when nothing compactable", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Everything up to and including the latest boundary is
|
|
// compressed; no uncompressed assistant follows, so the
|
|
// forced compact is skipped and the normal decision applies
|
|
// (after-compaction histories continue with an assistant
|
|
// generation, exactly as if no request were pending).
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("follow-up question")),
|
|
dbMessage(t, 5, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("follow-up answer")),
|
|
}
|
|
// Only messages up to the boundary: strip the follow-up.
|
|
requested, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: requestedChat,
|
|
messages: messages[:3],
|
|
})
|
|
require.NoError(t, err)
|
|
unrequested, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: database.Chat{},
|
|
messages: messages[:3],
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, unrequested.kind, requested.kind,
|
|
"stale request must not change the decision")
|
|
require.False(t, requested.forced)
|
|
|
|
// With an uncompressed assistant after the boundary the
|
|
// forced compact fires again.
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: requestedChat,
|
|
messages: messages,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionCompact, decision.kind)
|
|
require.True(t, decision.forced)
|
|
})
|
|
|
|
t.Run("no request follows normal decision", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
}
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: database.Chat{},
|
|
messages: messages,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionFinishTurn, decision.kind)
|
|
})
|
|
}
|
|
|
|
func TestBuildClearMessages_CompressedSentinelToolCallAndResult(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
built, err := buildClearMessages(buildClearMessagesInput{
|
|
modelConfigID: modelConfigID,
|
|
toolCallID: "chat_cleared_test",
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, built, 3)
|
|
for _, msg := range built {
|
|
require.True(t, msg.Compressed, "every boundary row must be compressed")
|
|
require.Equal(t, modelConfigID, msg.ModelConfigID.UUID)
|
|
}
|
|
|
|
sentinel := built[0]
|
|
require.Equal(t, database.ChatMessageRoleUser, sentinel.Role)
|
|
require.Equal(t, database.ChatMessageVisibilityModel, sentinel.Visibility)
|
|
sentinelParts := parseMessageParts(t, sentinel.Role, sentinel.Content)
|
|
require.Len(t, sentinelParts, 1)
|
|
require.Equal(t, "Previous conversation context was cleared by the user.", sentinelParts[0].Text)
|
|
|
|
call := built[1]
|
|
require.Equal(t, database.ChatMessageRoleAssistant, call.Role)
|
|
require.Equal(t, database.ChatMessageVisibilityUser, call.Visibility)
|
|
callParts := parseMessageParts(t, call.Role, call.Content)
|
|
require.Len(t, callParts, 1)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, callParts[0].Type)
|
|
require.Equal(t, "chat_cleared", callParts[0].ToolName)
|
|
require.Equal(t, "chat_cleared_test", callParts[0].ToolCallID)
|
|
require.JSONEq(t, `{"source":"manual"}`, string(callParts[0].Args))
|
|
|
|
result := built[2]
|
|
require.Equal(t, database.ChatMessageRoleTool, result.Role)
|
|
require.Equal(t, database.ChatMessageVisibilityBoth, result.Visibility)
|
|
resultParts := parseMessageParts(t, result.Role, result.Content)
|
|
require.Len(t, resultParts, 1)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, resultParts[0].Type)
|
|
require.Equal(t, "chat_cleared", resultParts[0].ToolName)
|
|
require.JSONEq(t, `{"source":"manual"}`, string(resultParts[0].Result))
|
|
}
|
|
|
|
func clearBoundaryTriplet(t *testing.T, startID int64) []database.ChatMessage {
|
|
t.Helper()
|
|
sentinel := dbMessage(t, startID, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("cleared"))
|
|
sentinel.Visibility = database.ChatMessageVisibilityModel
|
|
return []database.ChatMessage{
|
|
sentinel,
|
|
dbMessage(t, startID+1, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("clear-1", "chat_cleared", json.RawMessage(`{"source":"manual"}`))),
|
|
dbMessage(t, startID+2, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("clear-1", "chat_cleared", json.RawMessage(`{"source":"manual"}`), false, false)),
|
|
}
|
|
}
|
|
|
|
func TestLatestContextBoundaryIndex_RecognizesClearAndSummarize(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question")),
|
|
dbMessage(t, 5, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
}
|
|
require.Equal(t, 2, latestContextBoundaryIndex(messages),
|
|
"compaction result row is the latest boundary")
|
|
|
|
messages = append(messages, clearBoundaryTriplet(t, 6)...)
|
|
require.Equal(t, 7, latestContextBoundaryIndex(messages),
|
|
"clear boundary supersedes the earlier compaction boundary")
|
|
|
|
// An uncompressed chat_cleared row is not a boundary.
|
|
require.False(t, isContextBoundaryMessage(
|
|
dbMessage(t, 9, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("clear-2", "chat_cleared", nil)),
|
|
))
|
|
}
|
|
|
|
func TestHasClearableMessageAfter(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
system := dbMessage(t, 1, database.ChatMessageRoleSystem, false, codersdk.ChatMessageText("system prompt"))
|
|
userVisible := dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("status note"))
|
|
userVisible.Visibility = database.ChatMessageVisibilityUser
|
|
conversation := dbMessage(t, 3, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question"))
|
|
|
|
require.False(t, hasClearableMessageAfter([]database.ChatMessage{system, userVisible}, -1),
|
|
"system prompts and user-visibility rows alone are not clearable")
|
|
require.True(t, hasClearableMessageAfter([]database.ChatMessage{system, userVisible, conversation}, -1))
|
|
require.False(t, hasClearableMessageAfter([]database.ChatMessage{conversation}, 0),
|
|
"nothing after the boundary index")
|
|
}
|
|
|
|
func TestDecisionForcedCompactionRespectsClearBoundary(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
requestedChat := database.Chat{
|
|
CompactionRequestedAt: sql.NullTime{Time: time.Now(), Valid: true},
|
|
}
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("question")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
}
|
|
messages = append(messages, clearBoundaryTriplet(t, 3)...)
|
|
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: requestedChat,
|
|
messages: messages,
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotEqual(t, generationActionCompact, decision.kind,
|
|
"a stale compaction request must not reach across a clear boundary")
|
|
require.False(t, decision.forced)
|
|
}
|
|
|
|
func TestCompactionStatusFromHistory(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
const thresholdPercent = int32(70)
|
|
|
|
t.Run("needed without boundary", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusNeeded, got)
|
|
})
|
|
|
|
t.Run("after compaction without post boundary history", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusAfterCompaction, got)
|
|
})
|
|
|
|
t.Run("needed after under limit post compaction assistant", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("continued")), 20, 100),
|
|
dbMessage(t, 5, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("next")),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusNeeded, got)
|
|
})
|
|
|
|
t.Run("still over limit from first post compaction assistant usage", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 80, 100),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusStillOverLimit, got)
|
|
})
|
|
|
|
t.Run("zero threshold compacts again instead of reporting still over limit", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 80, 100),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, 0, 100)
|
|
require.Equal(t, compactionStatusNeeded, got)
|
|
})
|
|
|
|
t.Run("still over limit includes prompt cache tokens", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsageTokens(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), fantasy.Usage{CacheReadTokens: 80}, 100),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusStillOverLimit, got)
|
|
})
|
|
|
|
t.Run("still over limit uses configured context limit", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 80, 200),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusStillOverLimit, got)
|
|
|
|
got = compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 200)
|
|
require.Equal(t, compactionStatusNeeded, got)
|
|
})
|
|
|
|
t.Run("summary under a different trigger compacts again", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", json.RawMessage(`{"threshold_percent":70}`))),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{"threshold_percent":70,"context_limit_tokens":1000,"trigger_context_limit_tokens":100}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 80, 100),
|
|
}
|
|
|
|
require.Equal(t, compactionStatusStillOverLimit, compactionStatusFromHistory(messages, compactionRequirementNeeded, 70, 100))
|
|
require.Equal(t, compactionStatusNeeded, compactionStatusFromHistory(messages, compactionRequirementNeeded, 40, 100))
|
|
require.Equal(t, compactionStatusNeeded, compactionStatusFromHistory(messages, compactionRequirementNeeded, 70, 90))
|
|
})
|
|
|
|
t.Run("summary without trigger limit compares context limit", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", json.RawMessage(`{"threshold_percent":70}`))),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{"threshold_percent":70,"context_limit_tokens":1000}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 800, 1000),
|
|
}
|
|
|
|
require.Equal(t, compactionStatusStillOverLimit, compactionStatusFromHistory(messages, compactionRequirementNeeded, 70, 1000))
|
|
require.Equal(t, compactionStatusNeeded, compactionStatusFromHistory(messages, compactionRequirementNeeded, 70, 500))
|
|
})
|
|
|
|
t.Run("still over limit includes exact threshold boundary", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("summary")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
withUsage(dbMessage(t, 4, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("read-1", "read_file", json.RawMessage(`{}`))), 70, 100),
|
|
}
|
|
|
|
got := compactionStatusFromHistory(messages, compactionRequirementNeeded, thresholdPercent, 100)
|
|
require.Equal(t, compactionStatusStillOverLimit, got)
|
|
})
|
|
}
|
|
|
|
func TestDecisionDetectsStopAfterToolFromCommittedHistory(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("plan")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("plan-1", "propose_plan", json.RawMessage(`{}`))),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("plan-1", "propose_plan", json.RawMessage(`{"ok":true}`), false, false)),
|
|
}
|
|
got, err := historyHasStopAfterToolResult(messages, map[string]struct{}{"propose_plan": {}})
|
|
require.NoError(t, err)
|
|
require.True(t, got)
|
|
|
|
messages[2] = dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("plan-1", "propose_plan", json.RawMessage(`{"error":"no"}`), true, false))
|
|
got, err = historyHasStopAfterToolResult(messages, map[string]struct{}{"propose_plan": {}})
|
|
require.NoError(t, err)
|
|
require.False(t, got)
|
|
}
|
|
|
|
func TestDecisionDetectsStopAfterToolAcrossHookContext(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hookContext := dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("hook context"))
|
|
hookContext.Visibility = database.ChatMessageVisibilityModel
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("plan")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("plan-1", "propose_plan", json.RawMessage(`{}`))),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("plan-1", "propose_plan", json.RawMessage(`{"ok":true}`), false, false)),
|
|
hookContext,
|
|
}
|
|
got, err := historyHasStopAfterToolResult(messages, map[string]struct{}{"propose_plan": {}})
|
|
require.NoError(t, err)
|
|
require.True(t, got)
|
|
}
|
|
|
|
func TestCurrentTurnAdvisorUses(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
advice, err := json.Marshal(chatadvisor.AdvisorResult{
|
|
Type: chatadvisor.ResultTypeAdvice,
|
|
Advice: strings.Repeat("long advice ", 1000),
|
|
})
|
|
require.NoError(t, err)
|
|
truncatedAdvice, truncated := chatloop.TruncateToolResultText(string(advice), 1024)
|
|
require.True(t, truncated)
|
|
truncatedPart := chatprompt.PartFromContent(fantasy.ToolResultContent{
|
|
ToolCallID: "advisor-truncated",
|
|
ToolName: chatadvisor.ToolName,
|
|
Result: fantasy.ToolResultOutputContentText{Text: truncatedAdvice},
|
|
})
|
|
interruptedPart := chatprompt.PartFromContent(fantasy.ToolResultContent{
|
|
ToolCallID: "advisor-interrupted",
|
|
ToolName: chatadvisor.ToolName,
|
|
Result: fantasy.ToolResultOutputContentError{Error: xerrors.New(interruptedToolResultErrorMessage)},
|
|
})
|
|
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("plan")),
|
|
dbMessage(t, 2, database.ChatMessageRoleTool, false, truncatedPart),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, interruptedPart),
|
|
}
|
|
logger := slogtest.Make(t, nil)
|
|
got, err := currentTurnAdvisorUses(context.Background(), logger, messages)
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, got, "truncated advice consumed a use; an interrupted call did not")
|
|
}
|
|
|
|
func TestDecisionDetectsCurrentHistoryCompletion(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
complete, err := currentHistoryComplete([]database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("hello")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("done")),
|
|
})
|
|
require.NoError(t, err)
|
|
require.True(t, complete)
|
|
|
|
complete, err = currentHistoryComplete([]database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("hello")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{}`))),
|
|
})
|
|
require.NoError(t, err)
|
|
require.False(t, complete)
|
|
|
|
complete, err = currentHistoryComplete([]database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("hello")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{}`))),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("call-1", "execute", json.RawMessage(`{"ok":true}`), false, false)),
|
|
})
|
|
require.NoError(t, err)
|
|
require.False(t, complete)
|
|
}
|
|
|
|
func TestDecisionObservesToolCallsAnsweredBeforeExecution(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// A pre_tool_use hook denied the first computer call at admission, so
|
|
// its result committed with the assistant step and only the last call is
|
|
// left to run. The chat loop still sees the denied call in its place,
|
|
// so it can halt the later computer call, while the pending client call
|
|
// stays out.
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("fill in the form")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false,
|
|
codersdk.ChatMessageToolCall("call-denied", "computer", json.RawMessage(`{"action":"left_click"}`)),
|
|
codersdk.ChatMessageToolCall("call-client", "client_tool", json.RawMessage(`{}`)),
|
|
codersdk.ChatMessageToolCall("call-allowed", "computer", json.RawMessage(`{"action":"type","text":"hi"}`)),
|
|
),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false,
|
|
codersdk.ChatMessageToolResult("call-denied", "computer", json.RawMessage(`{"error":"denied"}`), true, false)),
|
|
}
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
messages: messages,
|
|
dynamicToolNames: map[string]bool{"client_tool": true},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionExecuteLocalTools, decision.kind)
|
|
callIDs := func(calls []fantasy.ToolCallContent) []string {
|
|
ids := make([]string, 0, len(calls))
|
|
for _, call := range calls {
|
|
ids = append(ids, call.ToolCallID)
|
|
}
|
|
return ids
|
|
}
|
|
require.Equal(t, []string{"call-allowed"}, callIDs(decision.localToolCalls))
|
|
require.Equal(t, []string{"call-denied", "call-allowed"}, callIDs(decision.observedToolCalls))
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_NormalizesToolCallDeltasBeforeFinal(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
createdAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("partial ")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call-1", ToolName: "execute", ArgsDelta: `{"cmd":`}},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call-1", ToolName: "execute", ArgsDelta: `"ignored"}`}},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{"cmd":"pwd"}`))},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: createdAt,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 2)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
require.Len(t, assistantParts, 2)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeText, assistantParts[0].Type)
|
|
call := assistantParts[1]
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, call.Type)
|
|
require.Equal(t, "call-1", call.ToolCallID)
|
|
require.Empty(t, call.ArgsDelta)
|
|
require.JSONEq(t, `{"cmd":"pwd"}`, string(call.Args))
|
|
syntheticParts := parseMessageParts(t, got[1].Role, got[1].Content)
|
|
require.Len(t, syntheticParts, 1)
|
|
require.Equal(t, "call-1", syntheticParts[0].ToolCallID)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_CoalescesStreamedDeltas(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
// Whitespace-only deltas must stay inside the run because prompt replay
|
|
// drops whitespace-only text parts.
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageReasoning("think")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageReasoning("ing")},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("Hel")},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("lo,")},
|
|
{Seq: 5, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText(" ")},
|
|
{Seq: 6, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("world")},
|
|
{Seq: 7, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("\n\n")},
|
|
{Seq: 8, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{"cmd":"pwd"}`))},
|
|
{Seq: 9, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("after")},
|
|
{Seq: 10, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText(" the call")},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
|
|
})
|
|
require.NoError(t, err)
|
|
require.NotEmpty(t, got)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got[0].Role)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
|
|
var summary []string
|
|
for _, part := range assistantParts {
|
|
summary = append(summary, string(part.Type)+":"+part.Text)
|
|
}
|
|
require.Equal(t, []string{
|
|
"reasoning:thinking",
|
|
"text:Hello, world\n\n",
|
|
"tool-call:",
|
|
"text:after the call",
|
|
}, summary, "adjacent deltas of the same type must be persisted as one part")
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_ToolResultEndsTextRun(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("a")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{"cmd":"pwd"}`))},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("b")},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("c")},
|
|
{Seq: 5, Role: codersdk.ChatMessageRoleTool, MessagePart: codersdk.ChatMessageToolResult("call-1", "execute", json.RawMessage(`{"ok":true}`), false, false)},
|
|
{Seq: 6, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("d")},
|
|
{Seq: 7, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("e")},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
var summary [][]string
|
|
for _, msg := range got {
|
|
message := []string{string(msg.Role)}
|
|
for _, part := range parseMessageParts(t, msg.Role, msg.Content) {
|
|
message = append(message, string(part.Type)+":"+part.Text)
|
|
}
|
|
summary = append(summary, message)
|
|
}
|
|
require.Equal(t, [][]string{
|
|
{"assistant", "text:a", "tool-call:", "text:bc"},
|
|
{"tool", "tool-result:"},
|
|
{"assistant", "text:de"},
|
|
}, summary, "a tool result must end the open text run")
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_SplitsAdjacentReasoningBlocks(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
firstStart := time.Date(2026, 3, 4, 5, 6, 1, 0, time.UTC)
|
|
secondStart := time.Date(2026, 3, 4, 5, 6, 2, 0, time.UTC)
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: withCreatedAt(codersdk.ChatMessageReasoning("first "), firstStart)},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: withCreatedAt(codersdk.ChatMessageReasoning("thought"), firstStart)},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: withCreatedAt(codersdk.ChatMessageReasoning("second "), secondStart)},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: withCreatedAt(codersdk.ChatMessageReasoning("thought"), secondStart)},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 1)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
|
|
var summary []string
|
|
for _, part := range assistantParts {
|
|
summary = append(summary, part.Text+"@"+requireNotNilTime(t, part.CreatedAt).Format(time.TimeOnly))
|
|
}
|
|
require.Equal(t, []string{
|
|
"first thought@05:06:01",
|
|
"second thought@05:06:02",
|
|
}, summary, "each reasoning block must keep its own part and start time")
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_SplitsNarrationFromAnswer(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
narration := func(text string) codersdk.ChatMessagePart {
|
|
part := codersdk.ChatMessageText(text)
|
|
part.Narration = true
|
|
return part
|
|
}
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: narration("Checking the ")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: narration("repo.")},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("The answer ")},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("is 42.")},
|
|
{Seq: 5, Role: codersdk.ChatMessageRoleAssistant, MessagePart: narration("Next, the tests.")},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 1)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
|
|
var summary []string
|
|
for _, part := range assistantParts {
|
|
summary = append(summary, fmt.Sprintf("narration=%t:%s", part.Narration, part.Text))
|
|
}
|
|
require.Equal(t, []string{
|
|
"narration=true:Checking the repo.",
|
|
"narration=false:The answer is 42.",
|
|
"narration=true:Next, the tests.",
|
|
}, summary, "narration and the answer must be separate parts")
|
|
}
|
|
|
|
// BenchmarkBufferedPartsToPartialMessages_StreamedTextDeltas persists N
|
|
// seven-byte text deltas as one run; B/op should grow linearly with N. A 1 MiB
|
|
// episode holds about 2,475 such deltas, so N=2500 is close to the largest
|
|
// real input.
|
|
func BenchmarkBufferedPartsToPartialMessages_StreamedTextDeltas(b *testing.B) {
|
|
for _, n := range []int{1000, 2500} {
|
|
b.Run(strconv.Itoa(n), func(b *testing.B) {
|
|
parts := make([]messagepartbuffer.Part, n)
|
|
for i := range parts {
|
|
parts[i] = messagepartbuffer.Part{
|
|
Seq: int64(i + 1),
|
|
Role: codersdk.ChatMessageRoleAssistant,
|
|
MessagePart: codersdk.ChatMessageText("1234567"),
|
|
}
|
|
}
|
|
input := bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC),
|
|
}
|
|
b.ReportAllocs()
|
|
for b.Loop() {
|
|
if _, err := bufferedPartsToPartialMessages(input); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_AttachesAttemptRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("partial ")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{"cmd":"pwd"}`))},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
attemptRuntime: 1500 * time.Millisecond,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 2)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got[0].Role)
|
|
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, got[0].RuntimeMs)
|
|
require.Equal(t, database.ChatMessageRoleTool, got[1].Role)
|
|
require.False(t, got[1].RuntimeMs.Valid)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_DropsRuntimeWithoutAssistantContent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: nil,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
attemptRuntime: 1500 * time.Millisecond,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Empty(t, got)
|
|
|
|
// Tool execution publishes assistant-role file parts for
|
|
// attachments. A suffix containing only those is a tool batch,
|
|
// not model generation, so its span is not billed.
|
|
got, err = bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageFile(uuid.New(), "image/png", "screenshot.png")},
|
|
},
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
attemptRuntime: 1500 * time.Millisecond,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 1)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got[0].Role)
|
|
require.False(t, got[0].RuntimeMs.Valid)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_MergesToolCallDeltasWithoutFinal(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call-1", ToolName: "execute", ArgsDelta: `{"cmd":`}},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call-1", ToolName: "execute", ArgsDelta: `"pwd"}`}},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 2)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
require.Len(t, assistantParts, 1)
|
|
require.Empty(t, assistantParts[0].ArgsDelta)
|
|
require.JSONEq(t, `{"cmd":"pwd"}`, string(assistantParts[0].Args))
|
|
syntheticParts := parseMessageParts(t, got[1].Role, got[1].Content)
|
|
require.Len(t, syntheticParts, 1)
|
|
require.Equal(t, "call-1", syntheticParts[0].ToolCallID)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_DeltaOnlyToolResultDoesNotAnswer(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logSink := &partialConversionLogSink{}
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).AppendSinks(logSink)
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "advisor", json.RawMessage(`{}`))},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleTool, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolCallID: "call-1", ToolName: "advisor", ResultDelta: `{"type":"advice"}`}},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 2)
|
|
toolParts := parseMessageParts(t, got[1].Role, got[1].Content)
|
|
require.Len(t, toolParts, 1)
|
|
require.Equal(t, "call-1", toolParts[0].ToolCallID)
|
|
require.True(t, toolParts[0].IsError)
|
|
require.Empty(t, toolParts[0].ResultDelta)
|
|
require.JSONEq(t, `{"error":"tool call was interrupted before it produced a result"}`, string(toolParts[0].Result))
|
|
require.NotEmpty(t, logSink.entriesAtLevelWithMessage(slog.LevelWarn, "skipping buffered chat message part"))
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_ReasoningDeltaToolResultIsNotDurable(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logSink := &partialConversionLogSink{}
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).AppendSinks(logSink)
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "advisor", json.RawMessage(`{}`))},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleTool, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolCallID: "call-1", ToolName: "advisor", ReasoningDelta: "weighing "}},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleTool, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolCallID: "call-1", ToolName: "advisor", ReasoningDelta: "tradeoffs"}},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 2)
|
|
require.NotContains(t, string(got[1].Content.RawMessage), "weighing")
|
|
toolParts := parseMessageParts(t, got[1].Role, got[1].Content)
|
|
require.Len(t, toolParts, 1)
|
|
require.Equal(t, "call-1", toolParts[0].ToolCallID)
|
|
require.True(t, toolParts[0].IsError)
|
|
require.Empty(t, toolParts[0].ReasoningDelta)
|
|
require.JSONEq(t, `{"error":"tool call was interrupted before it produced a result"}`, string(toolParts[0].Result))
|
|
|
|
// Reasoning deltas are stream-only like result deltas: the interrupted
|
|
// stream is reported once at flush time, not per chunk as an invalid
|
|
// durable result.
|
|
skipped := logSink.entriesAtLevelWithMessage(slog.LevelWarn, "skipping buffered chat message part")
|
|
require.Len(t, skipped, 1)
|
|
var reason string
|
|
for _, field := range skipped[0].Fields {
|
|
if field.Name == "reason" {
|
|
reason, _ = field.Value.(string)
|
|
}
|
|
}
|
|
require.Equal(t, "streaming tool result delta is not durable", reason)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_LogsMalformedSkippedParts(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logSink := &partialConversionLogSink{}
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}).AppendSinks(logSink)
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleSystem, MessagePart: codersdk.ChatMessageText("bad role")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{}},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleTool, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolName: "execute", Result: json.RawMessage(`{"ok":true}`)}},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "bad-args", ToolName: "execute", ArgsDelta: `{"cmd":`}},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: logger,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Empty(t, got)
|
|
require.GreaterOrEqual(t, len(logSink.entriesAtLevelWithMessage(slog.LevelWarn, "skipping buffered chat message part")), 4)
|
|
}
|
|
|
|
func TestBufferedPartsToPartialMessages_SynthesizesMissingToolResults(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
createdAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC)
|
|
reasoningStartedAt := createdAt.Add(-2 * time.Second)
|
|
reasoningPart := codersdk.ChatMessageReasoning("partial thought")
|
|
reasoningPart.CreatedAt = &reasoningStartedAt
|
|
parts := []messagepartbuffer.Part{
|
|
{Seq: 1, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageText("partial ")},
|
|
{Seq: 2, Role: codersdk.ChatMessageRoleAssistant, MessagePart: reasoningPart},
|
|
{Seq: 3, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-1", "execute", json.RawMessage(`{}`))},
|
|
{Seq: 4, Role: codersdk.ChatMessageRoleAssistant, MessagePart: codersdk.ChatMessageToolCall("call-2", "read_file", json.RawMessage(`{}`))},
|
|
{Seq: 5, Role: codersdk.ChatMessageRoleTool, MessagePart: withCreatedAt(codersdk.ChatMessageToolResult("call-2", "read_file", json.RawMessage(`{"ok":true}`), false, false), createdAt)},
|
|
}
|
|
got, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: modelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
interruptedAt: createdAt,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got, 3)
|
|
require.Equal(t, database.ChatMessageRoleAssistant, got[0].Role)
|
|
assistantParts := parseMessageParts(t, got[0].Role, got[0].Content)
|
|
require.Len(t, assistantParts, 4)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeReasoning, assistantParts[1].Type)
|
|
require.Equal(t, "partial thought", assistantParts[1].Text)
|
|
require.Equal(t, reasoningStartedAt, requireNotNilTime(t, assistantParts[1].CreatedAt))
|
|
require.Equal(t, createdAt, requireNotNilTime(t, assistantParts[1].CompletedAt))
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, assistantParts[2].Type)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolCall, assistantParts[3].Type)
|
|
|
|
require.Equal(t, database.ChatMessageRoleTool, got[1].Role)
|
|
require.Equal(t, database.ChatMessageVisibilityBoth, got[1].Visibility)
|
|
toolParts := parseMessageParts(t, got[1].Role, got[1].Content)
|
|
require.Equal(t, "call-2", toolParts[0].ToolCallID)
|
|
require.JSONEq(t, `{"ok":true}`, string(toolParts[0].Result))
|
|
require.Equal(t, createdAt, requireNotNilTime(t, toolParts[0].CreatedAt))
|
|
|
|
require.Equal(t, database.ChatMessageRoleTool, got[2].Role)
|
|
syntheticParts := parseMessageParts(t, got[2].Role, got[2].Content)
|
|
require.Len(t, syntheticParts, 1)
|
|
require.Equal(t, "call-1", syntheticParts[0].ToolCallID)
|
|
require.Equal(t, "execute", syntheticParts[0].ToolName)
|
|
require.True(t, syntheticParts[0].IsError)
|
|
require.JSONEq(t, `{"error":"tool call was interrupted before it produced a result"}`, string(syntheticParts[0].Result))
|
|
require.Equal(t, createdAt, requireNotNilTime(t, syntheticParts[0].CreatedAt))
|
|
require.Equal(t, uuid.NullUUID{UUID: modelConfigID, Valid: true}, got[2].ModelConfigID)
|
|
}
|
|
|
|
func parseMessageParts(t *testing.T, role database.ChatMessageRole, raw pqtype.NullRawMessage) []codersdk.ChatMessagePart {
|
|
t.Helper()
|
|
parts, err := chatprompt.ParseContent(database.ChatMessage{
|
|
Role: role,
|
|
Content: raw,
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
})
|
|
require.NoError(t, err)
|
|
return parts
|
|
}
|
|
|
|
func dbMessage(t *testing.T, id int64, role database.ChatMessageRole, compressed bool, parts ...codersdk.ChatMessagePart) database.ChatMessage {
|
|
t.Helper()
|
|
raw, err := chatprompt.MarshalParts(parts)
|
|
require.NoError(t, err)
|
|
return database.ChatMessage{
|
|
ID: id,
|
|
Role: role,
|
|
Content: raw,
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
Visibility: database.ChatMessageVisibilityBoth,
|
|
Compressed: compressed,
|
|
}
|
|
}
|
|
|
|
func withUsage(msg database.ChatMessage, inputTokens int64, contextLimit int64) database.ChatMessage {
|
|
return withUsageTokens(msg, fantasy.Usage{InputTokens: inputTokens, TotalTokens: inputTokens}, contextLimit)
|
|
}
|
|
|
|
func withUsageTokens(msg database.ChatMessage, usage fantasy.Usage, contextLimit int64) database.ChatMessage {
|
|
msg.InputTokens = sql.NullInt64{Int64: usage.InputTokens, Valid: usage.InputTokens != 0}
|
|
msg.OutputTokens = sql.NullInt64{Int64: usage.OutputTokens, Valid: usage.OutputTokens != 0}
|
|
msg.TotalTokens = sql.NullInt64{Int64: usage.TotalTokens, Valid: usage.TotalTokens != 0}
|
|
msg.ReasoningTokens = sql.NullInt64{Int64: usage.ReasoningTokens, Valid: usage.ReasoningTokens != 0}
|
|
msg.CacheCreationTokens = sql.NullInt64{Int64: usage.CacheCreationTokens, Valid: usage.CacheCreationTokens != 0}
|
|
msg.CacheReadTokens = sql.NullInt64{Int64: usage.CacheReadTokens, Valid: usage.CacheReadTokens != 0}
|
|
msg.ContextLimit = sql.NullInt64{Int64: contextLimit, Valid: contextLimit != 0}
|
|
return msg
|
|
}
|
|
|
|
func requireNotNilTime(t *testing.T, value *time.Time) time.Time {
|
|
t.Helper()
|
|
require.NotNil(t, value)
|
|
return *value
|
|
}
|
|
|
|
func withCreatedAt(part codersdk.ChatMessagePart, createdAt time.Time) codersdk.ChatMessagePart {
|
|
part.CreatedAt = &createdAt
|
|
return part
|
|
}
|
|
|
|
type partialConversionLogSink struct {
|
|
mu sync.Mutex
|
|
entries []slog.SinkEntry
|
|
}
|
|
|
|
func (s *partialConversionLogSink) LogEntry(_ context.Context, entry slog.SinkEntry) {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
s.entries = append(s.entries, entry)
|
|
}
|
|
|
|
func (*partialConversionLogSink) Sync() {}
|
|
|
|
func (s *partialConversionLogSink) entriesAtLevelWithMessage(level slog.Level, message string) []slog.SinkEntry {
|
|
s.mu.Lock()
|
|
defer s.mu.Unlock()
|
|
|
|
entries := make([]slog.SinkEntry, 0, len(s.entries))
|
|
for _, entry := range s.entries {
|
|
if entry.Level == level && entry.Message == message {
|
|
entries = append(entries, entry)
|
|
}
|
|
}
|
|
return entries
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_MarksHookRewrittenToolCalls(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{
|
|
ToolCallID: "rewritten",
|
|
ToolName: "execute",
|
|
Input: `{"command":"echo admitted"}`,
|
|
},
|
|
fantasy.ToolCallContent{
|
|
ToolCallID: "untouched",
|
|
ToolName: "execute",
|
|
Input: `{"command":"echo original"}`,
|
|
},
|
|
},
|
|
},
|
|
hookRewrittenToolCalls: map[string]json.RawMessage{"rewritten": {}},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 1)
|
|
|
|
parts := parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)
|
|
require.Len(t, parts, 2)
|
|
require.Equal(t, "rewritten", parts[0].ToolCallID)
|
|
require.True(t, parts[0].HookRewritten)
|
|
require.Equal(t, "untouched", parts[1].ToolCallID)
|
|
require.False(t, parts[1].HookRewritten)
|
|
}
|
|
|
|
func TestBuildCommitStepMessages_SkipsProviderExecutedRewriteAttribution(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
got, err := buildCommitStepMessages(buildCommitStepMessagesInput{
|
|
modelConfigID: uuid.New(),
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: slog.Make(),
|
|
step: stepData{
|
|
Content: []fantasy.Content{
|
|
fantasy.ToolCallContent{
|
|
ToolCallID: "shared",
|
|
ToolName: "web_search",
|
|
Input: `{"query":"coder"}`,
|
|
ProviderExecuted: true,
|
|
},
|
|
fantasy.ToolCallContent{
|
|
ToolCallID: "shared",
|
|
ToolName: "execute",
|
|
Input: `{"command":"echo admitted"}`,
|
|
},
|
|
},
|
|
},
|
|
hookRewrittenToolCalls: map[string]json.RawMessage{"shared": {}},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 1)
|
|
|
|
parts := parseMessageParts(t, got.Messages[0].Role, got.Messages[0].Content)
|
|
require.Len(t, parts, 2)
|
|
require.True(t, parts[0].ProviderExecuted)
|
|
require.False(t, parts[0].HookRewritten)
|
|
require.False(t, parts[1].ProviderExecuted)
|
|
require.True(t, parts[1].HookRewritten)
|
|
}
|
|
|
|
func TestBuildCompactionMessages_ReplaysPendingUserMessages(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
modelConfigID := uuid.New()
|
|
pending := []database.ChatMessage{
|
|
dbMessage(t, 10, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("do final checks, do NOT start a review")),
|
|
dbMessage(t, 11, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("also update the scope doc")),
|
|
}
|
|
got, err := buildCompactionMessages(buildCompactionMessagesInput{
|
|
modelConfigID: modelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
toolCallID: "summary-1",
|
|
toolName: "chat_summarized",
|
|
compaction: chatloop.CompactionResult{
|
|
SystemSummary: "system summary",
|
|
SummaryReport: "user report",
|
|
ThresholdPercent: 70,
|
|
},
|
|
pendingUserMessages: pending,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, got.Messages, 5)
|
|
for i := range 3 {
|
|
require.True(t, got.Messages[i].Compressed, "boundary trio stays compressed")
|
|
}
|
|
for i, want := range []string{"do final checks, do NOT start a review", "also update the scope doc"} {
|
|
replay := got.Messages[3+i]
|
|
require.Equal(t, database.ChatMessageRoleUser, replay.Role)
|
|
require.Equal(t, database.ChatMessageVisibilityModel, replay.Visibility)
|
|
require.False(t, replay.Compressed, "replay rows must stay inside the post-boundary prompt window")
|
|
require.Equal(t, uuid.NullUUID{UUID: modelConfigID, Valid: true}, replay.ModelConfigID)
|
|
require.Equal(t, chatprompt.CurrentContentVersion, replay.ContentVersion)
|
|
require.Equal(t, want, parseMessageParts(t, replay.Role, replay.Content)[0].Text)
|
|
}
|
|
}
|
|
|
|
func TestPendingUserSegmentStart(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("TrailingUserRun", func(t *testing.T) {
|
|
t.Parallel()
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
dbMessage(t, 3, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending one")),
|
|
dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending two")),
|
|
}
|
|
require.Equal(t, 2, pendingUserSegmentStart(rows))
|
|
})
|
|
|
|
t.Run("NoTrailingUserMessages", func(t *testing.T) {
|
|
t.Parallel()
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
}
|
|
require.Equal(t, len(rows), pendingUserSegmentStart(rows))
|
|
})
|
|
|
|
t.Run("ToolRowStopsSegment", func(t *testing.T) {
|
|
t.Parallel()
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("call", "tool", nil)),
|
|
dbMessage(t, 3, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("call", "tool", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 4, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending")),
|
|
}
|
|
require.Equal(t, 3, pendingUserSegmentStart(rows))
|
|
})
|
|
|
|
t.Run("CompressedTrailingRowStopsSegment", func(t *testing.T) {
|
|
t.Parallel()
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageText("answer")),
|
|
dbMessage(t, 3, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("compacted summary")),
|
|
}
|
|
require.Equal(t, len(rows), pendingUserSegmentStart(rows))
|
|
})
|
|
|
|
t.Run("NoAssistantBeforeSegment", func(t *testing.T) {
|
|
t.Parallel()
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleSystem, false, codersdk.ChatMessageText("system")),
|
|
dbMessage(t, 2, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("first message")),
|
|
}
|
|
require.Equal(t, len(rows), pendingUserSegmentStart(rows))
|
|
})
|
|
|
|
t.Run("DroppedAssistantRowStillStopsSegment", func(t *testing.T) {
|
|
t.Parallel()
|
|
// An assistant row that prompt conversion drops (no replayable
|
|
// parts) must still terminate the segment so the older user
|
|
// message before it is never treated as pending.
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("older user message")),
|
|
dbMessage(t, 2, database.ChatMessageRoleAssistant, false),
|
|
dbMessage(t, 3, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending")),
|
|
}
|
|
require.Equal(t, 2, pendingUserSegmentStart(rows))
|
|
})
|
|
}
|
|
|
|
// Converting the head and the user-only pending tail separately must
|
|
// produce the same generation prompt as converting all rows at once,
|
|
// while keeping the older user message in the head even when an
|
|
// intervening assistant row is dropped by conversion.
|
|
func TestPendingUserSegmentConversionEquivalence(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
logger := slogtest.Make(t, nil)
|
|
rows := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleSystem, false, codersdk.ChatMessageText("system")),
|
|
dbMessage(t, 2, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("start")),
|
|
dbMessage(t, 3, database.ChatMessageRoleAssistant, false, codersdk.ChatMessageToolCall("call-1", "tool", json.RawMessage(`{}`))),
|
|
dbMessage(t, 4, database.ChatMessageRoleTool, false, codersdk.ChatMessageToolResult("call-1", "tool", json.RawMessage(`{}`), false, false)),
|
|
dbMessage(t, 5, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("older user message")),
|
|
dbMessage(t, 6, database.ChatMessageRoleAssistant, false),
|
|
dbMessage(t, 7, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending one")),
|
|
dbMessage(t, 8, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending two")),
|
|
}
|
|
|
|
start := pendingUserSegmentStart(rows)
|
|
require.Equal(t, 6, start)
|
|
|
|
full, err := chatprompt.ConvertMessagesWithFiles(context.Background(), rows, nil, logger, nil, uuid.NullUUID{})
|
|
require.NoError(t, err)
|
|
head, err := chatprompt.ConvertMessagesWithFiles(context.Background(), rows[:start], nil, logger, nil, uuid.NullUUID{})
|
|
require.NoError(t, err)
|
|
tail, err := chatprompt.ConvertMessagesWithFiles(context.Background(), rows[start:], nil, logger, nil, uuid.NullUUID{})
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, full, append(append([]fantasy.Message{}, head...), tail...))
|
|
|
|
// The dropped assistant row makes the older user message adjacent
|
|
// to the pending run in the converted prompt; it must stay in the
|
|
// head so the summarizer still sees it.
|
|
require.Equal(t, fantasy.MessageRoleUser, head[len(head)-1].Role)
|
|
require.Equal(t, "older user message", head[len(head)-1].Content[0].(fantasy.TextPart).Text)
|
|
require.Len(t, tail, 2)
|
|
for _, msg := range tail {
|
|
require.Equal(t, fantasy.MessageRoleUser, msg.Role)
|
|
}
|
|
}
|
|
|
|
func TestDecisionGeneratesAfterCompactionWithReplayedPendingUser(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
replayed := dbMessage(t, 5, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("pending instruction"))
|
|
replayed.Visibility = database.ChatMessageVisibilityModel
|
|
messages := []database.ChatMessage{
|
|
dbMessage(t, 1, database.ChatMessageRoleUser, false, codersdk.ChatMessageText("original prompt")),
|
|
dbMessage(t, 2, database.ChatMessageRoleUser, true, codersdk.ChatMessageText("compacted summary")),
|
|
dbMessage(t, 3, database.ChatMessageRoleAssistant, true, codersdk.ChatMessageToolCall("summary-1", "chat_summarized", nil)),
|
|
dbMessage(t, 4, database.ChatMessageRoleTool, true, codersdk.ChatMessageToolResult("summary-1", "chat_summarized", json.RawMessage(`{}`), false, false)),
|
|
replayed,
|
|
}
|
|
|
|
decision, err := decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
messages: messages,
|
|
compactionEnabled: true,
|
|
compactionNeeded: false,
|
|
compactionThresholdPercent: 70,
|
|
compactionContextLimit: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionGenerateAssistant, decision.kind)
|
|
|
|
decision, err = decideGenerationAction(generationDecisionInput{
|
|
maxSteps: codersdk.DefaultChatMaxStepsPerTurn,
|
|
chat: database.Chat{CompactionRequestedAt: sql.NullTime{Time: time.Now(), Valid: true}},
|
|
messages: messages,
|
|
compactionEnabled: true,
|
|
compactionNeeded: false,
|
|
compactionThresholdPercent: 70,
|
|
compactionContextLimit: 100,
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, generationActionGenerateAssistant, decision.kind)
|
|
}
|