Files
coder/coderd/x/chatd/tasks_test.go
T
Callum Styan d412c38afb perf: validate chat transitions from the locked row (#30886)
One of the main costs in terms of DB load from chats is extra round
trips during transactions. This results in locks being held for longer
periods of time -> an increase in lock waits for chats related queries.

One of the opportunities for reducing some of the round trips is to
eliminate extra chat row reads after we have just read the row in order
to lock it.

In this PR `ChatMachine.Update` now seeds the transaction struct with
the row returned by `LockChatAndBumpSnapshotVersion`, plus a queue count
taken right after it. We can then pass it into `Update` as an optional
parameter validate against it, so neither re-reads the chat row from the
database while the row lock is held.

`updateChat` (send, edit, promote, tool results, interrupt, compact,
clear, reconcile), hook failure, family archive, and the worker's
generation and task transitions use it instead of reloading the chat
under the lock.

This PR relative to main:
|measurement | main | this PR | percent change |
|-- | -- | -- | -- |
| Round trips under lock  | 180.9 | 142.0 | -39% |
| Statements under lock  | 86.1 | 66.6 | -23% |
| GetChatByID under lock | 54.1 | 35.8 | -34% |
| GetChatByID total | 75.5 | 57.1 | -25% |
| Statements  | 235.8 | 216.2 | -8% |

Generated by Coder Agents on behalf of @cstyan.
2026-11-13 09:59:39 -07:00

1946 lines
73 KiB
Go

//nolint:testpackage // These tests exercise package-private task seams.
package chatd
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"net/http"
"slices"
"strings"
"sync"
"sync/atomic"
"testing"
"time"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbfake"
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
"github.com/coder/coder/v2/coderd/x/chatd/chatretry"
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
"github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/codersdk/workspacesdk"
"github.com/coder/coder/v2/codersdk/workspacesdk/agentconnmock"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func TestRetryWrapper_ExpectedExitsDoNotRetry(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
sink := testutil.NewFakeSink(t)
calls := 0
err := runTaskWithRetry(ctx, retryWrapperOptions{
clock: quartz.NewMock(t),
logger: sink.Logger(),
initialDelay: time.Second,
maxDelay: time.Second,
}, taskKindInterrupt, retryWrapperTaskInfo{}, func(context.Context) error {
calls++
return errTaskExpectedExit
})
require.NoError(t, err)
require.Equal(t, 1, calls)
require.Empty(t, entriesWithMessage(sink, "chatworker task retrying"))
}
func TestRetryWrapper_UnexpectedErrorsRetry(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
trap := clock.Trap().NewTimer("chatworker", "task-retry-requires_action_timeout")
defer trap.Close()
ctx := testutil.Context(t, testutil.WaitLong)
sink := testutil.NewFakeSink(t)
calls := 0
done := make(chan error, 1)
go func() {
done <- runTaskWithRetry(ctx, retryWrapperOptions{
clock: clock,
logger: sink.Logger(),
initialDelay: time.Minute,
maxDelay: time.Minute,
}, taskKindRequiresActionTimeout, retryWrapperTaskInfo{}, func(context.Context) error {
calls++
if calls == 1 {
return xerrors.New("database unavailable")
}
return nil
})
}()
trap.MustWait(ctx).MustRelease(ctx)
clock.Advance(time.Minute).MustWait(ctx)
require.NoError(t, <-done)
require.Equal(t, 2, calls)
entries := entriesWithMessage(sink, "chatworker task retrying")
require.Len(t, entries, 1)
require.Equal(t, string(taskKindRequiresActionTimeout), sinkFieldValue(t, entries[0].Fields, "task_kind"))
require.Equal(t, time.Minute.String(), sinkFieldValue(t, entries[0].Fields, "delay"))
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), "database unavailable")
}
func TestRetryWrapper_PanicsRetry(t *testing.T) {
t.Parallel()
clock := quartz.NewMock(t)
trap := clock.Trap().NewTimer("chatworker", "task-retry-generation")
defer trap.Close()
ctx := testutil.Context(t, testutil.WaitLong)
sink := testutil.NewFakeSink(t)
calls := 0
done := make(chan error, 1)
go func() {
done <- runTaskWithRetry(ctx, retryWrapperOptions{
clock: clock,
logger: sink.Logger(),
initialDelay: time.Minute,
maxDelay: time.Minute,
}, taskKindGeneration, retryWrapperTaskInfo{}, func(context.Context) error {
calls++
if calls == 1 {
panic("database unavailable")
}
return nil
})
}()
trap.MustWait(ctx).MustRelease(ctx)
clock.Advance(time.Minute).MustWait(ctx)
require.NoError(t, <-done)
require.Equal(t, 2, calls)
entries := entriesWithMessage(sink, "chatworker task retrying")
require.Len(t, entries, 1)
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), "chatworker task panic: database unavailable")
}
// database/sql returns ctx.Err() from ctxDriverQuery, not
// context.Cause(ctx). This test checks that the retry logic
// doesn't classify such an error as an expected exit when
// task timeout is the cause of the cancellation.
func TestRetryWrapper_TaskTimeoutDBQueryCancellationRetries(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
clock := quartz.NewMock(t)
timeoutTrap := clock.Trap().AfterFunc("chatworker", "task-timeout-generation")
retryTrap := clock.Trap().NewTimer("chatworker", "task-retry-generation")
defer retryTrap.Close()
ctx := testutil.Context(t, testutil.WaitLong)
sink := testutil.NewFakeSink(t)
calls := 0
firstCallStarted := make(chan struct{})
var firstQueryErr error
var firstQueryCause error
done := make(chan error, 1)
go func() {
done <- runTaskWithRetry(ctx, retryWrapperOptions{
clock: clock,
logger: sink.Logger(),
initialDelay: time.Minute,
maxDelay: time.Minute,
}, taskKindGeneration, retryWrapperTaskInfo{}, func(ctx context.Context) error {
calls++
if calls == 1 {
close(firstCallStarted)
<-ctx.Done()
_, err := f.db.GetDatabaseNow(ctx)
firstQueryErr = err
firstQueryCause = context.Cause(ctx)
return normalizeTaskTransitionError(err, "db query")
}
return nil
})
}()
timeoutTrap.MustWait(ctx).MustRelease(ctx)
timeoutTrap.Close()
<-firstCallStarted
clock.Advance(defaultTaskTimeout).MustWait(ctx)
retryTrap.MustWait(ctx).MustRelease(ctx)
clock.Advance(time.Minute).MustWait(ctx)
require.NoError(t, <-done)
require.Equal(t, 2, calls)
require.ErrorIs(t, firstQueryErr, context.Canceled)
require.NotErrorIs(t, firstQueryErr, errTaskTimeout)
require.ErrorIs(t, firstQueryCause, errTaskTimeout)
entries := entriesWithMessage(sink, "chatworker task retrying")
require.Len(t, entries, 1)
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), errTaskTimeout.Error())
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), context.Canceled.Error())
}
func TestRetryWrapper_ContextCancellationDoesNotRetryOrLog(t *testing.T) {
t.Parallel()
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitLong))
cancel()
sink := testutil.NewFakeSink(t)
calls := 0
original := xerrors.New("database unavailable")
err := runTaskWithRetry(ctx, retryWrapperOptions{
clock: quartz.NewMock(t),
logger: sink.Logger(),
initialDelay: time.Second,
maxDelay: time.Second,
}, taskKindGeneration, retryWrapperTaskInfo{}, func(context.Context) error {
calls++
return original
})
require.NoError(t, err)
require.Equal(t, 1, calls)
require.Empty(t, entriesWithMessage(sink, "chatworker task retrying"))
}
func TestNormalizeTaskErrors_ContextCancellationIsExpectedExit(t *testing.T) {
t.Parallel()
err := normalizeTaskInfrastructureError(context.Canceled, "lock chat")
require.ErrorIs(t, err, errTaskExpectedExit)
require.ErrorIs(t, err, context.Canceled)
require.NotErrorIs(t, err, errTaskRetryable)
require.NotErrorIs(t, err, errTaskTimeout)
err = normalizeTaskTransitionError(context.Canceled, "commit chat")
require.ErrorIs(t, err, errTaskExpectedExit)
require.ErrorIs(t, err, context.Canceled)
require.NotErrorIs(t, err, errTaskRetryable)
require.NotErrorIs(t, err, errTaskTimeout)
}
func entriesWithMessage(sink *testutil.FakeSink, message string) []slog.SinkEntry {
return sink.Entries(func(e slog.SinkEntry) bool { return e.Message == message })
}
func sinkFieldValue(t *testing.T, fields slog.Map, name string) string {
t.Helper()
for _, f := range fields {
if f.Name == name {
return fmt.Sprint(f.Value)
}
}
t.Fatalf("missing log field %q", name)
return ""
}
func TestInterruptTask_FinishInterruptionOnly(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
buffer := starter.opts.MessagePartBuffer
key := messagepartbuffer.Key{
ChatID: chat.ID,
HistoryVersion: acquired.HistoryVersion,
GenerationAttempt: acquired.GenerationAttempt,
}
require.NoError(t, buffer.CreateEpisode(key))
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
interrupting := interruptChat(t, f, chat.ID)
require.Equal(t, database.ChatStatusInterrupting, interrupting.Status)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, latest.Status)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
recorder.requireInterruptionOutcome(t, chat.ID, database.ChatStatusRunning)
recorder.requireCleanupCount(t, 0)
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.GreaterOrEqual(t, len(messages), 3)
parts, err := chatprompt.ParseContent(messages[len(messages)-2])
require.NoError(t, err)
require.Equal(t, []codersdk.ChatMessagePart{codersdk.ChatMessageText("partial answer")}, parts)
require.Equal(t, database.ChatMessageRoleUser, messages[len(messages)-1].Role)
}
func TestInterruptTask_StaleFenceExits(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquireChat(t, f, chat.ID, workerID, runnerID)
interrupting := interruptChat(t, f, chat.ID)
otherWorkerID := uuid.New()
otherRunnerID := uuid.New()
acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.ErrorIs(t, err, errTaskExpectedExit)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusInterrupting, latest.Status)
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
recorder.requireStateHintCount(t, 0)
f.requireNoWatchEvents(t)
}
func TestInterruptTask_MissingEpisodePersistsNilPartials(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquireChat(t, f, chat.ID, workerID, runnerID)
interrupting := forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusWaiting, latest.Status)
recorder.requireInterruptionOutcome(t, chat.ID, database.ChatStatusWaiting)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.Len(t, messages, 1)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusWaiting)
}
func TestInterruptTask_BufferedPartsBecomePartialMessages(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
buffer := starter.opts.MessagePartBuffer
key := messagepartbuffer.Key{ChatID: chat.ID, HistoryVersion: acquired.HistoryVersion, GenerationAttempt: acquired.GenerationAttempt}
require.NoError(t, buffer.CreateEpisode(key))
callID := "call_" + uuid.NewString()
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessagePart{
Type: codersdk.ChatMessagePartTypeToolCall,
ToolCallID: callID,
ToolName: "local_tool",
Args: json.RawMessage(`{"value":1}`),
}))
interrupting := interruptChat(t, f, chat.ID)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.GreaterOrEqual(t, len(messages), 4)
assistant := messages[len(messages)-3]
tool := messages[len(messages)-2]
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
require.Equal(t, database.ChatMessageRoleTool, tool.Role)
toolParts, err := chatprompt.ParseContent(tool)
require.NoError(t, err)
require.Len(t, toolParts, 1)
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, toolParts[0].Type)
require.Equal(t, callID, toolParts[0].ToolCallID)
require.True(t, toolParts[0].IsError)
}
func TestInterruptTask_PartialAssistantKeepsAttemptRuntime(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
clock := quartz.NewMock(t)
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
buffer := starter.opts.MessagePartBuffer
key := messagepartbuffer.Key{
ChatID: chat.ID,
HistoryVersion: acquired.HistoryVersion,
GenerationAttempt: acquired.GenerationAttempt,
}
require.NoError(t, buffer.CreateEpisode(key))
// Prompt preparation and attempt bookkeeping run before the
// provider stream opens and must not be billed.
clock.Advance(3 * time.Second)
require.NoError(t, buffer.StartModelInvocation(key))
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
clock.Advance(1500 * time.Millisecond)
interrupting := interruptChat(t, f, chat.ID)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.GreaterOrEqual(t, len(messages), 3)
assistant := messages[len(messages)-2]
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, assistant.RuntimeMs)
}
func TestInterruptTask_PartialAssistantWithoutModelInvocationHasNoRuntime(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
clock := quartz.NewMock(t)
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
buffer := starter.opts.MessagePartBuffer
key := messagepartbuffer.Key{
ChatID: chat.ID,
HistoryVersion: acquired.HistoryVersion,
GenerationAttempt: acquired.GenerationAttempt,
}
require.NoError(t, buffer.CreateEpisode(key))
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
clock.Advance(1500 * time.Millisecond)
interrupting := interruptChat(t, f, chat.ID)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
require.NoError(t, err)
require.GreaterOrEqual(t, len(messages), 3)
assistant := messages[len(messages)-2]
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
require.False(t, assistant.RuntimeMs.Valid)
}
type interruptedBatch struct {
chat database.Chat
starter *taskStarter
clock *quartz.Mock
key messagepartbuffer.Key
workerID uuid.UUID
runnerID uuid.UUID
}
func interruptedBatchFixture(
t *testing.T,
f *workerTestFixture,
calls []codersdk.ChatMessagePart,
) interruptedBatch {
t.Helper()
chat := f.createRunningChat(t)
raw, err := chatprompt.MarshalParts(calls)
require.NoError(t, err)
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
_, err = machine.Update(testutil.Context(t, testutil.WaitShort), func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
_, err := tx.CommitStep(chatstate.CommitStepInput{Messages: []chatstate.Message{{
Role: database.ChatMessageRoleAssistant,
Content: raw,
Visibility: database.ChatMessageVisibilityBoth,
ContentVersion: chatprompt.CurrentContentVersion,
ModelConfigID: uuid.NullUUID{UUID: f.model.ID, Valid: true},
}}})
return err
})
require.NoError(t, err)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
clock := quartz.NewMock(t)
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
key := messagepartbuffer.Key{
ChatID: chat.ID,
HistoryVersion: acquired.HistoryVersion,
GenerationAttempt: acquired.GenerationAttempt,
}
require.NoError(t, starter.opts.MessagePartBuffer.CreateEpisode(key))
return interruptedBatch{
chat: chat,
starter: starter,
clock: clock,
key: key,
workerID: workerID,
runnerID: runnerID,
}
}
func (b interruptedBatch) interrupt(t *testing.T, f *workerTestFixture) []database.ChatMessage {
t.Helper()
interrupting := interruptChat(t, f, b.chat.ID)
err := b.starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: b.chat.ID,
WorkerID: b.workerID,
RunnerID: b.runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.NoError(t, err)
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: b.chat.ID})
require.NoError(t, err)
return messages
}
func findToolResultMessage(t *testing.T, messages []database.ChatMessage, toolCallID string) database.ChatMessage {
t.Helper()
for _, msg := range messages {
if msg.Role != database.ChatMessageRoleTool {
continue
}
parts, err := chatprompt.ParseContent(msg)
require.NoError(t, err)
for _, part := range parts {
if part.Type == codersdk.ChatMessagePartTypeToolResult && part.ToolCallID == toolCallID {
return msg
}
}
}
t.Fatalf("no tool result message for call %s", toolCallID)
return database.ChatMessage{}
}
// batchUsageRecords returns the model-only tool batch usage rows for the
// chat. Cancellation rows are user-visible, so the dedicated usage record
// only appears in the model-visibility history.
func batchUsageRecords(t *testing.T, f *workerTestFixture, chatID uuid.UUID) []database.ChatMessage {
t.Helper()
rows, err := f.db.GetChatMessagesForPromptByChatID(testutil.Context(t, testutil.WaitShort), chatID)
require.NoError(t, err)
var records []database.ChatMessage
for _, msg := range rows {
if msg.Role != database.ChatMessageRoleTool || msg.Visibility != database.ChatMessageVisibilityModel {
continue
}
records = append(records, msg)
}
return records
}
func requireSingleBatchUsageRecord(t *testing.T, f *workerTestFixture, chatID uuid.UUID, billedMs int64, billedCalls int) {
t.Helper()
records := batchUsageRecords(t, f, chatID)
require.Len(t, records, 1)
require.Equal(t, sql.NullInt64{Int64: billedMs, Valid: true}, records[0].RuntimeMs)
parts, err := chatprompt.ParseContent(records[0])
require.NoError(t, err)
require.Len(t, parts, 1)
require.Equal(t, toolBatchUsagePartType, parts[0].Type)
require.JSONEq(t,
fmt.Sprintf(`{"billed_ms":%d,"billed_calls":%d}`, billedMs, billedCalls),
string(parts[0].Result),
)
}
func TestInterruptTask_ToolBatchBillsPartialWindowOnUsageRecord(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
execCallID := "call_" + uuid.NewString()
waitCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
})
buffer := batch.starter.opts.MessagePartBuffer
// Keep advances below the buffer's 15-second cleanup tick.
batch.clock.Advance(2 * time.Second)
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
// Record a live completion without publishing a tool result.
batch.clock.Advance(3 * time.Second)
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(5 * time.Second)
messages := batch.interrupt(t, f)
execRow := findToolResultMessage(t, messages, execCallID)
require.False(t, execRow.RuntimeMs.Valid)
waitRow := findToolResultMessage(t, messages, waitCallID)
require.False(t, waitRow.RuntimeMs.Valid)
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 3_000, 1)
}
func TestInterruptTask_RunningBilledToolBillsUpToInterrupt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
execCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
})
batch.clock.Advance(2 * time.Second)
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(7 * time.Second)
messages := batch.interrupt(t, f)
execRow := findToolResultMessage(t, messages, execCallID)
require.False(t, execRow.RuntimeMs.Valid)
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 7_000, 1)
}
func TestInterruptTask_UnbilledOnlyBatchBillsNothingOnInterrupt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
waitCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
})
batch.clock.Advance(2 * time.Second)
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(10 * time.Second)
messages := batch.interrupt(t, f)
waitRow := findToolResultMessage(t, messages, waitCallID)
require.False(t, waitRow.RuntimeMs.Valid)
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
}
func TestInterruptTask_UnstartedSerialCallBillsNothingOnInterrupt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
waitCallID := "call_" + uuid.NewString()
serialCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: serialCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
})
buffer := batch.starter.opts.MessagePartBuffer
batch.clock.Advance(2 * time.Second)
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(10 * time.Second)
messages := batch.interrupt(t, f)
for _, msg := range messages {
if msg.Role == database.ChatMessageRoleTool {
require.False(t, msg.RuntimeMs.Valid,
"no cancellation row may bill: the billed call never began executing")
}
}
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID),
"no usage record may bill: the billed call never began executing")
}
func TestInterruptTask_StartedSerialCallBillsFromItsOwnStart(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
execCallID := "call_" + uuid.NewString()
waitCallID := "call_" + uuid.NewString()
serialCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: serialCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
})
buffer := batch.starter.opts.MessagePartBuffer
batch.clock.Advance(2 * time.Second)
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
require.NoError(t, buffer.RecordToolCompletion(batch.key, 1, batch.clock.Now()))
require.NoError(t, buffer.RecordToolStart(batch.key, 2, batch.clock.Now()))
batch.clock.Advance(2 * time.Second)
messages := batch.interrupt(t, f)
serialRow := findToolResultMessage(t, messages, serialCallID)
require.False(t, serialRow.RuntimeMs.Valid)
execRow := findToolResultMessage(t, messages, execCallID)
require.False(t, execRow.RuntimeMs.Valid)
waitRow := findToolResultMessage(t, messages, waitCallID)
require.False(t, waitRow.RuntimeMs.Valid)
// Completed execute [2s,5s] plus the serial call's own window
// [8s,10s]: the gap and the unbilled wait_agent do not count.
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 5_000, 2)
}
func TestInterruptTask_DuplicateCallIDsKeepOccurrenceStates(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
dupCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
})
buffer := batch.starter.opts.MessagePartBuffer
batch.clock.Advance(2 * time.Second)
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
messages := batch.interrupt(t, f)
for _, msg := range messages {
if msg.Role == database.ChatMessageRoleTool {
require.False(t, msg.RuntimeMs.Valid)
}
}
// The still-running occurrence bills the full window exactly once.
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 6_000, 2)
}
func TestInterruptTask_DuplicateCallIDCompletionStampsOwnOccurrence(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
dupCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
})
buffer := batch.starter.opts.MessagePartBuffer
batch.clock.Advance(2 * time.Second)
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
require.NoError(t, buffer.RecordToolCompletion(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(3 * time.Second)
messages := batch.interrupt(t, f)
for _, msg := range messages {
if msg.Role == database.ChatMessageRoleTool {
require.False(t, msg.RuntimeMs.Valid)
}
}
// The running execute occurrence bills to the interrupt; wait_agent's
// early completion must not end it at 3s.
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 6_000, 1)
}
func TestInterruptTask_RejectedDuplicateIDDoesNotStealDispatchedOccurrence(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
dupCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
})
batch.clock.Advance(2 * time.Second)
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(10 * time.Second)
messages := batch.interrupt(t, f)
for _, msg := range messages {
if msg.Role == database.ChatMessageRoleTool {
require.False(t, msg.RuntimeMs.Valid)
}
}
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID),
"no usage record may bill: only the unbilled wait_agent occurrence ran")
}
func TestInterruptTask_RejectedCallBillsNothingOnInterrupt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
execCallID := "call_" + uuid.NewString()
waitCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
})
batch.clock.Advance(2 * time.Second)
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
batch.clock.Advance(10 * time.Second)
messages := batch.interrupt(t, f)
execRow := findToolResultMessage(t, messages, execCallID)
require.False(t, execRow.RuntimeMs.Valid)
waitRow := findToolResultMessage(t, messages, waitCallID)
require.False(t, waitRow.RuntimeMs.Valid)
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
}
func TestInterruptTask_ToolCancellationWithoutLiveBatchHasNoRuntime(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
execCallID := "call_" + uuid.NewString()
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
})
batch.clock.Advance(10 * time.Second)
messages := batch.interrupt(t, f)
execRow := findToolResultMessage(t, messages, execCallID)
require.False(t, execRow.RuntimeMs.Valid)
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
}
func TestInterruptTask_CancelsUnresolvedToolCallsOnAgent(t *testing.T) {
t.Parallel()
saved := func(status int, body string) workspacesdk.CancelToolCallResponse {
return workspacesdk.CancelToolCallResponse{Received: true, Status: status, ContentType: "application/json", Body: []byte(body)}
}
exitCode := 0
tests := []struct {
name string
toolName string
notCancelable bool
cancel workspacesdk.CancelToolCallResponse
cancelErr error
cancelPanics bool
output *workspacesdk.ProcessOutputResponse
outputErr error
wantError bool
want string
}{
{name: "ExecuteNotReceived", toolName: "execute", outputErr: codersdk.NewError(http.StatusNotFound, codersdk.Response{}), wantError: true, want: `{"error":"tool call was canceled before it ran; no changes were made"}`},
// The agent has no record, but the cancel stopped the call's process.
{name: "ExecuteNoRecordProcessCanceled", toolName: "execute", output: &workspacesdk.ProcessOutputResponse{Output: "partial", Canceled: true}, want: `{"canceled":true,"error":"tool call was canceled while running","exit_code":-1,"output":"partial","success":false}`},
{name: "ExecuteNoRecordOutputError", toolName: "execute", outputErr: xerrors.New("connection reset"), wantError: true, want: interruptedToolResultErrorMessage},
{name: "EditNotReceived", toolName: "edit_files", wantError: true, want: "tool call was canceled before it ran; no changes were made"},
{name: "ExecuteStartError", toolName: "execute", cancel: saved(http.StatusInternalServerError, `{"message":"no shell"}`), want: `"error":"start process: unexpected status code 500: no shell"`},
{name: "EditError", toolName: "edit_files", cancel: saved(http.StatusBadRequest, `{"message":"old_text not found"}`), wantError: true, want: "old_text not found"},
{name: "ExecuteExited", toolName: "execute", cancel: saved(http.StatusOK, `{}`), output: &workspacesdk.ProcessOutputResponse{Output: "done", ExitCode: &exitCode}, want: `{"exit_code":0,"output":"done","success":true}`},
{name: "EditApplied", toolName: "edit_files", cancel: saved(http.StatusOK, `{"files":[{"path":"/a","diff":"d"}]}`), want: `{"files":[{"diff":"d","path":"/a"}],"ok":true}`},
// A result over the 64 KB default budget is capped and stays JSON.
{name: "EditAppliedTruncated", toolName: "edit_files", cancel: saved(http.StatusOK, `{"files":[{"path":"/a","diff":"`+strings.Repeat("d", 64<<10)+`"}]}`), want: "Coder truncated"},
{name: "CancelError", toolName: "write_file", cancelErr: xerrors.New("unexpected status code 404"), wantError: true, want: interruptedToolResultErrorMessage},
{name: "CancelPanic", toolName: "execute", cancelPanics: true, wantError: true, want: interruptedToolResultErrorMessage},
{name: "OtherTool", toolName: "read_file", notCancelable: true, wantError: true, want: interruptedToolResultErrorMessage},
}
f := newWorkerTestFixture(t)
calls := make([]codersdk.ChatMessagePart, 0, len(tests))
for _, tc := range tests {
calls = append(calls, codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call_" + tc.name, ToolName: tc.toolName, Args: json.RawMessage(`{}`)})
}
batch := interruptedBatchFixture(t, f, calls)
ctx := testutil.Context(t, testutil.WaitLong)
workspace := dbfake.WorkspaceBuild(t, f.db, database.WorkspaceTable{OrganizationID: f.org.ID, OwnerID: f.user.ID}).WithAgent().Do()
_, err := f.db.UpdateChatWorkspaceBinding(ctx, database.UpdateChatWorkspaceBindingParams{
ID: batch.chat.ID,
WorkspaceID: uuid.NullUUID{UUID: workspace.Workspace.ID, Valid: true},
BuildID: uuid.NullUUID{UUID: workspace.Build.ID, Valid: true},
AgentID: uuid.NullUUID{UUID: workspace.Agents[0].ID, Valid: true},
})
require.NoError(t, err)
messages, err := f.db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: batch.chat.ID})
require.NoError(t, err)
assistantID := messages[len(messages)-1].ID
// Each cancel waits until all cancels arrive, so sequential cancels
// would time out.
var pending atomic.Int64
for _, tc := range tests {
if !tc.notCancelable {
pending.Add(1)
}
}
allReceived := make(chan struct{})
conn := agentconnmock.NewMockAgentConn(gomock.NewController(t))
conn.EXPECT().SetExtraHeaders(gomock.Any())
for _, tc := range tests {
if tc.notCancelable {
continue
}
id := chattool.ToolCallID(batch.chat.ID, assistantID, "call_"+tc.name)
conn.EXPECT().CancelToolCall(gomock.Any(), id).DoAndReturn(func(ctx context.Context, _ uuid.UUID) (workspacesdk.CancelToolCallResponse, error) {
if pending.Add(-1) == 0 {
close(allReceived)
}
select {
case <-allReceived:
if tc.cancelPanics {
panic("cancel panicked")
}
return tc.cancel, tc.cancelErr
case <-ctx.Done():
return workspacesdk.CancelToolCallResponse{}, ctx.Err()
}
})
if tc.output != nil || tc.outputErr != nil {
var output workspacesdk.ProcessOutputResponse
if tc.output != nil {
output = *tc.output
}
conn.EXPECT().ProcessOutput(gomock.Any(), id.String(), &workspacesdk.ProcessOutputOptions{Wait: true}).Return(output, tc.outputErr)
}
}
batch.starter.server.agentConnFn = func(context.Context, uuid.UUID) (workspacesdk.AgentConn, func(), error) {
return conn, func() {}, nil
}
messages = batch.interrupt(t, f)
for _, tc := range tests {
parts, err := chatprompt.ParseContent(findToolResultMessage(t, messages, "call_"+tc.name))
require.NoError(t, err)
require.Len(t, parts, 1)
assert.Equal(t, tc.wantError, parts[0].IsError, tc.name)
// Compact the stored JSON with sorted keys.
var result any
require.NoError(t, json.Unmarshal(parts[0].Result, &result))
got, err := json.Marshal(result)
require.NoError(t, err)
assert.Contains(t, string(got), tc.want, tc.name)
}
}
func TestStartGeneration_CancelsDeletedToolCallsOnAgent(t *testing.T) {
t.Parallel()
call := func(toolName string) codersdk.ChatMessagePart {
return codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call_" + toolName, ToolName: toolName, Args: json.RawMessage(`{}`)}
}
result := func(toolName string) codersdk.ChatMessagePart {
return codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolCallID: "call_" + toolName, ToolName: toolName, Result: json.RawMessage(`{}`)}
}
tests := []struct {
name string
build func(t *testing.T, h *deletedToolCallsHistory)
cancelErr error
want []string // Tools whose calls are canceled.
}{
{
name: "DeletedUnresolvedCalls",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"), call("edit_files"), call("write_file"), call("read_file"))
h.editPrompt(t)
},
want: []string{"execute", "edit_files", "write_file"},
},
{
name: "DeletedResolvedCalls",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"), call("write_file"))
h.commit(t, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, result("execute"))
h.commit(t, database.ChatMessageRoleTool, database.ChatMessageVisibilityModel, result("write_file"))
h.editPrompt(t)
},
},
{
name: "SecondEditBeforeFirstStep",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"))
h.editPrompt(t)
h.editPrompt(t)
},
want: []string{"execute"},
},
{
// The unresolved call is in the active history, not deleted.
name: "TurnWithoutEdit",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"))
h.commit(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText("next"))
},
},
{
// EditMessage inserts a result for the earlier unresolved call
// between the deleted messages and the replacement.
name: "UnresolvedCallBeforeEditedMessage",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("read_file"))
h.commit(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText("next"))
h.assistant(t, call("execute"))
h.editPrompt(t)
},
want: []string{"execute"},
},
{
name: "SecondStep",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"))
h.editPrompt(t)
h.assistant(t, codersdk.ChatMessageText("done"))
},
},
{
name: "CancelError",
build: func(t *testing.T, h *deletedToolCallsHistory) {
h.assistant(t, call("execute"), call("edit_files"))
h.editPrompt(t)
},
cancelErr: xerrors.New("unexpected status code 404"),
want: []string{"execute", "edit_files"},
},
}
f := newWorkerTestFixture(t)
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitLong)
chat := f.createRunningChat(t)
workspace := dbfake.WorkspaceBuild(t, f.db, database.WorkspaceTable{OrganizationID: f.org.ID, OwnerID: f.user.ID}).WithAgent().Do()
_, err := f.db.UpdateChatWorkspaceBinding(ctx, database.UpdateChatWorkspaceBindingParams{
ID: chat.ID,
WorkspaceID: uuid.NullUUID{UUID: workspace.Workspace.ID, Valid: true},
BuildID: uuid.NullUUID{UUID: workspace.Build.ID, Valid: true},
AgentID: uuid.NullUUID{UUID: workspace.Agents[0].ID, Valid: true},
})
require.NoError(t, err)
workerID := uuid.New()
runnerID := uuid.New()
acquireChat(t, f, chat.ID, workerID, runnerID)
h := &deletedToolCallsHistory{f: f, machine: chatstate.NewChatMachine(f.db, f.pubsub, chat.ID), chatID: chat.ID}
tc.build(t, h)
conn := agentconnmock.NewMockAgentConn(gomock.NewController(t))
if len(tc.want) > 0 {
conn.EXPECT().SetExtraHeaders(gomock.Any())
}
for _, toolName := range tc.want {
id := chattool.ToolCallID(chat.ID, h.toolCallMessageID, "call_"+toolName)
conn.EXPECT().CancelToolCall(gomock.Any(), id).DoAndReturn(func(ctx context.Context, _ uuid.UUID) (workspacesdk.CancelToolCallResponse, error) {
// The model step fails in this fixture and sets the chat's
// error, so a running chat without one has not reached it.
current, err := f.db.GetChatByID(ctx, chat.ID)
if assert.NoError(t, err) {
assert.Equal(t, database.ChatStatusRunning, current.Status)
assert.False(t, current.LastError.Valid)
}
return workspacesdk.CancelToolCallResponse{}, tc.cancelErr
})
if toolName == "execute" && tc.cancelErr == nil {
conn.EXPECT().ProcessOutput(gomock.Any(), id.String(), gomock.Any()).Return(workspacesdk.ProcessOutputResponse{}, codersdk.NewError(http.StatusNotFound, codersdk.Response{}))
}
}
var dials atomic.Int64
sink := testutil.NewFakeSink(t)
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
starter.opts.Logger = sink.Logger(slog.LevelWarn)
starter.server.agentConnFn = func(context.Context, uuid.UUID) (workspacesdk.AgentConn, func(), error) {
dials.Add(1)
return conn, func() {}, nil
}
debugTurn := newRunnerDebugTurn(starter.opts.Logger)
defer debugTurn.Finalize(ctx)
current, err := f.db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
err = starter.StartGeneration(ctx, chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: current.HistoryVersion,
GenerationAttempt: current.GenerationAttempt,
Status: database.ChatStatusRunning,
DebugTurn: debugTurn,
})
require.NoError(t, err)
if len(tc.want) == 0 {
require.Zero(t, dials.Load(), "the agent must not be dialed without calls to cancel")
}
cancelWarnings := sink.Entries(func(e slog.SinkEntry) bool { return e.Message == "cancel tool call on agent" })
if tc.cancelErr != nil {
require.Len(t, cancelWarnings, len(tc.want))
} else {
require.Empty(t, cancelWarnings)
}
// The generation went on to the model step.
final, err := f.db.GetChatByID(ctx, chat.ID)
require.NoError(t, err)
require.Contains(t, string(final.LastError.RawMessage), "create model")
})
}
}
// deletedToolCallsHistory builds a running chat's history for
// TestStartGeneration_CancelsDeletedToolCallsOnAgent.
type deletedToolCallsHistory struct {
f *workerTestFixture
machine *chatstate.ChatMachine
chatID uuid.UUID
toolCallMessageID int64 // ID of the last committed assistant message with tool calls.
}
func (h *deletedToolCallsHistory) commit(t *testing.T, role database.ChatMessageRole, visibility database.ChatMessageVisibility, parts ...codersdk.ChatMessagePart) database.ChatMessage {
t.Helper()
raw, err := chatprompt.MarshalParts(parts)
require.NoError(t, err)
var inserted []database.ChatMessage
_, err = h.machine.Update(testutil.Context(t, testutil.WaitShort), func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
res, err := tx.CommitStep(chatstate.CommitStepInput{Messages: []chatstate.Message{{
Role: role,
Content: raw,
Visibility: visibility,
ContentVersion: chatprompt.CurrentContentVersion,
ModelConfigID: uuid.NullUUID{UUID: h.f.model.ID, Valid: true},
}}})
inserted = res.InsertedMessages
return err
})
require.NoError(t, err)
require.Len(t, inserted, 1)
return inserted[0]
}
func (h *deletedToolCallsHistory) assistant(t *testing.T, parts ...codersdk.ChatMessagePart) {
t.Helper()
msg := h.commit(t, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, parts...)
if slices.ContainsFunc(parts, func(p codersdk.ChatMessagePart) bool { return p.Type == codersdk.ChatMessagePartTypeToolCall }) {
h.toolCallMessageID = msg.ID
}
}
// editPrompt edits the latest user prompt, as the edit endpoint does.
func (h *deletedToolCallsHistory) editPrompt(t *testing.T) {
t.Helper()
ctx := testutil.Context(t, testutil.WaitShort)
messages, err := h.f.db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: h.chatID})
require.NoError(t, err)
prompt := messages[lastUserPromptIndex(messages)]
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText("edited")})
require.NoError(t, err)
_, err = h.machine.Update(ctx, func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
_, err := tx.EditMessage(chatstate.EditMessageInput{MessageID: prompt.ID, CreatedBy: h.f.user.ID, Content: content})
return err
})
require.NoError(t, err)
}
func TestRequiresActionTimeout_ExpiredCancelsOnly(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRequiresActionChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
expired := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRequiresAction,
RequiresActionDeadlineAt: expired.RequiresActionDeadlineAt,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, latest.Status)
require.False(t, latest.RequiresActionDeadlineAt.Valid)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
}
func TestRequiresActionTimeout_NullDeadlineCancelsImmediately(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRequiresActionChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
nullDeadline := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{})
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRequiresAction,
RequiresActionDeadlineAt: nullDeadline.RequiresActionDeadlineAt,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, latest.Status)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
}
// failingChatStatePubsub fails the first chat:update publish for one chat,
// which ChatMachine.Update reports only after the transaction commits.
type failingChatStatePubsub struct {
chatWorkerPubsub
channel string
failed atomic.Bool
}
func (p *failingChatStatePubsub) Publish(event string, message []byte) error {
if event == p.channel && p.failed.CompareAndSwap(false, true) {
return xerrors.New("injected chat state publish failure")
}
return p.chatWorkerPubsub.Publish(event, message)
}
func failFirstChatStatePublish(starter *taskStarter, chatID uuid.UUID) *failingChatStatePubsub {
p := &failingChatStatePubsub{
chatWorkerPubsub: starter.opts.Pubsub,
channel: coderdpubsub.ChatStateUpdateChannel(chatID),
}
starter.opts.Pubsub = p
return p
}
// A publish failure after the commit must not turn a committed transition
// into a task failure.
func TestInterruptTask_PublishFailureAfterCommitSucceeds(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquireChat(t, f, chat.ID, workerID, runnerID)
interrupting := forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
publisher := failFirstChatStatePublish(starter, chat.ID)
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: interrupting.HistoryVersion,
GenerationAttempt: interrupting.GenerationAttempt,
Status: database.ChatStatusInterrupting,
})
require.True(t, publisher.failed.Load(), "the chat state publish must have failed")
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusWaiting, latest.Status)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusWaiting)
}
func TestRequiresActionTimeout_PublishFailureAfterCommitSucceeds(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRequiresActionChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
nullDeadline := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{})
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
publisher := failFirstChatStatePublish(starter, chat.ID)
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRequiresAction,
RequiresActionDeadlineAt: nullDeadline.RequiresActionDeadlineAt,
})
require.True(t, publisher.failed.Load(), "the chat state publish must have failed")
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, latest.Status)
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
}
func TestRequiresActionTimeout_StaleFenceExitsAfterToolResult(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRequiresActionChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
expired := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
forceExecutionStateWithDeadline(t, f, chat.ID, database.ChatStatusRunning, false, &sql.NullTime{})
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRequiresAction,
RequiresActionDeadlineAt: expired.RequiresActionDeadlineAt,
})
require.ErrorIs(t, err, errTaskExpectedExit)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, database.ChatStatusRunning, latest.Status)
recorder.requireStateHintCount(t, 0)
f.requireNoWatchEvents(t)
}
func TestAbandonTask_AbandonOnly(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.False(t, latest.WorkerID.Valid)
require.False(t, latest.RunnerID.Valid)
recorder.requireCleanup(t, chat.ID, runnerID)
recorder.requireStateHintCount(t, 0)
f.requireNoWatchEvents(t)
}
func TestAbandonTask_OwnershipMismatchRequestsCleanup(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquireChat(t, f, chat.ID, workerID, runnerID)
otherWorkerID := uuid.New()
otherRunnerID := uuid.New()
latestOwner := acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: latestOwner.HistoryVersion,
Status: database.ChatStatusRunning,
})
require.NoError(t, err)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
recorder.requireCleanup(t, chat.ID, runnerID)
}
func TestAbandonTask_StaleStatusFenceExits(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusWaiting,
})
require.ErrorIs(t, err, errTaskExpectedExit)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.True(t, latest.WorkerID.Valid)
require.True(t, latest.RunnerID.Valid)
require.Equal(t, database.ChatStatusInterrupting, latest.Status)
recorder.requireCleanupCount(t, 0)
}
func TestGenerationTask_RecordRetryState(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
recorder := newTaskSideEffectRecorder()
starter := newTestTaskStarter(t, f, recorder)
attempt, err := starter.beginGenerationAttempt(
testutil.Context(t, testutil.WaitLong),
chatstate.NewChatMachine(f.db, f.pubsub, chat.ID),
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
)
require.NoError(t, err)
attempt.closeEpisode()
require.Equal(t, int64(1), attempt.number)
before, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.False(t, before.RetryState.Valid)
decision, err := starter.recordGenerationRetry(
testutil.Context(t, testutil.WaitLong),
chatstate.NewChatMachine(f.db, f.pubsub, chat.ID),
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
chaterror.ClassifiedError{
Message: "OpenAI is rate limiting requests.",
Kind: codersdk.ChatErrorKindRateLimit,
Provider: "openai",
Retryable: true,
StatusCode: 429,
},
)
require.NoError(t, err)
require.True(t, decision.retry)
require.Equal(t, int64(1), decision.generationAttempt)
require.Equal(t, chatretry.Delay(0), decision.delay)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.True(t, latest.RetryState.Valid)
require.Equal(t, latest.SnapshotVersion, latest.RetryStateVersion)
require.Greater(t, latest.RetryStateVersion, before.RetryStateVersion)
require.Equal(t, before.GenerationAttempt, latest.GenerationAttempt)
recorder.requireStateHintCount(t, 0)
var retryPayload codersdk.ChatStreamRetry
require.NoError(t, json.Unmarshal(latest.RetryState.RawMessage, &retryPayload))
require.Equal(t, 1, retryPayload.Attempt)
require.Equal(t, chatretry.Delay(0).Milliseconds(), retryPayload.DelayMs)
require.Equal(t, "OpenAI is rate limiting requests.", retryPayload.Error)
require.Equal(t, codersdk.ChatErrorKindRateLimit, retryPayload.Kind)
require.Equal(t, "openai", retryPayload.Provider)
require.Equal(t, 429, retryPayload.StatusCode)
require.False(t, retryPayload.RetryingAt.IsZero())
}
func TestGenerationTask_RecordRetryStateUsesDurableGenerationAttempt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
for range 3 {
attempt, err := starter.beginGenerationAttempt(
testutil.Context(t, testutil.WaitLong),
machine,
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
)
require.NoError(t, err)
attempt.closeEpisode()
require.Positive(t, attempt.number)
}
decision, err := starter.recordGenerationRetry(
testutil.Context(t, testutil.WaitLong),
machine,
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
chaterror.ClassifiedError{
Message: "OpenAI is temporarily unavailable.",
Kind: codersdk.ChatErrorKindTimeout,
Provider: "openai",
Retryable: true,
},
)
require.NoError(t, err)
require.True(t, decision.retry)
require.Equal(t, int64(3), decision.generationAttempt)
require.Equal(t, chatretry.Delay(2), decision.delay)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
var retryPayload codersdk.ChatStreamRetry
require.NoError(t, json.Unmarshal(latest.RetryState.RawMessage, &retryPayload))
require.Equal(t, 3, retryPayload.Attempt)
require.Equal(t, chatretry.Delay(2).Milliseconds(), retryPayload.DelayMs)
}
func TestGenerationTask_RecordRetryStateHonorsConfiguredRetries(t *testing.T) {
t.Parallel()
tests := []struct {
name string
limits Limits
maxRetries int
}{
{name: "Default", maxRetries: codersdk.DefaultChatMaxGenerationRetries},
{name: "ConfiguredOne", limits: Limits{MaxGenerationRetries: 1}, maxRetries: 1},
{name: "ConfiguredThree", limits: Limits{MaxGenerationRetries: 3}, maxRetries: 3},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder(), withInternalTestServerLimits(tt.limits))
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
input := chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
}
classified := chaterror.ClassifiedError{
Message: "OpenAI is temporarily unavailable.",
Kind: codersdk.ChatErrorKindTimeout,
Provider: "openai",
Retryable: true,
}
// The configured value is a retry count, so every failure up to
// it is retried and the next failure is not.
ctx := testutil.Context(t, testutil.WaitLong)
for failure := 1; failure <= tt.maxRetries+1; failure++ {
attempt, err := starter.beginGenerationAttempt(ctx, machine, input)
require.NoError(t, err)
attempt.closeEpisode()
decision, err := starter.recordGenerationRetry(ctx, machine, input, classified)
require.NoError(t, err)
require.Equal(t, failure <= tt.maxRetries, decision.retry, "failure %d", failure)
require.EqualValues(t, failure, decision.generationAttempt)
}
})
}
}
func TestGenerationTask_RecordRetryStateClearedByNextAttempt(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
input := chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
}
attempt, err := starter.beginGenerationAttempt(testutil.Context(t, testutil.WaitLong), machine, input)
require.NoError(t, err)
attempt.closeEpisode()
require.Equal(t, int64(1), attempt.number)
_, err = starter.recordGenerationRetry(
testutil.Context(t, testutil.WaitLong),
machine,
input,
chaterror.ClassifiedError{
Message: "OpenAI is temporarily unavailable.",
Kind: codersdk.ChatErrorKindTimeout,
Provider: "openai",
Retryable: true,
},
)
require.NoError(t, err)
withRetry, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.True(t, withRetry.RetryState.Valid)
attempt, err = starter.beginGenerationAttempt(testutil.Context(t, testutil.WaitLong), machine, input)
require.NoError(t, err)
attempt.closeEpisode()
require.Equal(t, int64(2), attempt.number)
after, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.False(t, after.RetryState.Valid)
require.Equal(t, after.SnapshotVersion, after.RetryStateVersion)
require.Greater(t, after.RetryStateVersion, withRetry.RetryStateVersion)
}
func TestGenerationTask_RecordRetryStateStaleFenceExits(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
workerID := uuid.New()
runnerID := uuid.New()
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
attempt, err := starter.beginGenerationAttempt(
testutil.Context(t, testutil.WaitLong),
machine,
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
)
require.NoError(t, err)
attempt.closeEpisode()
require.Equal(t, int64(1), attempt.number)
otherWorkerID := uuid.New()
otherRunnerID := uuid.New()
acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
_, err = starter.recordGenerationRetry(
testutil.Context(t, testutil.WaitLong),
machine,
chatWorkerTaskStartInput{
ChatID: chat.ID,
WorkerID: workerID,
RunnerID: runnerID,
HistoryVersion: acquired.HistoryVersion,
Status: database.ChatStatusRunning,
},
chaterror.ClassifiedError{
Message: "OpenAI is temporarily unavailable.",
Kind: codersdk.ChatErrorKindTimeout,
Provider: "openai",
Retryable: true,
},
)
require.ErrorIs(t, err, errTaskExpectedExit)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.False(t, latest.RetryState.Valid)
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
}
func TestRunner_StartsRealInterruptTask(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
worker := startRealTaskWorker(t, f)
waitOwnedChat(t, f, chat.ID, worker.opts.WorkerID)
interruptChat(t, f, chat.ID)
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
latest, err := f.db.GetChatByID(ctx, chat.ID)
return err == nil && latest.Status == database.ChatStatusRunning
}, testutil.IntervalFast)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.Equal(t, worker.opts.WorkerID, latest.WorkerID.UUID)
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
}
func TestRunner_StartsRealRequiresActionTimeoutTask(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRequiresActionChat(t)
f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
worker := startRealTaskWorker(t, f)
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
latest, err := f.db.GetChatByID(ctx, chat.ID)
return err == nil && latest.Status == database.ChatStatusRunning && latest.WorkerID.Valid && latest.WorkerID.UUID == worker.opts.WorkerID
}, testutil.IntervalFast)
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
require.NoError(t, err)
require.True(t, latest.RunnerID.Valid)
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
}
func TestRunner_StartsRealAbandonTask(t *testing.T) {
t.Parallel()
f := newWorkerTestFixture(t)
chat := f.createRunningChat(t)
worker := startRealTaskWorker(t, f)
waitOwnedChat(t, f, chat.ID, worker.opts.WorkerID)
updated := forceExecutionState(t, f, chat.ID, database.ChatStatusError, false)
publishChatUpdate(t, f, updated)
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
latest, err := f.db.GetChatByID(ctx, chat.ID)
return err == nil && !latest.WorkerID.Valid && !latest.RunnerID.Valid
}, testutil.IntervalFast)
}
func (f *workerTestFixture) setRequiresActionDeadline(t *testing.T, chatID uuid.UUID, deadline sql.NullTime) database.Chat {
t.Helper()
chat, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chatID)
require.NoError(t, err)
return forceExecutionStateWithDeadline(t, f, chatID, chat.Status, chat.Archived, &deadline)
}
func (f *workerTestFixture) requireWatchEvent(t *testing.T, chatID uuid.UUID, kind codersdk.ChatWatchEventKind) {
t.Helper()
// Watch events are published after the corresponding database update
// commits, so poll instead of asserting on a single snapshot.
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(_ context.Context) bool {
for _, event := range f.pubsub.watchEvents(t) {
if event.Kind == kind && event.Chat.ID == chatID {
return true
}
}
return false
}, testutil.IntervalFast)
}
func (f *workerTestFixture) requireNoWatchEvents(t *testing.T) {
t.Helper()
require.Empty(t, f.pubsub.watchEvents(t))
}
func startRealTaskWorker(t *testing.T, f *workerTestFixture) *chatWorker {
t.Helper()
buffer := messagepartbuffer.New(messagepartbuffer.Options{})
t.Cleanup(buffer.Close)
opts := testOptions(t, f, nil)
// Without a task starter, the worker builds its real one.
opts.TaskStarter = nil
opts.Logger = slog.Make()
opts.MessagePartBuffer = buffer
opts.TaskRetryInitialBackoff = time.Millisecond
opts.TaskRetryMaxBackoff = time.Millisecond
worker, err := newChatWorker(newUnstartedServer(t, f.pubsub.Pubsub, f.db), opts)
require.NoError(t, err)
require.NoError(t, worker.Start(context.Background()))
t.Cleanup(func() { require.NoError(t, worker.Close()) })
return worker
}
func waitOwnedChat(t *testing.T, f *workerTestFixture, chatID uuid.UUID, workerID uuid.UUID) database.Chat {
t.Helper()
var latest database.Chat
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
chat, err := f.db.GetChatByID(ctx, chatID)
if err != nil {
return false
}
latest = chat
return chat.WorkerID.Valid && chat.WorkerID.UUID == workerID && chat.RunnerID.Valid
}, testutil.IntervalFast)
return latest
}
type taskSideEffectRecorder struct {
mu sync.Mutex
hints []runnerStateUpdate
cleanups []runnerKey
interrupts []interruptionOutcome
}
func newTaskSideEffectRecorder() *taskSideEffectRecorder {
return &taskSideEffectRecorder{}
}
func (r *taskSideEffectRecorder) routeStateHint(_ context.Context, state runnerStateUpdate) {
r.mu.Lock()
r.hints = append(r.hints, state)
r.mu.Unlock()
}
func (r *taskSideEffectRecorder) requestCleanup(_ context.Context, key runnerKey) {
r.mu.Lock()
r.cleanups = append(r.cleanups, key)
r.mu.Unlock()
}
func (r *taskSideEffectRecorder) afterInterruptionOutcome(_ context.Context, outcome interruptionOutcome) error {
r.mu.Lock()
r.interrupts = append(r.interrupts, outcome)
r.mu.Unlock()
return nil
}
func (r *taskSideEffectRecorder) requireStateHint(t *testing.T, chatID uuid.UUID, snapshot int64, status database.ChatStatus) {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
for _, hint := range r.hints {
if hint.ChatID == chatID && hint.SnapshotVersion == snapshot && hint.Status == status {
return
}
}
t.Fatalf("missing state hint chat_id=%s snapshot=%d status=%s hints=%v", chatID, snapshot, status, r.hints)
}
func (r *taskSideEffectRecorder) requireStateHintCount(t *testing.T, count int) {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
require.Len(t, r.hints, count)
}
func (r *taskSideEffectRecorder) requireCleanup(t *testing.T, chatID uuid.UUID, runnerID uuid.UUID) {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
for _, cleanup := range r.cleanups {
if cleanup.ChatID == chatID && cleanup.RunnerID == runnerID {
return
}
}
t.Fatalf("missing cleanup chat_id=%s runner_id=%s cleanups=%v", chatID, runnerID, r.cleanups)
}
func (r *taskSideEffectRecorder) requireCleanupCount(t *testing.T, count int) {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
require.Len(t, r.cleanups, count)
}
func (r *taskSideEffectRecorder) requireInterruptionOutcome(t *testing.T, chatID uuid.UUID, status database.ChatStatus) {
t.Helper()
r.mu.Lock()
defer r.mu.Unlock()
for _, outcome := range r.interrupts {
if outcome.Chat.ID == chatID && outcome.Chat.Status == status {
return
}
}
t.Fatalf("missing interruption outcome chat_id=%s status=%s outcomes=%v", chatID, status, r.interrupts)
}
func newTestTaskStarter(t *testing.T, f *workerTestFixture, recorder *taskSideEffectRecorder, serverOpts ...internalTestServerOpt) *taskStarter {
t.Helper()
return newTestTaskStarterWithClock(t, f, recorder, quartz.NewReal(), serverOpts...)
}
// newTestTaskStarterWithClock shares the clock between the starter and its
// message part buffer, mirroring production wiring.
func newTestTaskStarterWithClock(
t *testing.T,
f *workerTestFixture,
recorder *taskSideEffectRecorder,
clock quartz.Clock,
serverOpts ...internalTestServerOpt,
) *taskStarter {
t.Helper()
buffer := messagepartbuffer.New(messagepartbuffer.Options{Clock: clock})
t.Cleanup(buffer.Close)
starter, err := newTaskStarter(newUnstartedServer(t, f.pubsub.Pubsub, f.db, serverOpts...), chatWorkerOptions{
Store: f.db,
Pubsub: f.pubsub,
Logger: slog.Make(),
Clock: clock,
MessagePartBuffer: buffer,
TaskRetryInitialBackoff: time.Millisecond,
TaskRetryMaxBackoff: time.Millisecond,
}, recorder.routeStateHint, recorder.requestCleanup)
require.NoError(t, err)
starter.afterInterruptionOutcome = recorder.afterInterruptionOutcome
return starter
}