One of the main costs in terms of DB load from chats is extra round trips during transactions. This results in locks being held for longer periods of time -> an increase in lock waits for chats related queries. One of the opportunities for reducing some of the round trips is to eliminate extra chat row reads after we have just read the row in order to lock it. In this PR `ChatMachine.Update` now seeds the transaction struct with the row returned by `LockChatAndBumpSnapshotVersion`, plus a queue count taken right after it. We can then pass it into `Update` as an optional parameter validate against it, so neither re-reads the chat row from the database while the row lock is held. `updateChat` (send, edit, promote, tool results, interrupt, compact, clear, reconcile), hook failure, family archive, and the worker's generation and task transitions use it instead of reloading the chat under the lock. This PR relative to main: |measurement | main | this PR | percent change | |-- | -- | -- | -- | | Round trips under lock | 180.9 | 142.0 | -39% | | Statements under lock | 86.1 | 66.6 | -23% | | GetChatByID under lock | 54.1 | 35.8 | -34% | | GetChatByID total | 75.5 | 57.1 | -25% | | Statements | 235.8 | 216.2 | -8% | Generated by Coder Agents on behalf of @cstyan.
1946 lines
73 KiB
Go
1946 lines
73 KiB
Go
//nolint:testpackage // These tests exercise package-private task seams.
|
|
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/mock/gomock"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbfake"
|
|
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatretry"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk/agentconnmock"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func TestRetryWrapper_ExpectedExitsDoNotRetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
sink := testutil.NewFakeSink(t)
|
|
calls := 0
|
|
err := runTaskWithRetry(ctx, retryWrapperOptions{
|
|
clock: quartz.NewMock(t),
|
|
logger: sink.Logger(),
|
|
initialDelay: time.Second,
|
|
maxDelay: time.Second,
|
|
}, taskKindInterrupt, retryWrapperTaskInfo{}, func(context.Context) error {
|
|
calls++
|
|
return errTaskExpectedExit
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, calls)
|
|
require.Empty(t, entriesWithMessage(sink, "chatworker task retrying"))
|
|
}
|
|
|
|
func TestRetryWrapper_UnexpectedErrorsRetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().NewTimer("chatworker", "task-retry-requires_action_timeout")
|
|
defer trap.Close()
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
sink := testutil.NewFakeSink(t)
|
|
calls := 0
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- runTaskWithRetry(ctx, retryWrapperOptions{
|
|
clock: clock,
|
|
logger: sink.Logger(),
|
|
initialDelay: time.Minute,
|
|
maxDelay: time.Minute,
|
|
}, taskKindRequiresActionTimeout, retryWrapperTaskInfo{}, func(context.Context) error {
|
|
calls++
|
|
if calls == 1 {
|
|
return xerrors.New("database unavailable")
|
|
}
|
|
return nil
|
|
})
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
clock.Advance(time.Minute).MustWait(ctx)
|
|
require.NoError(t, <-done)
|
|
require.Equal(t, 2, calls)
|
|
entries := entriesWithMessage(sink, "chatworker task retrying")
|
|
require.Len(t, entries, 1)
|
|
require.Equal(t, string(taskKindRequiresActionTimeout), sinkFieldValue(t, entries[0].Fields, "task_kind"))
|
|
require.Equal(t, time.Minute.String(), sinkFieldValue(t, entries[0].Fields, "delay"))
|
|
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), "database unavailable")
|
|
}
|
|
|
|
func TestRetryWrapper_PanicsRetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
clock := quartz.NewMock(t)
|
|
trap := clock.Trap().NewTimer("chatworker", "task-retry-generation")
|
|
defer trap.Close()
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
sink := testutil.NewFakeSink(t)
|
|
calls := 0
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- runTaskWithRetry(ctx, retryWrapperOptions{
|
|
clock: clock,
|
|
logger: sink.Logger(),
|
|
initialDelay: time.Minute,
|
|
maxDelay: time.Minute,
|
|
}, taskKindGeneration, retryWrapperTaskInfo{}, func(context.Context) error {
|
|
calls++
|
|
if calls == 1 {
|
|
panic("database unavailable")
|
|
}
|
|
return nil
|
|
})
|
|
}()
|
|
|
|
trap.MustWait(ctx).MustRelease(ctx)
|
|
clock.Advance(time.Minute).MustWait(ctx)
|
|
require.NoError(t, <-done)
|
|
require.Equal(t, 2, calls)
|
|
entries := entriesWithMessage(sink, "chatworker task retrying")
|
|
require.Len(t, entries, 1)
|
|
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), "chatworker task panic: database unavailable")
|
|
}
|
|
|
|
// database/sql returns ctx.Err() from ctxDriverQuery, not
|
|
// context.Cause(ctx). This test checks that the retry logic
|
|
// doesn't classify such an error as an expected exit when
|
|
// task timeout is the cause of the cancellation.
|
|
func TestRetryWrapper_TaskTimeoutDBQueryCancellationRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
clock := quartz.NewMock(t)
|
|
timeoutTrap := clock.Trap().AfterFunc("chatworker", "task-timeout-generation")
|
|
retryTrap := clock.Trap().NewTimer("chatworker", "task-retry-generation")
|
|
defer retryTrap.Close()
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
sink := testutil.NewFakeSink(t)
|
|
calls := 0
|
|
firstCallStarted := make(chan struct{})
|
|
var firstQueryErr error
|
|
var firstQueryCause error
|
|
done := make(chan error, 1)
|
|
go func() {
|
|
done <- runTaskWithRetry(ctx, retryWrapperOptions{
|
|
clock: clock,
|
|
logger: sink.Logger(),
|
|
initialDelay: time.Minute,
|
|
maxDelay: time.Minute,
|
|
}, taskKindGeneration, retryWrapperTaskInfo{}, func(ctx context.Context) error {
|
|
calls++
|
|
if calls == 1 {
|
|
close(firstCallStarted)
|
|
<-ctx.Done()
|
|
_, err := f.db.GetDatabaseNow(ctx)
|
|
firstQueryErr = err
|
|
firstQueryCause = context.Cause(ctx)
|
|
return normalizeTaskTransitionError(err, "db query")
|
|
}
|
|
return nil
|
|
})
|
|
}()
|
|
|
|
timeoutTrap.MustWait(ctx).MustRelease(ctx)
|
|
timeoutTrap.Close()
|
|
<-firstCallStarted
|
|
clock.Advance(defaultTaskTimeout).MustWait(ctx)
|
|
retryTrap.MustWait(ctx).MustRelease(ctx)
|
|
clock.Advance(time.Minute).MustWait(ctx)
|
|
require.NoError(t, <-done)
|
|
require.Equal(t, 2, calls)
|
|
require.ErrorIs(t, firstQueryErr, context.Canceled)
|
|
require.NotErrorIs(t, firstQueryErr, errTaskTimeout)
|
|
require.ErrorIs(t, firstQueryCause, errTaskTimeout)
|
|
entries := entriesWithMessage(sink, "chatworker task retrying")
|
|
require.Len(t, entries, 1)
|
|
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), errTaskTimeout.Error())
|
|
require.Contains(t, sinkFieldValue(t, entries[0].Fields, "error"), context.Canceled.Error())
|
|
}
|
|
|
|
func TestRetryWrapper_ContextCancellationDoesNotRetryOrLog(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitLong))
|
|
cancel()
|
|
sink := testutil.NewFakeSink(t)
|
|
calls := 0
|
|
original := xerrors.New("database unavailable")
|
|
err := runTaskWithRetry(ctx, retryWrapperOptions{
|
|
clock: quartz.NewMock(t),
|
|
logger: sink.Logger(),
|
|
initialDelay: time.Second,
|
|
maxDelay: time.Second,
|
|
}, taskKindGeneration, retryWrapperTaskInfo{}, func(context.Context) error {
|
|
calls++
|
|
return original
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, 1, calls)
|
|
require.Empty(t, entriesWithMessage(sink, "chatworker task retrying"))
|
|
}
|
|
|
|
func TestNormalizeTaskErrors_ContextCancellationIsExpectedExit(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
err := normalizeTaskInfrastructureError(context.Canceled, "lock chat")
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
require.NotErrorIs(t, err, errTaskRetryable)
|
|
require.NotErrorIs(t, err, errTaskTimeout)
|
|
|
|
err = normalizeTaskTransitionError(context.Canceled, "commit chat")
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
require.ErrorIs(t, err, context.Canceled)
|
|
require.NotErrorIs(t, err, errTaskRetryable)
|
|
require.NotErrorIs(t, err, errTaskTimeout)
|
|
}
|
|
|
|
func entriesWithMessage(sink *testutil.FakeSink, message string) []slog.SinkEntry {
|
|
return sink.Entries(func(e slog.SinkEntry) bool { return e.Message == message })
|
|
}
|
|
|
|
func sinkFieldValue(t *testing.T, fields slog.Map, name string) string {
|
|
t.Helper()
|
|
for _, f := range fields {
|
|
if f.Name == name {
|
|
return fmt.Sprint(f.Value)
|
|
}
|
|
}
|
|
t.Fatalf("missing log field %q", name)
|
|
return ""
|
|
}
|
|
|
|
func TestInterruptTask_FinishInterruptionOnly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
buffer := starter.opts.MessagePartBuffer
|
|
key := messagepartbuffer.Key{
|
|
ChatID: chat.ID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
GenerationAttempt: acquired.GenerationAttempt,
|
|
}
|
|
require.NoError(t, buffer.CreateEpisode(key))
|
|
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
|
|
interrupting := interruptChat(t, f, chat.ID)
|
|
require.Equal(t, database.ChatStatusInterrupting, interrupting.Status)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusRunning, latest.Status)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
|
|
recorder.requireInterruptionOutcome(t, chat.ID, database.ChatStatusRunning)
|
|
recorder.requireCleanupCount(t, 0)
|
|
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
|
|
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(messages), 3)
|
|
parts, err := chatprompt.ParseContent(messages[len(messages)-2])
|
|
require.NoError(t, err)
|
|
require.Equal(t, []codersdk.ChatMessagePart{codersdk.ChatMessageText("partial answer")}, parts)
|
|
require.Equal(t, database.ChatMessageRoleUser, messages[len(messages)-1].Role)
|
|
}
|
|
|
|
func TestInterruptTask_StaleFenceExits(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
interrupting := interruptChat(t, f, chat.ID)
|
|
otherWorkerID := uuid.New()
|
|
otherRunnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusInterrupting, latest.Status)
|
|
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
|
|
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
|
|
recorder.requireStateHintCount(t, 0)
|
|
f.requireNoWatchEvents(t)
|
|
}
|
|
|
|
func TestInterruptTask_MissingEpisodePersistsNilPartials(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
interrupting := forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusWaiting, latest.Status)
|
|
recorder.requireInterruptionOutcome(t, chat.ID, database.ChatStatusWaiting)
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
require.NoError(t, err)
|
|
require.Len(t, messages, 1)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusWaiting)
|
|
}
|
|
|
|
func TestInterruptTask_BufferedPartsBecomePartialMessages(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
buffer := starter.opts.MessagePartBuffer
|
|
key := messagepartbuffer.Key{ChatID: chat.ID, HistoryVersion: acquired.HistoryVersion, GenerationAttempt: acquired.GenerationAttempt}
|
|
require.NoError(t, buffer.CreateEpisode(key))
|
|
callID := "call_" + uuid.NewString()
|
|
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessagePart{
|
|
Type: codersdk.ChatMessagePartTypeToolCall,
|
|
ToolCallID: callID,
|
|
ToolName: "local_tool",
|
|
Args: json.RawMessage(`{"value":1}`),
|
|
}))
|
|
interrupting := interruptChat(t, f, chat.ID)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(messages), 4)
|
|
assistant := messages[len(messages)-3]
|
|
tool := messages[len(messages)-2]
|
|
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
|
|
require.Equal(t, database.ChatMessageRoleTool, tool.Role)
|
|
toolParts, err := chatprompt.ParseContent(tool)
|
|
require.NoError(t, err)
|
|
require.Len(t, toolParts, 1)
|
|
require.Equal(t, codersdk.ChatMessagePartTypeToolResult, toolParts[0].Type)
|
|
require.Equal(t, callID, toolParts[0].ToolCallID)
|
|
require.True(t, toolParts[0].IsError)
|
|
}
|
|
|
|
func TestInterruptTask_PartialAssistantKeepsAttemptRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
clock := quartz.NewMock(t)
|
|
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
|
|
buffer := starter.opts.MessagePartBuffer
|
|
key := messagepartbuffer.Key{
|
|
ChatID: chat.ID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
GenerationAttempt: acquired.GenerationAttempt,
|
|
}
|
|
require.NoError(t, buffer.CreateEpisode(key))
|
|
// Prompt preparation and attempt bookkeeping run before the
|
|
// provider stream opens and must not be billed.
|
|
clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.StartModelInvocation(key))
|
|
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
|
|
clock.Advance(1500 * time.Millisecond)
|
|
interrupting := interruptChat(t, f, chat.ID)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(messages), 3)
|
|
assistant := messages[len(messages)-2]
|
|
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
|
|
require.Equal(t, sql.NullInt64{Int64: 1500, Valid: true}, assistant.RuntimeMs)
|
|
}
|
|
|
|
func TestInterruptTask_PartialAssistantWithoutModelInvocationHasNoRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
clock := quartz.NewMock(t)
|
|
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
|
|
buffer := starter.opts.MessagePartBuffer
|
|
key := messagepartbuffer.Key{
|
|
ChatID: chat.ID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
GenerationAttempt: acquired.GenerationAttempt,
|
|
}
|
|
require.NoError(t, buffer.CreateEpisode(key))
|
|
require.NoError(t, buffer.AddPart(key, codersdk.ChatMessageRoleAssistant, codersdk.ChatMessageText("partial answer")))
|
|
clock.Advance(1500 * time.Millisecond)
|
|
interrupting := interruptChat(t, f, chat.ID)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
require.NoError(t, err)
|
|
require.GreaterOrEqual(t, len(messages), 3)
|
|
assistant := messages[len(messages)-2]
|
|
require.Equal(t, database.ChatMessageRoleAssistant, assistant.Role)
|
|
require.False(t, assistant.RuntimeMs.Valid)
|
|
}
|
|
|
|
type interruptedBatch struct {
|
|
chat database.Chat
|
|
starter *taskStarter
|
|
clock *quartz.Mock
|
|
key messagepartbuffer.Key
|
|
workerID uuid.UUID
|
|
runnerID uuid.UUID
|
|
}
|
|
|
|
func interruptedBatchFixture(
|
|
t *testing.T,
|
|
f *workerTestFixture,
|
|
calls []codersdk.ChatMessagePart,
|
|
) interruptedBatch {
|
|
t.Helper()
|
|
chat := f.createRunningChat(t)
|
|
raw, err := chatprompt.MarshalParts(calls)
|
|
require.NoError(t, err)
|
|
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
|
|
_, err = machine.Update(testutil.Context(t, testutil.WaitShort), func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
|
|
_, err := tx.CommitStep(chatstate.CommitStepInput{Messages: []chatstate.Message{{
|
|
Role: database.ChatMessageRoleAssistant,
|
|
Content: raw,
|
|
Visibility: database.ChatMessageVisibilityBoth,
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
ModelConfigID: uuid.NullUUID{UUID: f.model.ID, Valid: true},
|
|
}}})
|
|
return err
|
|
})
|
|
require.NoError(t, err)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
clock := quartz.NewMock(t)
|
|
starter := newTestTaskStarterWithClock(t, f, recorder, clock)
|
|
key := messagepartbuffer.Key{
|
|
ChatID: chat.ID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
GenerationAttempt: acquired.GenerationAttempt,
|
|
}
|
|
require.NoError(t, starter.opts.MessagePartBuffer.CreateEpisode(key))
|
|
return interruptedBatch{
|
|
chat: chat,
|
|
starter: starter,
|
|
clock: clock,
|
|
key: key,
|
|
workerID: workerID,
|
|
runnerID: runnerID,
|
|
}
|
|
}
|
|
|
|
func (b interruptedBatch) interrupt(t *testing.T, f *workerTestFixture) []database.ChatMessage {
|
|
t.Helper()
|
|
interrupting := interruptChat(t, f, b.chat.ID)
|
|
err := b.starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: b.chat.ID,
|
|
WorkerID: b.workerID,
|
|
RunnerID: b.runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.NoError(t, err)
|
|
messages, err := f.db.GetChatMessagesByChatID(testutil.Context(t, testutil.WaitShort), database.GetChatMessagesByChatIDParams{ChatID: b.chat.ID})
|
|
require.NoError(t, err)
|
|
return messages
|
|
}
|
|
|
|
func findToolResultMessage(t *testing.T, messages []database.ChatMessage, toolCallID string) database.ChatMessage {
|
|
t.Helper()
|
|
for _, msg := range messages {
|
|
if msg.Role != database.ChatMessageRoleTool {
|
|
continue
|
|
}
|
|
parts, err := chatprompt.ParseContent(msg)
|
|
require.NoError(t, err)
|
|
for _, part := range parts {
|
|
if part.Type == codersdk.ChatMessagePartTypeToolResult && part.ToolCallID == toolCallID {
|
|
return msg
|
|
}
|
|
}
|
|
}
|
|
t.Fatalf("no tool result message for call %s", toolCallID)
|
|
return database.ChatMessage{}
|
|
}
|
|
|
|
// batchUsageRecords returns the model-only tool batch usage rows for the
|
|
// chat. Cancellation rows are user-visible, so the dedicated usage record
|
|
// only appears in the model-visibility history.
|
|
func batchUsageRecords(t *testing.T, f *workerTestFixture, chatID uuid.UUID) []database.ChatMessage {
|
|
t.Helper()
|
|
rows, err := f.db.GetChatMessagesForPromptByChatID(testutil.Context(t, testutil.WaitShort), chatID)
|
|
require.NoError(t, err)
|
|
var records []database.ChatMessage
|
|
for _, msg := range rows {
|
|
if msg.Role != database.ChatMessageRoleTool || msg.Visibility != database.ChatMessageVisibilityModel {
|
|
continue
|
|
}
|
|
records = append(records, msg)
|
|
}
|
|
return records
|
|
}
|
|
|
|
func requireSingleBatchUsageRecord(t *testing.T, f *workerTestFixture, chatID uuid.UUID, billedMs int64, billedCalls int) {
|
|
t.Helper()
|
|
records := batchUsageRecords(t, f, chatID)
|
|
require.Len(t, records, 1)
|
|
require.Equal(t, sql.NullInt64{Int64: billedMs, Valid: true}, records[0].RuntimeMs)
|
|
parts, err := chatprompt.ParseContent(records[0])
|
|
require.NoError(t, err)
|
|
require.Len(t, parts, 1)
|
|
require.Equal(t, toolBatchUsagePartType, parts[0].Type)
|
|
require.JSONEq(t,
|
|
fmt.Sprintf(`{"billed_ms":%d,"billed_calls":%d}`, billedMs, billedCalls),
|
|
string(parts[0].Result),
|
|
)
|
|
}
|
|
|
|
func TestInterruptTask_ToolBatchBillsPartialWindowOnUsageRecord(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
execCallID := "call_" + uuid.NewString()
|
|
waitCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
})
|
|
buffer := batch.starter.opts.MessagePartBuffer
|
|
|
|
// Keep advances below the buffer's 15-second cleanup tick.
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
// Record a live completion without publishing a tool result.
|
|
batch.clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(5 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
execRow := findToolResultMessage(t, messages, execCallID)
|
|
require.False(t, execRow.RuntimeMs.Valid)
|
|
waitRow := findToolResultMessage(t, messages, waitCallID)
|
|
require.False(t, waitRow.RuntimeMs.Valid)
|
|
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 3_000, 1)
|
|
}
|
|
|
|
func TestInterruptTask_RunningBilledToolBillsUpToInterrupt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
execCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
})
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(7 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
execRow := findToolResultMessage(t, messages, execCallID)
|
|
require.False(t, execRow.RuntimeMs.Valid)
|
|
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 7_000, 1)
|
|
}
|
|
|
|
func TestInterruptTask_UnbilledOnlyBatchBillsNothingOnInterrupt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
waitCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
})
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(10 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
waitRow := findToolResultMessage(t, messages, waitCallID)
|
|
require.False(t, waitRow.RuntimeMs.Valid)
|
|
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
|
|
}
|
|
|
|
func TestInterruptTask_UnstartedSerialCallBillsNothingOnInterrupt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
waitCallID := "call_" + uuid.NewString()
|
|
serialCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: serialCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
})
|
|
buffer := batch.starter.opts.MessagePartBuffer
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(10 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
for _, msg := range messages {
|
|
if msg.Role == database.ChatMessageRoleTool {
|
|
require.False(t, msg.RuntimeMs.Valid,
|
|
"no cancellation row may bill: the billed call never began executing")
|
|
}
|
|
}
|
|
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID),
|
|
"no usage record may bill: the billed call never began executing")
|
|
}
|
|
|
|
func TestInterruptTask_StartedSerialCallBillsFromItsOwnStart(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
execCallID := "call_" + uuid.NewString()
|
|
waitCallID := "call_" + uuid.NewString()
|
|
serialCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: serialCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
})
|
|
buffer := batch.starter.opts.MessagePartBuffer
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.RecordToolCompletion(batch.key, 1, batch.clock.Now()))
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 2, batch.clock.Now()))
|
|
batch.clock.Advance(2 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
serialRow := findToolResultMessage(t, messages, serialCallID)
|
|
require.False(t, serialRow.RuntimeMs.Valid)
|
|
execRow := findToolResultMessage(t, messages, execCallID)
|
|
require.False(t, execRow.RuntimeMs.Valid)
|
|
waitRow := findToolResultMessage(t, messages, waitCallID)
|
|
require.False(t, waitRow.RuntimeMs.Valid)
|
|
// Completed execute [2s,5s] plus the serial call's own window
|
|
// [8s,10s]: the gap and the unbilled wait_agent do not count.
|
|
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 5_000, 2)
|
|
}
|
|
|
|
func TestInterruptTask_DuplicateCallIDsKeepOccurrenceStates(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
dupCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
})
|
|
buffer := batch.starter.opts.MessagePartBuffer
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.RecordToolCompletion(batch.key, 0, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
for _, msg := range messages {
|
|
if msg.Role == database.ChatMessageRoleTool {
|
|
require.False(t, msg.RuntimeMs.Valid)
|
|
}
|
|
}
|
|
// The still-running occurrence bills the full window exactly once.
|
|
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 6_000, 2)
|
|
}
|
|
|
|
func TestInterruptTask_DuplicateCallIDCompletionStampsOwnOccurrence(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
dupCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
})
|
|
buffer := batch.starter.opts.MessagePartBuffer
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 0, batch.clock.Now()))
|
|
require.NoError(t, buffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
require.NoError(t, buffer.RecordToolCompletion(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(3 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
for _, msg := range messages {
|
|
if msg.Role == database.ChatMessageRoleTool {
|
|
require.False(t, msg.RuntimeMs.Valid)
|
|
}
|
|
}
|
|
// The running execute occurrence bills to the interrupt; wait_agent's
|
|
// early completion must not end it at 3s.
|
|
requireSingleBatchUsageRecord(t, f, batch.chat.ID, 6_000, 1)
|
|
}
|
|
|
|
func TestInterruptTask_RejectedDuplicateIDDoesNotStealDispatchedOccurrence(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
dupCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: dupCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
})
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(10 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
for _, msg := range messages {
|
|
if msg.Role == database.ChatMessageRoleTool {
|
|
require.False(t, msg.RuntimeMs.Valid)
|
|
}
|
|
}
|
|
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID),
|
|
"no usage record may bill: only the unbilled wait_agent occurrence ran")
|
|
}
|
|
|
|
func TestInterruptTask_RejectedCallBillsNothingOnInterrupt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
execCallID := "call_" + uuid.NewString()
|
|
waitCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: waitCallID, ToolName: "wait_agent", Args: json.RawMessage(`{}`)},
|
|
})
|
|
|
|
batch.clock.Advance(2 * time.Second)
|
|
require.NoError(t, batch.starter.opts.MessagePartBuffer.RecordToolStart(batch.key, 1, batch.clock.Now()))
|
|
batch.clock.Advance(10 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
execRow := findToolResultMessage(t, messages, execCallID)
|
|
require.False(t, execRow.RuntimeMs.Valid)
|
|
waitRow := findToolResultMessage(t, messages, waitCallID)
|
|
require.False(t, waitRow.RuntimeMs.Valid)
|
|
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
|
|
}
|
|
|
|
func TestInterruptTask_ToolCancellationWithoutLiveBatchHasNoRuntime(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
execCallID := "call_" + uuid.NewString()
|
|
batch := interruptedBatchFixture(t, f, []codersdk.ChatMessagePart{
|
|
{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: execCallID, ToolName: "execute", Args: json.RawMessage(`{}`)},
|
|
})
|
|
|
|
batch.clock.Advance(10 * time.Second)
|
|
|
|
messages := batch.interrupt(t, f)
|
|
execRow := findToolResultMessage(t, messages, execCallID)
|
|
require.False(t, execRow.RuntimeMs.Valid)
|
|
require.Empty(t, batchUsageRecords(t, f, batch.chat.ID))
|
|
}
|
|
|
|
func TestInterruptTask_CancelsUnresolvedToolCallsOnAgent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
saved := func(status int, body string) workspacesdk.CancelToolCallResponse {
|
|
return workspacesdk.CancelToolCallResponse{Received: true, Status: status, ContentType: "application/json", Body: []byte(body)}
|
|
}
|
|
exitCode := 0
|
|
tests := []struct {
|
|
name string
|
|
toolName string
|
|
notCancelable bool
|
|
cancel workspacesdk.CancelToolCallResponse
|
|
cancelErr error
|
|
cancelPanics bool
|
|
output *workspacesdk.ProcessOutputResponse
|
|
outputErr error
|
|
wantError bool
|
|
want string
|
|
}{
|
|
{name: "ExecuteNotReceived", toolName: "execute", outputErr: codersdk.NewError(http.StatusNotFound, codersdk.Response{}), wantError: true, want: `{"error":"tool call was canceled before it ran; no changes were made"}`},
|
|
// The agent has no record, but the cancel stopped the call's process.
|
|
{name: "ExecuteNoRecordProcessCanceled", toolName: "execute", output: &workspacesdk.ProcessOutputResponse{Output: "partial", Canceled: true}, want: `{"canceled":true,"error":"tool call was canceled while running","exit_code":-1,"output":"partial","success":false}`},
|
|
{name: "ExecuteNoRecordOutputError", toolName: "execute", outputErr: xerrors.New("connection reset"), wantError: true, want: interruptedToolResultErrorMessage},
|
|
{name: "EditNotReceived", toolName: "edit_files", wantError: true, want: "tool call was canceled before it ran; no changes were made"},
|
|
{name: "ExecuteStartError", toolName: "execute", cancel: saved(http.StatusInternalServerError, `{"message":"no shell"}`), want: `"error":"start process: unexpected status code 500: no shell"`},
|
|
{name: "EditError", toolName: "edit_files", cancel: saved(http.StatusBadRequest, `{"message":"old_text not found"}`), wantError: true, want: "old_text not found"},
|
|
{name: "ExecuteExited", toolName: "execute", cancel: saved(http.StatusOK, `{}`), output: &workspacesdk.ProcessOutputResponse{Output: "done", ExitCode: &exitCode}, want: `{"exit_code":0,"output":"done","success":true}`},
|
|
{name: "EditApplied", toolName: "edit_files", cancel: saved(http.StatusOK, `{"files":[{"path":"/a","diff":"d"}]}`), want: `{"files":[{"diff":"d","path":"/a"}],"ok":true}`},
|
|
// A result over the 64 KB default budget is capped and stays JSON.
|
|
{name: "EditAppliedTruncated", toolName: "edit_files", cancel: saved(http.StatusOK, `{"files":[{"path":"/a","diff":"`+strings.Repeat("d", 64<<10)+`"}]}`), want: "Coder truncated"},
|
|
{name: "CancelError", toolName: "write_file", cancelErr: xerrors.New("unexpected status code 404"), wantError: true, want: interruptedToolResultErrorMessage},
|
|
{name: "CancelPanic", toolName: "execute", cancelPanics: true, wantError: true, want: interruptedToolResultErrorMessage},
|
|
{name: "OtherTool", toolName: "read_file", notCancelable: true, wantError: true, want: interruptedToolResultErrorMessage},
|
|
}
|
|
|
|
f := newWorkerTestFixture(t)
|
|
calls := make([]codersdk.ChatMessagePart, 0, len(tests))
|
|
for _, tc := range tests {
|
|
calls = append(calls, codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call_" + tc.name, ToolName: tc.toolName, Args: json.RawMessage(`{}`)})
|
|
}
|
|
batch := interruptedBatchFixture(t, f, calls)
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
workspace := dbfake.WorkspaceBuild(t, f.db, database.WorkspaceTable{OrganizationID: f.org.ID, OwnerID: f.user.ID}).WithAgent().Do()
|
|
_, err := f.db.UpdateChatWorkspaceBinding(ctx, database.UpdateChatWorkspaceBindingParams{
|
|
ID: batch.chat.ID,
|
|
WorkspaceID: uuid.NullUUID{UUID: workspace.Workspace.ID, Valid: true},
|
|
BuildID: uuid.NullUUID{UUID: workspace.Build.ID, Valid: true},
|
|
AgentID: uuid.NullUUID{UUID: workspace.Agents[0].ID, Valid: true},
|
|
})
|
|
require.NoError(t, err)
|
|
messages, err := f.db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: batch.chat.ID})
|
|
require.NoError(t, err)
|
|
assistantID := messages[len(messages)-1].ID
|
|
|
|
// Each cancel waits until all cancels arrive, so sequential cancels
|
|
// would time out.
|
|
var pending atomic.Int64
|
|
for _, tc := range tests {
|
|
if !tc.notCancelable {
|
|
pending.Add(1)
|
|
}
|
|
}
|
|
allReceived := make(chan struct{})
|
|
conn := agentconnmock.NewMockAgentConn(gomock.NewController(t))
|
|
conn.EXPECT().SetExtraHeaders(gomock.Any())
|
|
for _, tc := range tests {
|
|
if tc.notCancelable {
|
|
continue
|
|
}
|
|
id := chattool.ToolCallID(batch.chat.ID, assistantID, "call_"+tc.name)
|
|
conn.EXPECT().CancelToolCall(gomock.Any(), id).DoAndReturn(func(ctx context.Context, _ uuid.UUID) (workspacesdk.CancelToolCallResponse, error) {
|
|
if pending.Add(-1) == 0 {
|
|
close(allReceived)
|
|
}
|
|
select {
|
|
case <-allReceived:
|
|
if tc.cancelPanics {
|
|
panic("cancel panicked")
|
|
}
|
|
return tc.cancel, tc.cancelErr
|
|
case <-ctx.Done():
|
|
return workspacesdk.CancelToolCallResponse{}, ctx.Err()
|
|
}
|
|
})
|
|
if tc.output != nil || tc.outputErr != nil {
|
|
var output workspacesdk.ProcessOutputResponse
|
|
if tc.output != nil {
|
|
output = *tc.output
|
|
}
|
|
conn.EXPECT().ProcessOutput(gomock.Any(), id.String(), &workspacesdk.ProcessOutputOptions{Wait: true}).Return(output, tc.outputErr)
|
|
}
|
|
}
|
|
batch.starter.server.agentConnFn = func(context.Context, uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
|
return conn, func() {}, nil
|
|
}
|
|
|
|
messages = batch.interrupt(t, f)
|
|
for _, tc := range tests {
|
|
parts, err := chatprompt.ParseContent(findToolResultMessage(t, messages, "call_"+tc.name))
|
|
require.NoError(t, err)
|
|
require.Len(t, parts, 1)
|
|
assert.Equal(t, tc.wantError, parts[0].IsError, tc.name)
|
|
// Compact the stored JSON with sorted keys.
|
|
var result any
|
|
require.NoError(t, json.Unmarshal(parts[0].Result, &result))
|
|
got, err := json.Marshal(result)
|
|
require.NoError(t, err)
|
|
assert.Contains(t, string(got), tc.want, tc.name)
|
|
}
|
|
}
|
|
|
|
func TestStartGeneration_CancelsDeletedToolCallsOnAgent(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
call := func(toolName string) codersdk.ChatMessagePart {
|
|
return codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolCall, ToolCallID: "call_" + toolName, ToolName: toolName, Args: json.RawMessage(`{}`)}
|
|
}
|
|
result := func(toolName string) codersdk.ChatMessagePart {
|
|
return codersdk.ChatMessagePart{Type: codersdk.ChatMessagePartTypeToolResult, ToolCallID: "call_" + toolName, ToolName: toolName, Result: json.RawMessage(`{}`)}
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
build func(t *testing.T, h *deletedToolCallsHistory)
|
|
cancelErr error
|
|
want []string // Tools whose calls are canceled.
|
|
}{
|
|
{
|
|
name: "DeletedUnresolvedCalls",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"), call("edit_files"), call("write_file"), call("read_file"))
|
|
h.editPrompt(t)
|
|
},
|
|
want: []string{"execute", "edit_files", "write_file"},
|
|
},
|
|
{
|
|
name: "DeletedResolvedCalls",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"), call("write_file"))
|
|
h.commit(t, database.ChatMessageRoleTool, database.ChatMessageVisibilityBoth, result("execute"))
|
|
h.commit(t, database.ChatMessageRoleTool, database.ChatMessageVisibilityModel, result("write_file"))
|
|
h.editPrompt(t)
|
|
},
|
|
},
|
|
{
|
|
name: "SecondEditBeforeFirstStep",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"))
|
|
h.editPrompt(t)
|
|
h.editPrompt(t)
|
|
},
|
|
want: []string{"execute"},
|
|
},
|
|
{
|
|
// The unresolved call is in the active history, not deleted.
|
|
name: "TurnWithoutEdit",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"))
|
|
h.commit(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText("next"))
|
|
},
|
|
},
|
|
{
|
|
// EditMessage inserts a result for the earlier unresolved call
|
|
// between the deleted messages and the replacement.
|
|
name: "UnresolvedCallBeforeEditedMessage",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("read_file"))
|
|
h.commit(t, database.ChatMessageRoleUser, database.ChatMessageVisibilityBoth, codersdk.ChatMessageText("next"))
|
|
h.assistant(t, call("execute"))
|
|
h.editPrompt(t)
|
|
},
|
|
want: []string{"execute"},
|
|
},
|
|
{
|
|
name: "SecondStep",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"))
|
|
h.editPrompt(t)
|
|
h.assistant(t, codersdk.ChatMessageText("done"))
|
|
},
|
|
},
|
|
{
|
|
name: "CancelError",
|
|
build: func(t *testing.T, h *deletedToolCallsHistory) {
|
|
h.assistant(t, call("execute"), call("edit_files"))
|
|
h.editPrompt(t)
|
|
},
|
|
cancelErr: xerrors.New("unexpected status code 404"),
|
|
want: []string{"execute", "edit_files"},
|
|
},
|
|
}
|
|
|
|
f := newWorkerTestFixture(t)
|
|
for _, tc := range tests {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
chat := f.createRunningChat(t)
|
|
workspace := dbfake.WorkspaceBuild(t, f.db, database.WorkspaceTable{OrganizationID: f.org.ID, OwnerID: f.user.ID}).WithAgent().Do()
|
|
_, err := f.db.UpdateChatWorkspaceBinding(ctx, database.UpdateChatWorkspaceBindingParams{
|
|
ID: chat.ID,
|
|
WorkspaceID: uuid.NullUUID{UUID: workspace.Workspace.ID, Valid: true},
|
|
BuildID: uuid.NullUUID{UUID: workspace.Build.ID, Valid: true},
|
|
AgentID: uuid.NullUUID{UUID: workspace.Agents[0].ID, Valid: true},
|
|
})
|
|
require.NoError(t, err)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
h := &deletedToolCallsHistory{f: f, machine: chatstate.NewChatMachine(f.db, f.pubsub, chat.ID), chatID: chat.ID}
|
|
tc.build(t, h)
|
|
|
|
conn := agentconnmock.NewMockAgentConn(gomock.NewController(t))
|
|
if len(tc.want) > 0 {
|
|
conn.EXPECT().SetExtraHeaders(gomock.Any())
|
|
}
|
|
for _, toolName := range tc.want {
|
|
id := chattool.ToolCallID(chat.ID, h.toolCallMessageID, "call_"+toolName)
|
|
conn.EXPECT().CancelToolCall(gomock.Any(), id).DoAndReturn(func(ctx context.Context, _ uuid.UUID) (workspacesdk.CancelToolCallResponse, error) {
|
|
// The model step fails in this fixture and sets the chat's
|
|
// error, so a running chat without one has not reached it.
|
|
current, err := f.db.GetChatByID(ctx, chat.ID)
|
|
if assert.NoError(t, err) {
|
|
assert.Equal(t, database.ChatStatusRunning, current.Status)
|
|
assert.False(t, current.LastError.Valid)
|
|
}
|
|
return workspacesdk.CancelToolCallResponse{}, tc.cancelErr
|
|
})
|
|
if toolName == "execute" && tc.cancelErr == nil {
|
|
conn.EXPECT().ProcessOutput(gomock.Any(), id.String(), gomock.Any()).Return(workspacesdk.ProcessOutputResponse{}, codersdk.NewError(http.StatusNotFound, codersdk.Response{}))
|
|
}
|
|
}
|
|
var dials atomic.Int64
|
|
sink := testutil.NewFakeSink(t)
|
|
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
|
|
starter.opts.Logger = sink.Logger(slog.LevelWarn)
|
|
starter.server.agentConnFn = func(context.Context, uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
|
dials.Add(1)
|
|
return conn, func() {}, nil
|
|
}
|
|
debugTurn := newRunnerDebugTurn(starter.opts.Logger)
|
|
defer debugTurn.Finalize(ctx)
|
|
current, err := f.db.GetChatByID(ctx, chat.ID)
|
|
require.NoError(t, err)
|
|
err = starter.StartGeneration(ctx, chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: current.HistoryVersion,
|
|
GenerationAttempt: current.GenerationAttempt,
|
|
Status: database.ChatStatusRunning,
|
|
DebugTurn: debugTurn,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
if len(tc.want) == 0 {
|
|
require.Zero(t, dials.Load(), "the agent must not be dialed without calls to cancel")
|
|
}
|
|
cancelWarnings := sink.Entries(func(e slog.SinkEntry) bool { return e.Message == "cancel tool call on agent" })
|
|
if tc.cancelErr != nil {
|
|
require.Len(t, cancelWarnings, len(tc.want))
|
|
} else {
|
|
require.Empty(t, cancelWarnings)
|
|
}
|
|
// The generation went on to the model step.
|
|
final, err := f.db.GetChatByID(ctx, chat.ID)
|
|
require.NoError(t, err)
|
|
require.Contains(t, string(final.LastError.RawMessage), "create model")
|
|
})
|
|
}
|
|
}
|
|
|
|
// deletedToolCallsHistory builds a running chat's history for
|
|
// TestStartGeneration_CancelsDeletedToolCallsOnAgent.
|
|
type deletedToolCallsHistory struct {
|
|
f *workerTestFixture
|
|
machine *chatstate.ChatMachine
|
|
chatID uuid.UUID
|
|
toolCallMessageID int64 // ID of the last committed assistant message with tool calls.
|
|
}
|
|
|
|
func (h *deletedToolCallsHistory) commit(t *testing.T, role database.ChatMessageRole, visibility database.ChatMessageVisibility, parts ...codersdk.ChatMessagePart) database.ChatMessage {
|
|
t.Helper()
|
|
raw, err := chatprompt.MarshalParts(parts)
|
|
require.NoError(t, err)
|
|
var inserted []database.ChatMessage
|
|
_, err = h.machine.Update(testutil.Context(t, testutil.WaitShort), func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
|
|
res, err := tx.CommitStep(chatstate.CommitStepInput{Messages: []chatstate.Message{{
|
|
Role: role,
|
|
Content: raw,
|
|
Visibility: visibility,
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
ModelConfigID: uuid.NullUUID{UUID: h.f.model.ID, Valid: true},
|
|
}}})
|
|
inserted = res.InsertedMessages
|
|
return err
|
|
})
|
|
require.NoError(t, err)
|
|
require.Len(t, inserted, 1)
|
|
return inserted[0]
|
|
}
|
|
|
|
func (h *deletedToolCallsHistory) assistant(t *testing.T, parts ...codersdk.ChatMessagePart) {
|
|
t.Helper()
|
|
msg := h.commit(t, database.ChatMessageRoleAssistant, database.ChatMessageVisibilityBoth, parts...)
|
|
if slices.ContainsFunc(parts, func(p codersdk.ChatMessagePart) bool { return p.Type == codersdk.ChatMessagePartTypeToolCall }) {
|
|
h.toolCallMessageID = msg.ID
|
|
}
|
|
}
|
|
|
|
// editPrompt edits the latest user prompt, as the edit endpoint does.
|
|
func (h *deletedToolCallsHistory) editPrompt(t *testing.T) {
|
|
t.Helper()
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
messages, err := h.f.db.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: h.chatID})
|
|
require.NoError(t, err)
|
|
prompt := messages[lastUserPromptIndex(messages)]
|
|
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{codersdk.ChatMessageText("edited")})
|
|
require.NoError(t, err)
|
|
_, err = h.machine.Update(ctx, func(tx *chatstate.Tx, _ database.Store, _ *chatstate.ChatSnapshot) error {
|
|
_, err := tx.EditMessage(chatstate.EditMessageInput{MessageID: prompt.ID, CreatedBy: h.f.user.ID, Content: content})
|
|
return err
|
|
})
|
|
require.NoError(t, err)
|
|
}
|
|
|
|
func TestRequiresActionTimeout_ExpiredCancelsOnly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRequiresActionChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
expired := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRequiresAction,
|
|
RequiresActionDeadlineAt: expired.RequiresActionDeadlineAt,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusRunning, latest.Status)
|
|
require.False(t, latest.RequiresActionDeadlineAt.Valid)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
|
|
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
|
|
func TestRequiresActionTimeout_NullDeadlineCancelsImmediately(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRequiresActionChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
nullDeadline := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{})
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRequiresAction,
|
|
RequiresActionDeadlineAt: nullDeadline.RequiresActionDeadlineAt,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusRunning, latest.Status)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
|
|
}
|
|
|
|
// failingChatStatePubsub fails the first chat:update publish for one chat,
|
|
// which ChatMachine.Update reports only after the transaction commits.
|
|
type failingChatStatePubsub struct {
|
|
chatWorkerPubsub
|
|
channel string
|
|
failed atomic.Bool
|
|
}
|
|
|
|
func (p *failingChatStatePubsub) Publish(event string, message []byte) error {
|
|
if event == p.channel && p.failed.CompareAndSwap(false, true) {
|
|
return xerrors.New("injected chat state publish failure")
|
|
}
|
|
return p.chatWorkerPubsub.Publish(event, message)
|
|
}
|
|
|
|
func failFirstChatStatePublish(starter *taskStarter, chatID uuid.UUID) *failingChatStatePubsub {
|
|
p := &failingChatStatePubsub{
|
|
chatWorkerPubsub: starter.opts.Pubsub,
|
|
channel: coderdpubsub.ChatStateUpdateChannel(chatID),
|
|
}
|
|
starter.opts.Pubsub = p
|
|
return p
|
|
}
|
|
|
|
// A publish failure after the commit must not turn a committed transition
|
|
// into a task failure.
|
|
func TestInterruptTask_PublishFailureAfterCommitSucceeds(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
interrupting := forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
publisher := failFirstChatStatePublish(starter, chat.ID)
|
|
|
|
err := starter.StartInterrupt(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: interrupting.HistoryVersion,
|
|
GenerationAttempt: interrupting.GenerationAttempt,
|
|
Status: database.ChatStatusInterrupting,
|
|
})
|
|
require.True(t, publisher.failed.Load(), "the chat state publish must have failed")
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusWaiting, latest.Status)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusWaiting)
|
|
}
|
|
|
|
func TestRequiresActionTimeout_PublishFailureAfterCommitSucceeds(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRequiresActionChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
nullDeadline := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{})
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
publisher := failFirstChatStatePublish(starter, chat.ID)
|
|
|
|
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRequiresAction,
|
|
RequiresActionDeadlineAt: nullDeadline.RequiresActionDeadlineAt,
|
|
})
|
|
require.True(t, publisher.failed.Load(), "the chat state publish must have failed")
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusRunning, latest.Status)
|
|
recorder.requireStateHint(t, chat.ID, latest.SnapshotVersion, database.ChatStatusRunning)
|
|
}
|
|
|
|
func TestRequiresActionTimeout_StaleFenceExitsAfterToolResult(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRequiresActionChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
expired := f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
|
|
forceExecutionStateWithDeadline(t, f, chat.ID, database.ChatStatusRunning, false, &sql.NullTime{})
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartRequiresActionTimeout(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRequiresAction,
|
|
RequiresActionDeadlineAt: expired.RequiresActionDeadlineAt,
|
|
})
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, database.ChatStatusRunning, latest.Status)
|
|
recorder.requireStateHintCount(t, 0)
|
|
f.requireNoWatchEvents(t)
|
|
}
|
|
|
|
func TestAbandonTask_AbandonOnly(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.False(t, latest.WorkerID.Valid)
|
|
require.False(t, latest.RunnerID.Valid)
|
|
recorder.requireCleanup(t, chat.ID, runnerID)
|
|
recorder.requireStateHintCount(t, 0)
|
|
f.requireNoWatchEvents(t)
|
|
}
|
|
|
|
func TestAbandonTask_OwnershipMismatchRequestsCleanup(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
otherWorkerID := uuid.New()
|
|
otherRunnerID := uuid.New()
|
|
latestOwner := acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: latestOwner.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
|
|
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
|
|
recorder.requireCleanup(t, chat.ID, runnerID)
|
|
}
|
|
|
|
func TestAbandonTask_StaleStatusFenceExits(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
forceExecutionState(t, f, chat.ID, database.ChatStatusInterrupting, false)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
err := starter.StartAbandon(testutil.Context(t, testutil.WaitLong), chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusWaiting,
|
|
})
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, latest.WorkerID.Valid)
|
|
require.True(t, latest.RunnerID.Valid)
|
|
require.Equal(t, database.ChatStatusInterrupting, latest.Status)
|
|
recorder.requireCleanupCount(t, 0)
|
|
}
|
|
|
|
func TestGenerationTask_RecordRetryState(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
recorder := newTaskSideEffectRecorder()
|
|
starter := newTestTaskStarter(t, f, recorder)
|
|
|
|
attempt, err := starter.beginGenerationAttempt(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
chatstate.NewChatMachine(f.db, f.pubsub, chat.ID),
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
require.Equal(t, int64(1), attempt.number)
|
|
before, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.False(t, before.RetryState.Valid)
|
|
|
|
decision, err := starter.recordGenerationRetry(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
chatstate.NewChatMachine(f.db, f.pubsub, chat.ID),
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
chaterror.ClassifiedError{
|
|
Message: "OpenAI is rate limiting requests.",
|
|
Kind: codersdk.ChatErrorKindRateLimit,
|
|
Provider: "openai",
|
|
Retryable: true,
|
|
StatusCode: 429,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
require.True(t, decision.retry)
|
|
require.Equal(t, int64(1), decision.generationAttempt)
|
|
require.Equal(t, chatretry.Delay(0), decision.delay)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, latest.RetryState.Valid)
|
|
require.Equal(t, latest.SnapshotVersion, latest.RetryStateVersion)
|
|
require.Greater(t, latest.RetryStateVersion, before.RetryStateVersion)
|
|
require.Equal(t, before.GenerationAttempt, latest.GenerationAttempt)
|
|
recorder.requireStateHintCount(t, 0)
|
|
|
|
var retryPayload codersdk.ChatStreamRetry
|
|
require.NoError(t, json.Unmarshal(latest.RetryState.RawMessage, &retryPayload))
|
|
require.Equal(t, 1, retryPayload.Attempt)
|
|
require.Equal(t, chatretry.Delay(0).Milliseconds(), retryPayload.DelayMs)
|
|
require.Equal(t, "OpenAI is rate limiting requests.", retryPayload.Error)
|
|
require.Equal(t, codersdk.ChatErrorKindRateLimit, retryPayload.Kind)
|
|
require.Equal(t, "openai", retryPayload.Provider)
|
|
require.Equal(t, 429, retryPayload.StatusCode)
|
|
require.False(t, retryPayload.RetryingAt.IsZero())
|
|
}
|
|
|
|
func TestGenerationTask_RecordRetryStateUsesDurableGenerationAttempt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
|
|
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
|
|
|
|
for range 3 {
|
|
attempt, err := starter.beginGenerationAttempt(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
machine,
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
require.Positive(t, attempt.number)
|
|
}
|
|
|
|
decision, err := starter.recordGenerationRetry(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
machine,
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
chaterror.ClassifiedError{
|
|
Message: "OpenAI is temporarily unavailable.",
|
|
Kind: codersdk.ChatErrorKindTimeout,
|
|
Provider: "openai",
|
|
Retryable: true,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
require.True(t, decision.retry)
|
|
require.Equal(t, int64(3), decision.generationAttempt)
|
|
require.Equal(t, chatretry.Delay(2), decision.delay)
|
|
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
var retryPayload codersdk.ChatStreamRetry
|
|
require.NoError(t, json.Unmarshal(latest.RetryState.RawMessage, &retryPayload))
|
|
require.Equal(t, 3, retryPayload.Attempt)
|
|
require.Equal(t, chatretry.Delay(2).Milliseconds(), retryPayload.DelayMs)
|
|
}
|
|
|
|
func TestGenerationTask_RecordRetryStateHonorsConfiguredRetries(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
limits Limits
|
|
maxRetries int
|
|
}{
|
|
{name: "Default", maxRetries: codersdk.DefaultChatMaxGenerationRetries},
|
|
{name: "ConfiguredOne", limits: Limits{MaxGenerationRetries: 1}, maxRetries: 1},
|
|
{name: "ConfiguredThree", limits: Limits{MaxGenerationRetries: 3}, maxRetries: 3},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder(), withInternalTestServerLimits(tt.limits))
|
|
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
|
|
input := chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
}
|
|
classified := chaterror.ClassifiedError{
|
|
Message: "OpenAI is temporarily unavailable.",
|
|
Kind: codersdk.ChatErrorKindTimeout,
|
|
Provider: "openai",
|
|
Retryable: true,
|
|
}
|
|
|
|
// The configured value is a retry count, so every failure up to
|
|
// it is retried and the next failure is not.
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
for failure := 1; failure <= tt.maxRetries+1; failure++ {
|
|
attempt, err := starter.beginGenerationAttempt(ctx, machine, input)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
decision, err := starter.recordGenerationRetry(ctx, machine, input, classified)
|
|
require.NoError(t, err)
|
|
require.Equal(t, failure <= tt.maxRetries, decision.retry, "failure %d", failure)
|
|
require.EqualValues(t, failure, decision.generationAttempt)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestGenerationTask_RecordRetryStateClearedByNextAttempt(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
|
|
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
|
|
input := chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
}
|
|
|
|
attempt, err := starter.beginGenerationAttempt(testutil.Context(t, testutil.WaitLong), machine, input)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
require.Equal(t, int64(1), attempt.number)
|
|
_, err = starter.recordGenerationRetry(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
machine,
|
|
input,
|
|
chaterror.ClassifiedError{
|
|
Message: "OpenAI is temporarily unavailable.",
|
|
Kind: codersdk.ChatErrorKindTimeout,
|
|
Provider: "openai",
|
|
Retryable: true,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
withRetry, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, withRetry.RetryState.Valid)
|
|
|
|
attempt, err = starter.beginGenerationAttempt(testutil.Context(t, testutil.WaitLong), machine, input)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
require.Equal(t, int64(2), attempt.number)
|
|
after, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.False(t, after.RetryState.Valid)
|
|
require.Equal(t, after.SnapshotVersion, after.RetryStateVersion)
|
|
require.Greater(t, after.RetryStateVersion, withRetry.RetryStateVersion)
|
|
}
|
|
|
|
func TestGenerationTask_RecordRetryStateStaleFenceExits(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
workerID := uuid.New()
|
|
runnerID := uuid.New()
|
|
acquired := acquireChat(t, f, chat.ID, workerID, runnerID)
|
|
starter := newTestTaskStarter(t, f, newTaskSideEffectRecorder())
|
|
machine := chatstate.NewChatMachine(f.db, f.pubsub, chat.ID)
|
|
attempt, err := starter.beginGenerationAttempt(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
machine,
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
)
|
|
require.NoError(t, err)
|
|
attempt.closeEpisode()
|
|
require.Equal(t, int64(1), attempt.number)
|
|
|
|
otherWorkerID := uuid.New()
|
|
otherRunnerID := uuid.New()
|
|
acquireChat(t, f, chat.ID, otherWorkerID, otherRunnerID)
|
|
_, err = starter.recordGenerationRetry(
|
|
testutil.Context(t, testutil.WaitLong),
|
|
machine,
|
|
chatWorkerTaskStartInput{
|
|
ChatID: chat.ID,
|
|
WorkerID: workerID,
|
|
RunnerID: runnerID,
|
|
HistoryVersion: acquired.HistoryVersion,
|
|
Status: database.ChatStatusRunning,
|
|
},
|
|
chaterror.ClassifiedError{
|
|
Message: "OpenAI is temporarily unavailable.",
|
|
Kind: codersdk.ChatErrorKindTimeout,
|
|
Provider: "openai",
|
|
Retryable: true,
|
|
},
|
|
)
|
|
require.ErrorIs(t, err, errTaskExpectedExit)
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.False(t, latest.RetryState.Valid)
|
|
require.Equal(t, otherWorkerID, latest.WorkerID.UUID)
|
|
require.Equal(t, otherRunnerID, latest.RunnerID.UUID)
|
|
}
|
|
|
|
func TestRunner_StartsRealInterruptTask(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
worker := startRealTaskWorker(t, f)
|
|
waitOwnedChat(t, f, chat.ID, worker.opts.WorkerID)
|
|
|
|
interruptChat(t, f, chat.ID)
|
|
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
|
|
latest, err := f.db.GetChatByID(ctx, chat.ID)
|
|
return err == nil && latest.Status == database.ChatStatusRunning
|
|
}, testutil.IntervalFast)
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.Equal(t, worker.opts.WorkerID, latest.WorkerID.UUID)
|
|
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
|
|
func TestRunner_StartsRealRequiresActionTimeoutTask(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRequiresActionChat(t)
|
|
f.setRequiresActionDeadline(t, chat.ID, sql.NullTime{Time: time.Now().Add(-time.Minute), Valid: true})
|
|
worker := startRealTaskWorker(t, f)
|
|
|
|
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
|
|
latest, err := f.db.GetChatByID(ctx, chat.ID)
|
|
return err == nil && latest.Status == database.ChatStatusRunning && latest.WorkerID.Valid && latest.WorkerID.UUID == worker.opts.WorkerID
|
|
}, testutil.IntervalFast)
|
|
latest, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chat.ID)
|
|
require.NoError(t, err)
|
|
require.True(t, latest.RunnerID.Valid)
|
|
f.requireWatchEvent(t, chat.ID, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
|
|
func TestRunner_StartsRealAbandonTask(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
f := newWorkerTestFixture(t)
|
|
chat := f.createRunningChat(t)
|
|
worker := startRealTaskWorker(t, f)
|
|
waitOwnedChat(t, f, chat.ID, worker.opts.WorkerID)
|
|
|
|
updated := forceExecutionState(t, f, chat.ID, database.ChatStatusError, false)
|
|
publishChatUpdate(t, f, updated)
|
|
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
|
|
latest, err := f.db.GetChatByID(ctx, chat.ID)
|
|
return err == nil && !latest.WorkerID.Valid && !latest.RunnerID.Valid
|
|
}, testutil.IntervalFast)
|
|
}
|
|
|
|
func (f *workerTestFixture) setRequiresActionDeadline(t *testing.T, chatID uuid.UUID, deadline sql.NullTime) database.Chat {
|
|
t.Helper()
|
|
chat, err := f.db.GetChatByID(testutil.Context(t, testutil.WaitShort), chatID)
|
|
require.NoError(t, err)
|
|
return forceExecutionStateWithDeadline(t, f, chatID, chat.Status, chat.Archived, &deadline)
|
|
}
|
|
|
|
func (f *workerTestFixture) requireWatchEvent(t *testing.T, chatID uuid.UUID, kind codersdk.ChatWatchEventKind) {
|
|
t.Helper()
|
|
// Watch events are published after the corresponding database update
|
|
// commits, so poll instead of asserting on a single snapshot.
|
|
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(_ context.Context) bool {
|
|
for _, event := range f.pubsub.watchEvents(t) {
|
|
if event.Kind == kind && event.Chat.ID == chatID {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}, testutil.IntervalFast)
|
|
}
|
|
|
|
func (f *workerTestFixture) requireNoWatchEvents(t *testing.T) {
|
|
t.Helper()
|
|
require.Empty(t, f.pubsub.watchEvents(t))
|
|
}
|
|
|
|
func startRealTaskWorker(t *testing.T, f *workerTestFixture) *chatWorker {
|
|
t.Helper()
|
|
buffer := messagepartbuffer.New(messagepartbuffer.Options{})
|
|
t.Cleanup(buffer.Close)
|
|
opts := testOptions(t, f, nil)
|
|
// Without a task starter, the worker builds its real one.
|
|
opts.TaskStarter = nil
|
|
opts.Logger = slog.Make()
|
|
opts.MessagePartBuffer = buffer
|
|
opts.TaskRetryInitialBackoff = time.Millisecond
|
|
opts.TaskRetryMaxBackoff = time.Millisecond
|
|
worker, err := newChatWorker(newUnstartedServer(t, f.pubsub.Pubsub, f.db), opts)
|
|
require.NoError(t, err)
|
|
require.NoError(t, worker.Start(context.Background()))
|
|
t.Cleanup(func() { require.NoError(t, worker.Close()) })
|
|
return worker
|
|
}
|
|
|
|
func waitOwnedChat(t *testing.T, f *workerTestFixture, chatID uuid.UUID, workerID uuid.UUID) database.Chat {
|
|
t.Helper()
|
|
var latest database.Chat
|
|
testutil.Eventually(testutil.Context(t, testutil.WaitLong), t, func(ctx context.Context) bool {
|
|
chat, err := f.db.GetChatByID(ctx, chatID)
|
|
if err != nil {
|
|
return false
|
|
}
|
|
latest = chat
|
|
return chat.WorkerID.Valid && chat.WorkerID.UUID == workerID && chat.RunnerID.Valid
|
|
}, testutil.IntervalFast)
|
|
return latest
|
|
}
|
|
|
|
type taskSideEffectRecorder struct {
|
|
mu sync.Mutex
|
|
hints []runnerStateUpdate
|
|
cleanups []runnerKey
|
|
interrupts []interruptionOutcome
|
|
}
|
|
|
|
func newTaskSideEffectRecorder() *taskSideEffectRecorder {
|
|
return &taskSideEffectRecorder{}
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) routeStateHint(_ context.Context, state runnerStateUpdate) {
|
|
r.mu.Lock()
|
|
r.hints = append(r.hints, state)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requestCleanup(_ context.Context, key runnerKey) {
|
|
r.mu.Lock()
|
|
r.cleanups = append(r.cleanups, key)
|
|
r.mu.Unlock()
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) afterInterruptionOutcome(_ context.Context, outcome interruptionOutcome) error {
|
|
r.mu.Lock()
|
|
r.interrupts = append(r.interrupts, outcome)
|
|
r.mu.Unlock()
|
|
return nil
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requireStateHint(t *testing.T, chatID uuid.UUID, snapshot int64, status database.ChatStatus) {
|
|
t.Helper()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, hint := range r.hints {
|
|
if hint.ChatID == chatID && hint.SnapshotVersion == snapshot && hint.Status == status {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("missing state hint chat_id=%s snapshot=%d status=%s hints=%v", chatID, snapshot, status, r.hints)
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requireStateHintCount(t *testing.T, count int) {
|
|
t.Helper()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
require.Len(t, r.hints, count)
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requireCleanup(t *testing.T, chatID uuid.UUID, runnerID uuid.UUID) {
|
|
t.Helper()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, cleanup := range r.cleanups {
|
|
if cleanup.ChatID == chatID && cleanup.RunnerID == runnerID {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("missing cleanup chat_id=%s runner_id=%s cleanups=%v", chatID, runnerID, r.cleanups)
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requireCleanupCount(t *testing.T, count int) {
|
|
t.Helper()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
require.Len(t, r.cleanups, count)
|
|
}
|
|
|
|
func (r *taskSideEffectRecorder) requireInterruptionOutcome(t *testing.T, chatID uuid.UUID, status database.ChatStatus) {
|
|
t.Helper()
|
|
r.mu.Lock()
|
|
defer r.mu.Unlock()
|
|
for _, outcome := range r.interrupts {
|
|
if outcome.Chat.ID == chatID && outcome.Chat.Status == status {
|
|
return
|
|
}
|
|
}
|
|
t.Fatalf("missing interruption outcome chat_id=%s status=%s outcomes=%v", chatID, status, r.interrupts)
|
|
}
|
|
|
|
func newTestTaskStarter(t *testing.T, f *workerTestFixture, recorder *taskSideEffectRecorder, serverOpts ...internalTestServerOpt) *taskStarter {
|
|
t.Helper()
|
|
return newTestTaskStarterWithClock(t, f, recorder, quartz.NewReal(), serverOpts...)
|
|
}
|
|
|
|
// newTestTaskStarterWithClock shares the clock between the starter and its
|
|
// message part buffer, mirroring production wiring.
|
|
func newTestTaskStarterWithClock(
|
|
t *testing.T,
|
|
f *workerTestFixture,
|
|
recorder *taskSideEffectRecorder,
|
|
clock quartz.Clock,
|
|
serverOpts ...internalTestServerOpt,
|
|
) *taskStarter {
|
|
t.Helper()
|
|
buffer := messagepartbuffer.New(messagepartbuffer.Options{Clock: clock})
|
|
t.Cleanup(buffer.Close)
|
|
starter, err := newTaskStarter(newUnstartedServer(t, f.pubsub.Pubsub, f.db, serverOpts...), chatWorkerOptions{
|
|
Store: f.db,
|
|
Pubsub: f.pubsub,
|
|
Logger: slog.Make(),
|
|
Clock: clock,
|
|
MessagePartBuffer: buffer,
|
|
TaskRetryInitialBackoff: time.Millisecond,
|
|
TaskRetryMaxBackoff: time.Millisecond,
|
|
}, recorder.routeStateHint, recorder.requestCleanup)
|
|
require.NoError(t, err)
|
|
starter.afterInterruptionOutcome = recorder.afterInterruptionOutcome
|
|
return starter
|
|
}
|