One of the main costs in terms of DB load from chats is extra round trips during transactions. This results in locks being held for longer periods of time -> an increase in lock waits for chats related queries. One of the opportunities for reducing some of the round trips is to eliminate extra chat row reads after we have just read the row in order to lock it. In this PR `ChatMachine.Update` now seeds the transaction struct with the row returned by `LockChatAndBumpSnapshotVersion`, plus a queue count taken right after it. We can then pass it into `Update` as an optional parameter validate against it, so neither re-reads the chat row from the database while the row lock is held. `updateChat` (send, edit, promote, tool results, interrupt, compact, clear, reconcile), hook failure, family archive, and the worker's generation and task transitions use it instead of reloading the chat under the lock. This PR relative to main: |measurement | main | this PR | percent change | |-- | -- | -- | -- | | Round trips under lock | 180.9 | 142.0 | -39% | | Statements under lock | 86.1 | 66.6 | -23% | | GetChatByID under lock | 54.1 | 35.8 | -34% | | GetChatByID total | 75.5 | 57.1 | -25% | | Statements | 235.8 | 216.2 | -8% | Generated by Coder Agents on behalf of @cstyan.
903 lines
30 KiB
Go
903 lines
30 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"errors"
|
|
"net/http"
|
|
"slices"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/google/uuid"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatdebug"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
const (
|
|
postCommitWatchPublishTimeout = 10 * time.Second
|
|
defaultTaskTimeout = 15 * time.Minute
|
|
taskTimeoutMargin = 5 * time.Minute
|
|
// interruptCancelTimeout bounds the agent dial and all cancels of one
|
|
// interrupt. A cancel returns only after the tool call's running
|
|
// request finishes, so a longer timeout records more real results.
|
|
interruptCancelTimeout = 30 * time.Second
|
|
)
|
|
|
|
var (
|
|
errTaskExpectedExit = xerrors.New("chatworker task expected exit")
|
|
errTaskRetryable = xerrors.New("chatworker task retryable error")
|
|
errTaskTimeout = xerrors.New("chatworker task timeout")
|
|
)
|
|
|
|
type taskRetryableError struct {
|
|
err error
|
|
}
|
|
|
|
func (e taskRetryableError) Error() string {
|
|
if e.err == nil {
|
|
return errTaskRetryable.Error()
|
|
}
|
|
return e.err.Error()
|
|
}
|
|
|
|
func (e taskRetryableError) Unwrap() error {
|
|
if e.err == nil {
|
|
return errTaskRetryable
|
|
}
|
|
return errors.Join(errTaskRetryable, e.err)
|
|
}
|
|
|
|
type retryWrapperOptions struct {
|
|
clock quartz.Clock
|
|
logger slog.Logger
|
|
initialDelay time.Duration
|
|
maxDelay time.Duration
|
|
}
|
|
|
|
type retryWrapperTaskInfo struct {
|
|
ChatID uuid.UUID
|
|
WorkerID uuid.UUID
|
|
RunnerID uuid.UUID
|
|
}
|
|
|
|
// runTaskWithRetry ensures that a task doesn't exit until it completes
|
|
// successfully or gets canceled. It retries the task in case of any ephemeral errors.
|
|
// It's critical for the correct operation of the chat runner:
|
|
// this function is THE place that ensures task liveness within the runner.
|
|
func runTaskWithRetry(
|
|
ctx context.Context,
|
|
opts retryWrapperOptions,
|
|
kind taskKind,
|
|
info retryWrapperTaskInfo,
|
|
fn func(context.Context) error,
|
|
) error {
|
|
if opts.clock == nil {
|
|
opts.clock = quartz.NewReal()
|
|
}
|
|
if opts.initialDelay <= 0 {
|
|
opts.initialDelay = defaultTaskRetryInitialBackoff
|
|
}
|
|
if opts.maxDelay <= 0 {
|
|
opts.maxDelay = defaultTaskRetryMaxBackoff
|
|
}
|
|
if opts.maxDelay < opts.initialDelay {
|
|
opts.maxDelay = opts.initialDelay
|
|
}
|
|
delay := opts.initialDelay
|
|
for {
|
|
attemptCtx, cancelAttempt := taskAttemptContext(ctx, opts.clock, kind)
|
|
err := executeTaskSafely(attemptCtx, fn)
|
|
timedOut := errors.Is(context.Cause(attemptCtx), errTaskTimeout)
|
|
cancelAttempt()
|
|
if timedOut && err != nil {
|
|
if !errors.Is(err, errTaskExpectedExit) ||
|
|
errors.Is(err, context.Canceled) ||
|
|
errors.Is(err, context.DeadlineExceeded) ||
|
|
errors.Is(err, errTaskTimeout) {
|
|
err = taskRetryableError{err: errors.Join(errTaskTimeout, err)}
|
|
}
|
|
}
|
|
if err == nil {
|
|
// no log on success to avoid noise
|
|
return nil
|
|
}
|
|
|
|
exitReason := ""
|
|
switch {
|
|
case ctx.Err() != nil:
|
|
exitReason = "context_canceled"
|
|
case errors.Is(err, errTaskExpectedExit) && !errors.Is(err, errTaskRetryable):
|
|
exitReason = "expected_non_retryable_exit"
|
|
}
|
|
if exitReason != "" {
|
|
opts.logger.Debug(ctx, "chatworker task exited",
|
|
slog.F("task_kind", kind),
|
|
slog.F("reason", exitReason),
|
|
slog.F("chat_id", info.ChatID),
|
|
slog.F("worker_id", info.WorkerID),
|
|
slog.F("runner_id", info.RunnerID),
|
|
slogError(err),
|
|
)
|
|
return nil
|
|
}
|
|
|
|
opts.logger.Warn(ctx, "chatworker task retrying",
|
|
slog.F("task_kind", kind),
|
|
slog.F("delay", delay),
|
|
slog.F("chat_id", info.ChatID),
|
|
slog.F("worker_id", info.WorkerID),
|
|
slog.F("runner_id", info.RunnerID),
|
|
slogError(err),
|
|
)
|
|
timer := opts.clock.NewTimer(delay, "chatworker", "task-retry-"+string(kind))
|
|
select {
|
|
case <-timer.C:
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return nil
|
|
}
|
|
timer.Stop()
|
|
if delay < opts.maxDelay {
|
|
delay *= 2
|
|
if delay > opts.maxDelay {
|
|
delay = opts.maxDelay
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func taskAttemptContext(ctx context.Context, clock quartz.Clock, kind taskKind) (context.Context, func()) {
|
|
attemptCtx, cancelCause := context.WithCancelCause(ctx)
|
|
tag := "task-timeout-" + string(kind)
|
|
timer := clock.AfterFunc(defaultTaskTimeout, func() {
|
|
cancelCause(errTaskTimeout)
|
|
}, "chatworker", tag)
|
|
// A silent stream must fail through the silence guard rather than the
|
|
// watchdog, and a healthy long stream must not be cut off.
|
|
attemptCtx = chatloop.WithStreamWatchdog(attemptCtx, func(silence time.Duration) {
|
|
if silence < 0 {
|
|
timer.Stop()
|
|
return
|
|
}
|
|
timer.Reset(max(defaultTaskTimeout, silence+taskTimeoutMargin), "chatworker", tag)
|
|
})
|
|
return attemptCtx, func() {
|
|
timer.Stop()
|
|
cancelCause(nil)
|
|
}
|
|
}
|
|
|
|
func executeTaskSafely(ctx context.Context, fn func(context.Context) error) (err error) {
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
err = xerrors.Errorf("chatworker task panic: %v", recovered)
|
|
}
|
|
}()
|
|
return fn(ctx)
|
|
}
|
|
|
|
type interruptionOutcome struct {
|
|
Chat database.Chat
|
|
Kind runnerActionKind
|
|
}
|
|
|
|
type taskStarter struct {
|
|
server *Server
|
|
opts chatWorkerOptions
|
|
routeStateHint func(context.Context, runnerStateUpdate)
|
|
requestCleanup func(context.Context, runnerKey)
|
|
afterInterruptionOutcome func(context.Context, interruptionOutcome) error
|
|
}
|
|
|
|
func newTaskStarter(
|
|
server *Server,
|
|
opts chatWorkerOptions,
|
|
routeStateHint func(context.Context, runnerStateUpdate),
|
|
requestCleanup func(context.Context, runnerKey),
|
|
) (*taskStarter, error) {
|
|
if server == nil {
|
|
return nil, xerrors.New("chatworker: server is required")
|
|
}
|
|
if opts.Store == nil {
|
|
return nil, xerrors.New("chatworker: task store is required")
|
|
}
|
|
if opts.Pubsub == nil {
|
|
return nil, xerrors.New("chatworker: task pubsub is required")
|
|
}
|
|
if opts.MessagePartBuffer == nil {
|
|
return nil, xerrors.New("chatworker: message part buffer is required")
|
|
}
|
|
if opts.Clock == nil {
|
|
opts.Clock = quartz.NewReal()
|
|
}
|
|
if opts.TaskRetryInitialBackoff <= 0 {
|
|
opts.TaskRetryInitialBackoff = defaultTaskRetryInitialBackoff
|
|
}
|
|
if opts.TaskRetryMaxBackoff <= 0 {
|
|
opts.TaskRetryMaxBackoff = defaultTaskRetryMaxBackoff
|
|
}
|
|
if opts.TaskRetryMaxBackoff < opts.TaskRetryInitialBackoff {
|
|
opts.TaskRetryMaxBackoff = opts.TaskRetryInitialBackoff
|
|
}
|
|
if routeStateHint == nil {
|
|
return nil, xerrors.New("chatworker: route state hint callback is required")
|
|
}
|
|
if requestCleanup == nil {
|
|
return nil, xerrors.New("chatworker: cleanup callback is required")
|
|
}
|
|
return &taskStarter{
|
|
server: server,
|
|
opts: opts,
|
|
routeStateHint: routeStateHint,
|
|
requestCleanup: requestCleanup,
|
|
}, nil
|
|
}
|
|
|
|
func (o chatWorkerOptions) retryOptions() retryWrapperOptions {
|
|
return retryWrapperOptions{
|
|
clock: o.Clock,
|
|
logger: o.Logger,
|
|
initialDelay: o.TaskRetryInitialBackoff,
|
|
maxDelay: o.TaskRetryMaxBackoff,
|
|
}
|
|
}
|
|
|
|
func (s *taskStarter) StartInterrupt(ctx context.Context, input chatWorkerTaskStartInput) error {
|
|
machine := chatstate.NewChatMachine(s.opts.Store, s.opts.Pubsub, input.ChatID)
|
|
var (
|
|
chat database.Chat
|
|
cancelable []fantasy.ToolCallContent
|
|
toolCallIDs map[string]uuid.UUID
|
|
)
|
|
err := machine.ReadSnapshot(func(store database.Store) error {
|
|
loadedChat, err := loadChatForTask(ctx, store, input, database.ChatStatusInterrupting, taskFenceOptions{requireHistory: true})
|
|
if err != nil {
|
|
return xerrors.Errorf("load chat for task: %w", err)
|
|
}
|
|
chat = loadedChat
|
|
cancelable, toolCallIDs, err = s.cancelableToolCalls(ctx, store, chat)
|
|
return err
|
|
})
|
|
if err != nil {
|
|
return normalizeTaskInfrastructureError(err, "lock chat for interrupt")
|
|
}
|
|
// Mark before the commit: it can promote a queued prompt whose turn
|
|
// would otherwise close this one as abandoned.
|
|
stoppedTurn := input.TurnSpan.OpenToken(ctx)
|
|
input.TurnSpan.Invalidate(stoppedTurn, chatloop.TurnOutcomeInterrupted, nil)
|
|
|
|
key := messagepartbuffer.Key{
|
|
ChatID: input.ChatID,
|
|
HistoryVersion: input.HistoryVersion,
|
|
GenerationAttempt: chat.GenerationAttempt,
|
|
}
|
|
modelInvokedAt := s.opts.MessagePartBuffer.ModelInvokedAt(key)
|
|
toolCompletions := s.opts.MessagePartBuffer.ToolCompletions(key)
|
|
if err := s.opts.MessagePartBuffer.CloseEpisode(key); err != nil {
|
|
if ctx.Err() != nil {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("close message part episode: %w", err), ctx.Err())
|
|
}
|
|
return taskRetryableError{err: xerrors.Errorf("close message part episode: %w", err)}
|
|
}
|
|
parts, err := s.opts.MessagePartBuffer.GetParts(key)
|
|
if errors.Is(err, messagepartbuffer.ErrEpisodeNotFound) {
|
|
parts = nil
|
|
err = nil
|
|
}
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("get message part episode: %w", err), ctx.Err())
|
|
}
|
|
return taskRetryableError{err: xerrors.Errorf("get message part episode: %w", err)}
|
|
}
|
|
interruptedAt := s.opts.Clock.Now("chatworker", "interrupt")
|
|
var attemptRuntime time.Duration
|
|
if !modelInvokedAt.IsZero() {
|
|
attemptRuntime = interruptedAt.Sub(modelInvokedAt)
|
|
}
|
|
partialMessages, err := bufferedPartsToPartialMessages(bufferedPartsToPartialMessagesInput{
|
|
parts: parts,
|
|
modelConfigID: chat.LastModelConfigID,
|
|
contentVersion: chatprompt.CurrentContentVersion,
|
|
logger: s.opts.Logger,
|
|
interruptedAt: interruptedAt,
|
|
attemptRuntime: attemptRuntime,
|
|
})
|
|
if err != nil {
|
|
return xerrors.Errorf("convert buffered parts: %w", err)
|
|
}
|
|
|
|
canceled := s.cancelUnresolvedToolCalls(ctx, chat, cancelable, toolCallIDs)
|
|
|
|
var promotedQueuedAt time.Time
|
|
committed, err := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store, snapshot *chatstate.ChatSnapshot) error {
|
|
chat, err := verifyLockedTaskFence(snapshot, input, database.ChatStatusInterrupting, taskFenceOptions{requireHistory: true})
|
|
if err != nil {
|
|
return xerrors.Errorf("verify task fence: %w", err)
|
|
}
|
|
messages := partialMessages
|
|
// Reuse the captured interrupt instant so database delay does not
|
|
// inflate billing.
|
|
committedCancels, err := committedPendingLocalToolCancellationMessages(ctx, store, chat, interruptedAt, toolCompletions, canceled)
|
|
if err != nil {
|
|
return xerrors.Errorf("committed pending local tool cancellation messages: %w", err)
|
|
}
|
|
if len(committedCancels) > 0 {
|
|
messages = append(append([]chatstate.Message{}, partialMessages...), committedCancels...)
|
|
}
|
|
finishResult, err := tx.FinishInterruption(chatstate.FinishInterruptionInput{
|
|
Snapshot: snapshot,
|
|
PartialMessages: messages,
|
|
})
|
|
if err != nil {
|
|
return xerrors.Errorf("finish interruption: %w", err)
|
|
}
|
|
promotedQueuedAt = finishResult.PromotedQueuedAt
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
if current, ok := s.committedStateAfterUpdateError(ctx, committed); ok {
|
|
input.TurnSpan.Settle(stoppedTurn)
|
|
s.server.recordQueueWait(ctx, committed, promotedQueuedAt)
|
|
return s.publishWatchAndRoute(ctx, current, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
return normalizeTaskTransitionError(err, "finish interruption")
|
|
}
|
|
input.DebugTurn.RecordOutcome(chatdebug.StatusInterrupted)
|
|
input.TurnSpan.Settle(stoppedTurn)
|
|
s.server.recordQueueWait(ctx, committed, promotedQueuedAt)
|
|
if err := s.publishWatchAndRoute(ctx, committed, codersdk.ChatWatchEventKindStatusChange); err != nil {
|
|
return xerrors.Errorf("publish watch and route: %w", err)
|
|
}
|
|
return s.runAfterInterruptionOutcome(ctx, interruptionOutcome{
|
|
Chat: committed,
|
|
Kind: runnerActionKindFinishInterruption,
|
|
})
|
|
}
|
|
|
|
func (s *taskStarter) runAfterInterruptionOutcome(ctx context.Context, outcome interruptionOutcome) error {
|
|
afterOutcome := s.afterInterruptionOutcome
|
|
if afterOutcome == nil {
|
|
afterOutcome = s.server.afterInterruptionOutcome
|
|
}
|
|
if afterOutcome == nil {
|
|
return nil
|
|
}
|
|
if err := afterOutcome(ctx, outcome); err != nil {
|
|
return taskRetryableError{err: xerrors.Errorf("interruption post-outcome side effects: %w", err)}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (s *taskStarter) StartRequiresActionTimeout(ctx context.Context, input chatWorkerTaskStartInput) error {
|
|
machine := chatstate.NewChatMachine(s.opts.Store, s.opts.Pubsub, input.ChatID)
|
|
if s.server.disableCallerSuppliedTools {
|
|
return s.cancelRequiresAction(ctx, machine, input, "Tool execution canceled because caller-supplied tools are disabled")
|
|
}
|
|
for {
|
|
decision, err := decideRequiresActionTimeout(ctx, machine, input)
|
|
if err != nil {
|
|
return xerrors.Errorf("decide requires action timeout: %w", err)
|
|
}
|
|
if decision.cancel {
|
|
return s.cancelRequiresAction(ctx, machine, input, decision.reason)
|
|
}
|
|
if !decision.waitUntil.Valid {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("requires action deadline is missing"))
|
|
}
|
|
if err := s.waitUntil(ctx, decision.waitUntil.Time); err != nil {
|
|
return xerrors.Errorf("wait until: %w", err)
|
|
}
|
|
}
|
|
}
|
|
|
|
type requiresActionTimeoutDecision struct {
|
|
cancel bool
|
|
reason string
|
|
waitUntil sql.NullTime
|
|
}
|
|
|
|
func decideRequiresActionTimeout(
|
|
ctx context.Context,
|
|
machine *chatstate.ChatMachine,
|
|
input chatWorkerTaskStartInput,
|
|
) (requiresActionTimeoutDecision, error) {
|
|
var decision requiresActionTimeoutDecision
|
|
err := machine.ReadSnapshot(func(store database.Store) error {
|
|
chat, err := loadChatForTask(ctx, store, input, database.ChatStatusRequiresAction, taskFenceOptions{requireHistory: true})
|
|
if err != nil {
|
|
return xerrors.Errorf("load chat for task: %w", err)
|
|
}
|
|
if !chat.RequiresActionDeadlineAt.Valid {
|
|
decision.cancel = true
|
|
decision.reason = "Tool execution canceled because the action deadline was missing"
|
|
return nil
|
|
}
|
|
now, err := store.GetDatabaseNow(ctx)
|
|
if err != nil {
|
|
return xerrors.Errorf("get database time: %w", err)
|
|
}
|
|
if now.Before(chat.RequiresActionDeadlineAt.Time) {
|
|
decision.waitUntil = chat.RequiresActionDeadlineAt
|
|
return nil
|
|
}
|
|
decision.cancel = true
|
|
decision.reason = "Tool execution timed out"
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return requiresActionTimeoutDecision{}, normalizeTaskInfrastructureError(err, "lock chat for requires action timeout")
|
|
}
|
|
return decision, nil
|
|
}
|
|
|
|
func (s *taskStarter) waitUntil(ctx context.Context, deadline time.Time) error {
|
|
now := s.opts.Clock.Now("chatworker", "requires-action-timeout")
|
|
if !now.Before(deadline) {
|
|
return nil
|
|
}
|
|
timer := s.opts.Clock.NewTimer(deadline.Sub(now), "chatworker", "requires-action-timeout")
|
|
defer timer.Stop()
|
|
select {
|
|
case <-timer.C:
|
|
return nil
|
|
case <-ctx.Done():
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("wait until: %w", ctx.Err()))
|
|
}
|
|
}
|
|
|
|
func (s *taskStarter) cancelRequiresAction(
|
|
ctx context.Context,
|
|
machine *chatstate.ChatMachine,
|
|
input chatWorkerTaskStartInput,
|
|
reason string,
|
|
) error {
|
|
committed, err := machine.Update(ctx, func(tx *chatstate.Tx, store database.Store, snapshot *chatstate.ChatSnapshot) error {
|
|
chat, err := verifyLockedTaskFence(snapshot, input, database.ChatStatusRequiresAction, taskFenceOptions{requireHistory: true})
|
|
if err != nil {
|
|
return xerrors.Errorf("verify task fence: %w", err)
|
|
}
|
|
if !s.server.disableCallerSuppliedTools && chat.RequiresActionDeadlineAt.Valid {
|
|
now, err := store.GetDatabaseNow(ctx)
|
|
if err != nil {
|
|
return xerrors.Errorf("get database time: %w", err)
|
|
}
|
|
if now.Before(chat.RequiresActionDeadlineAt.Time) {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("requires action deadline is in the future"))
|
|
}
|
|
}
|
|
if _, err := tx.CancelRequiresAction(chatstate.CancelRequiresActionInput{Snapshot: snapshot, Reason: reason}); err != nil {
|
|
return xerrors.Errorf("cancel requires action: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
if current, ok := s.committedStateAfterUpdateError(ctx, committed); ok {
|
|
return s.publishWatchAndRoute(ctx, current, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
return normalizeTaskTransitionError(err, "cancel requires action")
|
|
}
|
|
return s.publishWatchAndRoute(ctx, committed, codersdk.ChatWatchEventKindStatusChange)
|
|
}
|
|
|
|
func (s *taskStarter) StartAbandon(ctx context.Context, input chatWorkerTaskStartInput) error {
|
|
machine := chatstate.NewChatMachine(s.opts.Store, s.opts.Pubsub, input.ChatID)
|
|
mismatch := false
|
|
_, err := machine.Update(ctx, func(tx *chatstate.Tx, _ database.Store, snapshot *chatstate.ChatSnapshot) error {
|
|
chat := snapshot.Chat
|
|
if !ownedByTask(chat, input) {
|
|
mismatch = true
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("chat not owned by task"))
|
|
}
|
|
if err := verifyTaskFence(chat, input, input.Status, taskFenceOptions{requireHistory: true, allowArchived: true}); err != nil {
|
|
return xerrors.Errorf("task fence mismatch: %w", err)
|
|
}
|
|
if _, err := tx.Abandon(chatstate.AbandonInput{Snapshot: snapshot}); err != nil {
|
|
return xerrors.Errorf("abandon chat: %w", err)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
if errors.Is(err, errTaskExpectedExit) && mismatch {
|
|
s.requestCleanup(ctx, runnerKey{ChatID: input.ChatID, RunnerID: input.RunnerID})
|
|
return nil
|
|
}
|
|
return normalizeTaskTransitionError(err, "abandon chat")
|
|
}
|
|
s.requestCleanup(ctx, runnerKey{ChatID: input.ChatID, RunnerID: input.RunnerID})
|
|
return nil
|
|
}
|
|
|
|
func (s *taskStarter) committedStateAfterUpdateError(ctx context.Context, committed database.Chat) (database.Chat, bool) {
|
|
if committed.ID == uuid.Nil {
|
|
return database.Chat{}, false
|
|
}
|
|
current, err := s.opts.Store.GetChatByID(ctx, committed.ID)
|
|
if err != nil {
|
|
return database.Chat{}, false
|
|
}
|
|
if current.SnapshotVersion != committed.SnapshotVersion ||
|
|
current.HistoryVersion != committed.HistoryVersion ||
|
|
current.QueueVersion != committed.QueueVersion ||
|
|
current.GenerationAttempt != committed.GenerationAttempt ||
|
|
current.Status != committed.Status ||
|
|
current.Archived != committed.Archived ||
|
|
current.WorkerID != committed.WorkerID ||
|
|
current.RunnerID != committed.RunnerID {
|
|
return database.Chat{}, false
|
|
}
|
|
return current, true
|
|
}
|
|
|
|
func (s *taskStarter) publishWatchAndRoute(
|
|
ctx context.Context,
|
|
chat database.Chat,
|
|
kind codersdk.ChatWatchEventKind,
|
|
) error {
|
|
watchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), postCommitWatchPublishTimeout)
|
|
defer cancel()
|
|
if err := s.publishWatchWithRetry(watchCtx, chat, kind); err != nil {
|
|
return xerrors.Errorf("publish watch with retry: %w", err)
|
|
}
|
|
s.routeStateHint(ctx, stateUpdateFromChat(chat))
|
|
return nil
|
|
}
|
|
|
|
func (s *taskStarter) publishWatchWithRetry(
|
|
ctx context.Context,
|
|
chat database.Chat,
|
|
kind codersdk.ChatWatchEventKind,
|
|
) error {
|
|
delay := s.opts.TaskRetryInitialBackoff
|
|
for {
|
|
if err := publishChatWatchEvent(s.opts.Pubsub, chat, kind); err == nil {
|
|
return nil
|
|
} else if ctx.Err() != nil {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("publishChatWatchEvent: %w", ctx.Err()))
|
|
}
|
|
timer := s.opts.Clock.NewTimer(delay, "chatworker", "watch-publish-retry")
|
|
select {
|
|
case <-timer.C:
|
|
case <-ctx.Done():
|
|
timer.Stop()
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("watch publish retry context done: %w", ctx.Err()))
|
|
}
|
|
timer.Stop()
|
|
if delay < s.opts.TaskRetryMaxBackoff {
|
|
delay *= 2
|
|
if delay > s.opts.TaskRetryMaxBackoff {
|
|
delay = s.opts.TaskRetryMaxBackoff
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func publishChatWatchEvent(pubsub chatWorkerPubsub, chat database.Chat, kind codersdk.ChatWatchEventKind) error {
|
|
event := codersdk.ChatWatchEvent{
|
|
Kind: kind,
|
|
Chat: chatWatchEventSDKChat(chat, nil),
|
|
}
|
|
payload, err := json.Marshal(event)
|
|
if err != nil {
|
|
return xerrors.Errorf("marshal chat watch event: %w", err)
|
|
}
|
|
if err := pubsub.Publish(coderdpubsub.ChatWatchEventChannel(chat.OwnerID), payload); err != nil {
|
|
return xerrors.Errorf("publish chat watch event: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
type taskFenceOptions struct {
|
|
requireHistory bool
|
|
allowArchived bool
|
|
}
|
|
|
|
// loadChatForTask loads the chat row and verifies the task fence in one
|
|
// step so call sites cannot skip the fence check. It returns an error
|
|
// wrapping errTaskExpectedExit when the chat no longer exists or the fence
|
|
// no longer matches.
|
|
func loadChatForTask(
|
|
ctx context.Context,
|
|
store database.Store,
|
|
input chatWorkerTaskStartInput,
|
|
status database.ChatStatus,
|
|
opts taskFenceOptions,
|
|
) (database.Chat, error) {
|
|
chat, err := store.GetChatByID(ctx, input.ChatID)
|
|
if err != nil {
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return database.Chat{}, errors.Join(errTaskExpectedExit, xerrors.Errorf("load chat: %w", err))
|
|
}
|
|
return database.Chat{}, xerrors.Errorf("load chat: %w", err)
|
|
}
|
|
if err := verifyTaskFence(chat, input, status, opts); err != nil {
|
|
return database.Chat{}, xerrors.Errorf("verifyTaskFence: %w", err)
|
|
}
|
|
return chat, nil
|
|
}
|
|
|
|
// verifyLockedTaskFence is loadChatForTask for [chatstate.ChatMachine.Update]
|
|
// callbacks: it fences against the snapshot of the locked row instead of
|
|
// reading the chat again while the lock is held.
|
|
func verifyLockedTaskFence(
|
|
snapshot *chatstate.ChatSnapshot,
|
|
input chatWorkerTaskStartInput,
|
|
status database.ChatStatus,
|
|
opts taskFenceOptions,
|
|
) (database.Chat, error) {
|
|
chat := snapshot.Chat
|
|
if err := verifyTaskFence(chat, input, status, opts); err != nil {
|
|
return database.Chat{}, xerrors.Errorf("verifyTaskFence: %w", err)
|
|
}
|
|
return chat, nil
|
|
}
|
|
|
|
func verifyTaskFence(
|
|
chat database.Chat,
|
|
input chatWorkerTaskStartInput,
|
|
status database.ChatStatus,
|
|
opts taskFenceOptions,
|
|
) error {
|
|
if !ownedByTask(chat, input) {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("chat not owned by task"))
|
|
}
|
|
if chat.Status != status {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("chat status mismatch: %s != %s", chat.Status, status))
|
|
}
|
|
if !opts.allowArchived && chat.Archived {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("chat archived"))
|
|
}
|
|
if opts.requireHistory && chat.HistoryVersion != input.HistoryVersion {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("chat history version mismatch: %d != %d", chat.HistoryVersion, input.HistoryVersion))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ownedByTask(chat database.Chat, input chatWorkerTaskStartInput) bool {
|
|
return chat.WorkerID.Valid && chat.WorkerID.UUID == input.WorkerID &&
|
|
chat.RunnerID.Valid && chat.RunnerID.UUID == input.RunnerID
|
|
}
|
|
|
|
func normalizeTaskInfrastructureError(err error, action string) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if errors.Is(err, errTaskExpectedExit) {
|
|
return err
|
|
}
|
|
if errors.Is(err, chatstate.ErrChatNotFound) || errors.Is(err, sql.ErrNoRows) || errors.Is(err, context.Canceled) {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("%s: %w", action, err))
|
|
}
|
|
return taskRetryableError{err: xerrors.Errorf("%s: %w", action, err)}
|
|
}
|
|
|
|
func normalizeTaskTransitionError(err error, action string) error {
|
|
if err == nil {
|
|
return nil
|
|
}
|
|
if errors.Is(err, errTaskExpectedExit) {
|
|
return err
|
|
}
|
|
if errors.Is(err, chatstate.ErrChatNotFound) || errors.Is(err, sql.ErrNoRows) || errors.Is(err, context.Canceled) {
|
|
return errors.Join(errTaskExpectedExit, xerrors.Errorf("%s: %w", action, err))
|
|
}
|
|
if errors.Is(err, chatstate.ErrTransitionNotAllowed) || errors.Is(err, chatstate.ErrInvalidState) {
|
|
return xerrors.Errorf("%s: %w", action, err)
|
|
}
|
|
return taskRetryableError{err: xerrors.Errorf("%s: %w", action, err)}
|
|
}
|
|
|
|
func dynamicToolNamesFromChat(chat database.Chat) map[string]bool {
|
|
if !chat.DynamicTools.Valid || len(chat.DynamicTools.RawMessage) == 0 {
|
|
return nil
|
|
}
|
|
var tools []codersdk.DynamicTool
|
|
if err := json.Unmarshal(chat.DynamicTools.RawMessage, &tools); err != nil {
|
|
return nil
|
|
}
|
|
names := make(map[string]bool, len(tools))
|
|
for _, tool := range tools {
|
|
name := strings.TrimSpace(tool.Name)
|
|
if name != "" {
|
|
names[name] = true
|
|
}
|
|
}
|
|
return names
|
|
}
|
|
|
|
// cancelableToolCalls returns the chat's unresolved tool calls that can be
|
|
// canceled on its agent, and their tool call IDs. store must be the chat's
|
|
// locked read, so the calls match the interrupted state. A message read
|
|
// error is returned so the interrupt retries; unparsable history yields no
|
|
// calls.
|
|
func (s *taskStarter) cancelableToolCalls(ctx context.Context, store database.Store, chat database.Chat) ([]fantasy.ToolCallContent, map[string]uuid.UUID, error) {
|
|
if s.server.agentConnFn == nil || !chat.AgentID.Valid {
|
|
return nil, nil, nil
|
|
}
|
|
messages, err := store.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ChatID: chat.ID})
|
|
if err != nil {
|
|
return nil, nil, xerrors.Errorf("load messages to cancel tool calls on agent: %w", err)
|
|
}
|
|
calls, ids := s.cancelableToolCallsFromHistory(ctx, chat, messages)
|
|
return calls, ids, nil
|
|
}
|
|
|
|
// cancelableToolCallsFromHistory returns the unresolved tool calls of the
|
|
// last assistant message in messages that can be canceled on the chat's
|
|
// agent, and their tool call IDs. Unparsable history yields no calls.
|
|
func (s *taskStarter) cancelableToolCallsFromHistory(ctx context.Context, chat database.Chat, messages []database.ChatMessage) ([]fantasy.ToolCallContent, map[string]uuid.UUID) {
|
|
// Every turn that offers dynamic tools also offers the cancelable
|
|
// built-ins, and a built-in takes precedence over a dynamic tool with
|
|
// the same name. So a call to a cancelable tool ran as the built-in even
|
|
// if the chat declares a dynamic tool with that name.
|
|
calls, _, messageID, err := unresolvedToolCallsFromHistory(messages, nil)
|
|
if err != nil {
|
|
s.opts.Logger.Warn(ctx, "find tool calls to cancel on agent", slog.F("chat_id", chat.ID), slog.Error(err))
|
|
return nil, nil
|
|
}
|
|
// Keep only cancelable calls with an ID, so the agent is not dialed
|
|
// without them.
|
|
ids := chattool.ToolCallIDs(chat.ID, messageID, calls)
|
|
calls = slices.DeleteFunc(calls, func(call fantasy.ToolCallContent) bool {
|
|
_, ok := ids[call.ToolCallID]
|
|
return !ok || !chattool.CanCancelToolCall(call.ToolName)
|
|
})
|
|
return calls, ids
|
|
}
|
|
|
|
// cancelUnresolvedToolCalls sends the chat's agent a cancel request for each
|
|
// call and returns the calls' results by provider tool call ID. It logs a
|
|
// warning for each call without a result.
|
|
func (s *taskStarter) cancelUnresolvedToolCalls(ctx context.Context, chat database.Chat, calls []fantasy.ToolCallContent, ids map[string]uuid.UUID) map[string]fantasy.ToolResponse {
|
|
if len(calls) == 0 {
|
|
return nil
|
|
}
|
|
logger := s.opts.Logger.With(slog.F("chat_id", chat.ID))
|
|
ctx, cancel := context.WithTimeout(ctx, interruptCancelTimeout)
|
|
defer cancel()
|
|
conn, release, err := s.server.agentConnFn(ctx, chat.AgentID.UUID)
|
|
if err != nil {
|
|
logger.Warn(ctx, "dial agent to cancel tool calls", slog.Error(err))
|
|
return nil
|
|
}
|
|
defer release()
|
|
// The agent keys tool call IDs by chat.
|
|
conn.SetExtraHeaders(http.Header{workspacesdk.CoderChatIDHeader: {chat.ID.String()}})
|
|
// Send the cancels concurrently so a slow call does not use up the
|
|
// timeout of the others.
|
|
responses := make([]fantasy.ToolResponse, len(calls))
|
|
errs := make([]error, len(calls))
|
|
var wg sync.WaitGroup
|
|
for i, call := range calls {
|
|
wg.Go(func() {
|
|
// executeTaskSafely recovers only the task goroutine, so a panic
|
|
// here would crash the process.
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
errs[i] = xerrors.Errorf("cancel panicked: %v", r)
|
|
}
|
|
}()
|
|
responses[i], errs[i] = chattool.CancelToolCall(ctx, conn, ids[call.ToolCallID], call.ToolName)
|
|
})
|
|
}
|
|
wg.Wait()
|
|
results := make(map[string]fantasy.ToolResponse, len(calls))
|
|
for i, call := range calls {
|
|
if errs[i] != nil {
|
|
logger.Warn(ctx, "cancel tool call on agent",
|
|
slog.F("tool_call_id", call.ToolCallID),
|
|
slog.F("tool_name", call.ToolName),
|
|
slog.Error(errs[i]),
|
|
)
|
|
continue
|
|
}
|
|
results[call.ToolCallID] = responses[i]
|
|
}
|
|
return results
|
|
}
|
|
|
|
func committedPendingLocalToolCancellationMessages(
|
|
ctx context.Context,
|
|
store database.Store,
|
|
chat database.Chat,
|
|
interruptedAt time.Time,
|
|
toolCompletions map[int]messagepartbuffer.ToolCompletion,
|
|
canceled map[string]fantasy.ToolResponse,
|
|
) ([]chatstate.Message, error) {
|
|
messages, err := store.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{
|
|
ChatID: chat.ID,
|
|
AfterID: 0,
|
|
})
|
|
if err != nil {
|
|
return nil, xerrors.Errorf("load committed messages for interruption: %w", err)
|
|
}
|
|
localCalls, _, _, err := unresolvedToolCallsFromHistory(messages, dynamicToolNamesFromChat(chat))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(localCalls) == 0 {
|
|
return nil, nil
|
|
}
|
|
var intervals []chatloop.BilledInterval
|
|
result := make([]chatstate.Message, 0, len(localCalls))
|
|
for i, call := range localCalls {
|
|
resp, ok := canceled[call.ToolCallID]
|
|
if !ok {
|
|
resp = fantasy.NewTextErrorResponse(interruptedToolResultErrorMessage)
|
|
}
|
|
// Cap the result as chatloop caps tool results. The default budget
|
|
// fits an execute result, whose output the agent already caps.
|
|
text, _ := chatloop.TruncateToolResultText(resp.Content, chatloop.ToolResultByteBudget(0))
|
|
var output fantasy.ToolResultOutputContent = fantasy.ToolResultOutputContentText{Text: text}
|
|
if resp.IsError {
|
|
output = fantasy.ToolResultOutputContentError{Error: xerrors.New(text)}
|
|
}
|
|
// PartFromContent wraps text that is not valid JSON, such as a
|
|
// truncated JSON result.
|
|
part := chatprompt.PartFromContent(fantasy.ToolResultContent{ToolCallID: call.ToolCallID, ToolName: call.ToolName, Result: output})
|
|
if !interruptedAt.IsZero() {
|
|
part.CreatedAt = &interruptedAt
|
|
}
|
|
content, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{part})
|
|
if err != nil {
|
|
return nil, xerrors.Errorf("marshal interrupted tool result part: %w", err)
|
|
}
|
|
result = append(result, chatstate.Message{
|
|
Role: database.ChatMessageRoleTool,
|
|
Content: content,
|
|
Visibility: database.ChatMessageVisibilityBoth,
|
|
ModelConfigID: uuid.NullUUID{UUID: chat.LastModelConfigID, Valid: chat.LastModelConfigID != uuid.Nil},
|
|
ContentVersion: chatprompt.CurrentContentVersion,
|
|
})
|
|
if unbilledSubagentToolNames[call.ToolName] {
|
|
continue
|
|
}
|
|
// Bill only started calls. Completed calls end at completion and
|
|
// running calls end at the interrupt.
|
|
occurrence, ok := toolCompletions[i]
|
|
if !ok {
|
|
continue
|
|
}
|
|
start := occurrence.StartedAt
|
|
if start.IsZero() {
|
|
continue
|
|
}
|
|
end := occurrence.CompletedAt
|
|
if end.IsZero() {
|
|
end = interruptedAt
|
|
}
|
|
intervals = append(intervals, chatloop.BilledInterval{Start: start, End: end})
|
|
}
|
|
// Bill the interval union once on a dedicated usage record so
|
|
// cancellation rows stay free of batch-level runtime.
|
|
stamp, ok, err := batchUsageMessage(
|
|
chat.LastModelConfigID,
|
|
chatprompt.CurrentContentVersion,
|
|
chatloop.BilledIntervalsDuration(intervals),
|
|
len(intervals),
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if ok {
|
|
result = append(result, stamp)
|
|
}
|
|
return result, nil
|
|
}
|