package chatd import ( "cmp" "context" "database/sql" "encoding/json" "errors" "net/http" "slices" "github.com/google/uuid" "golang.org/x/xerrors" "cdr.dev/slog/v3" "github.com/coder/coder/v2/coderd/database" "github.com/coder/coder/v2/coderd/database/db2sdk" coderdpubsub "github.com/coder/coder/v2/coderd/pubsub" "github.com/coder/coder/v2/coderd/x/chatd/chatstate" "github.com/coder/coder/v2/codersdk" ) type streamLoop struct { chatID uuid.UUID db database.Store logger slog.Logger state streamLocalState } type streamLocalState struct { snapshotVersion int64 historyVersion int64 queueVersion int64 retryVersion int64 knownMessages map[int64]int64 status database.ChatStatus errorHistoryVersion int64 actionRequiredHistoryVersion int64 workerID uuid.NullUUID generationAttempt int64 lastPartSeq int64 // attemptRetired is set once the stream observes that the current // generation attempt failed: a retry payload was recorded for it, or // the chat entered the error status. Its remaining in-flight parts // are dropped until a new history version or generation attempt // starts a fresh preview episode. It stays set when a pending retry // is later canceled (interrupt, error), since the failed attempt does // not come back. attemptRetired bool afterMessageID int64 initialMessageSyncDone bool // When openedWithSnapshotVersion is set, versioned events include // snapshot_version, and history_reset includes from_message_id instead // of being followed by the full history. Clients that open the stream // without snapshot_version expect the full history. openedWithSnapshotVersion bool } type streamSyncHint struct { snapshotVersion int64 historyVersion int64 queueVersion int64 retryVersion int64 status database.ChatStatus workerID uuid.NullUUID generationAttempt int64 } type streamDBSnapshot struct { chat database.Chat changedMessages []database.ChatMessage historyReset bool fullHistory []database.ChatMessage queue []database.ChatQueuedMessage actionRequired *codersdk.ChatStreamActionRequired } // newStreamLoop returns a loop whose first sync is bounded by afterMessageID // or, when snapshotVersion is set, by snapshotVersion. afterMessageID is then // ignored, because a client can have received a message without every // message with a lower ID. func newStreamLoop(chat database.Chat, db database.Store, logger slog.Logger, afterMessageID, snapshotVersion int64) *streamLoop { loop := &streamLoop{ chatID: chat.ID, db: db, logger: logger, state: streamLocalState{ knownMessages: make(map[int64]int64), afterMessageID: afterMessageID, }, } if snapshotVersion > 0 { // Starting one below snapshotVersion makes the first sync read the // messages and the queue that last changed at that version or later. // The version itself is included because a client can have received // only some of its events before the connection closed. loop.state.historyVersion = snapshotVersion - 1 loop.state.queueVersion = snapshotVersion - 1 loop.state.afterMessageID = 0 loop.state.openedWithSnapshotVersion = true } return loop } func streamSyncHintFromUpdate(update coderdpubsub.ChatStateUpdateMessage) streamSyncHint { hint := streamSyncHint{ snapshotVersion: update.SnapshotVersion, historyVersion: update.HistoryVersion, queueVersion: update.QueueVersion, retryVersion: update.RetryStateVersion, status: database.ChatStatus(update.Status), generationAttempt: update.GenerationAttempt, } if update.WorkerID != nil { hint.workerID = uuid.NullUUID{UUID: *update.WorkerID, Valid: true} } return hint } func (l *streamLoop) sync(ctx context.Context, hint streamSyncHint) ([]codersdk.ChatStreamEvent, streamRelayTarget, bool, error) { if !l.shouldFetch(hint) { return nil, l.currentRelayTarget(), false, nil } return l.syncDB(ctx) } func (l *streamLoop) syncDB(ctx context.Context) ([]codersdk.ChatStreamEvent, streamRelayTarget, bool, error) { snapshot, err := l.loadDBSnapshot(ctx) if err != nil { return nil, l.currentRelayTarget(), false, err } if snapshot.chat.SnapshotVersion <= l.state.snapshotVersion { return nil, l.currentRelayTarget(), false, nil } return l.applyDBSnapshot(snapshot), l.currentRelayTarget(), true, nil } func (l *streamLoop) shouldFetch(hint streamSyncHint) bool { if hint.snapshotVersion <= l.state.snapshotVersion { return false } if hint.historyVersion > l.state.historyVersion { return true } if hint.queueVersion > l.state.queueVersion { return true } if hint.retryVersion > l.state.retryVersion { return true } if hint.status != l.state.status { return true } if !sameNullUUID(hint.workerID, l.state.workerID) { return true } if hint.generationAttempt != l.state.generationAttempt { return true } return false } func (l *streamLoop) loadDBSnapshot(ctx context.Context) (streamDBSnapshot, error) { var snapshot streamDBSnapshot machine := chatstate.NewChatMachine(l.db, nil, l.chatID) err := machine.ReadSnapshot(func(tx database.Store) error { chat, err := tx.GetChatByID(ctx, l.chatID) if err != nil { return xerrors.Errorf("get chat for stream: %w", err) } snapshot.chat = chat if chat.HistoryVersion > l.state.historyVersion { afterRevision := l.state.historyVersion if !l.state.initialMessageSyncDone && l.state.afterMessageID > 0 { row, ok, err := l.initialSyncCursor(ctx, tx) if err != nil { return err } if ok { // Include the cursor's own revision: rows committed with it can // have higher IDs that the client has not seen. afterRevision = row.Revision - 1 } } snapshot.changedMessages, err = tx.GetChatMessagesByRevisionForStream(ctx, database.GetChatMessagesByRevisionForStreamParams{ ChatID: l.chatID, AfterRevision: afterRevision, }) if err != nil { return xerrors.Errorf("get changed chat messages: %w", err) } snapshot.historyReset = l.historyResetRequired(snapshot.changedMessages) if snapshot.historyReset && !l.state.openedWithSnapshotVersion { snapshot.fullHistory, err = tx.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ ChatID: l.chatID, AfterID: 0, }) if err != nil { return xerrors.Errorf("get full chat history: %w", err) } } } if chat.QueueVersion > l.state.queueVersion { snapshot.queue, err = tx.GetChatQueuedMessages(ctx, l.chatID) if err != nil { return xerrors.Errorf("get chat queue: %w", err) } } if chat.Status == database.ChatStatusRequiresAction { history := snapshot.fullHistory if len(history) == 0 { history, err = tx.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{ ChatID: l.chatID, AfterID: 0, }) if err != nil { return xerrors.Errorf("get requires_action history: %w", err) } } actionRequired, err := l.actionRequiredFromHistory(chat, history) if err != nil { return err } snapshot.actionRequired = actionRequired } return nil }) if err != nil { return streamDBSnapshot{}, err } return snapshot, nil } // historyResetRequired reports whether changed contains a deleted message the // client may still display. Messages are only deleted as a suffix of the chat: // an edit deletes the edited message and every later one. While message // after_id is live, a deleted message below it was deleted before message // after_id was written, and the client does not hold one above it. After the // initial sync, changed holds only rows written since the previous sync, so // message after_id is absent unless deleted and any deletion requires a reset. func (l *streamLoop) historyResetRequired(changed []database.ChatMessage) bool { hasDeleted := slices.ContainsFunc(changed, func(msg database.ChatMessage) bool { return msg.Deleted }) if !hasDeleted { return false } if l.state.afterMessageID <= 0 { return true } idx := slices.IndexFunc(changed, func(msg database.ChatMessage) bool { return msg.ID == l.state.afterMessageID }) return idx < 0 || changed[idx].Deleted } // initialSyncCursor returns the afterMessageID row and true only when that row // belongs to this chat, so the first fetch can be bounded by its revision. A // cursor from another chat also clears afterMessageID. func (l *streamLoop) initialSyncCursor(ctx context.Context, tx database.Store) (database.ChatMessage, bool, error) { cursor, err := tx.GetChatMessageByIDForStream(ctx, l.state.afterMessageID) if err != nil { if errors.Is(err, sql.ErrNoRows) { return database.ChatMessage{}, false, nil } return database.ChatMessage{}, false, xerrors.Errorf("get stream cursor message: %w", err) } if cursor.ChatID != l.chatID { // Another chat's ID says nothing about which messages the client holds. l.state.afterMessageID = 0 return database.ChatMessage{}, false, nil } return cursor, true, nil } func (*streamLoop) actionRequiredFromHistory(chat database.Chat, messages []database.ChatMessage) (*codersdk.ChatStreamActionRequired, error) { dynamicToolNames, err := parseDynamicToolNames(chat.DynamicTools) if err != nil { return nil, xerrors.Errorf("parse dynamic tools for stream: %w", err) } _, pending, _, err := unresolvedToolCallsFromHistory(messages, dynamicToolNames) if err != nil { return nil, xerrors.Errorf("derive pending dynamic tool calls: %w", err) } toolCalls := make([]codersdk.ChatStreamToolCall, 0, len(pending)) for _, call := range pending { toolCalls = append(toolCalls, codersdk.ChatStreamToolCall{ ToolCallID: call.ToolCallID, ToolName: call.ToolName, Args: call.Args, }) } return &codersdk.ChatStreamActionRequired{ToolCalls: toolCalls}, nil } func (l *streamLoop) applyDBSnapshot(snapshot streamDBSnapshot) []codersdk.ChatStreamEvent { chat := snapshot.chat events := make([]codersdk.ChatStreamEvent, 0) historyChanged := chat.HistoryVersion > l.state.historyVersion generationChanged := chat.GenerationAttempt != l.state.generationAttempt if historyChanged { events = append(events, l.messageEvents(snapshot)...) } if !l.state.initialMessageSyncDone { l.state.initialMessageSyncDone = true } if chat.QueueVersion > l.state.queueVersion { event := codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeQueueUpdate, ChatID: l.chatID, QueuedMessages: db2sdk.ChatQueuedMessages(snapshot.queue), } if l.state.openedWithSnapshotVersion { event.SnapshotVersion = chat.QueueVersion } events = append(events, event) } if l.state.openedWithSnapshotVersion { // A client resumes from the highest snapshot_version it received, so // every lower one must reach it first. Within one version the stable // sort keeps history_reset, then messages by ID, then queue_update. slices.SortStableFunc(events, func(a, b codersdk.ChatStreamEvent) int { return cmp.Compare(a.SnapshotVersion, b.SnapshotVersion) }) } if chat.Status != l.state.status { events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeStatus, ChatID: l.chatID, Status: &codersdk.ChatStreamStatus{Status: codersdk.ChatStatus(chat.Status)}, }) } if chat.Status == database.ChatStatusError && chat.HistoryVersion > l.state.errorHistoryVersion { events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeError, ChatID: l.chatID, Error: l.chatError(chat), }) l.state.errorHistoryVersion = chat.HistoryVersion } if chat.Status == database.ChatStatusRequiresAction && chat.HistoryVersion > l.state.actionRequiredHistoryVersion { actionRequired := snapshot.actionRequired if actionRequired == nil { actionRequired = &codersdk.ChatStreamActionRequired{} } events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeActionRequired, ChatID: l.chatID, ActionRequired: actionRequired, }) l.state.actionRequiredHistoryVersion = chat.HistoryVersion } // retry_state describes a pending retry of a running chat. Never // announce it for a chat that stopped running, even if a stale payload // is still stored. if chat.RetryStateVersion > l.state.retryVersion && chat.Status == database.ChatStatusRunning { if retry := l.retryEvent(chat); retry != nil { events = append(events, *retry) } } if historyChanged || generationChanged { l.state.attemptRetired = false } // A retry payload is only recorded after the current generation // attempt failed, and the chats trigger clears it when the attempt // changes. Clients drop their preview on the retry event, so parts of // the failed attempt that are still in flight must not reach them. // Within one episode retry_state only changes when a retry is recorded // for the current attempt or that pending retry is cleared, so an // advanced retry_state_version also retires the attempt when the // payload was cleared again before this sync. // // Clients also drop their preview on the error event, and an errored // chat records a new generation attempt before it streams again, so // any later part of the current attempt is from the failed one. retryChangedInEpisode := chat.RetryStateVersion > l.state.retryVersion && !historyChanged && !generationChanged if retryChangedInEpisode || (chat.RetryState.Valid && len(chat.RetryState.RawMessage) > 0) || chat.Status == database.ChatStatusError { l.state.attemptRetired = true } if historyChanged || (generationChanged && chat.GenerationAttempt != 0) { l.state.lastPartSeq = 0 events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypePreviewReset, ChatID: l.chatID, }) } l.state.snapshotVersion = chat.SnapshotVersion l.state.historyVersion = chat.HistoryVersion l.state.queueVersion = chat.QueueVersion l.state.retryVersion = chat.RetryStateVersion l.state.status = chat.Status l.state.workerID = chat.WorkerID l.state.generationAttempt = chat.GenerationAttempt return events } func (l *streamLoop) messageEvents(snapshot streamDBSnapshot) []codersdk.ChatStreamEvent { if l.state.openedWithSnapshotVersion { return l.versionedMessageEvents(snapshot.changedMessages) } if snapshot.historyReset { events := []codersdk.ChatStreamEvent{{ Type: codersdk.ChatStreamEventTypeHistoryReset, ChatID: l.chatID, }} clear(l.state.knownMessages) for _, msg := range snapshot.fullHistory { l.state.knownMessages[msg.ID] = msg.Revision sdkMsg := db2sdk.ChatMessage(msg) events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeMessage, ChatID: l.chatID, Message: &sdkMsg, }) } return events } events := make([]codersdk.ChatStreamEvent, 0, len(snapshot.changedMessages)) for _, msg := range snapshot.changedMessages { // Without a reset, deleted rows are ones the client does not hold. if msg.Deleted { continue } knownRevision := l.state.knownMessages[msg.ID] if knownRevision >= msg.Revision { continue } l.state.knownMessages[msg.ID] = msg.Revision if !l.state.initialMessageSyncDone && msg.ID <= l.state.afterMessageID { continue } sdkMsg := db2sdk.ChatMessage(msg) events = append(events, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeMessage, ChatID: l.chatID, Message: &sdkMsg, }) } return events } // versionedMessageEvents returns, for a stream opened with a snapshot // version, one history_reset per edit, from the lowest ID it deleted, then a // message event per live row. changed contains the rows written since the // previous sync or, on the first sync, at the stream's snapshot version or // later. No full history is needed: an edit deletes the edited message and // every later one, so each live row at or above that ID was written by the // edit or after it, and is among the changed rows. func (l *streamLoop) versionedMessageEvents(changed []database.ChatMessage) []codersdk.ChatStreamEvent { var resets, messages []codersdk.ChatStreamEvent resetVersions := make(map[int64]bool) for _, msg := range changed { if msg.Deleted { // A soft-deleted row keeps the revision of the edit that deleted // it, and changed is ordered by ID. if !resetVersions[msg.Revision] { resetVersions[msg.Revision] = true resets = append(resets, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeHistoryReset, ChatID: l.chatID, HistoryReset: &codersdk.ChatStreamHistoryReset{FromMessageID: msg.ID}, SnapshotVersion: msg.Revision, }) } continue } sdkMsg := db2sdk.ChatMessage(msg) messages = append(messages, codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeMessage, ChatID: l.chatID, Message: &sdkMsg, SnapshotVersion: msg.Revision, }) } return append(resets, messages...) } func (l *streamLoop) chatError(chat database.Chat) *codersdk.ChatError { if !chat.LastError.Valid || len(chat.LastError.RawMessage) == 0 { return &codersdk.ChatError{ Message: "The chat request failed unexpectedly.", Kind: codersdk.ChatErrorKindGeneric, } } var payload codersdk.ChatError if err := json.Unmarshal(chat.LastError.RawMessage, &payload); err != nil { l.logger.Warn(context.Background(), "failed to parse chat stream last_error", slog.F("chat_id", l.chatID), slog.Error(err), ) return &codersdk.ChatError{ Message: "The chat request failed unexpectedly.", Kind: codersdk.ChatErrorKindGeneric, } } if payload.Message == "" { payload.Message = "The chat request failed unexpectedly." } if payload.Kind == "" { payload.Kind = codersdk.ChatErrorKindGeneric } return &payload } func (l *streamLoop) retryEvent(chat database.Chat) *codersdk.ChatStreamEvent { if !chat.RetryState.Valid || len(chat.RetryState.RawMessage) == 0 { return nil } var retry codersdk.ChatStreamRetry if err := json.Unmarshal(chat.RetryState.RawMessage, &retry); err != nil { l.logger.Warn(context.Background(), "failed to parse chat stream retry_state", slog.F("chat_id", l.chatID), slog.Error(err), ) return nil } return &codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeRetry, ChatID: l.chatID, Retry: &retry, } } func (l *streamLoop) part(part streamPart) (event codersdk.ChatStreamEvent, accepted bool, err error) { if part.HistoryVersion != l.state.historyVersion || part.GenerationAttempt != l.state.generationAttempt { return codersdk.ChatStreamEvent{}, false, nil } if l.state.attemptRetired { return codersdk.ChatStreamEvent{}, false, nil } if part.Seq <= l.state.lastPartSeq { return codersdk.ChatStreamEvent{}, false, nil } if part.Seq != l.state.lastPartSeq+1 { err := xerrors.Errorf( "chat stream message part sequence gap: got %d after %d", part.Seq, l.state.lastPartSeq, ) l.logger.Error(context.Background(), "chat stream message part sequence gap", slog.F("chat_id", l.chatID), slog.F("history_version", part.HistoryVersion), slog.F("generation_attempt", part.GenerationAttempt), slog.F("last_seq", l.state.lastPartSeq), slog.F("seq", part.Seq), slog.Error(err), ) return codersdk.ChatStreamEvent{}, false, err } l.state.lastPartSeq = part.Seq return codersdk.ChatStreamEvent{ Type: codersdk.ChatStreamEventTypeMessagePart, ChatID: l.chatID, MessagePart: &codersdk.ChatStreamMessagePart{ Role: part.Role, Part: part.Part, HistoryVersion: part.HistoryVersion, GenerationAttempt: part.GenerationAttempt, Seq: part.Seq, }, }, true, nil } func (l *streamLoop) currentRelayTarget() streamRelayTarget { return streamRelayTarget{ workerID: l.state.workerID, historyVersion: l.state.historyVersion, generationAttempt: l.state.generationAttempt, } } func sameNullUUID(a, b uuid.NullUUID) bool { if a.Valid != b.Valid { return false } if !a.Valid { return true } return a.UUID == b.UUID } func cloneHeader(header http.Header) http.Header { if header == nil { return nil } return header.Clone() }