Files
coder/coderd/x/chatd/stream_loop.go
Mathias Fredriksson d639ced1e0 fix: send only changed chat history over the stream (#29662)
A client could resume the chat stream only with after_id, a message
ID. A message ID does not identify what a client has received. A
reconnect could skip a message that another client sent, send no
history_reset for an edit the client missed, or resend the whole
history.

A client can now resume with the chat's snapshot_version, which the
newest messages page returns and the stream's events then include.
The stream sends what changed at that version or later. For an edit
it sends the ID of the edited message and the messages written
since, not the whole history. A stream opened without
snapshot_version sends the same events as before.

A write outside a chat state transition does not increase
snapshot_version, and a stream can miss it. The queries that only
tests used for such writes are removed.

When the stream cannot be set up, the server now returns 500 before
the WebSocket upgrade. It used to upgrade and send an error event,
which the web client shows as a chat error.

The Tasks to Chats migration guide told clients to reconnect with
after_id. It now describes snapshot_version.
2026-11-13 13:02:38 +02:00

610 lines
20 KiB
Go

package chatd
import (
"cmp"
"context"
"database/sql"
"encoding/json"
"errors"
"net/http"
"slices"
"github.com/google/uuid"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/db2sdk"
coderdpubsub "github.com/coder/coder/v2/coderd/pubsub"
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
"github.com/coder/coder/v2/codersdk"
)
type streamLoop struct {
chatID uuid.UUID
db database.Store
logger slog.Logger
state streamLocalState
}
type streamLocalState struct {
snapshotVersion int64
historyVersion int64
queueVersion int64
retryVersion int64
knownMessages map[int64]int64
status database.ChatStatus
errorHistoryVersion int64
actionRequiredHistoryVersion int64
workerID uuid.NullUUID
generationAttempt int64
lastPartSeq int64
// attemptRetired is set once the stream observes that the current
// generation attempt failed: a retry payload was recorded for it, or
// the chat entered the error status. Its remaining in-flight parts
// are dropped until a new history version or generation attempt
// starts a fresh preview episode. It stays set when a pending retry
// is later canceled (interrupt, error), since the failed attempt does
// not come back.
attemptRetired bool
afterMessageID int64
initialMessageSyncDone bool
// When openedWithSnapshotVersion is set, versioned events include
// snapshot_version, and history_reset includes from_message_id instead
// of being followed by the full history. Clients that open the stream
// without snapshot_version expect the full history.
openedWithSnapshotVersion bool
}
type streamSyncHint struct {
snapshotVersion int64
historyVersion int64
queueVersion int64
retryVersion int64
status database.ChatStatus
workerID uuid.NullUUID
generationAttempt int64
}
type streamDBSnapshot struct {
chat database.Chat
changedMessages []database.ChatMessage
historyReset bool
fullHistory []database.ChatMessage
queue []database.ChatQueuedMessage
actionRequired *codersdk.ChatStreamActionRequired
}
// newStreamLoop returns a loop whose first sync is bounded by afterMessageID
// or, when snapshotVersion is set, by snapshotVersion. afterMessageID is then
// ignored, because a client can have received a message without every
// message with a lower ID.
func newStreamLoop(chat database.Chat, db database.Store, logger slog.Logger, afterMessageID, snapshotVersion int64) *streamLoop {
loop := &streamLoop{
chatID: chat.ID,
db: db,
logger: logger,
state: streamLocalState{
knownMessages: make(map[int64]int64),
afterMessageID: afterMessageID,
},
}
if snapshotVersion > 0 {
// Starting one below snapshotVersion makes the first sync read the
// messages and the queue that last changed at that version or later.
// The version itself is included because a client can have received
// only some of its events before the connection closed.
loop.state.historyVersion = snapshotVersion - 1
loop.state.queueVersion = snapshotVersion - 1
loop.state.afterMessageID = 0
loop.state.openedWithSnapshotVersion = true
}
return loop
}
func streamSyncHintFromUpdate(update coderdpubsub.ChatStateUpdateMessage) streamSyncHint {
hint := streamSyncHint{
snapshotVersion: update.SnapshotVersion,
historyVersion: update.HistoryVersion,
queueVersion: update.QueueVersion,
retryVersion: update.RetryStateVersion,
status: database.ChatStatus(update.Status),
generationAttempt: update.GenerationAttempt,
}
if update.WorkerID != nil {
hint.workerID = uuid.NullUUID{UUID: *update.WorkerID, Valid: true}
}
return hint
}
func (l *streamLoop) sync(ctx context.Context, hint streamSyncHint) ([]codersdk.ChatStreamEvent, streamRelayTarget, bool, error) {
if !l.shouldFetch(hint) {
return nil, l.currentRelayTarget(), false, nil
}
return l.syncDB(ctx)
}
func (l *streamLoop) syncDB(ctx context.Context) ([]codersdk.ChatStreamEvent, streamRelayTarget, bool, error) {
snapshot, err := l.loadDBSnapshot(ctx)
if err != nil {
return nil, l.currentRelayTarget(), false, err
}
if snapshot.chat.SnapshotVersion <= l.state.snapshotVersion {
return nil, l.currentRelayTarget(), false, nil
}
return l.applyDBSnapshot(snapshot), l.currentRelayTarget(), true, nil
}
func (l *streamLoop) shouldFetch(hint streamSyncHint) bool {
if hint.snapshotVersion <= l.state.snapshotVersion {
return false
}
if hint.historyVersion > l.state.historyVersion {
return true
}
if hint.queueVersion > l.state.queueVersion {
return true
}
if hint.retryVersion > l.state.retryVersion {
return true
}
if hint.status != l.state.status {
return true
}
if !sameNullUUID(hint.workerID, l.state.workerID) {
return true
}
if hint.generationAttempt != l.state.generationAttempt {
return true
}
return false
}
func (l *streamLoop) loadDBSnapshot(ctx context.Context) (streamDBSnapshot, error) {
var snapshot streamDBSnapshot
machine := chatstate.NewChatMachine(l.db, nil, l.chatID)
err := machine.ReadSnapshot(func(tx database.Store) error {
chat, err := tx.GetChatByID(ctx, l.chatID)
if err != nil {
return xerrors.Errorf("get chat for stream: %w", err)
}
snapshot.chat = chat
if chat.HistoryVersion > l.state.historyVersion {
afterRevision := l.state.historyVersion
if !l.state.initialMessageSyncDone && l.state.afterMessageID > 0 {
row, ok, err := l.initialSyncCursor(ctx, tx)
if err != nil {
return err
}
if ok {
// Include the cursor's own revision: rows committed with it can
// have higher IDs that the client has not seen.
afterRevision = row.Revision - 1
}
}
snapshot.changedMessages, err = tx.GetChatMessagesByRevisionForStream(ctx, database.GetChatMessagesByRevisionForStreamParams{
ChatID: l.chatID,
AfterRevision: afterRevision,
})
if err != nil {
return xerrors.Errorf("get changed chat messages: %w", err)
}
snapshot.historyReset = l.historyResetRequired(snapshot.changedMessages)
if snapshot.historyReset && !l.state.openedWithSnapshotVersion {
snapshot.fullHistory, err = tx.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{
ChatID: l.chatID,
AfterID: 0,
})
if err != nil {
return xerrors.Errorf("get full chat history: %w", err)
}
}
}
if chat.QueueVersion > l.state.queueVersion {
snapshot.queue, err = tx.GetChatQueuedMessages(ctx, l.chatID)
if err != nil {
return xerrors.Errorf("get chat queue: %w", err)
}
}
if chat.Status == database.ChatStatusRequiresAction {
history := snapshot.fullHistory
if len(history) == 0 {
history, err = tx.GetChatMessagesByChatID(ctx, database.GetChatMessagesByChatIDParams{
ChatID: l.chatID,
AfterID: 0,
})
if err != nil {
return xerrors.Errorf("get requires_action history: %w", err)
}
}
actionRequired, err := l.actionRequiredFromHistory(chat, history)
if err != nil {
return err
}
snapshot.actionRequired = actionRequired
}
return nil
})
if err != nil {
return streamDBSnapshot{}, err
}
return snapshot, nil
}
// historyResetRequired reports whether changed contains a deleted message the
// client may still display. Messages are only deleted as a suffix of the chat:
// an edit deletes the edited message and every later one. While message
// after_id is live, a deleted message below it was deleted before message
// after_id was written, and the client does not hold one above it. After the
// initial sync, changed holds only rows written since the previous sync, so
// message after_id is absent unless deleted and any deletion requires a reset.
func (l *streamLoop) historyResetRequired(changed []database.ChatMessage) bool {
hasDeleted := slices.ContainsFunc(changed, func(msg database.ChatMessage) bool {
return msg.Deleted
})
if !hasDeleted {
return false
}
if l.state.afterMessageID <= 0 {
return true
}
idx := slices.IndexFunc(changed, func(msg database.ChatMessage) bool {
return msg.ID == l.state.afterMessageID
})
return idx < 0 || changed[idx].Deleted
}
// initialSyncCursor returns the afterMessageID row and true only when that row
// belongs to this chat, so the first fetch can be bounded by its revision. A
// cursor from another chat also clears afterMessageID.
func (l *streamLoop) initialSyncCursor(ctx context.Context, tx database.Store) (database.ChatMessage, bool, error) {
cursor, err := tx.GetChatMessageByIDForStream(ctx, l.state.afterMessageID)
if err != nil {
if errors.Is(err, sql.ErrNoRows) {
return database.ChatMessage{}, false, nil
}
return database.ChatMessage{}, false, xerrors.Errorf("get stream cursor message: %w", err)
}
if cursor.ChatID != l.chatID {
// Another chat's ID says nothing about which messages the client holds.
l.state.afterMessageID = 0
return database.ChatMessage{}, false, nil
}
return cursor, true, nil
}
func (*streamLoop) actionRequiredFromHistory(chat database.Chat, messages []database.ChatMessage) (*codersdk.ChatStreamActionRequired, error) {
dynamicToolNames, err := parseDynamicToolNames(chat.DynamicTools)
if err != nil {
return nil, xerrors.Errorf("parse dynamic tools for stream: %w", err)
}
_, pending, _, err := unresolvedToolCallsFromHistory(messages, dynamicToolNames)
if err != nil {
return nil, xerrors.Errorf("derive pending dynamic tool calls: %w", err)
}
toolCalls := make([]codersdk.ChatStreamToolCall, 0, len(pending))
for _, call := range pending {
toolCalls = append(toolCalls, codersdk.ChatStreamToolCall{
ToolCallID: call.ToolCallID,
ToolName: call.ToolName,
Args: call.Args,
})
}
return &codersdk.ChatStreamActionRequired{ToolCalls: toolCalls}, nil
}
func (l *streamLoop) applyDBSnapshot(snapshot streamDBSnapshot) []codersdk.ChatStreamEvent {
chat := snapshot.chat
events := make([]codersdk.ChatStreamEvent, 0)
historyChanged := chat.HistoryVersion > l.state.historyVersion
generationChanged := chat.GenerationAttempt != l.state.generationAttempt
if historyChanged {
events = append(events, l.messageEvents(snapshot)...)
}
if !l.state.initialMessageSyncDone {
l.state.initialMessageSyncDone = true
}
if chat.QueueVersion > l.state.queueVersion {
event := codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeQueueUpdate,
ChatID: l.chatID,
QueuedMessages: db2sdk.ChatQueuedMessages(snapshot.queue),
}
if l.state.openedWithSnapshotVersion {
event.SnapshotVersion = chat.QueueVersion
}
events = append(events, event)
}
if l.state.openedWithSnapshotVersion {
// A client resumes from the highest snapshot_version it received, so
// every lower one must reach it first. Within one version the stable
// sort keeps history_reset, then messages by ID, then queue_update.
slices.SortStableFunc(events, func(a, b codersdk.ChatStreamEvent) int {
return cmp.Compare(a.SnapshotVersion, b.SnapshotVersion)
})
}
if chat.Status != l.state.status {
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeStatus,
ChatID: l.chatID,
Status: &codersdk.ChatStreamStatus{Status: codersdk.ChatStatus(chat.Status)},
})
}
if chat.Status == database.ChatStatusError && chat.HistoryVersion > l.state.errorHistoryVersion {
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeError,
ChatID: l.chatID,
Error: l.chatError(chat),
})
l.state.errorHistoryVersion = chat.HistoryVersion
}
if chat.Status == database.ChatStatusRequiresAction && chat.HistoryVersion > l.state.actionRequiredHistoryVersion {
actionRequired := snapshot.actionRequired
if actionRequired == nil {
actionRequired = &codersdk.ChatStreamActionRequired{}
}
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeActionRequired,
ChatID: l.chatID,
ActionRequired: actionRequired,
})
l.state.actionRequiredHistoryVersion = chat.HistoryVersion
}
// retry_state describes a pending retry of a running chat. Never
// announce it for a chat that stopped running, even if a stale payload
// is still stored.
if chat.RetryStateVersion > l.state.retryVersion && chat.Status == database.ChatStatusRunning {
if retry := l.retryEvent(chat); retry != nil {
events = append(events, *retry)
}
}
if historyChanged || generationChanged {
l.state.attemptRetired = false
}
// A retry payload is only recorded after the current generation
// attempt failed, and the chats trigger clears it when the attempt
// changes. Clients drop their preview on the retry event, so parts of
// the failed attempt that are still in flight must not reach them.
// Within one episode retry_state only changes when a retry is recorded
// for the current attempt or that pending retry is cleared, so an
// advanced retry_state_version also retires the attempt when the
// payload was cleared again before this sync.
//
// Clients also drop their preview on the error event, and an errored
// chat records a new generation attempt before it streams again, so
// any later part of the current attempt is from the failed one.
retryChangedInEpisode := chat.RetryStateVersion > l.state.retryVersion && !historyChanged && !generationChanged
if retryChangedInEpisode || (chat.RetryState.Valid && len(chat.RetryState.RawMessage) > 0) || chat.Status == database.ChatStatusError {
l.state.attemptRetired = true
}
if historyChanged || (generationChanged && chat.GenerationAttempt != 0) {
l.state.lastPartSeq = 0
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypePreviewReset,
ChatID: l.chatID,
})
}
l.state.snapshotVersion = chat.SnapshotVersion
l.state.historyVersion = chat.HistoryVersion
l.state.queueVersion = chat.QueueVersion
l.state.retryVersion = chat.RetryStateVersion
l.state.status = chat.Status
l.state.workerID = chat.WorkerID
l.state.generationAttempt = chat.GenerationAttempt
return events
}
func (l *streamLoop) messageEvents(snapshot streamDBSnapshot) []codersdk.ChatStreamEvent {
if l.state.openedWithSnapshotVersion {
return l.versionedMessageEvents(snapshot.changedMessages)
}
if snapshot.historyReset {
events := []codersdk.ChatStreamEvent{{
Type: codersdk.ChatStreamEventTypeHistoryReset,
ChatID: l.chatID,
}}
clear(l.state.knownMessages)
for _, msg := range snapshot.fullHistory {
l.state.knownMessages[msg.ID] = msg.Revision
sdkMsg := db2sdk.ChatMessage(msg)
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeMessage,
ChatID: l.chatID,
Message: &sdkMsg,
})
}
return events
}
events := make([]codersdk.ChatStreamEvent, 0, len(snapshot.changedMessages))
for _, msg := range snapshot.changedMessages {
// Without a reset, deleted rows are ones the client does not hold.
if msg.Deleted {
continue
}
knownRevision := l.state.knownMessages[msg.ID]
if knownRevision >= msg.Revision {
continue
}
l.state.knownMessages[msg.ID] = msg.Revision
if !l.state.initialMessageSyncDone && msg.ID <= l.state.afterMessageID {
continue
}
sdkMsg := db2sdk.ChatMessage(msg)
events = append(events, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeMessage,
ChatID: l.chatID,
Message: &sdkMsg,
})
}
return events
}
// versionedMessageEvents returns, for a stream opened with a snapshot
// version, one history_reset per edit, from the lowest ID it deleted, then a
// message event per live row. changed contains the rows written since the
// previous sync or, on the first sync, at the stream's snapshot version or
// later. No full history is needed: an edit deletes the edited message and
// every later one, so each live row at or above that ID was written by the
// edit or after it, and is among the changed rows.
func (l *streamLoop) versionedMessageEvents(changed []database.ChatMessage) []codersdk.ChatStreamEvent {
var resets, messages []codersdk.ChatStreamEvent
resetVersions := make(map[int64]bool)
for _, msg := range changed {
if msg.Deleted {
// A soft-deleted row keeps the revision of the edit that deleted
// it, and changed is ordered by ID.
if !resetVersions[msg.Revision] {
resetVersions[msg.Revision] = true
resets = append(resets, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeHistoryReset,
ChatID: l.chatID,
HistoryReset: &codersdk.ChatStreamHistoryReset{FromMessageID: msg.ID},
SnapshotVersion: msg.Revision,
})
}
continue
}
sdkMsg := db2sdk.ChatMessage(msg)
messages = append(messages, codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeMessage,
ChatID: l.chatID,
Message: &sdkMsg,
SnapshotVersion: msg.Revision,
})
}
return append(resets, messages...)
}
func (l *streamLoop) chatError(chat database.Chat) *codersdk.ChatError {
if !chat.LastError.Valid || len(chat.LastError.RawMessage) == 0 {
return &codersdk.ChatError{
Message: "The chat request failed unexpectedly.",
Kind: codersdk.ChatErrorKindGeneric,
}
}
var payload codersdk.ChatError
if err := json.Unmarshal(chat.LastError.RawMessage, &payload); err != nil {
l.logger.Warn(context.Background(), "failed to parse chat stream last_error",
slog.F("chat_id", l.chatID),
slog.Error(err),
)
return &codersdk.ChatError{
Message: "The chat request failed unexpectedly.",
Kind: codersdk.ChatErrorKindGeneric,
}
}
if payload.Message == "" {
payload.Message = "The chat request failed unexpectedly."
}
if payload.Kind == "" {
payload.Kind = codersdk.ChatErrorKindGeneric
}
return &payload
}
func (l *streamLoop) retryEvent(chat database.Chat) *codersdk.ChatStreamEvent {
if !chat.RetryState.Valid || len(chat.RetryState.RawMessage) == 0 {
return nil
}
var retry codersdk.ChatStreamRetry
if err := json.Unmarshal(chat.RetryState.RawMessage, &retry); err != nil {
l.logger.Warn(context.Background(), "failed to parse chat stream retry_state",
slog.F("chat_id", l.chatID),
slog.Error(err),
)
return nil
}
return &codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeRetry,
ChatID: l.chatID,
Retry: &retry,
}
}
func (l *streamLoop) part(part streamPart) (event codersdk.ChatStreamEvent, accepted bool, err error) {
if part.HistoryVersion != l.state.historyVersion || part.GenerationAttempt != l.state.generationAttempt {
return codersdk.ChatStreamEvent{}, false, nil
}
if l.state.attemptRetired {
return codersdk.ChatStreamEvent{}, false, nil
}
if part.Seq <= l.state.lastPartSeq {
return codersdk.ChatStreamEvent{}, false, nil
}
if part.Seq != l.state.lastPartSeq+1 {
err := xerrors.Errorf(
"chat stream message part sequence gap: got %d after %d",
part.Seq,
l.state.lastPartSeq,
)
l.logger.Error(context.Background(), "chat stream message part sequence gap",
slog.F("chat_id", l.chatID),
slog.F("history_version", part.HistoryVersion),
slog.F("generation_attempt", part.GenerationAttempt),
slog.F("last_seq", l.state.lastPartSeq),
slog.F("seq", part.Seq),
slog.Error(err),
)
return codersdk.ChatStreamEvent{}, false, err
}
l.state.lastPartSeq = part.Seq
return codersdk.ChatStreamEvent{
Type: codersdk.ChatStreamEventTypeMessagePart,
ChatID: l.chatID,
MessagePart: &codersdk.ChatStreamMessagePart{
Role: part.Role,
Part: part.Part,
HistoryVersion: part.HistoryVersion,
GenerationAttempt: part.GenerationAttempt,
Seq: part.Seq,
},
}, true, nil
}
func (l *streamLoop) currentRelayTarget() streamRelayTarget {
return streamRelayTarget{
workerID: l.state.workerID,
historyVersion: l.state.historyVersion,
generationAttempt: l.state.generationAttempt,
}
}
func sameNullUUID(a, b uuid.NullUUID) bool {
if a.Valid != b.Valid {
return false
}
if !a.Valid {
return true
}
return a.UUID == b.UUID
}
func cloneHeader(header http.Header) http.Header {
if header == nil {
return nil
}
return header.Clone()
}