The owner of a chat can edit a queued message through
`PATCH /api/v2/chats/{chat}/queue/{queuedMessage}`. chatd never sends
a row under edit (`editing_since`): rows ahead of it still send in
order, and a turn that ends at it puts the chat in the new `paused`
status. Ending the head's edit, deleting it, or promoting a queued row
continues the chat. Interrupting sends still run ahead of the queue.
The chat state machine classifies a queue whose head is under edit as
its own sub-state (R1P, I1P, A1P, E1P, XE1P, P), so every path to
`paused` is in the transition matrix. A chat whose family has a paused
member cannot be archived, and auto-archive skips that family. Queue
edit and delete are owner-only, because both can send the next message
with the owner's credentials.
`ChatQueuedMessage` gains `editing_since` and `reasoning_effort`, and
`ChatStatus` gains `paused`.
Closes CODAGT-1028
Refs CODAGT-971
390 lines
16 KiB
Go
390 lines
16 KiB
Go
package chatloop
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"time"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
|
)
|
|
|
|
const (
|
|
metricsNamespace = "coderd"
|
|
metricsSubsystem = "chatd"
|
|
|
|
// Label values for Chats.
|
|
StateStreaming = "streaming"
|
|
|
|
// Label values for CompactionTotal.
|
|
CompactionResultSuccess = "success"
|
|
CompactionResultError = "error"
|
|
CompactionResultTimeout = "timeout"
|
|
)
|
|
|
|
// StageAnomaly is the `reason` label value of stageAnomaliesTotal.
|
|
type StageAnomaly string
|
|
|
|
// StageAnomaly values. FutureStart and StaleAnchor samples are clamped;
|
|
// the others are dropped.
|
|
const (
|
|
StageAnomalyNegativeElapsed StageAnomaly = "negative_elapsed"
|
|
StageAnomalyInvertedWindow StageAnomaly = "inverted_window"
|
|
StageAnomalyMissingTimestamp StageAnomaly = "missing_timestamp"
|
|
StageAnomalyFutureStart StageAnomaly = "future_start"
|
|
StageAnomalyStaleAnchor StageAnomaly = "stale_anchor"
|
|
)
|
|
|
|
// observedStages get histogram samples; other stages are span-only.
|
|
// The stage_duration_seconds help text must list the same set.
|
|
var observedStages = map[Stage]struct{}{
|
|
StageChatTurn: {},
|
|
StageQueueWait: {},
|
|
StageAcquisition: {},
|
|
StageMCPConnect: {},
|
|
StageStream: {},
|
|
StageTimeToFirstToken: {},
|
|
StageProviderAttempt: {},
|
|
StageToolCall: {},
|
|
StageCommit: {},
|
|
StageRetryBackoff: {},
|
|
}
|
|
|
|
// modelStages measure provider work on a model. Per-model TTFT is
|
|
// TTFTSeconds.
|
|
var modelStages = map[Stage]struct{}{
|
|
StageStream: {},
|
|
StageProviderAttempt: {},
|
|
}
|
|
|
|
var stageDurationBuckets = []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 1800, 3600}
|
|
|
|
// MetricsOptions configures optional metric families.
|
|
type MetricsOptions struct {
|
|
// StageMetrics registers the stage families. When false they are
|
|
// built unregistered so recorders still work.
|
|
StageMetrics bool
|
|
}
|
|
|
|
// Metrics holds Prometheus metrics for the chatd subsystem.
|
|
type Metrics struct {
|
|
Chats *prometheus.GaugeVec
|
|
MessageCount *prometheus.HistogramVec
|
|
PromptSizeBytes *prometheus.HistogramVec
|
|
ResponseSizeBytes *prometheus.HistogramVec
|
|
ToolResultSizeBytes *prometheus.HistogramVec
|
|
ToolResultTruncatedTotal *prometheus.CounterVec
|
|
ToolErrorsTotal *prometheus.CounterVec
|
|
TTFTSeconds *prometheus.HistogramVec
|
|
stageDurationSeconds *prometheus.HistogramVec
|
|
modelStageDurationSeconds *prometheus.HistogramVec
|
|
TurnOutcomesTotal *prometheus.CounterVec
|
|
stageAnomaliesTotal *prometheus.CounterVec
|
|
CompactionTotal *prometheus.CounterVec
|
|
StepsTotal *prometheus.CounterVec
|
|
StreamRetriesTotal *prometheus.CounterVec
|
|
FindToolsCallsTotal prometheus.Counter
|
|
FindToolsEmptyTotal prometheus.Counter
|
|
FindToolsMatchCount prometheus.Histogram
|
|
FindToolsActivationsTotal prometheus.Counter
|
|
}
|
|
|
|
// NewMetrics creates a new Metrics instance registered with the
|
|
// given registerer, with the optional metric families disabled.
|
|
func NewMetrics(reg prometheus.Registerer) *Metrics {
|
|
return NewMetricsWithOptions(reg, MetricsOptions{})
|
|
}
|
|
|
|
// NewMetricsWithOptions creates a new Metrics instance registered with
|
|
// the given registerer.
|
|
func NewMetricsWithOptions(reg prometheus.Registerer, opts MetricsOptions) *Metrics {
|
|
factory := promauto.With(reg)
|
|
stageFactory := factory
|
|
if !opts.StageMetrics {
|
|
stageFactory = promauto.With(nil)
|
|
}
|
|
return &Metrics{
|
|
Chats: factory.NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "chats",
|
|
Help: "Number of chats being processed, by state.",
|
|
}, []string{"state"}),
|
|
MessageCount: factory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "message_count",
|
|
Help: "Number of messages in the prompt per LLM request.",
|
|
Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1, 2, 4, ..., 1024
|
|
}, []string{"provider", "model"}),
|
|
PromptSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "prompt_size_bytes",
|
|
Help: "Estimated byte size of the prompt per LLM request.",
|
|
Buckets: prometheus.ExponentialBuckets(1024, 4, 10), // 1KB .. 256MB
|
|
}, []string{"provider", "model"}),
|
|
ResponseSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "response_size_bytes",
|
|
Help: "Byte size of streamed model response content (assistant text, reasoning, and tool-call input deltas) per LLM request.",
|
|
Buckets: prometheus.ExponentialBuckets(64, 4, 11), // 64B .. 64MB
|
|
}, []string{"provider", "model"}),
|
|
ToolResultSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "tool_result_size_bytes",
|
|
Help: "Size in bytes of each tool execution result.",
|
|
Buckets: prometheus.ExponentialBuckets(64, 4, 9), // 64B .. 4MB
|
|
}, []string{"provider", "model", "tool_name"}),
|
|
ToolResultTruncatedTotal: factory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "tool_result_truncated_total",
|
|
Help: "Total tool results truncated to fit the model context window.",
|
|
}, []string{"provider", "model", "tool_name"}),
|
|
ToolErrorsTotal: factory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "tool_errors_total",
|
|
Help: "Total tool calls that returned an error result.",
|
|
}, []string{"provider", "model", "tool_name"}),
|
|
TTFTSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "ttft_seconds",
|
|
Help: "Time-to-first-token: wall time from LLM request to the first streamed output part: any part other than warnings, keepalives, finish, or error, including start markers such as text_start. Attempts that end before an output part, by failing or by finishing empty, are not observed. The time_to_first_token stage of stage_duration_seconds measures the same window without a model label and is registered only with the chat-stage-metrics experiment.",
|
|
Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
|
|
}, []string{"provider", "model"}),
|
|
stageDurationSeconds: stageFactory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "stage_duration_seconds",
|
|
Help: "Wall time spent in each chat lifecycle stage. Stages overlap, so this is a stage-time profile, not a partition of the turn. scope separates latency attributable to a prompt (stages inside a chat turn, and queue_wait before it) from detached background work; chat_kind is empty without a known chat. Observed: chat_turn, queue_wait, acquisition, mcp_connect, stream, time_to_first_token, provider_attempt, tool_call, commit, retry_backoff; other stages are span-only. Failed windows are observed, except time_to_first_token, which observes only windows closed by an output part, and chat_turn, which observes only completed turns. chat_turn runs from its trigger (the prompt, a submitted client tool result, or a compaction request) to the end of its turn, includes acquisition, and excludes queue_wait and time waiting in requires_action, so a prompt that waits in requires_action is observed once per segment. queue_wait runs from a queued message's creation to its promotion, so it includes time the message spent under edit. Registered only with the chat-stage-metrics experiment.",
|
|
Buckets: stageDurationBuckets,
|
|
}, []string{"stage", "scope", "chat_kind"}),
|
|
modelStageDurationSeconds: stageFactory.NewHistogramVec(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "model_stage_duration_seconds",
|
|
Help: "Wall time of the stages that are a provider's work on a model: stream (open to close) and provider_attempt (one HTTP round trip, closed on response headers). Time to first token per model is ttft_seconds. Failed stream and provider_attempt windows are observed. Observed only for turn-scoped stages, which are also observed on stage_duration_seconds; background-scoped model stages appear on stage_duration_seconds only. provider_type is the configured AI provider type (for example bedrock), not the wire protocol other chatd metrics report as provider. Registered only with the chat-stage-metrics experiment.",
|
|
Buckets: stageDurationBuckets,
|
|
}, []string{"stage", "provider_type", "chat_kind", "model"}),
|
|
TurnOutcomesTotal: stageFactory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "turn_outcomes_total",
|
|
Help: "Total closed chat turns by outcome: completed (the turn finished, including a turn that ends when the chat enters requires_action; the submitted tool result opens a new turn), interrupted (an interrupt request from a user, an API client, or a parent agent stopped it), error (a failure stopped it), abandoned (a newer prompt, such as a message edit, or runner exit closed it before it finished; an edit sent while the chat is interrupting can close the stopped turn as abandoned rather than interrupted). Every closed turn is counted exactly once. Registered only with the chat-stage-metrics experiment.",
|
|
}, []string{"outcome", "chat_kind"}),
|
|
stageAnomaliesTotal: stageFactory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "stage_anomalies_total",
|
|
Help: "Chat lifecycle stage observations dropped or adjusted, by reason: negative_elapsed and inverted_window (end before start, clock skew), missing_timestamp (unset start or end), future_start (start ahead of this replica's clock, span started now), stale_anchor (trigger not after the previous turn's anchor, turn started now). Registered only with the chat-stage-metrics experiment.",
|
|
}, []string{"reason"}),
|
|
CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "compaction_total",
|
|
Help: "Total compaction outcomes (only recorded when compaction was triggered or failed).",
|
|
}, []string{"provider", "model", "result"}),
|
|
StepsTotal: factory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "steps_total",
|
|
Help: "Total agentic loop steps across all chats.",
|
|
}, []string{"provider", "model"}),
|
|
StreamRetriesTotal: factory.NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "stream_retries_total",
|
|
Help: "Total LLM stream retries.",
|
|
}, []string{"provider", "model", "kind"}),
|
|
FindToolsCallsTotal: factory.NewCounter(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "find_tools_calls_total",
|
|
Help: "Total find_tools calls.",
|
|
}),
|
|
FindToolsEmptyTotal: factory.NewCounter(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "find_tools_empty_total",
|
|
Help: "Total find_tools calls with no matches.",
|
|
}),
|
|
FindToolsMatchCount: factory.NewHistogram(prometheus.HistogramOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "find_tools_match_count",
|
|
Help: "Number of matches returned by find_tools calls.",
|
|
Buckets: prometheus.LinearBuckets(0, 2, 11),
|
|
}),
|
|
FindToolsActivationsTotal: factory.NewCounter(prometheus.CounterOpts{
|
|
Namespace: metricsNamespace,
|
|
Subsystem: metricsSubsystem,
|
|
Name: "find_tools_activations_total",
|
|
Help: "Total deferred tool activations returned by find_tools.",
|
|
}),
|
|
}
|
|
}
|
|
|
|
// NopMetrics returns a Metrics instance that discards all data.
|
|
// Useful for tests and when metrics collection is not desired.
|
|
func NopMetrics() *Metrics {
|
|
return NewMetrics(prometheus.NewRegistry())
|
|
}
|
|
|
|
func (m *Metrics) recordStageDuration(stage Stage, scope Scope, chatKind ChatKind, model StageModel, elapsed time.Duration) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
if _, ok := observedStages[stage]; !ok {
|
|
return
|
|
}
|
|
if elapsed < 0 {
|
|
m.recordStageAnomaly(StageAnomalyNegativeElapsed)
|
|
return
|
|
}
|
|
seconds := elapsed.Seconds()
|
|
m.stageDurationSeconds.WithLabelValues(string(stage), string(scope), string(chatKind)).Observe(seconds)
|
|
if scope != ScopeTurn || model.Model == "" {
|
|
return
|
|
}
|
|
if _, modelStage := modelStages[stage]; modelStage {
|
|
m.modelStageDurationSeconds.WithLabelValues(string(stage), model.ProviderType, string(chatKind), model.Model).Observe(seconds)
|
|
}
|
|
}
|
|
|
|
func (m *Metrics) recordStageAnomaly(reason StageAnomaly) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.stageAnomaliesTotal.WithLabelValues(string(reason)).Inc()
|
|
}
|
|
|
|
// RecordTurnOutcome counts a closed turn by outcome.
|
|
func (m *Metrics) RecordTurnOutcome(outcome TurnOutcome, chatKind ChatKind) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.TurnOutcomesTotal.WithLabelValues(string(outcome), string(chatKind)).Inc()
|
|
}
|
|
|
|
// RecordCompaction classifies and records a compaction attempt.
|
|
// It is a no-op when m is nil.
|
|
func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err error) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
switch {
|
|
case err != nil && errors.Is(err, context.DeadlineExceeded):
|
|
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultTimeout).Inc()
|
|
case err != nil && errors.Is(err, context.Canceled):
|
|
// User interruption, not a compaction failure.
|
|
return
|
|
case err != nil:
|
|
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultError).Inc()
|
|
case compacted:
|
|
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultSuccess).Inc()
|
|
// !compacted && err == nil means threshold not reached -- not
|
|
// recorded.
|
|
}
|
|
}
|
|
|
|
// RecordStreamRetry increments stream_retries_total. The caller
|
|
// must obtain classified via chaterror.Classify (non-empty Kind).
|
|
// No-op when m is nil.
|
|
func (m *Metrics) RecordStreamRetry(provider, model string, classified chaterror.ClassifiedError) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.StreamRetriesTotal.WithLabelValues(
|
|
provider,
|
|
model,
|
|
string(classified.Kind),
|
|
).Inc()
|
|
}
|
|
|
|
// RecordToolError increments tool_errors_total for the given
|
|
// tool. No-op when m is nil.
|
|
func (m *Metrics) RecordToolError(provider, model, toolLabel string) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
m.ToolErrorsTotal.WithLabelValues(provider, model, toolLabel).Inc()
|
|
}
|
|
|
|
// RecordToolResultTruncated increments tool_result_truncated_total for
|
|
// the given tool. No-op when m is nil. An empty tool label is
|
|
// normalized to "unknown" to match the other tool metrics.
|
|
func (m *Metrics) RecordToolResultTruncated(provider, model, toolLabel string) {
|
|
if m == nil {
|
|
return
|
|
}
|
|
if toolLabel == "" {
|
|
toolLabel = "unknown"
|
|
}
|
|
m.ToolResultTruncatedTotal.WithLabelValues(provider, model, toolLabel).Inc()
|
|
}
|
|
|
|
// EstimatePromptSize returns a cheap byte-size estimate of a
|
|
// fantasy prompt by summing the text content lengths of all
|
|
// message parts. This avoids JSON marshaling overhead.
|
|
func EstimatePromptSize(messages []fantasy.Message) int {
|
|
var size int
|
|
for _, msg := range messages {
|
|
for _, part := range msg.Content {
|
|
size += ContentPartSize(part)
|
|
}
|
|
}
|
|
return size
|
|
}
|
|
|
|
// ContentPartSize returns the byte length of a MessagePart's
|
|
// primary text or data field.
|
|
func ContentPartSize(part fantasy.MessagePart) int {
|
|
switch p := part.(type) {
|
|
case fantasy.TextPart:
|
|
return len(p.Text)
|
|
case fantasy.ReasoningPart:
|
|
return len(p.Text)
|
|
case fantasy.FilePart:
|
|
return len(p.Data)
|
|
case fantasy.ToolCallPart:
|
|
return len(p.Input)
|
|
case fantasy.ToolResultPart:
|
|
return toolResultOutputSize(p.Output)
|
|
default:
|
|
return 0
|
|
}
|
|
}
|
|
|
|
// ToolResultSize returns the byte length of a
|
|
// ToolResultContent's primary text or data field.
|
|
func ToolResultSize(r fantasy.ToolResultContent) int {
|
|
return toolResultOutputSize(r.Result)
|
|
}
|
|
|
|
func toolResultOutputSize(output fantasy.ToolResultOutputContent) int {
|
|
if output == nil {
|
|
return 0
|
|
}
|
|
switch v := output.(type) {
|
|
case fantasy.ToolResultOutputContentText:
|
|
return len(v.Text)
|
|
case fantasy.ToolResultOutputContentError:
|
|
if v.Error != nil {
|
|
return len(v.Error.Error())
|
|
}
|
|
return 0
|
|
case fantasy.ToolResultOutputContentMedia:
|
|
return len(v.Data)
|
|
default:
|
|
return 0
|
|
}
|
|
}
|