Files
coder/coderd/x/chatd/chatloop/metrics.go
T
Mathias Fredriksson c0f7663e95 feat: support editing queued chat messages (#29215)
The owner of a chat can edit a queued message through
`PATCH /api/v2/chats/{chat}/queue/{queuedMessage}`. chatd never sends
a row under edit (`editing_since`): rows ahead of it still send in
order, and a turn that ends at it puts the chat in the new `paused`
status. Ending the head's edit, deleting it, or promoting a queued row
continues the chat. Interrupting sends still run ahead of the queue.

The chat state machine classifies a queue whose head is under edit as
its own sub-state (R1P, I1P, A1P, E1P, XE1P, P), so every path to
`paused` is in the transition matrix. A chat whose family has a paused
member cannot be archived, and auto-archive skips that family. Queue
edit and delete are owner-only, because both can send the next message
with the owner's credentials.

`ChatQueuedMessage` gains `editing_since` and `reasoning_effort`, and
`ChatStatus` gains `paused`.

Closes CODAGT-1028
Refs CODAGT-971
2026-10-30 15:52:22 +02:00

390 lines
16 KiB
Go

package chatloop
import (
"context"
"errors"
"time"
"charm.land/fantasy"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
)
const (
metricsNamespace = "coderd"
metricsSubsystem = "chatd"
// Label values for Chats.
StateStreaming = "streaming"
// Label values for CompactionTotal.
CompactionResultSuccess = "success"
CompactionResultError = "error"
CompactionResultTimeout = "timeout"
)
// StageAnomaly is the `reason` label value of stageAnomaliesTotal.
type StageAnomaly string
// StageAnomaly values. FutureStart and StaleAnchor samples are clamped;
// the others are dropped.
const (
StageAnomalyNegativeElapsed StageAnomaly = "negative_elapsed"
StageAnomalyInvertedWindow StageAnomaly = "inverted_window"
StageAnomalyMissingTimestamp StageAnomaly = "missing_timestamp"
StageAnomalyFutureStart StageAnomaly = "future_start"
StageAnomalyStaleAnchor StageAnomaly = "stale_anchor"
)
// observedStages get histogram samples; other stages are span-only.
// The stage_duration_seconds help text must list the same set.
var observedStages = map[Stage]struct{}{
StageChatTurn: {},
StageQueueWait: {},
StageAcquisition: {},
StageMCPConnect: {},
StageStream: {},
StageTimeToFirstToken: {},
StageProviderAttempt: {},
StageToolCall: {},
StageCommit: {},
StageRetryBackoff: {},
}
// modelStages measure provider work on a model. Per-model TTFT is
// TTFTSeconds.
var modelStages = map[Stage]struct{}{
StageStream: {},
StageProviderAttempt: {},
}
var stageDurationBuckets = []float64{0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 1800, 3600}
// MetricsOptions configures optional metric families.
type MetricsOptions struct {
// StageMetrics registers the stage families. When false they are
// built unregistered so recorders still work.
StageMetrics bool
}
// Metrics holds Prometheus metrics for the chatd subsystem.
type Metrics struct {
Chats *prometheus.GaugeVec
MessageCount *prometheus.HistogramVec
PromptSizeBytes *prometheus.HistogramVec
ResponseSizeBytes *prometheus.HistogramVec
ToolResultSizeBytes *prometheus.HistogramVec
ToolResultTruncatedTotal *prometheus.CounterVec
ToolErrorsTotal *prometheus.CounterVec
TTFTSeconds *prometheus.HistogramVec
stageDurationSeconds *prometheus.HistogramVec
modelStageDurationSeconds *prometheus.HistogramVec
TurnOutcomesTotal *prometheus.CounterVec
stageAnomaliesTotal *prometheus.CounterVec
CompactionTotal *prometheus.CounterVec
StepsTotal *prometheus.CounterVec
StreamRetriesTotal *prometheus.CounterVec
FindToolsCallsTotal prometheus.Counter
FindToolsEmptyTotal prometheus.Counter
FindToolsMatchCount prometheus.Histogram
FindToolsActivationsTotal prometheus.Counter
}
// NewMetrics creates a new Metrics instance registered with the
// given registerer, with the optional metric families disabled.
func NewMetrics(reg prometheus.Registerer) *Metrics {
return NewMetricsWithOptions(reg, MetricsOptions{})
}
// NewMetricsWithOptions creates a new Metrics instance registered with
// the given registerer.
func NewMetricsWithOptions(reg prometheus.Registerer, opts MetricsOptions) *Metrics {
factory := promauto.With(reg)
stageFactory := factory
if !opts.StageMetrics {
stageFactory = promauto.With(nil)
}
return &Metrics{
Chats: factory.NewGaugeVec(prometheus.GaugeOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "chats",
Help: "Number of chats being processed, by state.",
}, []string{"state"}),
MessageCount: factory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "message_count",
Help: "Number of messages in the prompt per LLM request.",
Buckets: prometheus.ExponentialBuckets(1, 2, 11), // 1, 2, 4, ..., 1024
}, []string{"provider", "model"}),
PromptSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "prompt_size_bytes",
Help: "Estimated byte size of the prompt per LLM request.",
Buckets: prometheus.ExponentialBuckets(1024, 4, 10), // 1KB .. 256MB
}, []string{"provider", "model"}),
ResponseSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "response_size_bytes",
Help: "Byte size of streamed model response content (assistant text, reasoning, and tool-call input deltas) per LLM request.",
Buckets: prometheus.ExponentialBuckets(64, 4, 11), // 64B .. 64MB
}, []string{"provider", "model"}),
ToolResultSizeBytes: factory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "tool_result_size_bytes",
Help: "Size in bytes of each tool execution result.",
Buckets: prometheus.ExponentialBuckets(64, 4, 9), // 64B .. 4MB
}, []string{"provider", "model", "tool_name"}),
ToolResultTruncatedTotal: factory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "tool_result_truncated_total",
Help: "Total tool results truncated to fit the model context window.",
}, []string{"provider", "model", "tool_name"}),
ToolErrorsTotal: factory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "tool_errors_total",
Help: "Total tool calls that returned an error result.",
}, []string{"provider", "model", "tool_name"}),
TTFTSeconds: factory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "ttft_seconds",
Help: "Time-to-first-token: wall time from LLM request to the first streamed output part: any part other than warnings, keepalives, finish, or error, including start markers such as text_start. Attempts that end before an output part, by failing or by finishing empty, are not observed. The time_to_first_token stage of stage_duration_seconds measures the same window without a model label and is registered only with the chat-stage-metrics experiment.",
Buckets: []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
}, []string{"provider", "model"}),
stageDurationSeconds: stageFactory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "stage_duration_seconds",
Help: "Wall time spent in each chat lifecycle stage. Stages overlap, so this is a stage-time profile, not a partition of the turn. scope separates latency attributable to a prompt (stages inside a chat turn, and queue_wait before it) from detached background work; chat_kind is empty without a known chat. Observed: chat_turn, queue_wait, acquisition, mcp_connect, stream, time_to_first_token, provider_attempt, tool_call, commit, retry_backoff; other stages are span-only. Failed windows are observed, except time_to_first_token, which observes only windows closed by an output part, and chat_turn, which observes only completed turns. chat_turn runs from its trigger (the prompt, a submitted client tool result, or a compaction request) to the end of its turn, includes acquisition, and excludes queue_wait and time waiting in requires_action, so a prompt that waits in requires_action is observed once per segment. queue_wait runs from a queued message's creation to its promotion, so it includes time the message spent under edit. Registered only with the chat-stage-metrics experiment.",
Buckets: stageDurationBuckets,
}, []string{"stage", "scope", "chat_kind"}),
modelStageDurationSeconds: stageFactory.NewHistogramVec(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "model_stage_duration_seconds",
Help: "Wall time of the stages that are a provider's work on a model: stream (open to close) and provider_attempt (one HTTP round trip, closed on response headers). Time to first token per model is ttft_seconds. Failed stream and provider_attempt windows are observed. Observed only for turn-scoped stages, which are also observed on stage_duration_seconds; background-scoped model stages appear on stage_duration_seconds only. provider_type is the configured AI provider type (for example bedrock), not the wire protocol other chatd metrics report as provider. Registered only with the chat-stage-metrics experiment.",
Buckets: stageDurationBuckets,
}, []string{"stage", "provider_type", "chat_kind", "model"}),
TurnOutcomesTotal: stageFactory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "turn_outcomes_total",
Help: "Total closed chat turns by outcome: completed (the turn finished, including a turn that ends when the chat enters requires_action; the submitted tool result opens a new turn), interrupted (an interrupt request from a user, an API client, or a parent agent stopped it), error (a failure stopped it), abandoned (a newer prompt, such as a message edit, or runner exit closed it before it finished; an edit sent while the chat is interrupting can close the stopped turn as abandoned rather than interrupted). Every closed turn is counted exactly once. Registered only with the chat-stage-metrics experiment.",
}, []string{"outcome", "chat_kind"}),
stageAnomaliesTotal: stageFactory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "stage_anomalies_total",
Help: "Chat lifecycle stage observations dropped or adjusted, by reason: negative_elapsed and inverted_window (end before start, clock skew), missing_timestamp (unset start or end), future_start (start ahead of this replica's clock, span started now), stale_anchor (trigger not after the previous turn's anchor, turn started now). Registered only with the chat-stage-metrics experiment.",
}, []string{"reason"}),
CompactionTotal: factory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "compaction_total",
Help: "Total compaction outcomes (only recorded when compaction was triggered or failed).",
}, []string{"provider", "model", "result"}),
StepsTotal: factory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "steps_total",
Help: "Total agentic loop steps across all chats.",
}, []string{"provider", "model"}),
StreamRetriesTotal: factory.NewCounterVec(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "stream_retries_total",
Help: "Total LLM stream retries.",
}, []string{"provider", "model", "kind"}),
FindToolsCallsTotal: factory.NewCounter(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "find_tools_calls_total",
Help: "Total find_tools calls.",
}),
FindToolsEmptyTotal: factory.NewCounter(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "find_tools_empty_total",
Help: "Total find_tools calls with no matches.",
}),
FindToolsMatchCount: factory.NewHistogram(prometheus.HistogramOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "find_tools_match_count",
Help: "Number of matches returned by find_tools calls.",
Buckets: prometheus.LinearBuckets(0, 2, 11),
}),
FindToolsActivationsTotal: factory.NewCounter(prometheus.CounterOpts{
Namespace: metricsNamespace,
Subsystem: metricsSubsystem,
Name: "find_tools_activations_total",
Help: "Total deferred tool activations returned by find_tools.",
}),
}
}
// NopMetrics returns a Metrics instance that discards all data.
// Useful for tests and when metrics collection is not desired.
func NopMetrics() *Metrics {
return NewMetrics(prometheus.NewRegistry())
}
func (m *Metrics) recordStageDuration(stage Stage, scope Scope, chatKind ChatKind, model StageModel, elapsed time.Duration) {
if m == nil {
return
}
if _, ok := observedStages[stage]; !ok {
return
}
if elapsed < 0 {
m.recordStageAnomaly(StageAnomalyNegativeElapsed)
return
}
seconds := elapsed.Seconds()
m.stageDurationSeconds.WithLabelValues(string(stage), string(scope), string(chatKind)).Observe(seconds)
if scope != ScopeTurn || model.Model == "" {
return
}
if _, modelStage := modelStages[stage]; modelStage {
m.modelStageDurationSeconds.WithLabelValues(string(stage), model.ProviderType, string(chatKind), model.Model).Observe(seconds)
}
}
func (m *Metrics) recordStageAnomaly(reason StageAnomaly) {
if m == nil {
return
}
m.stageAnomaliesTotal.WithLabelValues(string(reason)).Inc()
}
// RecordTurnOutcome counts a closed turn by outcome.
func (m *Metrics) RecordTurnOutcome(outcome TurnOutcome, chatKind ChatKind) {
if m == nil {
return
}
m.TurnOutcomesTotal.WithLabelValues(string(outcome), string(chatKind)).Inc()
}
// RecordCompaction classifies and records a compaction attempt.
// It is a no-op when m is nil.
func (m *Metrics) RecordCompaction(provider, model string, compacted bool, err error) {
if m == nil {
return
}
switch {
case err != nil && errors.Is(err, context.DeadlineExceeded):
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultTimeout).Inc()
case err != nil && errors.Is(err, context.Canceled):
// User interruption, not a compaction failure.
return
case err != nil:
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultError).Inc()
case compacted:
m.CompactionTotal.WithLabelValues(provider, model, CompactionResultSuccess).Inc()
// !compacted && err == nil means threshold not reached -- not
// recorded.
}
}
// RecordStreamRetry increments stream_retries_total. The caller
// must obtain classified via chaterror.Classify (non-empty Kind).
// No-op when m is nil.
func (m *Metrics) RecordStreamRetry(provider, model string, classified chaterror.ClassifiedError) {
if m == nil {
return
}
m.StreamRetriesTotal.WithLabelValues(
provider,
model,
string(classified.Kind),
).Inc()
}
// RecordToolError increments tool_errors_total for the given
// tool. No-op when m is nil.
func (m *Metrics) RecordToolError(provider, model, toolLabel string) {
if m == nil {
return
}
m.ToolErrorsTotal.WithLabelValues(provider, model, toolLabel).Inc()
}
// RecordToolResultTruncated increments tool_result_truncated_total for
// the given tool. No-op when m is nil. An empty tool label is
// normalized to "unknown" to match the other tool metrics.
func (m *Metrics) RecordToolResultTruncated(provider, model, toolLabel string) {
if m == nil {
return
}
if toolLabel == "" {
toolLabel = "unknown"
}
m.ToolResultTruncatedTotal.WithLabelValues(provider, model, toolLabel).Inc()
}
// EstimatePromptSize returns a cheap byte-size estimate of a
// fantasy prompt by summing the text content lengths of all
// message parts. This avoids JSON marshaling overhead.
func EstimatePromptSize(messages []fantasy.Message) int {
var size int
for _, msg := range messages {
for _, part := range msg.Content {
size += ContentPartSize(part)
}
}
return size
}
// ContentPartSize returns the byte length of a MessagePart's
// primary text or data field.
func ContentPartSize(part fantasy.MessagePart) int {
switch p := part.(type) {
case fantasy.TextPart:
return len(p.Text)
case fantasy.ReasoningPart:
return len(p.Text)
case fantasy.FilePart:
return len(p.Data)
case fantasy.ToolCallPart:
return len(p.Input)
case fantasy.ToolResultPart:
return toolResultOutputSize(p.Output)
default:
return 0
}
}
// ToolResultSize returns the byte length of a
// ToolResultContent's primary text or data field.
func ToolResultSize(r fantasy.ToolResultContent) int {
return toolResultOutputSize(r.Result)
}
func toolResultOutputSize(output fantasy.ToolResultOutputContent) int {
if output == nil {
return 0
}
switch v := output.(type) {
case fantasy.ToolResultOutputContentText:
return len(v.Text)
case fantasy.ToolResultOutputContentError:
if v.Error != nil {
return len(v.Error.Error())
}
return 0
case fantasy.ToolResultOutputContentMedia:
return len(v.Data)
default:
return 0
}
}