Files
coder/coderd/x/chatd/modelcall_internal_test.go
T
J. Scott Miller e988ddb11d feat: trace chatd provider stages and count only output parts as first token (#29770)
**TL;DR:** Traces the model provider call: a `stream` stage, a nested
`time_to_first_token` stage, and one `provider_attempt` per HTTP round
trip, labelled with provider type and model. Behavior change:
`coderd_chatd_ttft_seconds` now counts only a real output part as the
first token, so error, warning, and finish parts no longer count.

Instruments the provider call.

**Stream and time to first token.** `GenerateAssistant` records a
`stream` stage from stream open to close and a nested
`time_to_first_token` stage. The TTFT window closes once, on the first
streamed output part: any part other than warnings, finish, or error,
including start markers such as `text_start`. A closed window observes
the stage and the pre-existing `coderd_chatd_ttft_seconds` with the same
elapsed time. A failed open, an error part, or a release without an
output part ends the TTFT span with the error (a silence timeout is
classified as such) and observes nothing. The `stream` span ends once,
with the error `GenerateAssistant` returns, so content-filter and
no-output attempts end errored. Failed `stream` windows are observed
like successful ones. The attempt is released before the stream span
ends, so the TTFT window always falls inside its stream. The compaction
summary stream passes a nil tracer, so it records neither.

Without #29771, production emits only `provider_attempt`; #29771 passes
the tracer to `GenerateAssistant`.

**Provider attempts.** The provider HTTP transport is wrapped in
`stageSpanRoundTripper`, which emits one `provider_attempt` stage per
round trip, so retries each get a span. The span closes on response
headers; a status >= 400 marks it errored without changing what the
client sees.

`resolveModelCall` resolves the `StageModel` (wire provider, configured
`provider_type`, model, effective reasoning effort) and threads it to
the transport, so span attributes and labels match the call.
`provider_type` is read from `ai_providers.type`, so types outside the
fantasy set (copilot) keep their configured name. `chatd.New` builds the
tracer with the server clock.

**Behavior change to `coderd_chatd_ttft_seconds`:** only an output part
counts as a first token. Error parts no longer close the window as a
success, and warnings and finish parts are skipped; the help text says
so.

`StageSpan.End` returns the span's elapsed window, which `ttft_seconds`
observes. Callers pass `Stages` and `StageModel` through
`GenerateAssistantOptions`; a nil tracer discards (TTFT still records
`ttft_seconds`). `Server` gains the `stages` tracer and a
`TracerProvider` option, wired from `coderd`.

<details>
<summary>Review decisions</summary>

- Start markers close the TTFT window: they are the first sign the model
is producing output, and waiting for a delta would fold hidden reasoning
time into TTFT.
- Failed `stream` and `provider_attempt` windows are observed: that time
was spent. TTFT is the exception because a first token that never
arrived has no duration.
- `http_host` was dropped from `provider_attempt`; every request goes
through the local AI Gateway transport, so it was constant.
- The advisor tool's nested model call is intentionally not instrumented
as `stream`/TTFT.

</details>

Supersedes #28970 and #28971.

🤖 This PR was generated by Coder Agents on behalf of @jscottmiller.
2026-09-30 11:53:58 -05:00

199 lines
7.5 KiB
Go

package chatd
import (
"encoding/json"
"io"
"net/http"
"strings"
"testing"
"charm.land/fantasy"
fantasyopenai "charm.land/fantasy/providers/openai"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/attribute"
"go.uber.org/mock/gomock"
"cdr.dev/slog/v3/sloggers/slogtest"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/util/ptr"
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func modelCallSentinelOptions(t *testing.T, user string) json.RawMessage {
t.Helper()
raw, err := json.Marshal(codersdk.ChatModelCallConfig{
ProviderOptions: &codersdk.ChatModelProviderOptions{
OpenAI: &codersdk.ChatModelOpenAIProviderOptions{
User: ptr.Ref(user),
},
},
})
require.NoError(t, err)
return raw
}
// The transport decides which of the two OpenAI option shapes derivation
// produces, so both are accepted.
func requireOpenAIUserOption(t *testing.T, options fantasy.ProviderOptions, user string) {
t.Helper()
switch opts := options[fantasyopenai.Name].(type) {
case *fantasyopenai.ResponsesProviderOptions:
require.NotNil(t, opts.User)
require.Equal(t, user, *opts.User)
case *fantasyopenai.ProviderOptions:
require.NotNil(t, opts.User)
require.Equal(t, user, *opts.User)
default:
t.Fatalf("unexpected openai provider options type %T", opts)
}
}
func requireOpenAIReasoningEffort(t *testing.T, options fantasy.ProviderOptions, effort string) {
t.Helper()
switch opts := options[fantasyopenai.Name].(type) {
case *fantasyopenai.ResponsesProviderOptions:
require.NotNil(t, opts.ReasoningEffort)
require.Equal(t, fantasyopenai.ReasoningEffort(effort), *opts.ReasoningEffort)
case *fantasyopenai.ProviderOptions:
require.NotNil(t, opts.ReasoningEffort)
require.Equal(t, fantasyopenai.ReasoningEffort(effort), *opts.ReasoningEffort)
default:
t.Fatalf("unexpected openai provider options type %T", opts)
}
}
func TestResolveModelCallDerivesProviderOptions(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
chat, _ := titleOverrideTestChatAndMessages(t)
providerID := uuid.New()
config := titleOverrideModelConfig("gpt-4o-mini", true)
config.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true}
config.Options = modelCallSentinelOptions(t, "summary-options-sentinel")
chat.LastModelConfigID = config.ID
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), config.ID).Return(config, nil)
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary-openai", database.AIProviderTypeOpenai), nil).AnyTimes()
db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return([]database.AIProviderKey{{
ProviderID: providerID,
APIKey: "test-key",
}}, nil).AnyTimes()
server := titleOverrideTestServer(db, logger)
resolved, err := server.resolveModelCall(ctx, modelCallSpec{
purpose: "chat_summary",
chat: chat,
buildOptions: modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
})
require.NoError(t, err)
requireOpenAIUserOption(t, resolved.providerOptions, "summary-options-sentinel")
requireOpenAIUserOption(t, summaryObjectCall(resolved).ProviderOptions, "summary-options-sentinel")
}
func TestResolveModelCallStageModel(t *testing.T) {
t.Parallel()
tests := []struct {
name string
providerType database.AIProviderType
reasoningEffort *codersdk.ChatModelReasoningEffortConfig
requestedEffort *string
wantEffort string
}{
{
name: "ClampedToMax",
providerType: database.AIProviderTypeOpenai,
reasoningEffort: &codersdk.ChatModelReasoningEffortConfig{Default: ptr.Ref("low"), Max: ptr.Ref("medium")},
requestedEffort: ptr.Ref("high"),
wantEffort: "medium",
},
{
name: "EmptyWithoutConfig",
providerType: database.AIProviderTypeOpenai,
wantEffort: "",
},
{
// gpt-5 infers openai, so the provider type must come from the
// configured provider.
name: "CopilotProviderType",
providerType: database.AIProviderTypeCopilot,
reasoningEffort: &codersdk.ChatModelReasoningEffortConfig{Default: ptr.Ref("low")},
wantEffort: "low",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
ctrl := gomock.NewController(t)
db := dbmock.NewMockStore(ctrl)
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
chat, _ := titleOverrideTestChatAndMessages(t)
providerID := uuid.New()
config := titleOverrideModelConfig("gpt-5", true)
config.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true}
options, err := json.Marshal(codersdk.ChatModelCallConfig{ReasoningEffort: tt.reasoningEffort})
require.NoError(t, err)
config.Options = options
chat.LastModelConfigID = config.ID
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), config.ID).Return(config, nil)
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary", tt.providerType), nil).AnyTimes()
db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return([]database.AIProviderKey{{
ProviderID: providerID,
APIKey: "test-key",
}}, nil).AnyTimes()
tracer, recorder := newStageTestTracer(t)
server := titleOverrideTestServer(db, logger)
server.stages = tracer
server.aibridgeTransportFactory = aibridgeTestFactoryPointer(&aibridgeTestFactory{
rt: roundTripFunc(func(req *http.Request) (*http.Response, error) {
return &http.Response{
StatusCode: http.StatusBadRequest,
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rejected"}}`)),
Request: req,
}, nil
}),
})
resolved, err := server.resolveModelCall(ctx, modelCallSpec{
purpose: "chat_turn",
chat: chat,
requestedEffort: tt.requestedEffort,
buildOptions: modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
})
require.NoError(t, err)
require.Equal(t, tt.wantEffort, resolved.resolvedEffort)
require.NotEmpty(t, resolved.model.Provider())
wantStageModel := chatloop.StageModel{Provider: resolved.model.Provider(), ProviderType: string(tt.providerType), Model: "gpt-5", Effort: tt.wantEffort}
require.Equal(t, wantStageModel, resolved.stageModel())
if tt.providerType == database.AIProviderTypeOpenai && tt.wantEffort != "" {
requireOpenAIReasoningEffort(t, resolved.providerOptions, tt.wantEffort)
}
_, err = resolved.model.LanguageModel().Generate(ctx, fantasy.Call{Prompt: []fantasy.Message{{
Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "hello"}},
}}})
require.Error(t, err)
ended := recorder.Ended()
require.Len(t, ended, 1)
require.Equal(t, string(chatloop.StageProviderAttempt), ended[0].Name())
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrProvider, wantStageModel.Provider))
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrProviderType, wantStageModel.ProviderType))
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrModel, wantStageModel.Model))
if wantStageModel.Effort != "" {
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrReasoningEffort, wantStageModel.Effort))
}
})
}
}