**TL;DR:** Traces the model provider call: a `stream` stage, a nested `time_to_first_token` stage, and one `provider_attempt` per HTTP round trip, labelled with provider type and model. Behavior change: `coderd_chatd_ttft_seconds` now counts only a real output part as the first token, so error, warning, and finish parts no longer count. Instruments the provider call. **Stream and time to first token.** `GenerateAssistant` records a `stream` stage from stream open to close and a nested `time_to_first_token` stage. The TTFT window closes once, on the first streamed output part: any part other than warnings, finish, or error, including start markers such as `text_start`. A closed window observes the stage and the pre-existing `coderd_chatd_ttft_seconds` with the same elapsed time. A failed open, an error part, or a release without an output part ends the TTFT span with the error (a silence timeout is classified as such) and observes nothing. The `stream` span ends once, with the error `GenerateAssistant` returns, so content-filter and no-output attempts end errored. Failed `stream` windows are observed like successful ones. The attempt is released before the stream span ends, so the TTFT window always falls inside its stream. The compaction summary stream passes a nil tracer, so it records neither. Without #29771, production emits only `provider_attempt`; #29771 passes the tracer to `GenerateAssistant`. **Provider attempts.** The provider HTTP transport is wrapped in `stageSpanRoundTripper`, which emits one `provider_attempt` stage per round trip, so retries each get a span. The span closes on response headers; a status >= 400 marks it errored without changing what the client sees. `resolveModelCall` resolves the `StageModel` (wire provider, configured `provider_type`, model, effective reasoning effort) and threads it to the transport, so span attributes and labels match the call. `provider_type` is read from `ai_providers.type`, so types outside the fantasy set (copilot) keep their configured name. `chatd.New` builds the tracer with the server clock. **Behavior change to `coderd_chatd_ttft_seconds`:** only an output part counts as a first token. Error parts no longer close the window as a success, and warnings and finish parts are skipped; the help text says so. `StageSpan.End` returns the span's elapsed window, which `ttft_seconds` observes. Callers pass `Stages` and `StageModel` through `GenerateAssistantOptions`; a nil tracer discards (TTFT still records `ttft_seconds`). `Server` gains the `stages` tracer and a `TracerProvider` option, wired from `coderd`. <details> <summary>Review decisions</summary> - Start markers close the TTFT window: they are the first sign the model is producing output, and waiting for a delta would fold hidden reasoning time into TTFT. - Failed `stream` and `provider_attempt` windows are observed: that time was spent. TTFT is the exception because a first token that never arrived has no duration. - `http_host` was dropped from `provider_attempt`; every request goes through the local AI Gateway transport, so it was constant. - The advisor tool's nested model call is intentionally not instrumented as `stream`/TTFT. </details> Supersedes #28970 and #28971. 🤖 This PR was generated by Coder Agents on behalf of @jscottmiller.
199 lines
7.5 KiB
Go
199 lines
7.5 KiB
Go
package chatd
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
|
|
"charm.land/fantasy"
|
|
fantasyopenai "charm.land/fantasy/providers/openai"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/require"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
"go.uber.org/mock/gomock"
|
|
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbmock"
|
|
"github.com/coder/coder/v2/coderd/util/ptr"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
func modelCallSentinelOptions(t *testing.T, user string) json.RawMessage {
|
|
t.Helper()
|
|
raw, err := json.Marshal(codersdk.ChatModelCallConfig{
|
|
ProviderOptions: &codersdk.ChatModelProviderOptions{
|
|
OpenAI: &codersdk.ChatModelOpenAIProviderOptions{
|
|
User: ptr.Ref(user),
|
|
},
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
return raw
|
|
}
|
|
|
|
// The transport decides which of the two OpenAI option shapes derivation
|
|
// produces, so both are accepted.
|
|
func requireOpenAIUserOption(t *testing.T, options fantasy.ProviderOptions, user string) {
|
|
t.Helper()
|
|
switch opts := options[fantasyopenai.Name].(type) {
|
|
case *fantasyopenai.ResponsesProviderOptions:
|
|
require.NotNil(t, opts.User)
|
|
require.Equal(t, user, *opts.User)
|
|
case *fantasyopenai.ProviderOptions:
|
|
require.NotNil(t, opts.User)
|
|
require.Equal(t, user, *opts.User)
|
|
default:
|
|
t.Fatalf("unexpected openai provider options type %T", opts)
|
|
}
|
|
}
|
|
|
|
func requireOpenAIReasoningEffort(t *testing.T, options fantasy.ProviderOptions, effort string) {
|
|
t.Helper()
|
|
switch opts := options[fantasyopenai.Name].(type) {
|
|
case *fantasyopenai.ResponsesProviderOptions:
|
|
require.NotNil(t, opts.ReasoningEffort)
|
|
require.Equal(t, fantasyopenai.ReasoningEffort(effort), *opts.ReasoningEffort)
|
|
case *fantasyopenai.ProviderOptions:
|
|
require.NotNil(t, opts.ReasoningEffort)
|
|
require.Equal(t, fantasyopenai.ReasoningEffort(effort), *opts.ReasoningEffort)
|
|
default:
|
|
t.Fatalf("unexpected openai provider options type %T", opts)
|
|
}
|
|
}
|
|
|
|
func TestResolveModelCallDerivesProviderOptions(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
chat, _ := titleOverrideTestChatAndMessages(t)
|
|
providerID := uuid.New()
|
|
config := titleOverrideModelConfig("gpt-4o-mini", true)
|
|
config.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true}
|
|
config.Options = modelCallSentinelOptions(t, "summary-options-sentinel")
|
|
chat.LastModelConfigID = config.ID
|
|
|
|
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), config.ID).Return(config, nil)
|
|
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary-openai", database.AIProviderTypeOpenai), nil).AnyTimes()
|
|
db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return([]database.AIProviderKey{{
|
|
ProviderID: providerID,
|
|
APIKey: "test-key",
|
|
}}, nil).AnyTimes()
|
|
|
|
server := titleOverrideTestServer(db, logger)
|
|
resolved, err := server.resolveModelCall(ctx, modelCallSpec{
|
|
purpose: "chat_summary",
|
|
chat: chat,
|
|
buildOptions: modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
|
|
})
|
|
require.NoError(t, err)
|
|
requireOpenAIUserOption(t, resolved.providerOptions, "summary-options-sentinel")
|
|
requireOpenAIUserOption(t, summaryObjectCall(resolved).ProviderOptions, "summary-options-sentinel")
|
|
}
|
|
|
|
func TestResolveModelCallStageModel(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
providerType database.AIProviderType
|
|
reasoningEffort *codersdk.ChatModelReasoningEffortConfig
|
|
requestedEffort *string
|
|
wantEffort string
|
|
}{
|
|
{
|
|
name: "ClampedToMax",
|
|
providerType: database.AIProviderTypeOpenai,
|
|
reasoningEffort: &codersdk.ChatModelReasoningEffortConfig{Default: ptr.Ref("low"), Max: ptr.Ref("medium")},
|
|
requestedEffort: ptr.Ref("high"),
|
|
wantEffort: "medium",
|
|
},
|
|
{
|
|
name: "EmptyWithoutConfig",
|
|
providerType: database.AIProviderTypeOpenai,
|
|
wantEffort: "",
|
|
},
|
|
{
|
|
// gpt-5 infers openai, so the provider type must come from the
|
|
// configured provider.
|
|
name: "CopilotProviderType",
|
|
providerType: database.AIProviderTypeCopilot,
|
|
reasoningEffort: &codersdk.ChatModelReasoningEffortConfig{Default: ptr.Ref("low")},
|
|
wantEffort: "low",
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
ctrl := gomock.NewController(t)
|
|
db := dbmock.NewMockStore(ctrl)
|
|
logger := slogtest.Make(t, &slogtest.Options{IgnoreErrors: true})
|
|
chat, _ := titleOverrideTestChatAndMessages(t)
|
|
providerID := uuid.New()
|
|
config := titleOverrideModelConfig("gpt-5", true)
|
|
config.AIProviderID = uuid.NullUUID{UUID: providerID, Valid: true}
|
|
options, err := json.Marshal(codersdk.ChatModelCallConfig{ReasoningEffort: tt.reasoningEffort})
|
|
require.NoError(t, err)
|
|
config.Options = options
|
|
chat.LastModelConfigID = config.ID
|
|
|
|
db.EXPECT().GetEnabledChatModelConfigByID(gomock.Any(), config.ID).Return(config, nil)
|
|
db.EXPECT().GetAIProviderByID(gomock.Any(), providerID).Return(aibridgeTestAIProvider(providerID, "primary", tt.providerType), nil).AnyTimes()
|
|
db.EXPECT().GetAIProviderKeysByProviderID(gomock.Any(), providerID).Return([]database.AIProviderKey{{
|
|
ProviderID: providerID,
|
|
APIKey: "test-key",
|
|
}}, nil).AnyTimes()
|
|
|
|
tracer, recorder := newStageTestTracer(t)
|
|
server := titleOverrideTestServer(db, logger)
|
|
server.stages = tracer
|
|
server.aibridgeTransportFactory = aibridgeTestFactoryPointer(&aibridgeTestFactory{
|
|
rt: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
|
return &http.Response{
|
|
StatusCode: http.StatusBadRequest,
|
|
Body: io.NopCloser(strings.NewReader(`{"error":{"message":"rejected"}}`)),
|
|
Request: req,
|
|
}, nil
|
|
}),
|
|
})
|
|
resolved, err := server.resolveModelCall(ctx, modelCallSpec{
|
|
purpose: "chat_turn",
|
|
chat: chat,
|
|
requestedEffort: tt.requestedEffort,
|
|
buildOptions: modelBuildOptions{ActiveAPIKeyID: uuid.NewString()},
|
|
})
|
|
require.NoError(t, err)
|
|
require.Equal(t, tt.wantEffort, resolved.resolvedEffort)
|
|
require.NotEmpty(t, resolved.model.Provider())
|
|
wantStageModel := chatloop.StageModel{Provider: resolved.model.Provider(), ProviderType: string(tt.providerType), Model: "gpt-5", Effort: tt.wantEffort}
|
|
require.Equal(t, wantStageModel, resolved.stageModel())
|
|
if tt.providerType == database.AIProviderTypeOpenai && tt.wantEffort != "" {
|
|
requireOpenAIReasoningEffort(t, resolved.providerOptions, tt.wantEffort)
|
|
}
|
|
|
|
_, err = resolved.model.LanguageModel().Generate(ctx, fantasy.Call{Prompt: []fantasy.Message{{
|
|
Role: fantasy.MessageRoleUser, Content: []fantasy.MessagePart{fantasy.TextPart{Text: "hello"}},
|
|
}}})
|
|
require.Error(t, err)
|
|
ended := recorder.Ended()
|
|
require.Len(t, ended, 1)
|
|
require.Equal(t, string(chatloop.StageProviderAttempt), ended[0].Name())
|
|
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrProvider, wantStageModel.Provider))
|
|
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrProviderType, wantStageModel.ProviderType))
|
|
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrModel, wantStageModel.Model))
|
|
if wantStageModel.Effort != "" {
|
|
require.Contains(t, ended[0].Attributes(), attribute.String(chatloop.AttrReasoningEffort, wantStageModel.Effort))
|
|
}
|
|
})
|
|
}
|
|
}
|