Files
coder/coderd/telemetry/telemetry_test.go
T
Michael Suchacz 9cd38b5373 feat!: remove the Tasks API from coderd and codersdk (#28789)
Removes the Tasks API and task orchestration from coderd and codersdk:
the `/api/v2/tasks` handlers, task middleware and search filters, task
name generation, autobuild pause/resume logic, task notification
enqueues, telemetry, and the codersdk task types. Regenerates apidoc,
the API reference docs, and `typesGenerated.ts`.

**Breaking change:** the Tasks API is removed; requests to
`/api/v2/tasks` fail. Tasks was deprecated in v2.34.

**Stack context:** Coder Tasks removal stack (CODAGT-642): #28743
frontend → #28788 CLI → #28789 API → #28790 provisioner → #29084
notifications → #29085 build reasons → #29086 tables → #28750
permissions. The last four PRs replace the single data-drop migration
that used to live in #28750; each carries one concern and its own
migration number.

**Why:** with the UI (#28743) and CLI (#28788) gone, this deletes the
server surface. Database tables, RBAC, and notification templates stay
until the top of the stack so every layer builds and migrates
independently.

**What to scrutinize:**
- The `enable-ai-tasks` option is removed outright (review feedback):
configs still carrying `enableAITasks` fail to parse until the key is
deleted.
- provisionerdserver now writes `HasAITask: false` on template/build
flags; the column and these lines are dropped in #29085.
- `codersdk.ResourceTypeTask` stays so audit logs recorded before the
removal keep rendering.

**Validation:** `go build ./...`, `go vet ./...`, `go test
./codersdk/...`, site biome and tsc, CLI goldens regenerated, pre-commit
hooks.

**Size:** +242/-10,792 over 76 files; additions are regenerated
docs/goldens and test fixture updates.

> Created by Xum (AI agent) on behalf of @ibetitsmike.
2026-09-11 01:48:56 +00:00

1821 lines
69 KiB
Go

package telemetry_test
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"sort"
"testing"
"time"
"github.com/go-chi/chi/v5"
"github.com/google/uuid"
"github.com/sqlc-dev/pqtype"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"go.uber.org/mock/gomock"
"github.com/coder/coder/v2/buildinfo"
"github.com/coder/coder/v2/coderd/boundaryusage"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbgen"
"github.com/coder/coder/v2/coderd/database/dbmock"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/idpsync"
"github.com/coder/coder/v2/coderd/runtimeconfig"
"github.com/coder/coder/v2/coderd/telemetry"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
"github.com/coder/quartz"
)
func TestMain(m *testing.M) {
goleak.VerifyTestMain(m, testutil.GoleakOptions...)
}
func TestConvertTemplateAgentsAllowed(t *testing.T) {
t.Parallel()
for _, tt := range []struct {
name string
allowed bool
}{
{name: "Allowed", allowed: true},
{name: "Disallowed", allowed: false},
} {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
got := telemetry.ConvertTemplate(database.Template{AgentsAllowed: tt.allowed})
require.Equal(t, tt.allowed, got.AgentsAllowed)
})
}
}
func TestTelemetry(t *testing.T) {
t.Parallel()
t.Run("Snapshot", func(t *testing.T) {
t.Parallel()
var err error
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
now := dbtime.Now()
org, err := db.GetDefaultOrganization(ctx)
require.NoError(t, err)
user := dbgen.User(t, db, database.User{})
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{
UserID: user.ID,
OrganizationID: org.ID,
})
require.NoError(t, err)
_, _ = dbgen.APIKey(t, db, database.APIKey{
UserID: user.ID,
})
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
Provisioner: database.ProvisionerTypeTerraform,
StorageMethod: database.ProvisionerStorageMethodFile,
Type: database.ProvisionerJobTypeTemplateVersionDryRun,
OrganizationID: org.ID,
})
tpl := dbgen.Template(t, db, database.Template{
Provisioner: database.ProvisionerTypeTerraform,
OrganizationID: org.ID,
CreatedBy: user.ID,
})
sourceExampleID := uuid.NewString()
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
SourceExampleID: sql.NullString{String: sourceExampleID, Valid: true},
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
CreatedBy: user.ID,
JobID: job.ID,
})
_ = dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
CreatedBy: user.ID,
JobID: job.ID,
})
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: tpl.ID,
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonAutostart,
WorkspaceID: ws.ID,
TemplateVersionID: tv.ID,
JobID: job.ID,
})
wsresource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{
JobID: job.ID,
})
wsagent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{
ResourceID: wsresource.ID,
})
_ = dbgen.WorkspaceApp(t, db, database.WorkspaceApp{
SharingLevel: database.AppSharingLevelOwner,
Health: database.WorkspaceAppHealthDisabled,
OpenIn: database.WorkspaceAppOpenInSlimWindow,
AgentID: wsagent.ID,
})
group := dbgen.Group(t, db, database.Group{
OrganizationID: org.ID,
})
_ = dbgen.TelemetryItem(t, db, database.TelemetryItem{
Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt),
Value: time.Now().Format(time.RFC3339),
})
_ = dbgen.GroupMember(t, db, database.GroupMemberTable{UserID: user.ID, GroupID: group.ID})
// Update the workspace agent to have a valid subsystem.
err = db.UpdateWorkspaceAgentStartupByID(ctx, database.UpdateWorkspaceAgentStartupByIDParams{
ID: wsagent.ID,
Version: wsagent.Version,
ExpandedDirectory: wsagent.ExpandedDirectory,
Subsystems: []database.WorkspaceAgentSubsystem{
database.WorkspaceAgentSubsystemEnvbox,
database.WorkspaceAgentSubsystemExectrace,
},
})
require.NoError(t, err)
_ = dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{
ConnectionMedianLatencyMS: 1,
// Names from the same family, one of them an alias, so the fixed
// session count fields cover the app name folding.
SessionCounts: dbgen.SessionCounts(t, map[string]int64{"vscode": 1, "cursor": 2, "zed": 3}),
})
_, err = db.InsertLicense(ctx, database.InsertLicenseParams{
UploadedAt: dbtime.Now(),
JWT: "",
Exp: dbtime.Now().Add(time.Hour),
UUID: uuid.New(),
})
assert.NoError(t, err)
_, _ = dbgen.WorkspaceProxy(t, db, database.WorkspaceProxy{})
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
JobID: job.ID,
})
_ = dbgen.WorkspaceAgentMemoryResourceMonitor(t, db, database.WorkspaceAgentMemoryResourceMonitor{
AgentID: wsagent.ID,
})
_ = dbgen.WorkspaceAgentVolumeResourceMonitor(t, db, database.WorkspaceAgentVolumeResourceMonitor{
AgentID: wsagent.ID,
})
previousAIBridgeInterceptionPeriod := now.Truncate(time.Hour)
user2 := dbgen.User(t, db, database.User{})
aiBridgeInterception1 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
InitiatorID: user.ID,
Provider: "anthropic",
Model: "deanseek",
StartedAt: previousAIBridgeInterceptionPeriod.Add(-30 * time.Minute),
}, nil)
_ = dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
InterceptionID: aiBridgeInterception1.ID,
InputTokens: 100,
OutputTokens: 200,
CacheReadInputTokens: 300,
CacheWriteInputTokens: 400,
Metadata: json.RawMessage(`{"cache_read_input":300,"cache_creation_input":400}`),
})
_ = dbgen.AIBridgeUserPrompt(t, db, database.InsertAIBridgeUserPromptParams{
InterceptionID: aiBridgeInterception1.ID,
})
_ = dbgen.AIBridgeToolUsage(t, db, database.InsertAIBridgeToolUsageParams{
InterceptionID: aiBridgeInterception1.ID,
Injected: true,
InvocationError: sql.NullString{String: "error1", Valid: true},
})
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: aiBridgeInterception1.ID,
EndedAt: aiBridgeInterception1.StartedAt.Add(1 * time.Minute), // 1 minute duration
})
require.NoError(t, err)
aiBridgeInterception2 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
InitiatorID: user2.ID,
Provider: aiBridgeInterception1.Provider,
Model: aiBridgeInterception1.Model,
StartedAt: aiBridgeInterception1.StartedAt,
}, nil)
_ = dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
InterceptionID: aiBridgeInterception2.ID,
InputTokens: 100,
OutputTokens: 200,
CacheReadInputTokens: 300,
CacheWriteInputTokens: 400,
Metadata: json.RawMessage(`{"cache_read_input":300,"cache_creation_input":400}`),
})
_ = dbgen.AIBridgeUserPrompt(t, db, database.InsertAIBridgeUserPromptParams{
InterceptionID: aiBridgeInterception2.ID,
})
_ = dbgen.AIBridgeToolUsage(t, db, database.InsertAIBridgeToolUsageParams{
InterceptionID: aiBridgeInterception2.ID,
Injected: false,
})
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: aiBridgeInterception2.ID,
EndedAt: aiBridgeInterception2.StartedAt.Add(2 * time.Minute), // 2 minute duration
})
require.NoError(t, err)
aiBridgeInterception3 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
InitiatorID: user2.ID,
Provider: "openai",
Model: "gpt-5",
StartedAt: aiBridgeInterception1.StartedAt,
}, nil)
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
ID: aiBridgeInterception3.ID,
EndedAt: aiBridgeInterception3.StartedAt.Add(3 * time.Minute), // 3 minute duration
})
require.NoError(t, err)
_ = dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
InitiatorID: user2.ID,
Provider: "openai",
Model: "gpt-5",
StartedAt: aiBridgeInterception1.StartedAt,
}, nil)
// not ended, so it should not affect summaries
clock := quartz.NewMock(t)
clock.Set(now)
deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
var agentsExperiments map[string]json.RawMessage
require.NoError(t, json.Unmarshal(deployment.AgentsExperiments, &agentsExperiments))
require.Contains(t, agentsExperiments, "virtual_desktop")
require.Contains(t, agentsExperiments, "advisor")
require.Len(t, snapshot.ProvisionerJobs, 1)
require.Len(t, snapshot.Licenses, 1)
require.Len(t, snapshot.Templates, 1)
require.Len(t, snapshot.TemplateVersions, 2)
require.Len(t, snapshot.Users, 2)
require.Len(t, snapshot.Groups, 2)
// 1 member in the everyone group + 1 member in the custom group
require.Len(t, snapshot.GroupMembers, 2)
require.Len(t, snapshot.Workspaces, 1)
require.Len(t, snapshot.WorkspaceApps, 1)
require.Len(t, snapshot.WorkspaceAgents, 1)
require.Len(t, snapshot.WorkspaceBuilds, 1)
require.Len(t, snapshot.WorkspaceResources, 1)
require.Len(t, snapshot.WorkspaceAgentStats, 1)
require.Equal(t, int64(3), snapshot.WorkspaceAgentStats[0].SessionCountVSCode)
require.Equal(t, int64(3), snapshot.WorkspaceAgentStats[0].SessionCountSSH)
require.Len(t, snapshot.WorkspaceProxies, 1)
require.Len(t, snapshot.WorkspaceModules, 1)
require.Len(t, snapshot.Organizations, 1)
telemetryItemKeys := slice.Convert(snapshot.TelemetryItems, func(item telemetry.TelemetryItem) string {
return item.Key
})
require.ElementsMatch(t, []string{
string(telemetry.TelemetryItemKeyHTMLFirstServedAt),
string(telemetry.TelemetryItemKeyTelemetryEnabled),
}, telemetryItemKeys)
require.Len(t, snapshot.WorkspaceAgentMemoryResourceMonitors, 1)
require.Len(t, snapshot.WorkspaceAgentVolumeResourceMonitors, 1)
wsa := snapshot.WorkspaceAgents[0]
require.Len(t, wsa.Subsystems, 2)
require.Equal(t, string(database.WorkspaceAgentSubsystemEnvbox), wsa.Subsystems[0])
require.Equal(t, string(database.WorkspaceAgentSubsystemExectrace), wsa.Subsystems[1])
tvs := snapshot.TemplateVersions
sort.Slice(tvs, func(i, j int) bool {
// Sort by SourceExampleID presence (non-nil comes before nil)
if (tvs[i].SourceExampleID != nil) != (tvs[j].SourceExampleID != nil) {
return tvs[i].SourceExampleID != nil
}
return false
})
require.Equal(t, tvs[0].SourceExampleID, &sourceExampleID)
require.Nil(t, tvs[1].SourceExampleID)
for _, entity := range snapshot.Workspaces {
require.Equal(t, entity.OrganizationID, org.ID)
}
for _, entity := range snapshot.ProvisionerJobs {
require.Equal(t, entity.OrganizationID, org.ID)
}
for _, entity := range snapshot.TemplateVersions {
require.Equal(t, entity.OrganizationID, org.ID)
}
for _, entity := range snapshot.Templates {
require.Equal(t, entity.OrganizationID, org.ID)
}
// 2 unique provider + model + client combinations
require.Len(t, snapshot.AIBridgeInterceptionsSummaries, 2)
snapshot1 := snapshot.AIBridgeInterceptionsSummaries[0]
snapshot2 := snapshot.AIBridgeInterceptionsSummaries[1]
if snapshot1.Provider != aiBridgeInterception1.Provider {
snapshot1, snapshot2 = snapshot2, snapshot1
}
require.Equal(t, snapshot1.Provider, aiBridgeInterception1.Provider)
require.Equal(t, snapshot1.Model, aiBridgeInterception1.Model)
require.Equal(t, snapshot1.Client, "Unknown") // no client info yet
require.EqualValues(t, snapshot1.InterceptionCount, 2)
require.EqualValues(t, snapshot1.InterceptionsByRoute, map[string]int64{}) // no route info yet
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P50, 90_000)
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P90, 114_000)
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P95, 117_000)
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P99, 119_400)
require.EqualValues(t, snapshot1.UniqueInitiatorCount, 2)
require.EqualValues(t, snapshot1.UserPromptsCount, 2)
require.EqualValues(t, snapshot1.TokenUsagesCount, 2)
require.EqualValues(t, snapshot1.TokenCount.Input, 200)
require.EqualValues(t, snapshot1.TokenCount.Output, 400)
require.EqualValues(t, snapshot1.TokenCount.CachedRead, 600)
require.EqualValues(t, snapshot1.TokenCount.CachedWritten, 800)
require.EqualValues(t, snapshot1.ToolCallsCount.Injected, 1)
require.EqualValues(t, snapshot1.ToolCallsCount.NonInjected, 1)
require.EqualValues(t, snapshot1.InjectedToolCallErrorCount, 1)
require.Equal(t, snapshot2.Provider, aiBridgeInterception3.Provider)
require.Equal(t, snapshot2.Model, aiBridgeInterception3.Model)
require.Equal(t, snapshot2.Client, "Unknown") // no client info yet
require.EqualValues(t, snapshot2.InterceptionCount, 1)
require.EqualValues(t, snapshot2.InterceptionsByRoute, map[string]int64{}) // no route info yet
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P50, 180_000)
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P90, 180_000)
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P95, 180_000)
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P99, 180_000)
require.EqualValues(t, snapshot2.UniqueInitiatorCount, 1)
require.EqualValues(t, snapshot2.UserPromptsCount, 0)
require.EqualValues(t, snapshot2.TokenUsagesCount, 0)
require.EqualValues(t, snapshot2.TokenCount.Input, 0)
require.EqualValues(t, snapshot2.TokenCount.Output, 0)
require.EqualValues(t, snapshot2.TokenCount.CachedRead, 0)
require.EqualValues(t, snapshot2.TokenCount.CachedWritten, 0)
require.EqualValues(t, snapshot2.ToolCallsCount.Injected, 0)
require.EqualValues(t, snapshot2.ToolCallsCount.NonInjected, 0)
})
t.Run("HashedEmail", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
_ = dbgen.User(t, db, database.User{
Email: "kyle@coder.com",
})
_, snapshot := collectSnapshot(ctx, t, db, nil)
require.Len(t, snapshot.Users, 1)
require.Equal(t, snapshot.Users[0].EmailHashed, "bb44bf07cf9a2db0554bba63a03d822c927deae77df101874496df5a6a3e896d@coder.com")
})
t.Run("HashedModule", func(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
pj := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{})
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
JobID: pj.ID,
Source: "registry.coder.com/terraform/aws",
Version: "1.0.0",
})
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
JobID: pj.ID,
Source: "https://internal-url.com/some-module",
Version: "1.0.0",
})
_, snapshot := collectSnapshot(ctx, t, db, nil)
require.Len(t, snapshot.WorkspaceModules, 2)
modules := snapshot.WorkspaceModules
sort.Slice(modules, func(i, j int) bool {
return modules[i].Source < modules[j].Source
})
require.Equal(t, modules[0].Source, "ed662ec0396db67e77119f14afcb9253574cc925b04a51d4374bcb1eae299f5d")
require.Equal(t, modules[0].Version, "92521fc3cbd964bdc9f584a991b89fddaa5754ed1cc96d6d42445338669c1305")
require.Equal(t, modules[0].SourceType, telemetry.ModuleSourceTypeHTTP)
require.Equal(t, modules[1].Source, "registry.coder.com/terraform/aws")
require.Equal(t, modules[1].Version, "1.0.0")
require.Equal(t, modules[1].SourceType, telemetry.ModuleSourceTypeCoderRegistry)
})
t.Run("ModuleSourceType", func(t *testing.T) {
t.Parallel()
cases := []struct {
source string
want telemetry.ModuleSourceType
}{
// Local relative paths
{source: "./modules/terraform-aws-vpc", want: telemetry.ModuleSourceTypeLocal},
{source: "../shared/modules/vpc", want: telemetry.ModuleSourceTypeLocal},
{source: " ./my-module ", want: telemetry.ModuleSourceTypeLocal}, // with whitespace
// Local absolute paths
{source: "/opt/terraform/modules/vpc", want: telemetry.ModuleSourceTypeLocalAbs},
{source: "/Users/dev/modules/app", want: telemetry.ModuleSourceTypeLocalAbs},
{source: "/etc/terraform/modules/network", want: telemetry.ModuleSourceTypeLocalAbs},
// Public registry
{source: "hashicorp/consul/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "registry.terraform.io/hashicorp/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "terraform-aws-modules/vpc/aws", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "hashicorp/consul/aws//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
{source: "hashicorp/co-nsul/aw_s//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
// Private registry
{source: "app.terraform.io/company/vpc/aws", want: telemetry.ModuleSourceTypePrivateRegistry},
{source: "localterraform.com/org/module", want: telemetry.ModuleSourceTypePrivateRegistry},
{source: "APP.TERRAFORM.IO/test/module", want: telemetry.ModuleSourceTypePrivateRegistry}, // case insensitive
// Coder registry
{source: "registry.coder.com/terraform/aws", want: telemetry.ModuleSourceTypeCoderRegistry},
{source: "registry.coder.com/modules/base", want: telemetry.ModuleSourceTypeCoderRegistry},
{source: "REGISTRY.CODER.COM/test/module", want: telemetry.ModuleSourceTypeCoderRegistry}, // case insensitive
// GitHub
{source: "github.com/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeGitHub},
{source: "git::https://github.com/org/repo.git", want: telemetry.ModuleSourceTypeGitHub},
{source: "git::https://github.com/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeGitHub},
// Bitbucket
{source: "bitbucket.org/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeBitbucket},
{source: "git::https://bitbucket.org/org/repo.git", want: telemetry.ModuleSourceTypeBitbucket},
{source: "https://bitbucket.org/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeBitbucket},
// Generic Git
{source: "git::ssh://git.internal.com/repo.git", want: telemetry.ModuleSourceTypeGit},
{source: "git@gitlab.com:org/repo.git", want: telemetry.ModuleSourceTypeGit},
{source: "git::https://git.internal.com/repo.git?ref=v1.0.0", want: telemetry.ModuleSourceTypeGit},
// Mercurial
{source: "hg::https://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
{source: "hg::http://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
{source: "hg::ssh://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
// HTTP
{source: "https://example.com/vpc-module.zip", want: telemetry.ModuleSourceTypeHTTP},
{source: "http://example.com/modules/vpc", want: telemetry.ModuleSourceTypeHTTP},
{source: "https://internal.network/terraform/modules", want: telemetry.ModuleSourceTypeHTTP},
// S3
{source: "s3::https://s3-eu-west-1.amazonaws.com/bucket/vpc", want: telemetry.ModuleSourceTypeS3},
{source: "s3::https://bucket.s3.amazonaws.com/vpc", want: telemetry.ModuleSourceTypeS3},
{source: "s3::http://bucket.s3.amazonaws.com/vpc?version=1", want: telemetry.ModuleSourceTypeS3},
// GCS
{source: "gcs::https://www.googleapis.com/storage/v1/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
{source: "gcs::https://storage.googleapis.com/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
{source: "gcs::https://bucket.storage.googleapis.com/vpc", want: telemetry.ModuleSourceTypeGCS},
// Unknown
{source: "custom://example.com/vpc", want: telemetry.ModuleSourceTypeUnknown},
{source: "something-random", want: telemetry.ModuleSourceTypeUnknown},
{source: "", want: telemetry.ModuleSourceTypeUnknown},
}
for _, c := range cases {
require.Equal(t, c.want, telemetry.GetModuleSourceType(c.source))
}
})
t.Run("IDPOrgSync", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// 1. No org sync settings
deployment, _ := collectSnapshot(ctx, t, db, nil)
require.False(t, *deployment.IDPOrgSync)
// 2. Org sync settings set in server flags
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.DeploymentConfig = &codersdk.DeploymentValues{
OIDC: codersdk.OIDCConfig{
OrganizationField: "organizations",
},
}
return opts
})
require.True(t, *deployment.IDPOrgSync)
// 3. Org sync settings set in runtime config
org, err := db.GetDefaultOrganization(ctx)
require.NoError(t, err)
sync := idpsync.NewAGPLSync(testutil.Logger(t), runtimeconfig.NewManager(), idpsync.DeploymentSyncSettings{})
err = sync.UpdateOrganizationSyncSettings(ctx, db, idpsync.OrganizationSyncSettings{
Field: "organizations",
Mapping: map[string][]uuid.UUID{
"first": {org.ID},
},
AssignDefault: true,
})
require.NoError(t, err)
deployment, _ = collectSnapshot(ctx, t, db, nil)
require.True(t, *deployment.IDPOrgSync)
})
t.Run("SCIM", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// 1. Default Options: both flags false (and reported as such).
deployment, _ := collectSnapshot(ctx, t, db, nil)
require.NotNil(t, deployment.SCIMEnabled)
require.False(t, *deployment.SCIMEnabled)
require.NotNil(t, deployment.SCIMUseLegacy)
require.False(t, *deployment.SCIMUseLegacy)
// 2. Both Options flags true: both reported true.
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.SCIMEnabled = true
opts.SCIMUseLegacy = true
return opts
})
require.True(t, *deployment.SCIMEnabled)
require.True(t, *deployment.SCIMUseLegacy)
// 3. Enabled only: enabled true, legacy false.
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.SCIMEnabled = true
return opts
})
require.True(t, *deployment.SCIMEnabled)
require.False(t, *deployment.SCIMUseLegacy)
// 4. The reporter never reads DeploymentConfig.SCIMAPIKey directly:
// even if a non-empty key sneaks through (it would not in production
// because of WithoutSecrets), SCIMEnabled reflects only Options.
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.DeploymentConfig = &codersdk.DeploymentValues{
SCIMAPIKey: "a-secret-bearer-token",
}
return opts
})
require.False(t, *deployment.SCIMEnabled)
})
}
// nolint:paralleltest
func TestTelemetryInstallSource(t *testing.T) {
t.Setenv("CODER_TELEMETRY_INSTALL_SOURCE", "aws_marketplace")
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
deployment, _ := collectSnapshot(ctx, t, db, nil)
require.Equal(t, "aws_marketplace", deployment.InstallSource)
}
func TestTelemetryItem(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
key := testutil.GetRandomName(t)
value := time.Now().Format(time.RFC3339)
err := db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{
Key: key,
Value: value,
})
require.NoError(t, err)
item, err := db.GetTelemetryItem(ctx, key)
require.NoError(t, err)
require.Equal(t, item.Key, key)
require.Equal(t, item.Value, value)
// Inserting a new value should not update the existing value
err = db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{
Key: key,
Value: "new_value",
})
require.NoError(t, err)
item, err = db.GetTelemetryItem(ctx, key)
require.NoError(t, err)
require.Equal(t, item.Value, value)
// Upserting a new value should update the existing value
err = db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{
Key: key,
Value: "new_value",
})
require.NoError(t, err)
item, err = db.GetTelemetryItem(ctx, key)
require.NoError(t, err)
require.Equal(t, item.Value, "new_value")
}
func TestPrebuiltWorkspacesTelemetry(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
cases := []struct {
name string
storeFn func(store database.Store) database.Store
expectedSnapshotEntries int
expectedCreated int
expectedFailed int
expectedClaimed int
}{
{
name: "prebuilds enabled",
storeFn: func(store database.Store) database.Store {
return &mockDB{Store: store}
},
expectedSnapshotEntries: 3,
expectedCreated: 5,
expectedFailed: 2,
expectedClaimed: 3,
},
{
name: "prebuilds not used",
storeFn: func(store database.Store) database.Store {
return &emptyMockDB{Store: store}
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitShort)
deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Database = tc.storeFn(db)
return opts
})
require.NotNil(t, deployment)
require.NotNil(t, snapshot)
require.Len(t, snapshot.PrebuiltWorkspaces, tc.expectedSnapshotEntries)
eventCounts := make(map[telemetry.PrebuiltWorkspaceEventType]int)
for _, event := range snapshot.PrebuiltWorkspaces {
eventCounts[event.EventType] = event.Count
require.NotEqual(t, uuid.Nil, event.ID)
require.False(t, event.CreatedAt.IsZero())
}
require.Equal(t, tc.expectedCreated, eventCounts[telemetry.PrebuiltWorkspaceEventTypeCreated])
require.Equal(t, tc.expectedFailed, eventCounts[telemetry.PrebuiltWorkspaceEventTypeFailed])
require.Equal(t, tc.expectedClaimed, eventCounts[telemetry.PrebuiltWorkspaceEventTypeClaimed])
})
}
}
type mockDB struct {
database.Store
}
func (*mockDB) GetPrebuildMetrics(context.Context) ([]database.GetPrebuildMetricsRow, error) {
return []database.GetPrebuildMetricsRow{
{
TemplateName: "template1",
PresetName: "preset1",
OrganizationName: "org1",
CreatedCount: 3,
FailedCount: 1,
ClaimedCount: 2,
},
{
TemplateName: "template2",
PresetName: "preset2",
OrganizationName: "org1",
CreatedCount: 2,
FailedCount: 1,
ClaimedCount: 1,
},
}, nil
}
type emptyMockDB struct {
database.Store
}
func (*emptyMockDB) GetPrebuildMetrics(context.Context) ([]database.GetPrebuildMetricsRow, error) {
return []database.GetPrebuildMetricsRow{}, nil
}
func TestShouldReportTelemetryDisabled(t *testing.T) {
t.Parallel()
// Description | telemetryEnabled (db) | telemetryEnabled (is) | Report Telemetry Disabled |
//----------------------------------------|-----------------------|-----------------------|---------------------------|
// New deployment | <null> | true | No |
// New deployment with telemetry disabled | <null> | false | No |
// Telemetry was enabled, and still is | true | true | No |
// Telemetry was enabled but now disabled | true | false | Yes |
// Telemetry was disabled, now is enabled | false | true | No |
// Telemetry was disabled, still disabled | false | false | No |
boolTrue := true
boolFalse := false
require.False(t, telemetry.ShouldReportTelemetryDisabled(nil, true))
require.False(t, telemetry.ShouldReportTelemetryDisabled(nil, false))
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolTrue, true))
require.True(t, telemetry.ShouldReportTelemetryDisabled(&boolTrue, false))
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolFalse, true))
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolFalse, false))
}
func TestRecordTelemetryStatus(t *testing.T) {
t.Parallel()
for _, testCase := range []struct {
name string
recordedTelemetryEnabled string
telemetryEnabled bool
shouldReport bool
}{
{name: "New deployment", recordedTelemetryEnabled: "nil", telemetryEnabled: true, shouldReport: false},
{name: "Telemetry disabled", recordedTelemetryEnabled: "nil", telemetryEnabled: false, shouldReport: false},
{name: "Telemetry was enabled and still is", recordedTelemetryEnabled: "true", telemetryEnabled: true, shouldReport: false},
{name: "Telemetry was enabled but now disabled", recordedTelemetryEnabled: "true", telemetryEnabled: false, shouldReport: true},
{name: "Telemetry was disabled now is enabled", recordedTelemetryEnabled: "false", telemetryEnabled: true, shouldReport: false},
{name: "Telemetry was disabled still disabled", recordedTelemetryEnabled: "false", telemetryEnabled: false, shouldReport: false},
{name: "Telemetry was disabled still disabled, invalid value", recordedTelemetryEnabled: "invalid", telemetryEnabled: false, shouldReport: false},
} {
t.Run(testCase.name, func(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
logger := testutil.Logger(t)
if testCase.recordedTelemetryEnabled != "nil" {
db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{
Key: string(telemetry.TelemetryItemKeyTelemetryEnabled),
Value: testCase.recordedTelemetryEnabled,
})
}
snapshot1, err := telemetry.RecordTelemetryStatus(ctx, logger, db, testCase.telemetryEnabled)
require.NoError(t, err)
if testCase.shouldReport {
require.NotNil(t, snapshot1)
require.Equal(t, snapshot1.TelemetryItems[0].Key, string(telemetry.TelemetryItemKeyTelemetryEnabled))
require.Equal(t, snapshot1.TelemetryItems[0].Value, "false")
} else {
require.Nil(t, snapshot1)
}
for range 3 {
// Whatever happens, subsequent calls should not report if telemetryEnabled didn't change
snapshot2, err := telemetry.RecordTelemetryStatus(ctx, logger, db, testCase.telemetryEnabled)
require.NoError(t, err)
require.Nil(t, snapshot2)
}
})
}
}
func mockTelemetryServer(ctx context.Context, t *testing.T) (*url.URL, chan *telemetry.Deployment, chan *telemetry.Snapshot) {
t.Helper()
deployment := make(chan *telemetry.Deployment, 64)
snapshot := make(chan *telemetry.Snapshot, 64)
r := chi.NewRouter()
r.Post("/deployment", func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, buildinfo.Version(), r.Header.Get(telemetry.VersionHeader))
dd := &telemetry.Deployment{}
err := json.NewDecoder(r.Body).Decode(dd)
require.NoError(t, err)
ok := testutil.AssertSend(ctx, t, deployment, dd)
if !ok {
w.WriteHeader(http.StatusInternalServerError)
return
}
// Ensure the header is sent only after deployment is sent
w.WriteHeader(http.StatusAccepted)
})
r.Post("/snapshot", func(w http.ResponseWriter, r *http.Request) {
require.Equal(t, buildinfo.Version(), r.Header.Get(telemetry.VersionHeader))
ss := &telemetry.Snapshot{}
err := json.NewDecoder(r.Body).Decode(ss)
require.NoError(t, err)
ok := testutil.AssertSend(ctx, t, snapshot, ss)
if !ok {
w.WriteHeader(http.StatusInternalServerError)
return
}
// Ensure the header is sent only after snapshot is sent
w.WriteHeader(http.StatusAccepted)
})
server := httptest.NewServer(r)
t.Cleanup(server.Close)
serverURL, err := url.Parse(server.URL)
require.NoError(t, err)
return serverURL, deployment, snapshot
}
func collectSnapshot(
ctx context.Context,
t *testing.T,
db database.Store,
addOptionsFn func(opts telemetry.Options) telemetry.Options,
) (*telemetry.Deployment, *telemetry.Snapshot) {
t.Helper()
serverURL, deployment, snapshot := mockTelemetryServer(ctx, t)
options := telemetry.Options{
Database: db,
Logger: testutil.Logger(t),
URL: serverURL,
DeploymentID: uuid.NewString(),
}
if addOptionsFn != nil {
options = addOptionsFn(options)
}
reporter, err := telemetry.New(options)
require.NoError(t, err)
t.Cleanup(reporter.Close)
return testutil.RequireReceive(ctx, t, deployment), testutil.RequireReceive(ctx, t, snapshot)
}
func TestTelemetry_BoundaryUsageSummary(t *testing.T) {
t.Parallel()
t.Run("IncludedInSnapshot", func(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
tracker := boundaryusage.NewTracker()
workspace1, workspace2 := uuid.New(), uuid.New()
user1, user2 := uuid.New(), uuid.New()
replicaID := uuid.New()
tracker.Track(workspace1, user1, 10, 2)
tracker.Track(workspace2, user1, 5, 1)
tracker.Track(workspace2, user2, 3, 0)
// Flush the tracker to the database.
err := tracker.FlushToDB(ctx, db, replicaID)
require.NoError(t, err)
// Collect a snapshot and verify boundary usage is included.
clock := quartz.NewMock(t)
clock.Set(dbtime.Now())
_, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
require.NotNil(t, snapshot.BoundaryUsageSummary)
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.UniqueWorkspaces)
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.UniqueUsers)
require.Equal(t, int64(10+5+3), snapshot.BoundaryUsageSummary.AllowedRequests)
require.Equal(t, int64(2+1+0), snapshot.BoundaryUsageSummary.DeniedRequests)
require.Equal(t, clock.Now().Add(-telemetry.DefaultSnapshotFrequency), snapshot.BoundaryUsageSummary.PeriodStart)
require.Equal(t, int64(telemetry.DefaultSnapshotFrequency/time.Millisecond), snapshot.BoundaryUsageSummary.PeriodDurationMilliseconds)
})
t.Run("ResetAfterCollection", func(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
tracker := boundaryusage.NewTracker()
replicaID := uuid.New()
tracker.Track(uuid.New(), uuid.New(), 5, 1)
err := tracker.FlushToDB(ctx, db, replicaID)
require.NoError(t, err)
clock := quartz.NewMock(t)
clock.Set(dbtime.Now())
// First snapshot should have the data.
_, snapshot1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
require.NotNil(t, snapshot1.BoundaryUsageSummary)
require.Equal(t, int64(5), snapshot1.BoundaryUsageSummary.AllowedRequests)
// Advance clock to next snapshot period to avoid lock conflict.
clock.Advance(30 * time.Minute)
// Second snapshot should have no data (stats were reset).
_, snapshot2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
// Summary should be nil or have zero values since stats were reset.
if snapshot2.BoundaryUsageSummary != nil {
require.Equal(t, int64(0), snapshot2.BoundaryUsageSummary.AllowedRequests)
}
})
t.Run("OnlyOneReplicaCollects", func(t *testing.T) {
t.Parallel()
db, _ := dbtestutil.NewDB(t)
ctx := testutil.Context(t, testutil.WaitMedium)
// Set up boundary usage stats from two replicas.
tracker1 := boundaryusage.NewTracker()
tracker2 := boundaryusage.NewTracker()
replica1ID := uuid.New()
replica2ID := uuid.New()
tracker1.Track(uuid.New(), uuid.New(), 10, 1)
tracker2.Track(uuid.New(), uuid.New(), 20, 2)
err := tracker1.FlushToDB(ctx, db, replica1ID)
require.NoError(t, err)
err = tracker2.FlushToDB(ctx, db, replica2ID)
require.NoError(t, err)
clock := quartz.NewMock(t)
clock.Set(dbtime.Now())
// First snapshot collects and resets.
_, snapshot1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
require.NotNil(t, snapshot1.BoundaryUsageSummary)
require.Equal(t, int64(10+20), snapshot1.BoundaryUsageSummary.AllowedRequests)
// Second snapshot in same period should skip (lock already claimed).
_, snapshot2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
// The second snapshot should have nil because another "replica" already
// claimed the lock for this period.
require.Nil(t, snapshot2.BoundaryUsageSummary)
})
}
func TestChatsTelemetry(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
user := dbgen.User(t, db, database.User{})
// Create chat providers (required FK for model configs).
anthropicProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
Provider: "anthropic",
DisplayName: "Anthropic",
})
openaiProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
Provider: "openai",
DisplayName: "OpenAI",
})
// Create a model config.
modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
Model: "claude-sonnet-4-20250514",
DisplayName: "Claude Sonnet",
IsDefault: true,
ContextLimit: 200000,
})
org2 := dbgen.Organization(t, db, database.Organization{})
modelCfg2 := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
AIProviderID: uuid.NullUUID{UUID: openaiProvider.ID, Valid: true},
OrganizationID: org2.ID,
Model: "gpt-4o",
DisplayName: "GPT-4o",
})
// Soft-deleted model configurations must not appear in telemetry.
deletedCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
Model: "claude-deleted",
DisplayName: "Deleted Model",
ContextLimit: 100000,
})
_, err := db.DeleteChatModelConfigByID(ctx, deletedCfg.ID)
require.NoError(t, err)
// Create a root chat with a workspace.
org, err := db.GetDefaultOrganization(ctx)
require.NoError(t, err)
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
OrganizationID: org.ID,
Type: database.ProvisionerJobTypeTemplateVersionDryRun,
})
tpl := dbgen.Template(t, db, database.Template{
OrganizationID: org.ID,
CreatedBy: user.ID,
})
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
OrganizationID: org.ID,
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
CreatedBy: user.ID,
JobID: job.ID,
})
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
OwnerID: user.ID,
OrganizationID: org.ID,
TemplateID: tpl.ID,
})
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
Transition: database.WorkspaceTransitionStart,
Reason: database.BuildReasonInitiator,
WorkspaceID: ws.ID,
TemplateVersionID: tv.ID,
JobID: job.ID,
})
rootChat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: modelCfg.ID,
Title: "Root Chat",
Status: database.ChatStatusRunning,
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
Mode: database.NullChatMode{ChatMode: database.ChatModeComputerUse, Valid: true},
})
// Create a child chat (has parent + root).
childChat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: modelCfg2.ID,
Title: "Child Chat",
Status: database.ChatStatusWaiting,
ParentChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true},
RootChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true},
})
// Associate a PR with the root chat so PullRequestState is populated.
rootChatNow := dbtime.Now()
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
ChatID: rootChat.ID,
PullRequestState: sql.NullString{String: "merged", Valid: true},
RefreshedAt: rootChatNow,
StaleAt: rootChatNow,
})
require.NoError(t, err)
// Insert messages for root chat: 2 user, 2 assistant, 1 tool.
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"hello"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 100, Valid: true},
TotalTokens: sql.NullInt64{Int64: 100, Valid: true},
CacheCreationTokens: sql.NullInt64{Int64: 50, Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
})
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleAssistant,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"hi"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 200, Valid: true},
OutputTokens: sql.NullInt64{Int64: 50, Valid: true},
TotalTokens: sql.NullInt64{Int64: 250, Valid: true},
ReasoningTokens: sql.NullInt64{Int64: 10, Valid: true},
CacheReadTokens: sql.NullInt64{Int64: 25, Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
RuntimeMs: sql.NullInt64{Int64: 500, Valid: true},
ProviderResponseID: sql.NullString{String: "resp-1", Valid: true},
})
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"help"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 150, Valid: true},
TotalTokens: sql.NullInt64{Int64: 150, Valid: true},
CacheCreationTokens: sql.NullInt64{Int64: 30, Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
})
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleAssistant,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"sure"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 300, Valid: true},
OutputTokens: sql.NullInt64{Int64: 100, Valid: true},
TotalTokens: sql.NullInt64{Int64: 400, Valid: true},
ReasoningTokens: sql.NullInt64{Int64: 20, Valid: true},
CacheReadTokens: sql.NullInt64{Int64: 40, Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
RuntimeMs: sql.NullInt64{Int64: 800, Valid: true},
ProviderResponseID: sql.NullString{String: "resp-2", Valid: true},
})
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleTool,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"result"}]`), Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
RuntimeMs: sql.NullInt64{Int64: 100, Valid: true},
})
// Insert messages for child chat: 1 user, 1 assistant (compressed).
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: childChat.ID,
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
ModelConfigID: uuid.NullUUID{UUID: modelCfg2.ID, Valid: true},
Role: database.ChatMessageRoleUser,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"q"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 500, Valid: true},
TotalTokens: sql.NullInt64{Int64: 500, Valid: true},
CacheCreationTokens: sql.NullInt64{Int64: 100, Valid: true},
ContextLimit: sql.NullInt64{Int64: 128000, Valid: true},
})
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: childChat.ID,
ModelConfigID: uuid.NullUUID{UUID: modelCfg2.ID, Valid: true},
Role: database.ChatMessageRoleAssistant,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"a"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 600, Valid: true},
OutputTokens: sql.NullInt64{Int64: 200, Valid: true},
TotalTokens: sql.NullInt64{Int64: 800, Valid: true},
ReasoningTokens: sql.NullInt64{Int64: 50, Valid: true},
CacheReadTokens: sql.NullInt64{Int64: 75, Valid: true},
ContextLimit: sql.NullInt64{Int64: 128000, Valid: true},
Compressed: true,
RuntimeMs: sql.NullInt64{Int64: 1200, Valid: true},
ProviderResponseID: sql.NullString{String: "resp-3", Valid: true},
})
// Large token values expose a missing soft-delete filter in the totals.
poisonMsg := dbgen.ChatMessage(t, db, database.ChatMessage{
ChatID: rootChat.ID,
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
Role: database.ChatMessageRoleAssistant,
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"poison"}]`), Valid: true},
InputTokens: sql.NullInt64{Int64: 999999, Valid: true},
OutputTokens: sql.NullInt64{Int64: 999999, Valid: true},
TotalTokens: sql.NullInt64{Int64: 999999, Valid: true},
ReasoningTokens: sql.NullInt64{Int64: 999999, Valid: true},
CacheCreationTokens: sql.NullInt64{Int64: 999999, Valid: true},
CacheReadTokens: sql.NullInt64{Int64: 999999, Valid: true},
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
RuntimeMs: sql.NullInt64{Int64: 999999, Valid: true},
})
err = db.SoftDeleteChatMessageByID(ctx, poisonMsg.ID)
require.NoError(t, err)
_, snapshot := collectSnapshot(ctx, t, db, nil)
// --- Assert Chats ---
require.Len(t, snapshot.Chats, 2)
// Find root and child by HasParent flag.
var foundRoot, foundChild *telemetry.Chat
for i := range snapshot.Chats {
if !snapshot.Chats[i].HasParent {
foundRoot = &snapshot.Chats[i]
} else {
foundChild = &snapshot.Chats[i]
}
}
require.NotNil(t, foundRoot, "expected root chat")
require.NotNil(t, foundChild, "expected child chat")
// Root chat assertions.
assert.Equal(t, rootChat.ID, foundRoot.ID)
assert.Equal(t, user.ID, foundRoot.OwnerID)
assert.Equal(t, org.ID, foundRoot.OrganizationID)
assert.Equal(t, "running", foundRoot.Status)
assert.False(t, foundRoot.HasParent)
assert.Nil(t, foundRoot.RootChatID)
require.NotNil(t, foundRoot.WorkspaceID)
assert.Equal(t, ws.ID, *foundRoot.WorkspaceID)
assert.Equal(t, modelCfg.ID, foundRoot.LastModelConfigID)
require.NotNil(t, foundRoot.Mode)
assert.Equal(t, "computer_use", *foundRoot.Mode)
assert.False(t, foundRoot.Archived)
assert.Equal(t, "ui", foundRoot.ClientType)
require.NotNil(t, foundRoot.PullRequestState)
assert.Equal(t, "merged", *foundRoot.PullRequestState)
// Child chat assertions.
assert.Equal(t, childChat.ID, foundChild.ID)
assert.Equal(t, user.ID, foundChild.OwnerID)
assert.True(t, foundChild.HasParent)
require.NotNil(t, foundChild.RootChatID)
assert.Equal(t, rootChat.ID, *foundChild.RootChatID)
assert.Nil(t, foundChild.WorkspaceID)
assert.Equal(t, "waiting", foundChild.Status)
assert.Equal(t, modelCfg2.ID, foundChild.LastModelConfigID)
assert.Nil(t, foundChild.Mode)
assert.False(t, foundChild.Archived)
assert.Equal(t, "ui", foundChild.ClientType)
assert.Nil(t, foundChild.PullRequestState)
// --- Assert ChatMessageSummaries ---
require.Len(t, snapshot.ChatMessageSummaries, 2)
summaryMap := make(map[uuid.UUID]telemetry.ChatMessageSummary)
for _, s := range snapshot.ChatMessageSummaries {
summaryMap[s.ChatID] = s
}
// Root chat summary: 2 user + 2 assistant + 1 tool = 5 messages.
rootSummary, ok := summaryMap[rootChat.ID]
require.True(t, ok, "expected summary for root chat")
assert.Equal(t, int64(5), rootSummary.MessageCount)
assert.Equal(t, int64(2), rootSummary.UserMessageCount)
assert.Equal(t, int64(2), rootSummary.AssistantMessageCount)
assert.Equal(t, int64(1), rootSummary.ToolMessageCount)
assert.Equal(t, int64(0), rootSummary.SystemMessageCount)
assert.Equal(t, int64(750), rootSummary.TotalInputTokens) // 100+200+150+300+0
assert.Equal(t, int64(150), rootSummary.TotalOutputTokens) // 0+50+0+100+0
assert.Equal(t, int64(30), rootSummary.TotalReasoningTokens) // 0+10+0+20+0
assert.Equal(t, int64(80), rootSummary.TotalCacheCreationTokens) // 50+0+30+0+0
assert.Equal(t, int64(65), rootSummary.TotalCacheReadTokens) // 0+25+0+40+0
assert.Equal(t, int64(1400), rootSummary.TotalRuntimeMs) // 0+500+0+800+100
assert.Equal(t, int64(1), rootSummary.DistinctModelCount)
assert.Equal(t, int64(0), rootSummary.CompressedMessageCount)
// Child chat summary: 1 user + 1 assistant = 2 messages, 1 compressed.
childSummary, ok := summaryMap[childChat.ID]
require.True(t, ok, "expected summary for child chat")
assert.Equal(t, int64(2), childSummary.MessageCount)
assert.Equal(t, int64(1), childSummary.UserMessageCount)
assert.Equal(t, int64(1), childSummary.AssistantMessageCount)
assert.Equal(t, int64(1100), childSummary.TotalInputTokens) // 500+600
assert.Equal(t, int64(200), childSummary.TotalOutputTokens) // 0+200
assert.Equal(t, int64(50), childSummary.TotalReasoningTokens) // 0+50
assert.Equal(t, int64(0), childSummary.ToolMessageCount)
assert.Equal(t, int64(0), childSummary.SystemMessageCount)
assert.Equal(t, int64(100), childSummary.TotalCacheCreationTokens) // 100+0
assert.Equal(t, int64(75), childSummary.TotalCacheReadTokens) // 0+75
assert.Equal(t, int64(1200), childSummary.TotalRuntimeMs) // 0+1200
assert.Equal(t, int64(1), childSummary.DistinctModelCount)
assert.Equal(t, int64(1), childSummary.CompressedMessageCount)
// --- Assert ChatModelConfigs ---
require.Len(t, snapshot.ChatModelConfigs, 2)
configMap := make(map[uuid.UUID]telemetry.ChatModelConfig)
for _, c := range snapshot.ChatModelConfigs {
configMap[c.ID] = c
}
cfg1, ok := configMap[modelCfg.ID]
require.True(t, ok)
assert.Equal(t, org.ID, cfg1.OrganizationID)
assert.Equal(t, "anthropic", cfg1.Provider)
assert.Equal(t, "claude-sonnet-4-20250514", cfg1.Model)
assert.Equal(t, int64(200000), cfg1.ContextLimit)
assert.True(t, cfg1.Enabled)
assert.True(t, cfg1.IsDefault)
cfg2, ok := configMap[modelCfg2.ID]
require.True(t, ok)
assert.Equal(t, org2.ID, cfg2.OrganizationID)
assert.Equal(t, "openai", cfg2.Provider)
assert.Equal(t, "gpt-4o", cfg2.Model)
assert.Equal(t, int64(128000), cfg2.ContextLimit)
assert.True(t, cfg2.Enabled)
assert.False(t, cfg2.IsDefault)
}
func TestChatDiffStatusSummaryTelemetry(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// Verify zero counts when no chat_diff_statuses exist.
_, emptySnapshot := collectSnapshot(ctx, t, db, nil)
require.NotNil(t, emptySnapshot.ChatDiffStatusSummary)
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Total)
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Open)
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Merged)
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Closed)
// Set up minimal FK chain: provider -> model config -> chat.
user := dbgen.User(t, db, database.User{})
org, err := db.GetDefaultOrganization(ctx)
require.NoError(t, err)
anthropicProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
Provider: "anthropic",
DisplayName: "Anthropic",
})
modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
Model: "claude-sonnet-4-20250514",
DisplayName: "Claude Sonnet",
IsDefault: true,
ContextLimit: 200000,
})
// Helper to create a chat and upsert its diff status.
insertChatWithDiffStatus := func(prURL, state string) uuid.UUID {
t.Helper()
chat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: modelCfg.ID,
Title: "Chat " + state,
Status: database.ChatStatusWaiting,
})
now := dbtime.Now()
_, chatErr := db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
ChatID: chat.ID,
Url: sql.NullString{String: prURL, Valid: prURL != ""},
PullRequestState: sql.NullString{String: state, Valid: true},
RefreshedAt: now,
StaleAt: now,
})
require.NoError(t, chatErr)
return chat.ID
}
// Insert: 1 merged, 1 open, 1 closed (each with unique URLs).
// For pull/1, first insert an older chat with stale "open" state,
// then a newer chat with refreshed "merged" state. The dedup
// query orders by cds.updated_at DESC, so "merged" should win.
insertChatWithDiffStatus("https://github.com/org/repo/pull/1", "open")
insertChatWithDiffStatus("https://github.com/org/repo/pull/1", "merged")
openChatID := insertChatWithDiffStatus("https://github.com/org/repo/pull/2", "open")
insertChatWithDiffStatus("https://github.com/org/repo/pull/3", "closed")
// Insert a chat with NULL pull_request_state (no PR yet).
// This should be excluded from all counts.
noPRChat := dbgen.Chat(t, db, database.Chat{
OrganizationID: org.ID,
OwnerID: user.ID,
LastModelConfigID: modelCfg.ID,
Title: "Chat no PR",
Status: database.ChatStatusRunning,
})
now := dbtime.Now()
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
ChatID: noPRChat.ID,
RefreshedAt: now,
StaleAt: now,
})
require.NoError(t, err)
_, snapshot := collectSnapshot(ctx, t, db, nil)
// 3 unique PRs (deduped by URL), not 4 chat_diff_statuses rows.
require.NotNil(t, snapshot.ChatDiffStatusSummary)
assert.Equal(t, int64(3), snapshot.ChatDiffStatusSummary.Total)
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Open)
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Merged)
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Closed)
// Transition the "open" PR to "merged" via upsert on the same
// chat_id. The aggregate should reflect the new state.
now = dbtime.Now()
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
ChatID: openChatID,
Url: sql.NullString{String: "https://github.com/org/repo/pull/2", Valid: true},
PullRequestState: sql.NullString{String: "merged", Valid: true},
RefreshedAt: now,
StaleAt: now,
})
require.NoError(t, err)
_, snapshot2 := collectSnapshot(ctx, t, db, nil)
require.NotNil(t, snapshot2.ChatDiffStatusSummary)
assert.Equal(t, int64(3), snapshot2.ChatDiffStatusSummary.Total)
assert.Equal(t, int64(0), snapshot2.ChatDiffStatusSummary.Open)
assert.Equal(t, int64(2), snapshot2.ChatDiffStatusSummary.Merged)
assert.Equal(t, int64(1), snapshot2.ChatDiffStatusSummary.Closed)
}
func TestUserSecretsTelemetry(t *testing.T) {
t.Parallel()
t.Run("Empty", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// Empty deployment should report a non-nil summary with zeros.
_, snap := collectSnapshot(ctx, t, db, nil)
require.Equal(t, &telemetry.UserSecretsSummary{}, snap.UserSecretsSummary)
})
t.Run("ConfigurationBreakdown", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
userA := dbgen.User(t, db, database.User{})
userB := dbgen.User(t, db, database.User{})
// userA: env-only and file-only. dbgen.UserSecret defaults
// EnvName and FilePath to non-empty, so use mutators to clear
// them where the test wants empty values.
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: userA.ID,
Name: "a-env",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = "A_ENV"
p.FilePath = ""
})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: userA.ID,
Name: "a-file",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = "/home/coder/a.file"
})
// userB: both and neither.
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: userB.ID,
Name: "b-both",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = "B_BOTH"
p.FilePath = "/home/coder/b.both"
})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: userB.ID,
Name: "b-neither",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = ""
// A target-less secret must be disabled to satisfy the
// user_secrets_enabled_requires_target constraint. Disabled
// secrets are still counted in the telemetry breakdown.
p.Enabled = false
})
_, snap := collectSnapshot(ctx, t, db, nil)
// Each user has exactly two secrets, so every percentile and
// the max collapse to 2.
require.Equal(t, &telemetry.UserSecretsSummary{
UsersWithSecrets: 2,
TotalSecrets: 4,
EnvNameOnly: 1,
FilePathOnly: 1,
Both: 1,
Neither: 1,
SecretsPerUserMax: 2,
SecretsPerUserP25: 2,
SecretsPerUserP50: 2,
SecretsPerUserP75: 2,
SecretsPerUserP90: 2,
}, snap.UserSecretsSummary)
})
t.Run("PercentileDistribution", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// Five users have secret counts 1, 2, 4, 8, 16 and five other
// users have zero secrets. Including the zero-secret users in
// the distribution gives a sorted vector of length 10:
// [0, 0, 0, 0, 0, 1, 2, 4, 8, 16]
// percentile_disc(p) returns the value at the smallest
// 1-indexed position i where i/n >= p, so the buckets land at:
// p25 -> position 3 -> 0
// p50 -> position 5 -> 0
// p75 -> position 8 -> 4
// p90 -> position 9 -> 8
adopters := []int{1, 2, 4, 8, 16}
for _, n := range adopters {
u := dbgen.User(t, db, database.User{})
for i := 0; i < n; i++ {
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: u.ID,
Name: fmt.Sprintf("secret-%d", i),
}, func(p *database.CreateUserSecretParams) {
// Clear EnvName and FilePath so the unique
// (user_id, env_name) and (user_id, file_path)
// indexes don't collide across multiple secrets
// for the same user. Target-less secrets must be
// disabled to satisfy the
// user_secrets_enabled_requires_target constraint.
p.EnvName = ""
p.FilePath = ""
p.Enabled = false
})
}
}
for i := 0; i < 5; i++ {
_ = dbgen.User(t, db, database.User{})
}
_, snap := collectSnapshot(ctx, t, db, nil)
require.Equal(t, &telemetry.UserSecretsSummary{
UsersWithSecrets: 5,
TotalSecrets: 31,
EnvNameOnly: 0,
FilePathOnly: 0,
Both: 0,
Neither: 31,
SecretsPerUserMax: 16,
SecretsPerUserP25: 0,
SecretsPerUserP50: 0,
SecretsPerUserP75: 4,
SecretsPerUserP90: 8,
}, snap.UserSecretsSummary)
})
t.Run("FilterSkipsInactiveUsers", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// Active user with two secrets contributes the only entries
// to UsersWithSecrets, TotalSecrets, and the percentile
// distribution.
active := dbgen.User(t, db, database.User{})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: active.ID,
Name: "active-env",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = "ACTIVE_ENV"
p.FilePath = ""
})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: active.ID,
Name: "active-file",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = "/home/coder/active.file"
})
// User secret owned by a dormant user should be excluded.
dormant := dbgen.User(t, db, database.User{Status: database.UserStatusDormant})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: dormant.ID,
Name: "dormant-secret",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = "DORMANT_ENV"
p.FilePath = ""
})
// User secret owned by a suspended user should be excluded.
suspended := dbgen.User(t, db, database.User{Status: database.UserStatusSuspended})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: suspended.ID,
Name: "suspended-secret",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = "/home/coder/suspended.file"
})
// System user. Only its UUID is needed. Tying a secret to it
// proves the is_system filter excludes it.
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: database.PrebuildsSystemUserID,
Name: "prebuilds-secret",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = "/home/coder/prebuilds.file"
})
_, snap := collectSnapshot(ctx, t, db, nil)
require.Equal(t, &telemetry.UserSecretsSummary{
UsersWithSecrets: 1,
TotalSecrets: 2,
EnvNameOnly: 1,
FilePathOnly: 1,
Both: 0,
Neither: 0,
SecretsPerUserMax: 2,
SecretsPerUserP25: 2,
SecretsPerUserP50: 2,
SecretsPerUserP75: 2,
SecretsPerUserP90: 2,
}, snap.UserSecretsSummary)
})
t.Run("OnlyOneReplicaCollects", func(t *testing.T) {
t.Parallel()
ctx := testutil.Context(t, testutil.WaitMedium)
db, _ := dbtestutil.NewDB(t)
// Seed one user with one secret so the summary would normally
// be populated. The user_secrets_summary aggregate has no
// natural per-row UUID for the telemetry server to dedupe on,
// so a telemetry lock elects a single replica per period.
u := dbgen.User(t, db, database.User{})
_ = dbgen.UserSecret(t, db, database.UserSecret{
UserID: u.ID,
Name: "only-secret",
}, func(p *database.CreateUserSecretParams) {
p.EnvName = ""
p.FilePath = ""
// Target-less secrets must be disabled to satisfy the
// user_secrets_enabled_requires_target constraint.
p.Enabled = false
})
clock := quartz.NewMock(t)
clock.Set(dbtime.Now())
// First snapshot claims the lock and reports the summary.
_, snap1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
require.Equal(t, &telemetry.UserSecretsSummary{
UsersWithSecrets: 1,
TotalSecrets: 1,
EnvNameOnly: 0,
FilePathOnly: 0,
Both: 0,
Neither: 1,
SecretsPerUserMax: 1,
SecretsPerUserP25: 1,
SecretsPerUserP50: 1,
SecretsPerUserP75: 1,
SecretsPerUserP90: 1,
}, snap1.UserSecretsSummary)
// A second snapshot in the same period simulates a second
// replica racing to claim the lock; it should observe the
// unique violation and skip reporting.
_, snap2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
opts.Clock = clock
return opts
})
require.Nil(t, snap2.UserSecretsSummary)
})
}
func TestCollectAgentsVirtualDesktop(t *testing.T) {
t.Parallel()
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsVirtualDesktopTelemetry {
t.Helper()
var payload telemetry.AgentsVirtualDesktopTelemetry
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsVirtualDesktop(context.Background(), opts), &payload))
return payload
}
t.Run("Default", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", nil)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.False(t, payload.Enabled)
require.EqualValues(t, codersdk.ChatComputerUseProviderAnthropic, payload.ComputerUse.Provider)
require.Equal(t, "default", payload.ComputerUse.ProviderSource)
})
t.Run("Configured", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("openai", nil)
payload := collect(t, telemetry.Options{
Database: db,
Logger: testutil.Logger(t),
Experiments: codersdk.Experiments{codersdk.ExperimentChatVirtualDesktop},
})
require.True(t, payload.Enabled)
require.Equal(t, "openai", payload.ComputerUse.Provider)
require.Equal(t, "configured", payload.ComputerUse.ProviderSource)
})
t.Run("QueryError", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatComputerUseProvider(gomock.Any()).Return("", sql.ErrConnDone)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.Provider)
require.Equal(t, telemetry.AgentsExperimentUnknown, payload.ComputerUse.ProviderSource)
})
}
func TestCollectAgentsAdvisor(t *testing.T) {
t.Parallel()
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsAdvisorTelemetry {
t.Helper()
var payload telemetry.AgentsAdvisorTelemetry
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsAdvisor(context.Background(), opts), &payload))
return payload
}
marshalConfig := func(t *testing.T, cfg codersdk.AdvisorConfig) string {
t.Helper()
raw, err := json.Marshal(cfg)
require.NoError(t, err)
return string(raw)
}
expectOverrides := func(db *dbmock.MockStore, rows ...database.GetChatOrganizationModelOverridesByContextRow) {
db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), string(codersdk.ChatModelOverrideContextAdvisor)).Return(rows, nil)
}
t.Run("Defaults", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
expectOverrides(db)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.False(t, payload.Enabled)
require.Zero(t, payload.MaxUsesPerRun)
require.Zero(t, payload.MaxOutputTokens)
require.Empty(t, payload.Overrides)
})
t.Run("ModelOverrides", func(t *testing.T) {
t.Parallel()
org := database.Organization{ID: uuid.New()}
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{MaxUsesPerRun: 7, MaxOutputTokens: 2048}), nil)
expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
OrganizationID: org.ID,
ModelAvailable: true,
Model: "gpt-6-preview",
ProviderType: string(database.AIProviderTypeOpenai),
})
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t), Experiments: codersdk.Experiments{codersdk.ExperimentChatAdvisor}})
require.True(t, payload.Enabled)
require.Equal(t, 7, payload.MaxUsesPerRun)
require.Equal(t, int64(2048), payload.MaxOutputTokens)
require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
OrganizationID: org.ID.String(), Provider: string(database.AIProviderTypeOpenai), Model: "gpt-6-preview",
}}, payload.Overrides)
})
t.Run("InactiveModel", func(t *testing.T) {
t.Parallel()
org := database.Organization{ID: uuid.New()}
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
OrganizationID: org.ID,
ModelAvailable: false,
})
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
OrganizationID: org.ID.String(),
Provider: telemetry.AgentsExperimentAdvisorReuseChatModel,
Model: telemetry.AgentsExperimentAdvisorReuseChatModel,
}}, payload.Overrides)
})
t.Run("NoOverride", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
expectOverrides(db)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Empty(t, payload.Overrides)
})
t.Run("MalformedConfig", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("not-json", nil)
expectOverrides(db)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Zero(t, payload.MaxUsesPerRun)
require.Zero(t, payload.MaxOutputTokens)
})
t.Run("ClampsNegativeLimits", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(`{"max_uses_per_run":-3,"max_output_tokens":-99}`, nil)
expectOverrides(db)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Zero(t, payload.MaxUsesPerRun)
require.Zero(t, payload.MaxOutputTokens)
})
t.Run("ConfigFetchError", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("", sql.ErrConnDone)
expectOverrides(db)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Empty(t, payload.Overrides)
})
t.Run("OverridesFetchError", func(t *testing.T) {
t.Parallel()
db := dbmock.NewMockStore(gomock.NewController(t))
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), gomock.Any()).Return(nil, sql.ErrConnDone)
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
require.Empty(t, payload.Overrides)
})
}