A client could resume the chat stream only with after_id, a message ID. A message ID does not identify what a client has received. A reconnect could skip a message that another client sent, send no history_reset for an edit the client missed, or resend the whole history. A client can now resume with the chat's snapshot_version, which the newest messages page returns and the stream's events then include. The stream sends what changed at that version or later. For an edit it sends the ID of the edited message and the messages written since, not the whole history. A stream opened without snapshot_version sends the same events as before. A write outside a chat state transition does not increase snapshot_version, and a stream can miss it. The queries that only tests used for such writes are removed. When the stream cannot be set up, the server now returns 500 before the WebSocket upgrade. It used to upgrade and send an error event, which the web client shows as a chat error. The Tasks to Chats migration guide told clients to reconnect with after_id. It now describes snapshot_version.
2133 lines
80 KiB
Go
2133 lines
80 KiB
Go
package telemetry_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"sort"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/go-chi/chi/v5"
|
|
"github.com/google/uuid"
|
|
"github.com/sqlc-dev/pqtype"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
"go.uber.org/goleak"
|
|
"go.uber.org/mock/gomock"
|
|
|
|
"github.com/coder/coder/v2/buildinfo"
|
|
"github.com/coder/coder/v2/coderd/boundaryusage"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbgen"
|
|
"github.com/coder/coder/v2/coderd/database/dbmock"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
"github.com/coder/coder/v2/coderd/database/dbtime"
|
|
"github.com/coder/coder/v2/coderd/idpsync"
|
|
"github.com/coder/coder/v2/coderd/runtimeconfig"
|
|
"github.com/coder/coder/v2/coderd/telemetry"
|
|
"github.com/coder/coder/v2/coderd/util/slice"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
"github.com/coder/quartz"
|
|
)
|
|
|
|
func TestMain(m *testing.M) {
|
|
goleak.VerifyTestMain(m, testutil.GoleakOptions...)
|
|
}
|
|
|
|
func TestConvertTemplateAgentsAllowed(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
for _, tt := range []struct {
|
|
name string
|
|
allowed bool
|
|
}{
|
|
{name: "Allowed", allowed: true},
|
|
{name: "Disallowed", allowed: false},
|
|
} {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
got := telemetry.ConvertTemplate(database.Template{AgentsAllowed: tt.allowed})
|
|
require.Equal(t, tt.allowed, got.AgentsAllowed)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTelemetry(t *testing.T) {
|
|
t.Parallel()
|
|
t.Run("Snapshot", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
var err error
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
now := dbtime.Now()
|
|
|
|
org, err := db.GetDefaultOrganization(ctx)
|
|
require.NoError(t, err)
|
|
|
|
user := dbgen.User(t, db, database.User{})
|
|
_ = dbgen.OrganizationMember(t, db, database.OrganizationMember{
|
|
UserID: user.ID,
|
|
OrganizationID: org.ID,
|
|
})
|
|
require.NoError(t, err)
|
|
_, _ = dbgen.APIKey(t, db, database.APIKey{
|
|
UserID: user.ID,
|
|
})
|
|
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
|
|
Provisioner: database.ProvisionerTypeTerraform,
|
|
StorageMethod: database.ProvisionerStorageMethodFile,
|
|
Type: database.ProvisionerJobTypeTemplateVersionDryRun,
|
|
OrganizationID: org.ID,
|
|
})
|
|
tpl := dbgen.Template(t, db, database.Template{
|
|
Provisioner: database.ProvisionerTypeTerraform,
|
|
OrganizationID: org.ID,
|
|
CreatedBy: user.ID,
|
|
})
|
|
sourceExampleID := uuid.NewString()
|
|
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
|
SourceExampleID: sql.NullString{String: sourceExampleID, Valid: true},
|
|
OrganizationID: org.ID,
|
|
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
|
|
CreatedBy: user.ID,
|
|
JobID: job.ID,
|
|
})
|
|
_ = dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
|
OrganizationID: org.ID,
|
|
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
|
|
CreatedBy: user.ID,
|
|
JobID: job.ID,
|
|
})
|
|
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
|
|
OwnerID: user.ID,
|
|
OrganizationID: org.ID,
|
|
TemplateID: tpl.ID,
|
|
})
|
|
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
|
|
Transition: database.WorkspaceTransitionStart,
|
|
Reason: database.BuildReasonAutostart,
|
|
WorkspaceID: ws.ID,
|
|
TemplateVersionID: tv.ID,
|
|
JobID: job.ID,
|
|
})
|
|
wsresource := dbgen.WorkspaceResource(t, db, database.WorkspaceResource{
|
|
JobID: job.ID,
|
|
})
|
|
wsagent := dbgen.WorkspaceAgent(t, db, database.WorkspaceAgent{
|
|
ResourceID: wsresource.ID,
|
|
})
|
|
_ = dbgen.WorkspaceApp(t, db, database.WorkspaceApp{
|
|
SharingLevel: database.AppSharingLevelOwner,
|
|
Health: database.WorkspaceAppHealthDisabled,
|
|
OpenIn: database.WorkspaceAppOpenInSlimWindow,
|
|
AgentID: wsagent.ID,
|
|
})
|
|
|
|
group := dbgen.Group(t, db, database.Group{
|
|
OrganizationID: org.ID,
|
|
})
|
|
_ = dbgen.TelemetryItem(t, db, database.TelemetryItem{
|
|
Key: string(telemetry.TelemetryItemKeyHTMLFirstServedAt),
|
|
Value: time.Now().Format(time.RFC3339),
|
|
})
|
|
_ = dbgen.GroupMember(t, db, database.GroupMemberTable{UserID: user.ID, GroupID: group.ID})
|
|
// Update the workspace agent to have a valid subsystem.
|
|
err = db.UpdateWorkspaceAgentStartupByID(ctx, database.UpdateWorkspaceAgentStartupByIDParams{
|
|
ID: wsagent.ID,
|
|
Version: wsagent.Version,
|
|
ExpandedDirectory: wsagent.ExpandedDirectory,
|
|
Subsystems: []database.WorkspaceAgentSubsystem{
|
|
database.WorkspaceAgentSubsystemEnvbox,
|
|
database.WorkspaceAgentSubsystemExectrace,
|
|
},
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_ = dbgen.WorkspaceAgentStat(t, db, database.WorkspaceAgentStat{
|
|
ConnectionMedianLatencyMS: 1,
|
|
SessionCounts: dbgen.SessionCounts(t, map[string]int64{
|
|
"cursor": 2,
|
|
"vscode": 1,
|
|
"zed": 3,
|
|
"unknown": 4,
|
|
}),
|
|
})
|
|
_, err = db.InsertLicense(ctx, database.InsertLicenseParams{
|
|
UploadedAt: dbtime.Now(),
|
|
JWT: "",
|
|
Exp: dbtime.Now().Add(time.Hour),
|
|
UUID: uuid.New(),
|
|
})
|
|
assert.NoError(t, err)
|
|
_, _ = dbgen.WorkspaceProxy(t, db, database.WorkspaceProxy{})
|
|
|
|
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
|
|
JobID: job.ID,
|
|
})
|
|
_ = dbgen.WorkspaceAgentMemoryResourceMonitor(t, db, database.WorkspaceAgentMemoryResourceMonitor{
|
|
AgentID: wsagent.ID,
|
|
})
|
|
_ = dbgen.WorkspaceAgentVolumeResourceMonitor(t, db, database.WorkspaceAgentVolumeResourceMonitor{
|
|
AgentID: wsagent.ID,
|
|
})
|
|
|
|
previousAIBridgeInterceptionPeriod := now.Truncate(time.Hour)
|
|
user2 := dbgen.User(t, db, database.User{})
|
|
aiBridgeInterception1 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
|
InitiatorID: user.ID,
|
|
Provider: "anthropic",
|
|
Model: "deanseek",
|
|
StartedAt: previousAIBridgeInterceptionPeriod.Add(-30 * time.Minute),
|
|
}, nil)
|
|
_ = dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
|
|
InterceptionID: aiBridgeInterception1.ID,
|
|
InputTokens: 100,
|
|
OutputTokens: 200,
|
|
CacheReadInputTokens: 300,
|
|
CacheWriteInputTokens: 400,
|
|
Metadata: json.RawMessage(`{"cache_read_input":300,"cache_creation_input":400}`),
|
|
})
|
|
_ = dbgen.AIBridgeUserPrompt(t, db, database.InsertAIBridgeUserPromptParams{
|
|
InterceptionID: aiBridgeInterception1.ID,
|
|
})
|
|
_ = dbgen.AIBridgeToolUsage(t, db, database.InsertAIBridgeToolUsageParams{
|
|
InterceptionID: aiBridgeInterception1.ID,
|
|
Injected: true,
|
|
InvocationError: sql.NullString{String: "error1", Valid: true},
|
|
})
|
|
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
|
|
ID: aiBridgeInterception1.ID,
|
|
EndedAt: aiBridgeInterception1.StartedAt.Add(1 * time.Minute), // 1 minute duration
|
|
})
|
|
require.NoError(t, err)
|
|
aiBridgeInterception2 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
|
InitiatorID: user2.ID,
|
|
Provider: aiBridgeInterception1.Provider,
|
|
Model: aiBridgeInterception1.Model,
|
|
StartedAt: aiBridgeInterception1.StartedAt,
|
|
}, nil)
|
|
_ = dbgen.AIBridgeTokenUsage(t, db, database.InsertAIBridgeTokenUsageParams{
|
|
InterceptionID: aiBridgeInterception2.ID,
|
|
InputTokens: 100,
|
|
OutputTokens: 200,
|
|
CacheReadInputTokens: 300,
|
|
CacheWriteInputTokens: 400,
|
|
Metadata: json.RawMessage(`{"cache_read_input":300,"cache_creation_input":400}`),
|
|
})
|
|
_ = dbgen.AIBridgeUserPrompt(t, db, database.InsertAIBridgeUserPromptParams{
|
|
InterceptionID: aiBridgeInterception2.ID,
|
|
})
|
|
_ = dbgen.AIBridgeToolUsage(t, db, database.InsertAIBridgeToolUsageParams{
|
|
InterceptionID: aiBridgeInterception2.ID,
|
|
Injected: false,
|
|
})
|
|
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
|
|
ID: aiBridgeInterception2.ID,
|
|
EndedAt: aiBridgeInterception2.StartedAt.Add(2 * time.Minute), // 2 minute duration
|
|
})
|
|
require.NoError(t, err)
|
|
aiBridgeInterception3 := dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
|
InitiatorID: user2.ID,
|
|
Provider: "openai",
|
|
Model: "gpt-5",
|
|
StartedAt: aiBridgeInterception1.StartedAt,
|
|
}, nil)
|
|
_, err = db.UpdateAIBridgeInterceptionEnded(ctx, database.UpdateAIBridgeInterceptionEndedParams{
|
|
ID: aiBridgeInterception3.ID,
|
|
EndedAt: aiBridgeInterception3.StartedAt.Add(3 * time.Minute), // 3 minute duration
|
|
})
|
|
require.NoError(t, err)
|
|
_ = dbgen.AIBridgeInterception(t, db, database.InsertAIBridgeInterceptionParams{
|
|
InitiatorID: user2.ID,
|
|
Provider: "openai",
|
|
Model: "gpt-5",
|
|
StartedAt: aiBridgeInterception1.StartedAt,
|
|
}, nil)
|
|
// not ended, so it should not affect summaries
|
|
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(now)
|
|
|
|
deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
var agentsExperiments map[string]json.RawMessage
|
|
require.NoError(t, json.Unmarshal(deployment.AgentsExperiments, &agentsExperiments))
|
|
require.Contains(t, agentsExperiments, "virtual_desktop")
|
|
require.Contains(t, agentsExperiments, "advisor")
|
|
require.Len(t, snapshot.ProvisionerJobs, 1)
|
|
require.Len(t, snapshot.Licenses, 1)
|
|
require.Len(t, snapshot.Templates, 1)
|
|
require.Len(t, snapshot.TemplateVersions, 2)
|
|
require.Len(t, snapshot.Users, 2)
|
|
require.Len(t, snapshot.Groups, 2)
|
|
// 1 member in the everyone group + 1 member in the custom group
|
|
require.Len(t, snapshot.GroupMembers, 2)
|
|
require.Len(t, snapshot.Workspaces, 1)
|
|
require.Len(t, snapshot.WorkspaceApps, 1)
|
|
require.Len(t, snapshot.WorkspaceAgents, 1)
|
|
require.Len(t, snapshot.WorkspaceBuilds, 1)
|
|
require.Len(t, snapshot.WorkspaceResources, 1)
|
|
require.Len(t, snapshot.WorkspaceAgentStats, 1)
|
|
require.Equal(t, map[string]int64{
|
|
"cursor": 2,
|
|
"vscode": 1,
|
|
"zed": 3,
|
|
"unknown": 4,
|
|
}, snapshot.WorkspaceAgentStats[0].SessionCounts)
|
|
require.Equal(t, int64(3), snapshot.WorkspaceAgentStats[0].SessionCountVSCode)
|
|
require.Equal(t, int64(3), snapshot.WorkspaceAgentStats[0].SessionCountSSH)
|
|
require.Len(t, snapshot.WorkspaceProxies, 1)
|
|
require.Len(t, snapshot.WorkspaceModules, 1)
|
|
require.Len(t, snapshot.Organizations, 1)
|
|
telemetryItemKeys := slice.Convert(snapshot.TelemetryItems, func(item telemetry.TelemetryItem) string {
|
|
return item.Key
|
|
})
|
|
require.ElementsMatch(t, []string{
|
|
string(telemetry.TelemetryItemKeyHTMLFirstServedAt),
|
|
string(telemetry.TelemetryItemKeyTelemetryEnabled),
|
|
}, telemetryItemKeys)
|
|
require.Len(t, snapshot.WorkspaceAgentMemoryResourceMonitors, 1)
|
|
require.Len(t, snapshot.WorkspaceAgentVolumeResourceMonitors, 1)
|
|
wsa := snapshot.WorkspaceAgents[0]
|
|
require.Len(t, wsa.Subsystems, 2)
|
|
require.Equal(t, string(database.WorkspaceAgentSubsystemEnvbox), wsa.Subsystems[0])
|
|
require.Equal(t, string(database.WorkspaceAgentSubsystemExectrace), wsa.Subsystems[1])
|
|
tvs := snapshot.TemplateVersions
|
|
sort.Slice(tvs, func(i, j int) bool {
|
|
// Sort by SourceExampleID presence (non-nil comes before nil)
|
|
if (tvs[i].SourceExampleID != nil) != (tvs[j].SourceExampleID != nil) {
|
|
return tvs[i].SourceExampleID != nil
|
|
}
|
|
return false
|
|
})
|
|
require.Equal(t, tvs[0].SourceExampleID, &sourceExampleID)
|
|
require.Nil(t, tvs[1].SourceExampleID)
|
|
|
|
for _, entity := range snapshot.Workspaces {
|
|
require.Equal(t, entity.OrganizationID, org.ID)
|
|
}
|
|
for _, entity := range snapshot.ProvisionerJobs {
|
|
require.Equal(t, entity.OrganizationID, org.ID)
|
|
}
|
|
for _, entity := range snapshot.TemplateVersions {
|
|
require.Equal(t, entity.OrganizationID, org.ID)
|
|
}
|
|
for _, entity := range snapshot.Templates {
|
|
require.Equal(t, entity.OrganizationID, org.ID)
|
|
}
|
|
|
|
// 2 unique provider + model + client combinations
|
|
require.Len(t, snapshot.AIBridgeInterceptionsSummaries, 2)
|
|
snapshot1 := snapshot.AIBridgeInterceptionsSummaries[0]
|
|
snapshot2 := snapshot.AIBridgeInterceptionsSummaries[1]
|
|
if snapshot1.Provider != aiBridgeInterception1.Provider {
|
|
snapshot1, snapshot2 = snapshot2, snapshot1
|
|
}
|
|
|
|
require.Equal(t, snapshot1.Provider, aiBridgeInterception1.Provider)
|
|
require.Equal(t, snapshot1.Model, aiBridgeInterception1.Model)
|
|
require.Equal(t, snapshot1.Client, "Unknown") // no client info yet
|
|
require.EqualValues(t, snapshot1.InterceptionCount, 2)
|
|
require.EqualValues(t, snapshot1.InterceptionsByRoute, map[string]int64{}) // no route info yet
|
|
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P50, 90_000)
|
|
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P90, 114_000)
|
|
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P95, 117_000)
|
|
require.EqualValues(t, snapshot1.InterceptionDurationMillis.P99, 119_400)
|
|
require.EqualValues(t, snapshot1.UniqueInitiatorCount, 2)
|
|
require.EqualValues(t, snapshot1.UserPromptsCount, 2)
|
|
require.EqualValues(t, snapshot1.TokenUsagesCount, 2)
|
|
require.EqualValues(t, snapshot1.TokenCount.Input, 200)
|
|
require.EqualValues(t, snapshot1.TokenCount.Output, 400)
|
|
require.EqualValues(t, snapshot1.TokenCount.CachedRead, 600)
|
|
require.EqualValues(t, snapshot1.TokenCount.CachedWritten, 800)
|
|
require.EqualValues(t, snapshot1.ToolCallsCount.Injected, 1)
|
|
require.EqualValues(t, snapshot1.ToolCallsCount.NonInjected, 1)
|
|
require.EqualValues(t, snapshot1.InjectedToolCallErrorCount, 1)
|
|
|
|
require.Equal(t, snapshot2.Provider, aiBridgeInterception3.Provider)
|
|
require.Equal(t, snapshot2.Model, aiBridgeInterception3.Model)
|
|
require.Equal(t, snapshot2.Client, "Unknown") // no client info yet
|
|
require.EqualValues(t, snapshot2.InterceptionCount, 1)
|
|
require.EqualValues(t, snapshot2.InterceptionsByRoute, map[string]int64{}) // no route info yet
|
|
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P50, 180_000)
|
|
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P90, 180_000)
|
|
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P95, 180_000)
|
|
require.EqualValues(t, snapshot2.InterceptionDurationMillis.P99, 180_000)
|
|
require.EqualValues(t, snapshot2.UniqueInitiatorCount, 1)
|
|
require.EqualValues(t, snapshot2.UserPromptsCount, 0)
|
|
require.EqualValues(t, snapshot2.TokenUsagesCount, 0)
|
|
require.EqualValues(t, snapshot2.TokenCount.Input, 0)
|
|
require.EqualValues(t, snapshot2.TokenCount.Output, 0)
|
|
require.EqualValues(t, snapshot2.TokenCount.CachedRead, 0)
|
|
require.EqualValues(t, snapshot2.TokenCount.CachedWritten, 0)
|
|
require.EqualValues(t, snapshot2.ToolCallsCount.Injected, 0)
|
|
require.EqualValues(t, snapshot2.ToolCallsCount.NonInjected, 0)
|
|
})
|
|
t.Run("HashedEmail", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
_ = dbgen.User(t, db, database.User{
|
|
Email: "kyle@coder.com",
|
|
})
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
require.Len(t, snapshot.Users, 1)
|
|
require.Equal(t, snapshot.Users[0].EmailHashed, "bb44bf07cf9a2db0554bba63a03d822c927deae77df101874496df5a6a3e896d@coder.com")
|
|
})
|
|
t.Run("HashedModule", func(t *testing.T) {
|
|
t.Parallel()
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
pj := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{})
|
|
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
|
|
JobID: pj.ID,
|
|
Source: "registry.coder.com/terraform/aws",
|
|
Version: "1.0.0",
|
|
})
|
|
_ = dbgen.WorkspaceModule(t, db, database.WorkspaceModule{
|
|
JobID: pj.ID,
|
|
Source: "https://internal-url.com/some-module",
|
|
Version: "1.0.0",
|
|
})
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
require.Len(t, snapshot.WorkspaceModules, 2)
|
|
modules := snapshot.WorkspaceModules
|
|
sort.Slice(modules, func(i, j int) bool {
|
|
return modules[i].Source < modules[j].Source
|
|
})
|
|
require.Equal(t, modules[0].Source, "ed662ec0396db67e77119f14afcb9253574cc925b04a51d4374bcb1eae299f5d")
|
|
require.Equal(t, modules[0].Version, "92521fc3cbd964bdc9f584a991b89fddaa5754ed1cc96d6d42445338669c1305")
|
|
require.Equal(t, modules[0].SourceType, telemetry.ModuleSourceTypeHTTP)
|
|
require.Equal(t, modules[1].Source, "registry.coder.com/terraform/aws")
|
|
require.Equal(t, modules[1].Version, "1.0.0")
|
|
require.Equal(t, modules[1].SourceType, telemetry.ModuleSourceTypeCoderRegistry)
|
|
})
|
|
t.Run("ModuleSourceType", func(t *testing.T) {
|
|
t.Parallel()
|
|
cases := []struct {
|
|
source string
|
|
want telemetry.ModuleSourceType
|
|
}{
|
|
// Local relative paths
|
|
{source: "./modules/terraform-aws-vpc", want: telemetry.ModuleSourceTypeLocal},
|
|
{source: "../shared/modules/vpc", want: telemetry.ModuleSourceTypeLocal},
|
|
{source: " ./my-module ", want: telemetry.ModuleSourceTypeLocal}, // with whitespace
|
|
|
|
// Local absolute paths
|
|
{source: "/opt/terraform/modules/vpc", want: telemetry.ModuleSourceTypeLocalAbs},
|
|
{source: "/Users/dev/modules/app", want: telemetry.ModuleSourceTypeLocalAbs},
|
|
{source: "/etc/terraform/modules/network", want: telemetry.ModuleSourceTypeLocalAbs},
|
|
|
|
// Public registry
|
|
{source: "hashicorp/consul/aws", want: telemetry.ModuleSourceTypePublicRegistry},
|
|
{source: "registry.terraform.io/hashicorp/aws", want: telemetry.ModuleSourceTypePublicRegistry},
|
|
{source: "terraform-aws-modules/vpc/aws", want: telemetry.ModuleSourceTypePublicRegistry},
|
|
{source: "hashicorp/consul/aws//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
|
|
{source: "hashicorp/co-nsul/aw_s//modules/consul-cluster", want: telemetry.ModuleSourceTypePublicRegistry},
|
|
|
|
// Private registry
|
|
{source: "app.terraform.io/company/vpc/aws", want: telemetry.ModuleSourceTypePrivateRegistry},
|
|
{source: "localterraform.com/org/module", want: telemetry.ModuleSourceTypePrivateRegistry},
|
|
{source: "APP.TERRAFORM.IO/test/module", want: telemetry.ModuleSourceTypePrivateRegistry}, // case insensitive
|
|
|
|
// Coder registry
|
|
{source: "registry.coder.com/terraform/aws", want: telemetry.ModuleSourceTypeCoderRegistry},
|
|
{source: "registry.coder.com/modules/base", want: telemetry.ModuleSourceTypeCoderRegistry},
|
|
{source: "REGISTRY.CODER.COM/test/module", want: telemetry.ModuleSourceTypeCoderRegistry}, // case insensitive
|
|
|
|
// GitHub
|
|
{source: "github.com/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeGitHub},
|
|
{source: "git::https://github.com/org/repo.git", want: telemetry.ModuleSourceTypeGitHub},
|
|
{source: "git::https://github.com/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeGitHub},
|
|
|
|
// Bitbucket
|
|
{source: "bitbucket.org/hashicorp/terraform-aws-vpc", want: telemetry.ModuleSourceTypeBitbucket},
|
|
{source: "git::https://bitbucket.org/org/repo.git", want: telemetry.ModuleSourceTypeBitbucket},
|
|
{source: "https://bitbucket.org/org/repo//modules/vpc", want: telemetry.ModuleSourceTypeBitbucket},
|
|
|
|
// Generic Git
|
|
{source: "git::ssh://git.internal.com/repo.git", want: telemetry.ModuleSourceTypeGit},
|
|
{source: "git@gitlab.com:org/repo.git", want: telemetry.ModuleSourceTypeGit},
|
|
{source: "git::https://git.internal.com/repo.git?ref=v1.0.0", want: telemetry.ModuleSourceTypeGit},
|
|
|
|
// Mercurial
|
|
{source: "hg::https://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
|
|
{source: "hg::http://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
|
|
{source: "hg::ssh://example.com/vpc.hg", want: telemetry.ModuleSourceTypeMercurial},
|
|
|
|
// HTTP
|
|
{source: "https://example.com/vpc-module.zip", want: telemetry.ModuleSourceTypeHTTP},
|
|
{source: "http://example.com/modules/vpc", want: telemetry.ModuleSourceTypeHTTP},
|
|
{source: "https://internal.network/terraform/modules", want: telemetry.ModuleSourceTypeHTTP},
|
|
|
|
// S3
|
|
{source: "s3::https://s3-eu-west-1.amazonaws.com/bucket/vpc", want: telemetry.ModuleSourceTypeS3},
|
|
{source: "s3::https://bucket.s3.amazonaws.com/vpc", want: telemetry.ModuleSourceTypeS3},
|
|
{source: "s3::http://bucket.s3.amazonaws.com/vpc?version=1", want: telemetry.ModuleSourceTypeS3},
|
|
|
|
// GCS
|
|
{source: "gcs::https://www.googleapis.com/storage/v1/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
|
|
{source: "gcs::https://storage.googleapis.com/bucket/vpc", want: telemetry.ModuleSourceTypeGCS},
|
|
{source: "gcs::https://bucket.storage.googleapis.com/vpc", want: telemetry.ModuleSourceTypeGCS},
|
|
|
|
// Unknown
|
|
{source: "custom://example.com/vpc", want: telemetry.ModuleSourceTypeUnknown},
|
|
{source: "something-random", want: telemetry.ModuleSourceTypeUnknown},
|
|
{source: "", want: telemetry.ModuleSourceTypeUnknown},
|
|
}
|
|
for _, c := range cases {
|
|
require.Equal(t, c.want, telemetry.GetModuleSourceType(c.source))
|
|
}
|
|
})
|
|
t.Run("IDPOrgSync", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// 1. No org sync settings
|
|
deployment, _ := collectSnapshot(ctx, t, db, nil)
|
|
require.False(t, *deployment.IDPOrgSync)
|
|
|
|
// 2. Org sync settings set in server flags
|
|
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.DeploymentConfig = &codersdk.DeploymentValues{
|
|
OIDC: codersdk.OIDCConfig{
|
|
OrganizationField: "organizations",
|
|
},
|
|
}
|
|
return opts
|
|
})
|
|
require.True(t, *deployment.IDPOrgSync)
|
|
|
|
// 3. Org sync settings set in runtime config
|
|
org, err := db.GetDefaultOrganization(ctx)
|
|
require.NoError(t, err)
|
|
sync := idpsync.NewAGPLSync(testutil.Logger(t), runtimeconfig.NewManager(), idpsync.DeploymentSyncSettings{})
|
|
err = sync.UpdateOrganizationSyncSettings(ctx, db, idpsync.OrganizationSyncSettings{
|
|
Field: "organizations",
|
|
Mapping: map[string][]uuid.UUID{
|
|
"first": {org.ID},
|
|
},
|
|
AssignDefault: true,
|
|
})
|
|
require.NoError(t, err)
|
|
deployment, _ = collectSnapshot(ctx, t, db, nil)
|
|
require.True(t, *deployment.IDPOrgSync)
|
|
})
|
|
t.Run("SCIM", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// 1. Default Options: both flags false (and reported as such).
|
|
deployment, _ := collectSnapshot(ctx, t, db, nil)
|
|
require.NotNil(t, deployment.SCIMEnabled)
|
|
require.False(t, *deployment.SCIMEnabled)
|
|
require.NotNil(t, deployment.SCIMUseLegacy)
|
|
require.False(t, *deployment.SCIMUseLegacy)
|
|
|
|
// 2. Both Options flags true: both reported true.
|
|
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.SCIMEnabled = true
|
|
opts.SCIMUseLegacy = true
|
|
return opts
|
|
})
|
|
require.True(t, *deployment.SCIMEnabled)
|
|
require.True(t, *deployment.SCIMUseLegacy)
|
|
|
|
// 3. Enabled only: enabled true, legacy false.
|
|
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.SCIMEnabled = true
|
|
return opts
|
|
})
|
|
require.True(t, *deployment.SCIMEnabled)
|
|
require.False(t, *deployment.SCIMUseLegacy)
|
|
|
|
// 4. The reporter never reads DeploymentConfig.SCIMAPIKey directly:
|
|
// even if a non-empty key sneaks through (it would not in production
|
|
// because of WithoutSecrets), SCIMEnabled reflects only Options.
|
|
deployment, _ = collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.DeploymentConfig = &codersdk.DeploymentValues{
|
|
SCIMAPIKey: "a-secret-bearer-token",
|
|
}
|
|
return opts
|
|
})
|
|
require.False(t, *deployment.SCIMEnabled)
|
|
})
|
|
}
|
|
|
|
// nolint:paralleltest
|
|
func TestTelemetryInstallSource(t *testing.T) {
|
|
t.Setenv("CODER_TELEMETRY_INSTALL_SOURCE", "aws_marketplace")
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
deployment, _ := collectSnapshot(ctx, t, db, nil)
|
|
require.Equal(t, "aws_marketplace", deployment.InstallSource)
|
|
}
|
|
|
|
// TestTelemetryExcludesExperimentRuleConditions guards against condition
|
|
// text, which only the rules API may return, leaking into telemetry.
|
|
func TestTelemetryExcludesExperimentRuleConditions(t *testing.T) {
|
|
t.Parallel()
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
sentinel := "telemetry-sentinel-" + uuid.NewString()
|
|
value, err := json.Marshal(map[string]any{"mode": "condition", "condition": fmt.Sprintf("user.email == %q", sentinel), "revision": 1})
|
|
require.NoError(t, err)
|
|
require.NoError(t, db.UpsertExperimentRule(ctx, database.UpsertExperimentRuleParams{
|
|
Experiment: string(codersdk.ExperimentExample),
|
|
Value: string(value),
|
|
}))
|
|
|
|
deployment, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
for _, report := range []any{deployment, snapshot} {
|
|
data, err := json.Marshal(report)
|
|
require.NoError(t, err)
|
|
require.NotContains(t, string(data), sentinel)
|
|
}
|
|
}
|
|
|
|
func TestTelemetryItem(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
key := testutil.GetRandomName(t)
|
|
value := time.Now().Format(time.RFC3339)
|
|
|
|
err := db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{
|
|
Key: key,
|
|
Value: value,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
item, err := db.GetTelemetryItem(ctx, key)
|
|
require.NoError(t, err)
|
|
require.Equal(t, item.Key, key)
|
|
require.Equal(t, item.Value, value)
|
|
|
|
// Inserting a new value should not update the existing value
|
|
err = db.InsertTelemetryItemIfNotExists(ctx, database.InsertTelemetryItemIfNotExistsParams{
|
|
Key: key,
|
|
Value: "new_value",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
item, err = db.GetTelemetryItem(ctx, key)
|
|
require.NoError(t, err)
|
|
require.Equal(t, item.Value, value)
|
|
|
|
// Upserting a new value should update the existing value
|
|
err = db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{
|
|
Key: key,
|
|
Value: "new_value",
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
item, err = db.GetTelemetryItem(ctx, key)
|
|
require.NoError(t, err)
|
|
require.Equal(t, item.Value, "new_value")
|
|
}
|
|
|
|
func TestPrebuiltWorkspacesTelemetry(t *testing.T) {
|
|
t.Parallel()
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
cases := []struct {
|
|
name string
|
|
storeFn func(store database.Store) database.Store
|
|
expectedSnapshotEntries int
|
|
expectedCreated int
|
|
expectedFailed int
|
|
expectedClaimed int
|
|
}{
|
|
{
|
|
name: "prebuilds enabled",
|
|
storeFn: func(store database.Store) database.Store {
|
|
return &mockDB{Store: store}
|
|
},
|
|
expectedSnapshotEntries: 3,
|
|
expectedCreated: 5,
|
|
expectedFailed: 2,
|
|
expectedClaimed: 3,
|
|
},
|
|
{
|
|
name: "prebuilds not used",
|
|
storeFn: func(store database.Store) database.Store {
|
|
return &emptyMockDB{Store: store}
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitShort)
|
|
|
|
deployment, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Database = tc.storeFn(db)
|
|
return opts
|
|
})
|
|
|
|
require.NotNil(t, deployment)
|
|
require.NotNil(t, snapshot)
|
|
|
|
require.Len(t, snapshot.PrebuiltWorkspaces, tc.expectedSnapshotEntries)
|
|
|
|
eventCounts := make(map[telemetry.PrebuiltWorkspaceEventType]int)
|
|
for _, event := range snapshot.PrebuiltWorkspaces {
|
|
eventCounts[event.EventType] = event.Count
|
|
require.NotEqual(t, uuid.Nil, event.ID)
|
|
require.False(t, event.CreatedAt.IsZero())
|
|
}
|
|
|
|
require.Equal(t, tc.expectedCreated, eventCounts[telemetry.PrebuiltWorkspaceEventTypeCreated])
|
|
require.Equal(t, tc.expectedFailed, eventCounts[telemetry.PrebuiltWorkspaceEventTypeFailed])
|
|
require.Equal(t, tc.expectedClaimed, eventCounts[telemetry.PrebuiltWorkspaceEventTypeClaimed])
|
|
})
|
|
}
|
|
}
|
|
|
|
type mockDB struct {
|
|
database.Store
|
|
}
|
|
|
|
func (*mockDB) GetPrebuildMetrics(context.Context) ([]database.GetPrebuildMetricsRow, error) {
|
|
return []database.GetPrebuildMetricsRow{
|
|
{
|
|
TemplateName: "template1",
|
|
PresetName: "preset1",
|
|
OrganizationName: "org1",
|
|
CreatedCount: 3,
|
|
FailedCount: 1,
|
|
ClaimedCount: 2,
|
|
},
|
|
{
|
|
TemplateName: "template2",
|
|
PresetName: "preset2",
|
|
OrganizationName: "org1",
|
|
CreatedCount: 2,
|
|
FailedCount: 1,
|
|
ClaimedCount: 1,
|
|
},
|
|
}, nil
|
|
}
|
|
|
|
type emptyMockDB struct {
|
|
database.Store
|
|
}
|
|
|
|
func (*emptyMockDB) GetPrebuildMetrics(context.Context) ([]database.GetPrebuildMetricsRow, error) {
|
|
return []database.GetPrebuildMetricsRow{}, nil
|
|
}
|
|
|
|
func TestShouldReportTelemetryDisabled(t *testing.T) {
|
|
t.Parallel()
|
|
// Description | telemetryEnabled (db) | telemetryEnabled (is) | Report Telemetry Disabled |
|
|
//----------------------------------------|-----------------------|-----------------------|---------------------------|
|
|
// New deployment | <null> | true | No |
|
|
// New deployment with telemetry disabled | <null> | false | No |
|
|
// Telemetry was enabled, and still is | true | true | No |
|
|
// Telemetry was enabled but now disabled | true | false | Yes |
|
|
// Telemetry was disabled, now is enabled | false | true | No |
|
|
// Telemetry was disabled, still disabled | false | false | No |
|
|
boolTrue := true
|
|
boolFalse := false
|
|
require.False(t, telemetry.ShouldReportTelemetryDisabled(nil, true))
|
|
require.False(t, telemetry.ShouldReportTelemetryDisabled(nil, false))
|
|
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolTrue, true))
|
|
require.True(t, telemetry.ShouldReportTelemetryDisabled(&boolTrue, false))
|
|
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolFalse, true))
|
|
require.False(t, telemetry.ShouldReportTelemetryDisabled(&boolFalse, false))
|
|
}
|
|
|
|
func TestRecordTelemetryStatus(t *testing.T) {
|
|
t.Parallel()
|
|
for _, testCase := range []struct {
|
|
name string
|
|
recordedTelemetryEnabled string
|
|
telemetryEnabled bool
|
|
shouldReport bool
|
|
}{
|
|
{name: "New deployment", recordedTelemetryEnabled: "nil", telemetryEnabled: true, shouldReport: false},
|
|
{name: "Telemetry disabled", recordedTelemetryEnabled: "nil", telemetryEnabled: false, shouldReport: false},
|
|
{name: "Telemetry was enabled and still is", recordedTelemetryEnabled: "true", telemetryEnabled: true, shouldReport: false},
|
|
{name: "Telemetry was enabled but now disabled", recordedTelemetryEnabled: "true", telemetryEnabled: false, shouldReport: true},
|
|
{name: "Telemetry was disabled now is enabled", recordedTelemetryEnabled: "false", telemetryEnabled: true, shouldReport: false},
|
|
{name: "Telemetry was disabled still disabled", recordedTelemetryEnabled: "false", telemetryEnabled: false, shouldReport: false},
|
|
{name: "Telemetry was disabled still disabled, invalid value", recordedTelemetryEnabled: "invalid", telemetryEnabled: false, shouldReport: false},
|
|
} {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
logger := testutil.Logger(t)
|
|
if testCase.recordedTelemetryEnabled != "nil" {
|
|
db.UpsertTelemetryItem(ctx, database.UpsertTelemetryItemParams{
|
|
Key: string(telemetry.TelemetryItemKeyTelemetryEnabled),
|
|
Value: testCase.recordedTelemetryEnabled,
|
|
})
|
|
}
|
|
snapshot1, err := telemetry.RecordTelemetryStatus(ctx, logger, db, testCase.telemetryEnabled)
|
|
require.NoError(t, err)
|
|
|
|
if testCase.shouldReport {
|
|
require.NotNil(t, snapshot1)
|
|
require.Equal(t, snapshot1.TelemetryItems[0].Key, string(telemetry.TelemetryItemKeyTelemetryEnabled))
|
|
require.Equal(t, snapshot1.TelemetryItems[0].Value, "false")
|
|
} else {
|
|
require.Nil(t, snapshot1)
|
|
}
|
|
|
|
for range 3 {
|
|
// Whatever happens, subsequent calls should not report if telemetryEnabled didn't change
|
|
snapshot2, err := telemetry.RecordTelemetryStatus(ctx, logger, db, testCase.telemetryEnabled)
|
|
require.NoError(t, err)
|
|
require.Nil(t, snapshot2)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func mockTelemetryServer(ctx context.Context, t *testing.T) (*url.URL, chan *telemetry.Deployment, chan *telemetry.Snapshot) {
|
|
t.Helper()
|
|
deployment := make(chan *telemetry.Deployment, 64)
|
|
snapshot := make(chan *telemetry.Snapshot, 64)
|
|
r := chi.NewRouter()
|
|
r.Post("/deployment", func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, buildinfo.Version(), r.Header.Get(telemetry.VersionHeader))
|
|
dd := &telemetry.Deployment{}
|
|
err := json.NewDecoder(r.Body).Decode(dd)
|
|
require.NoError(t, err)
|
|
ok := testutil.AssertSend(ctx, t, deployment, dd)
|
|
if !ok {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
return
|
|
}
|
|
// Ensure the header is sent only after deployment is sent
|
|
w.WriteHeader(http.StatusAccepted)
|
|
})
|
|
r.Post("/snapshot", func(w http.ResponseWriter, r *http.Request) {
|
|
require.Equal(t, buildinfo.Version(), r.Header.Get(telemetry.VersionHeader))
|
|
ss := &telemetry.Snapshot{}
|
|
err := json.NewDecoder(r.Body).Decode(ss)
|
|
require.NoError(t, err)
|
|
ok := testutil.AssertSend(ctx, t, snapshot, ss)
|
|
if !ok {
|
|
w.WriteHeader(http.StatusInternalServerError)
|
|
return
|
|
}
|
|
// Ensure the header is sent only after snapshot is sent
|
|
w.WriteHeader(http.StatusAccepted)
|
|
})
|
|
server := httptest.NewServer(r)
|
|
t.Cleanup(server.Close)
|
|
serverURL, err := url.Parse(server.URL)
|
|
require.NoError(t, err)
|
|
|
|
return serverURL, deployment, snapshot
|
|
}
|
|
|
|
func collectSnapshot(
|
|
ctx context.Context,
|
|
t *testing.T,
|
|
db database.Store,
|
|
addOptionsFn func(opts telemetry.Options) telemetry.Options,
|
|
) (*telemetry.Deployment, *telemetry.Snapshot) {
|
|
t.Helper()
|
|
|
|
serverURL, deployment, snapshot := mockTelemetryServer(ctx, t)
|
|
|
|
options := telemetry.Options{
|
|
Database: db,
|
|
Logger: testutil.Logger(t),
|
|
URL: serverURL,
|
|
DeploymentID: uuid.NewString(),
|
|
}
|
|
if addOptionsFn != nil {
|
|
options = addOptionsFn(options)
|
|
}
|
|
|
|
reporter, err := telemetry.New(options)
|
|
require.NoError(t, err)
|
|
t.Cleanup(reporter.Close)
|
|
|
|
return testutil.RequireReceive(ctx, t, deployment), testutil.RequireReceive(ctx, t, snapshot)
|
|
}
|
|
|
|
func TestTelemetry_BoundaryUsageSummary(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("IncludedInSnapshot", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
|
|
tracker := boundaryusage.NewTracker()
|
|
workspace1, workspace2 := uuid.New(), uuid.New()
|
|
user1, user2 := uuid.New(), uuid.New()
|
|
replicaID := uuid.New()
|
|
|
|
tracker.Track(workspace1, user1, 10, 2)
|
|
tracker.Track(workspace2, user1, 5, 1)
|
|
tracker.Track(workspace2, user2, 3, 0)
|
|
|
|
// Flush the tracker to the database.
|
|
err := tracker.FlushToDB(ctx, db, replicaID)
|
|
require.NoError(t, err)
|
|
|
|
// Collect a snapshot and verify boundary usage is included.
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(dbtime.Now())
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
|
|
require.NotNil(t, snapshot.BoundaryUsageSummary)
|
|
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.UniqueWorkspaces)
|
|
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.UniqueUsers)
|
|
require.Equal(t, int64(10+5+3), snapshot.BoundaryUsageSummary.AllowedRequests)
|
|
require.Equal(t, int64(2+1+0), snapshot.BoundaryUsageSummary.DeniedRequests)
|
|
require.Equal(t, clock.Now().Add(-telemetry.DefaultSnapshotFrequency), snapshot.BoundaryUsageSummary.PeriodStart)
|
|
require.Equal(t, int64(telemetry.DefaultSnapshotFrequency/time.Millisecond), snapshot.BoundaryUsageSummary.PeriodDurationMilliseconds)
|
|
})
|
|
|
|
t.Run("ResetAfterCollection", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
|
|
tracker := boundaryusage.NewTracker()
|
|
replicaID := uuid.New()
|
|
|
|
tracker.Track(uuid.New(), uuid.New(), 5, 1)
|
|
err := tracker.FlushToDB(ctx, db, replicaID)
|
|
require.NoError(t, err)
|
|
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(dbtime.Now())
|
|
|
|
// First snapshot should have the data.
|
|
_, snapshot1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
require.NotNil(t, snapshot1.BoundaryUsageSummary)
|
|
require.Equal(t, int64(5), snapshot1.BoundaryUsageSummary.AllowedRequests)
|
|
|
|
// Advance clock to next snapshot period to avoid lock conflict.
|
|
clock.Advance(30 * time.Minute)
|
|
|
|
// Second snapshot should have no data (stats were reset).
|
|
_, snapshot2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
// Summary should be nil or have zero values since stats were reset.
|
|
if snapshot2.BoundaryUsageSummary != nil {
|
|
require.Equal(t, int64(0), snapshot2.BoundaryUsageSummary.AllowedRequests)
|
|
}
|
|
})
|
|
|
|
t.Run("OnlyOneReplicaCollects", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
|
|
// Set up boundary usage stats from two replicas.
|
|
tracker1 := boundaryusage.NewTracker()
|
|
tracker2 := boundaryusage.NewTracker()
|
|
replica1ID := uuid.New()
|
|
replica2ID := uuid.New()
|
|
|
|
tracker1.Track(uuid.New(), uuid.New(), 10, 1)
|
|
tracker2.Track(uuid.New(), uuid.New(), 20, 2)
|
|
|
|
err := tracker1.FlushToDB(ctx, db, replica1ID)
|
|
require.NoError(t, err)
|
|
err = tracker2.FlushToDB(ctx, db, replica2ID)
|
|
require.NoError(t, err)
|
|
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(dbtime.Now())
|
|
|
|
// First snapshot collects and resets.
|
|
_, snapshot1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
require.NotNil(t, snapshot1.BoundaryUsageSummary)
|
|
require.Equal(t, int64(10+20), snapshot1.BoundaryUsageSummary.AllowedRequests)
|
|
|
|
// Second snapshot in same period should skip (lock already claimed).
|
|
_, snapshot2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
// The second snapshot should have nil because another "replica" already
|
|
// claimed the lock for this period.
|
|
require.Nil(t, snapshot2.BoundaryUsageSummary)
|
|
})
|
|
}
|
|
|
|
func TestChatsTelemetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
user := dbgen.User(t, db, database.User{})
|
|
|
|
// Create chat providers (required FK for model configs).
|
|
anthropicProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
|
|
Provider: "anthropic",
|
|
DisplayName: "Anthropic",
|
|
})
|
|
openaiProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
|
|
Provider: "openai",
|
|
DisplayName: "OpenAI",
|
|
})
|
|
|
|
// Create a model config.
|
|
modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
|
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
|
|
Model: "claude-sonnet-4-20250514",
|
|
DisplayName: "Claude Sonnet",
|
|
IsDefault: true,
|
|
ContextLimit: 200000,
|
|
})
|
|
|
|
org2 := dbgen.Organization(t, db, database.Organization{})
|
|
modelCfg2 := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
|
AIProviderID: uuid.NullUUID{UUID: openaiProvider.ID, Valid: true},
|
|
OrganizationID: org2.ID,
|
|
Model: "gpt-4o",
|
|
DisplayName: "GPT-4o",
|
|
})
|
|
|
|
// Soft-deleted model configurations must not appear in telemetry.
|
|
deletedCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
|
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
|
|
Model: "claude-deleted",
|
|
DisplayName: "Deleted Model",
|
|
ContextLimit: 100000,
|
|
})
|
|
_, err := db.DeleteChatModelConfigByID(ctx, deletedCfg.ID)
|
|
require.NoError(t, err)
|
|
|
|
// Create a root chat with a workspace.
|
|
org, err := db.GetDefaultOrganization(ctx)
|
|
require.NoError(t, err)
|
|
job := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
|
|
OrganizationID: org.ID,
|
|
Type: database.ProvisionerJobTypeTemplateVersionDryRun,
|
|
})
|
|
tpl := dbgen.Template(t, db, database.Template{
|
|
OrganizationID: org.ID,
|
|
CreatedBy: user.ID,
|
|
})
|
|
tv := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
|
OrganizationID: org.ID,
|
|
TemplateID: uuid.NullUUID{UUID: tpl.ID, Valid: true},
|
|
CreatedBy: user.ID,
|
|
JobID: job.ID,
|
|
})
|
|
ws := dbgen.Workspace(t, db, database.WorkspaceTable{
|
|
OwnerID: user.ID,
|
|
OrganizationID: org.ID,
|
|
TemplateID: tpl.ID,
|
|
})
|
|
_ = dbgen.WorkspaceBuild(t, db, database.WorkspaceBuild{
|
|
Transition: database.WorkspaceTransitionStart,
|
|
Reason: database.BuildReasonInitiator,
|
|
WorkspaceID: ws.ID,
|
|
TemplateVersionID: tv.ID,
|
|
JobID: job.ID,
|
|
})
|
|
|
|
rootChat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
LastModelConfigID: modelCfg.ID,
|
|
Title: "Root Chat",
|
|
Status: database.ChatStatusRunning,
|
|
WorkspaceID: uuid.NullUUID{UUID: ws.ID, Valid: true},
|
|
Mode: database.NullChatMode{ChatMode: database.ChatModeComputerUse, Valid: true},
|
|
})
|
|
|
|
// Create a child chat (has parent + root).
|
|
childChat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
LastModelConfigID: modelCfg2.ID,
|
|
Title: "Child Chat",
|
|
Status: database.ChatStatusWaiting,
|
|
ParentChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true},
|
|
RootChatID: uuid.NullUUID{UUID: rootChat.ID, Valid: true},
|
|
})
|
|
|
|
// Associate a PR with the root chat so PullRequestState is populated.
|
|
rootChatNow := dbtime.Now()
|
|
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
|
|
ChatID: rootChat.ID,
|
|
PullRequestState: sql.NullString{String: "merged", Valid: true},
|
|
RefreshedAt: rootChatNow,
|
|
StaleAt: rootChatNow,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
// Insert messages for root chat: 2 user, 2 assistant, 1 tool.
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleUser,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"hello"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 100, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 100, Valid: true},
|
|
CacheCreationTokens: sql.NullInt64{Int64: 50, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
})
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleAssistant,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"hi"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 200, Valid: true},
|
|
OutputTokens: sql.NullInt64{Int64: 50, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 250, Valid: true},
|
|
ReasoningTokens: sql.NullInt64{Int64: 10, Valid: true},
|
|
CacheReadTokens: sql.NullInt64{Int64: 25, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
RuntimeMs: sql.NullInt64{Int64: 500, Valid: true},
|
|
ProviderResponseID: sql.NullString{String: "resp-1", Valid: true},
|
|
})
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleUser,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"help"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 150, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 150, Valid: true},
|
|
CacheCreationTokens: sql.NullInt64{Int64: 30, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
})
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleAssistant,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"sure"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 300, Valid: true},
|
|
OutputTokens: sql.NullInt64{Int64: 100, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 400, Valid: true},
|
|
ReasoningTokens: sql.NullInt64{Int64: 20, Valid: true},
|
|
CacheReadTokens: sql.NullInt64{Int64: 40, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
RuntimeMs: sql.NullInt64{Int64: 800, Valid: true},
|
|
ProviderResponseID: sql.NullString{String: "resp-2", Valid: true},
|
|
})
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleTool,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"result"}]`), Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
RuntimeMs: sql.NullInt64{Int64: 100, Valid: true},
|
|
})
|
|
|
|
// Insert messages for child chat: 1 user, 1 assistant (compressed).
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: childChat.ID,
|
|
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg2.ID, Valid: true},
|
|
Role: database.ChatMessageRoleUser,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"q"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 500, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 500, Valid: true},
|
|
CacheCreationTokens: sql.NullInt64{Int64: 100, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 128000, Valid: true},
|
|
})
|
|
_ = dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: childChat.ID,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg2.ID, Valid: true},
|
|
Role: database.ChatMessageRoleAssistant,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"a"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 600, Valid: true},
|
|
OutputTokens: sql.NullInt64{Int64: 200, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 800, Valid: true},
|
|
ReasoningTokens: sql.NullInt64{Int64: 50, Valid: true},
|
|
CacheReadTokens: sql.NullInt64{Int64: 75, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 128000, Valid: true},
|
|
Compressed: true,
|
|
RuntimeMs: sql.NullInt64{Int64: 1200, Valid: true},
|
|
ProviderResponseID: sql.NullString{String: "resp-3", Valid: true},
|
|
})
|
|
|
|
// Large token values expose a missing soft-delete filter in the totals.
|
|
poisonMsg := dbgen.ChatMessage(t, db, database.ChatMessage{
|
|
ChatID: rootChat.ID,
|
|
ModelConfigID: uuid.NullUUID{UUID: modelCfg.ID, Valid: true},
|
|
Role: database.ChatMessageRoleAssistant,
|
|
Content: pqtype.NullRawMessage{RawMessage: json.RawMessage(`[{"type":"text","text":"poison"}]`), Valid: true},
|
|
InputTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
OutputTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
TotalTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
ReasoningTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
CacheCreationTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
CacheReadTokens: sql.NullInt64{Int64: 999999, Valid: true},
|
|
ContextLimit: sql.NullInt64{Int64: 200000, Valid: true},
|
|
RuntimeMs: sql.NullInt64{Int64: 999999, Valid: true},
|
|
})
|
|
err = db.SoftDeleteChatMessagesAfterID(ctx, database.SoftDeleteChatMessagesAfterIDParams{
|
|
ChatID: rootChat.ID,
|
|
AfterID: poisonMsg.ID - 1,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
|
|
// --- Assert Chats ---
|
|
require.Len(t, snapshot.Chats, 2)
|
|
|
|
// Find root and child by HasParent flag.
|
|
var foundRoot, foundChild *telemetry.Chat
|
|
for i := range snapshot.Chats {
|
|
if !snapshot.Chats[i].HasParent {
|
|
foundRoot = &snapshot.Chats[i]
|
|
} else {
|
|
foundChild = &snapshot.Chats[i]
|
|
}
|
|
}
|
|
require.NotNil(t, foundRoot, "expected root chat")
|
|
require.NotNil(t, foundChild, "expected child chat")
|
|
|
|
// Root chat assertions.
|
|
assert.Equal(t, rootChat.ID, foundRoot.ID)
|
|
assert.Equal(t, user.ID, foundRoot.OwnerID)
|
|
assert.Equal(t, org.ID, foundRoot.OrganizationID)
|
|
assert.Equal(t, "running", foundRoot.Status)
|
|
assert.False(t, foundRoot.HasParent)
|
|
assert.Nil(t, foundRoot.RootChatID)
|
|
require.NotNil(t, foundRoot.WorkspaceID)
|
|
assert.Equal(t, ws.ID, *foundRoot.WorkspaceID)
|
|
assert.Equal(t, modelCfg.ID, foundRoot.LastModelConfigID)
|
|
require.NotNil(t, foundRoot.Mode)
|
|
assert.Equal(t, "computer_use", *foundRoot.Mode)
|
|
assert.False(t, foundRoot.Archived)
|
|
assert.Equal(t, "ui", foundRoot.ClientType)
|
|
require.NotNil(t, foundRoot.PullRequestState)
|
|
assert.Equal(t, "merged", *foundRoot.PullRequestState)
|
|
|
|
// Child chat assertions.
|
|
|
|
assert.Equal(t, childChat.ID, foundChild.ID)
|
|
assert.Equal(t, user.ID, foundChild.OwnerID)
|
|
assert.True(t, foundChild.HasParent)
|
|
require.NotNil(t, foundChild.RootChatID)
|
|
assert.Equal(t, rootChat.ID, *foundChild.RootChatID)
|
|
assert.Nil(t, foundChild.WorkspaceID)
|
|
assert.Equal(t, "waiting", foundChild.Status)
|
|
assert.Equal(t, modelCfg2.ID, foundChild.LastModelConfigID)
|
|
assert.Nil(t, foundChild.Mode)
|
|
assert.False(t, foundChild.Archived)
|
|
assert.Equal(t, "ui", foundChild.ClientType)
|
|
assert.Nil(t, foundChild.PullRequestState)
|
|
|
|
// --- Assert ChatMessageSummaries ---
|
|
|
|
require.Len(t, snapshot.ChatMessageSummaries, 2)
|
|
|
|
summaryMap := make(map[uuid.UUID]telemetry.ChatMessageSummary)
|
|
for _, s := range snapshot.ChatMessageSummaries {
|
|
summaryMap[s.ChatID] = s
|
|
}
|
|
|
|
// Root chat summary: 2 user + 2 assistant + 1 tool = 5 messages.
|
|
rootSummary, ok := summaryMap[rootChat.ID]
|
|
require.True(t, ok, "expected summary for root chat")
|
|
assert.Equal(t, int64(5), rootSummary.MessageCount)
|
|
assert.Equal(t, int64(2), rootSummary.UserMessageCount)
|
|
assert.Equal(t, int64(2), rootSummary.AssistantMessageCount)
|
|
assert.Equal(t, int64(1), rootSummary.ToolMessageCount)
|
|
assert.Equal(t, int64(0), rootSummary.SystemMessageCount)
|
|
assert.Equal(t, int64(750), rootSummary.TotalInputTokens) // 100+200+150+300+0
|
|
assert.Equal(t, int64(150), rootSummary.TotalOutputTokens) // 0+50+0+100+0
|
|
assert.Equal(t, int64(30), rootSummary.TotalReasoningTokens) // 0+10+0+20+0
|
|
assert.Equal(t, int64(80), rootSummary.TotalCacheCreationTokens) // 50+0+30+0+0
|
|
assert.Equal(t, int64(65), rootSummary.TotalCacheReadTokens) // 0+25+0+40+0
|
|
assert.Equal(t, int64(1400), rootSummary.TotalRuntimeMs) // 0+500+0+800+100
|
|
assert.Equal(t, int64(1), rootSummary.DistinctModelCount)
|
|
assert.Equal(t, int64(0), rootSummary.CompressedMessageCount)
|
|
|
|
// Child chat summary: 1 user + 1 assistant = 2 messages, 1 compressed.
|
|
childSummary, ok := summaryMap[childChat.ID]
|
|
require.True(t, ok, "expected summary for child chat")
|
|
assert.Equal(t, int64(2), childSummary.MessageCount)
|
|
assert.Equal(t, int64(1), childSummary.UserMessageCount)
|
|
assert.Equal(t, int64(1), childSummary.AssistantMessageCount)
|
|
assert.Equal(t, int64(1100), childSummary.TotalInputTokens) // 500+600
|
|
assert.Equal(t, int64(200), childSummary.TotalOutputTokens) // 0+200
|
|
assert.Equal(t, int64(50), childSummary.TotalReasoningTokens) // 0+50
|
|
assert.Equal(t, int64(0), childSummary.ToolMessageCount)
|
|
assert.Equal(t, int64(0), childSummary.SystemMessageCount)
|
|
assert.Equal(t, int64(100), childSummary.TotalCacheCreationTokens) // 100+0
|
|
assert.Equal(t, int64(75), childSummary.TotalCacheReadTokens) // 0+75
|
|
assert.Equal(t, int64(1200), childSummary.TotalRuntimeMs) // 0+1200
|
|
assert.Equal(t, int64(1), childSummary.DistinctModelCount)
|
|
assert.Equal(t, int64(1), childSummary.CompressedMessageCount)
|
|
|
|
// --- Assert ChatModelConfigs ---
|
|
require.Len(t, snapshot.ChatModelConfigs, 2)
|
|
|
|
configMap := make(map[uuid.UUID]telemetry.ChatModelConfig)
|
|
for _, c := range snapshot.ChatModelConfigs {
|
|
configMap[c.ID] = c
|
|
}
|
|
|
|
cfg1, ok := configMap[modelCfg.ID]
|
|
require.True(t, ok)
|
|
assert.Equal(t, org.ID, cfg1.OrganizationID)
|
|
assert.Equal(t, "anthropic", cfg1.Provider)
|
|
assert.Equal(t, "claude-sonnet-4-20250514", cfg1.Model)
|
|
assert.Equal(t, int64(200000), cfg1.ContextLimit)
|
|
assert.True(t, cfg1.Enabled)
|
|
assert.True(t, cfg1.IsDefault)
|
|
|
|
cfg2, ok := configMap[modelCfg2.ID]
|
|
require.True(t, ok)
|
|
assert.Equal(t, org2.ID, cfg2.OrganizationID)
|
|
assert.Equal(t, "openai", cfg2.Provider)
|
|
assert.Equal(t, "gpt-4o", cfg2.Model)
|
|
assert.Equal(t, int64(128000), cfg2.ContextLimit)
|
|
assert.True(t, cfg2.Enabled)
|
|
assert.False(t, cfg2.IsDefault)
|
|
}
|
|
|
|
func TestChatDiffStatusSummaryTelemetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// Verify zero counts when no chat_diff_statuses exist.
|
|
_, emptySnapshot := collectSnapshot(ctx, t, db, nil)
|
|
require.NotNil(t, emptySnapshot.ChatDiffStatusSummary)
|
|
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Total)
|
|
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Open)
|
|
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Merged)
|
|
assert.Equal(t, int64(0), emptySnapshot.ChatDiffStatusSummary.Closed)
|
|
|
|
// Set up minimal FK chain: provider -> model config -> chat.
|
|
user := dbgen.User(t, db, database.User{})
|
|
org, err := db.GetDefaultOrganization(ctx)
|
|
require.NoError(t, err)
|
|
|
|
anthropicProvider := dbgen.ChatProvider(t, db, database.ChatProvider{
|
|
Provider: "anthropic",
|
|
DisplayName: "Anthropic",
|
|
})
|
|
|
|
modelCfg := dbgen.ChatModelConfig(t, db, database.ChatModelConfig{
|
|
AIProviderID: uuid.NullUUID{UUID: anthropicProvider.ID, Valid: true},
|
|
Model: "claude-sonnet-4-20250514",
|
|
DisplayName: "Claude Sonnet",
|
|
IsDefault: true,
|
|
ContextLimit: 200000,
|
|
})
|
|
|
|
// Helper to create a chat and upsert its diff status.
|
|
insertChatWithDiffStatus := func(prURL, state string) uuid.UUID {
|
|
t.Helper()
|
|
chat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
LastModelConfigID: modelCfg.ID,
|
|
Title: "Chat " + state,
|
|
Status: database.ChatStatusWaiting,
|
|
})
|
|
now := dbtime.Now()
|
|
_, chatErr := db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
|
|
ChatID: chat.ID,
|
|
Url: sql.NullString{String: prURL, Valid: prURL != ""},
|
|
PullRequestState: sql.NullString{String: state, Valid: true},
|
|
RefreshedAt: now,
|
|
StaleAt: now,
|
|
})
|
|
require.NoError(t, chatErr)
|
|
return chat.ID
|
|
}
|
|
|
|
// Insert: 1 merged, 1 open, 1 closed (each with unique URLs).
|
|
// For pull/1, first insert an older chat with stale "open" state,
|
|
// then a newer chat with refreshed "merged" state. The dedup
|
|
// query orders by cds.updated_at DESC, so "merged" should win.
|
|
insertChatWithDiffStatus("https://github.com/org/repo/pull/1", "open")
|
|
insertChatWithDiffStatus("https://github.com/org/repo/pull/1", "merged")
|
|
openChatID := insertChatWithDiffStatus("https://github.com/org/repo/pull/2", "open")
|
|
insertChatWithDiffStatus("https://github.com/org/repo/pull/3", "closed")
|
|
|
|
// Insert a chat with NULL pull_request_state (no PR yet).
|
|
// This should be excluded from all counts.
|
|
noPRChat := dbgen.Chat(t, db, database.Chat{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
LastModelConfigID: modelCfg.ID,
|
|
Title: "Chat no PR",
|
|
Status: database.ChatStatusRunning,
|
|
})
|
|
now := dbtime.Now()
|
|
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
|
|
ChatID: noPRChat.ID,
|
|
RefreshedAt: now,
|
|
StaleAt: now,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
|
|
// 3 unique PRs (deduped by URL), not 4 chat_diff_statuses rows.
|
|
require.NotNil(t, snapshot.ChatDiffStatusSummary)
|
|
assert.Equal(t, int64(3), snapshot.ChatDiffStatusSummary.Total)
|
|
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Open)
|
|
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Merged)
|
|
assert.Equal(t, int64(1), snapshot.ChatDiffStatusSummary.Closed)
|
|
|
|
// Transition the "open" PR to "merged" via upsert on the same
|
|
// chat_id. The aggregate should reflect the new state.
|
|
now = dbtime.Now()
|
|
_, err = db.UpsertChatDiffStatus(ctx, database.UpsertChatDiffStatusParams{
|
|
ChatID: openChatID,
|
|
Url: sql.NullString{String: "https://github.com/org/repo/pull/2", Valid: true},
|
|
PullRequestState: sql.NullString{String: "merged", Valid: true},
|
|
RefreshedAt: now,
|
|
StaleAt: now,
|
|
})
|
|
require.NoError(t, err)
|
|
|
|
_, snapshot2 := collectSnapshot(ctx, t, db, nil)
|
|
|
|
require.NotNil(t, snapshot2.ChatDiffStatusSummary)
|
|
assert.Equal(t, int64(3), snapshot2.ChatDiffStatusSummary.Total)
|
|
assert.Equal(t, int64(0), snapshot2.ChatDiffStatusSummary.Open)
|
|
assert.Equal(t, int64(2), snapshot2.ChatDiffStatusSummary.Merged)
|
|
assert.Equal(t, int64(1), snapshot2.ChatDiffStatusSummary.Closed)
|
|
}
|
|
|
|
func TestUserSecretsTelemetry(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("Empty", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// Empty deployment should report a non-nil summary with zeros.
|
|
_, snap := collectSnapshot(ctx, t, db, nil)
|
|
require.Equal(t, &telemetry.UserSecretsSummary{}, snap.UserSecretsSummary)
|
|
})
|
|
|
|
t.Run("ConfigurationBreakdown", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
userA := dbgen.User(t, db, database.User{})
|
|
userB := dbgen.User(t, db, database.User{})
|
|
|
|
// userA: env-only and file-only. dbgen.UserSecret defaults
|
|
// EnvName and FilePath to non-empty, so use mutators to clear
|
|
// them where the test wants empty values.
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: userA.ID,
|
|
Name: "a-env",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = "A_ENV"
|
|
p.FilePath = ""
|
|
})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: userA.ID,
|
|
Name: "a-file",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = "/home/coder/a.file"
|
|
})
|
|
// userB: both and neither.
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: userB.ID,
|
|
Name: "b-both",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = "B_BOTH"
|
|
p.FilePath = "/home/coder/b.both"
|
|
})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: userB.ID,
|
|
Name: "b-neither",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = ""
|
|
// A target-less secret must be disabled to satisfy the
|
|
// user_secrets_enabled_requires_target constraint. Disabled
|
|
// secrets are still counted in the telemetry breakdown.
|
|
p.Enabled = false
|
|
})
|
|
|
|
_, snap := collectSnapshot(ctx, t, db, nil)
|
|
// Each user has exactly two secrets, so every percentile and
|
|
// the max collapse to 2.
|
|
require.Equal(t, &telemetry.UserSecretsSummary{
|
|
UsersWithSecrets: 2,
|
|
TotalSecrets: 4,
|
|
EnvNameOnly: 1,
|
|
FilePathOnly: 1,
|
|
Both: 1,
|
|
Neither: 1,
|
|
SecretsPerUserMax: 2,
|
|
SecretsPerUserP25: 2,
|
|
SecretsPerUserP50: 2,
|
|
SecretsPerUserP75: 2,
|
|
SecretsPerUserP90: 2,
|
|
}, snap.UserSecretsSummary)
|
|
})
|
|
|
|
t.Run("PercentileDistribution", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// Five users have secret counts 1, 2, 4, 8, 16 and five other
|
|
// users have zero secrets. Including the zero-secret users in
|
|
// the distribution gives a sorted vector of length 10:
|
|
// [0, 0, 0, 0, 0, 1, 2, 4, 8, 16]
|
|
// percentile_disc(p) returns the value at the smallest
|
|
// 1-indexed position i where i/n >= p, so the buckets land at:
|
|
// p25 -> position 3 -> 0
|
|
// p50 -> position 5 -> 0
|
|
// p75 -> position 8 -> 4
|
|
// p90 -> position 9 -> 8
|
|
adopters := []int{1, 2, 4, 8, 16}
|
|
for _, n := range adopters {
|
|
u := dbgen.User(t, db, database.User{})
|
|
for i := 0; i < n; i++ {
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: u.ID,
|
|
Name: fmt.Sprintf("secret-%d", i),
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
// Clear EnvName and FilePath so the unique
|
|
// (user_id, env_name) and (user_id, file_path)
|
|
// indexes don't collide across multiple secrets
|
|
// for the same user. Target-less secrets must be
|
|
// disabled to satisfy the
|
|
// user_secrets_enabled_requires_target constraint.
|
|
p.EnvName = ""
|
|
p.FilePath = ""
|
|
p.Enabled = false
|
|
})
|
|
}
|
|
}
|
|
for i := 0; i < 5; i++ {
|
|
_ = dbgen.User(t, db, database.User{})
|
|
}
|
|
|
|
_, snap := collectSnapshot(ctx, t, db, nil)
|
|
require.Equal(t, &telemetry.UserSecretsSummary{
|
|
UsersWithSecrets: 5,
|
|
TotalSecrets: 31,
|
|
EnvNameOnly: 0,
|
|
FilePathOnly: 0,
|
|
Both: 0,
|
|
Neither: 31,
|
|
SecretsPerUserMax: 16,
|
|
SecretsPerUserP25: 0,
|
|
SecretsPerUserP50: 0,
|
|
SecretsPerUserP75: 4,
|
|
SecretsPerUserP90: 8,
|
|
}, snap.UserSecretsSummary)
|
|
})
|
|
|
|
t.Run("FilterSkipsInactiveUsers", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// Active user with two secrets contributes the only entries
|
|
// to UsersWithSecrets, TotalSecrets, and the percentile
|
|
// distribution.
|
|
active := dbgen.User(t, db, database.User{})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: active.ID,
|
|
Name: "active-env",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = "ACTIVE_ENV"
|
|
p.FilePath = ""
|
|
})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: active.ID,
|
|
Name: "active-file",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = "/home/coder/active.file"
|
|
})
|
|
|
|
// User secret owned by a dormant user should be excluded.
|
|
dormant := dbgen.User(t, db, database.User{Status: database.UserStatusDormant})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: dormant.ID,
|
|
Name: "dormant-secret",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = "DORMANT_ENV"
|
|
p.FilePath = ""
|
|
})
|
|
|
|
// User secret owned by a suspended user should be excluded.
|
|
suspended := dbgen.User(t, db, database.User{Status: database.UserStatusSuspended})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: suspended.ID,
|
|
Name: "suspended-secret",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = "/home/coder/suspended.file"
|
|
})
|
|
|
|
// System user. Only its UUID is needed. Tying a secret to it
|
|
// proves the is_system filter excludes it.
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: database.PrebuildsSystemUserID,
|
|
Name: "prebuilds-secret",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = "/home/coder/prebuilds.file"
|
|
})
|
|
|
|
_, snap := collectSnapshot(ctx, t, db, nil)
|
|
require.Equal(t, &telemetry.UserSecretsSummary{
|
|
UsersWithSecrets: 1,
|
|
TotalSecrets: 2,
|
|
EnvNameOnly: 1,
|
|
FilePathOnly: 1,
|
|
Both: 0,
|
|
Neither: 0,
|
|
SecretsPerUserMax: 2,
|
|
SecretsPerUserP25: 2,
|
|
SecretsPerUserP50: 2,
|
|
SecretsPerUserP75: 2,
|
|
SecretsPerUserP90: 2,
|
|
}, snap.UserSecretsSummary)
|
|
})
|
|
|
|
t.Run("OnlyOneReplicaCollects", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// Seed one user with one secret so the summary would normally
|
|
// be populated. The user_secrets_summary aggregate has no
|
|
// natural per-row UUID for the telemetry server to dedupe on,
|
|
// so a telemetry lock elects a single replica per period.
|
|
u := dbgen.User(t, db, database.User{})
|
|
_ = dbgen.UserSecret(t, db, database.UserSecret{
|
|
UserID: u.ID,
|
|
Name: "only-secret",
|
|
}, func(p *database.CreateUserSecretParams) {
|
|
p.EnvName = ""
|
|
p.FilePath = ""
|
|
// Target-less secrets must be disabled to satisfy the
|
|
// user_secrets_enabled_requires_target constraint.
|
|
p.Enabled = false
|
|
})
|
|
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(dbtime.Now())
|
|
|
|
// First snapshot claims the lock and reports the summary.
|
|
_, snap1 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
require.Equal(t, &telemetry.UserSecretsSummary{
|
|
UsersWithSecrets: 1,
|
|
TotalSecrets: 1,
|
|
EnvNameOnly: 0,
|
|
FilePathOnly: 0,
|
|
Both: 0,
|
|
Neither: 1,
|
|
SecretsPerUserMax: 1,
|
|
SecretsPerUserP25: 1,
|
|
SecretsPerUserP50: 1,
|
|
SecretsPerUserP75: 1,
|
|
SecretsPerUserP90: 1,
|
|
}, snap1.UserSecretsSummary)
|
|
|
|
// A second snapshot in the same period simulates a second
|
|
// replica racing to claim the lock; it should observe the
|
|
// unique violation and skip reporting.
|
|
_, snap2 := collectSnapshot(ctx, t, db, func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
return opts
|
|
})
|
|
require.Nil(t, snap2.UserSecretsSummary)
|
|
})
|
|
}
|
|
|
|
func TestTelemetryScriptOrderSummary(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
org, err := db.GetDefaultOrganization(ctx)
|
|
require.NoError(t, err)
|
|
user := dbgen.User(t, db, database.User{})
|
|
now := dbtime.Now()
|
|
|
|
// Seed all fixtures into one database, then verify the combined
|
|
// telemetry summary below.
|
|
for _, tt := range []struct {
|
|
name string
|
|
jobType database.ProvisionerJobType
|
|
dataSources int32
|
|
rules int32
|
|
completed bool
|
|
failed bool
|
|
canceled bool
|
|
inactive bool
|
|
deleted bool
|
|
unattached bool
|
|
}{
|
|
// These three templates contribute to the summary totals. A
|
|
// script-order declaration counts even without literal rules.
|
|
{name: "FirstTemplate", dataSources: 2, rules: 3, completed: true},
|
|
{name: "SecondTemplate", dataSources: 1, rules: 2, completed: true},
|
|
{name: "NoLiteralRules", dataSources: 1, completed: true},
|
|
// These must not contribute to the summary totals.
|
|
{name: "NoScriptOrder", completed: true},
|
|
{name: "InactiveVersion", dataSources: 9, rules: 9, completed: true, inactive: true},
|
|
{name: "DeletedTemplate", dataSources: 9, rules: 9, completed: true, deleted: true},
|
|
{name: "UnattachedVersion", dataSources: 9, rules: 9, completed: true, unattached: true},
|
|
{name: "FailedImport", dataSources: 9, rules: 9, completed: true, failed: true},
|
|
{name: "CanceledImport", dataSources: 9, rules: 9, completed: true, canceled: true},
|
|
{name: "IncompleteImport", dataSources: 9, rules: 9},
|
|
{name: "NonImportJob", jobType: database.ProvisionerJobTypeTemplateVersionDryRun, dataSources: 9, rules: 9, completed: true},
|
|
} {
|
|
job := database.ProvisionerJob{
|
|
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
|
OrganizationID: org.ID,
|
|
InitiatorID: user.ID,
|
|
}
|
|
if tt.jobType != "" {
|
|
job.Type = tt.jobType
|
|
}
|
|
if tt.completed {
|
|
job.CompletedAt = sql.NullTime{Time: now, Valid: true}
|
|
}
|
|
if tt.failed {
|
|
job.Error = sql.NullString{String: "import failed", Valid: true}
|
|
}
|
|
if tt.canceled {
|
|
job.CanceledAt = sql.NullTime{Time: now, Valid: true}
|
|
}
|
|
job = dbgen.ProvisionerJob(t, db, nil, job)
|
|
var templateID uuid.NullUUID
|
|
if !tt.unattached {
|
|
template := dbgen.Template(t, db, database.Template{
|
|
OrganizationID: org.ID,
|
|
CreatedBy: user.ID,
|
|
})
|
|
templateID = uuid.NullUUID{UUID: template.ID, Valid: true}
|
|
if tt.deleted {
|
|
err := db.UpdateTemplateDeletedByID(ctx, database.UpdateTemplateDeletedByIDParams{
|
|
ID: template.ID,
|
|
Deleted: true,
|
|
UpdatedAt: now,
|
|
})
|
|
require.NoError(t, err, tt.name)
|
|
}
|
|
}
|
|
// Active versions count regardless of when they were imported.
|
|
version := dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
|
CreatedAt: now.Add(-24 * time.Hour),
|
|
CreatedBy: user.ID,
|
|
OrganizationID: org.ID,
|
|
TemplateID: templateID,
|
|
JobID: job.ID,
|
|
ScriptOrderDataSourceCount: tt.dataSources,
|
|
ScriptOrderRuleCount: tt.rules,
|
|
})
|
|
if tt.inactive {
|
|
// A newer version without script ordering replaces one that uses it.
|
|
activeJob := dbgen.ProvisionerJob(t, db, nil, database.ProvisionerJob{
|
|
Type: database.ProvisionerJobTypeTemplateVersionImport,
|
|
OrganizationID: org.ID,
|
|
InitiatorID: user.ID,
|
|
CompletedAt: sql.NullTime{Time: now, Valid: true},
|
|
})
|
|
version = dbgen.TemplateVersion(t, db, database.TemplateVersion{
|
|
CreatedBy: user.ID,
|
|
OrganizationID: org.ID,
|
|
TemplateID: templateID,
|
|
JobID: activeJob.ID,
|
|
// No script order counts.
|
|
})
|
|
}
|
|
if templateID.Valid {
|
|
err := db.UpdateTemplateActiveVersionByID(ctx, database.UpdateTemplateActiveVersionByIDParams{
|
|
ID: templateID.UUID,
|
|
ActiveVersionID: version.ID,
|
|
UpdatedAt: now,
|
|
})
|
|
require.NoError(t, err, tt.name)
|
|
}
|
|
}
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
|
|
// Keep totals distinct so swapped telemetry fields fail this test.
|
|
require.Equal(t, &telemetry.ScriptOrderSummary{
|
|
TemplatesWithScriptOrder: 3,
|
|
DataSourceCount: 4,
|
|
RuleCount: 5,
|
|
}, snapshot.ScriptOrderSummary)
|
|
data, err := json.Marshal(snapshot.ScriptOrderSummary)
|
|
require.NoError(t, err)
|
|
|
|
// Verify the serialized telemetry contains only aggregate counts,
|
|
// with no identifiers or rule contents.
|
|
require.JSONEq(t, `{
|
|
"templates_with_script_order":3,"data_source_count":4,"rule_count":5
|
|
}`, string(data))
|
|
|
|
// Verify script-order counts are excluded from individual
|
|
// template-version telemetry records and reported only in the
|
|
// deployment-wide summary.
|
|
versionData, err := json.Marshal(telemetry.ConvertTemplateVersion(database.TemplateVersion{
|
|
ScriptOrderDataSourceCount: 1,
|
|
ScriptOrderRuleCount: 2,
|
|
}))
|
|
require.NoError(t, err)
|
|
var fields map[string]json.RawMessage
|
|
require.NoError(t, json.Unmarshal(versionData, &fields))
|
|
require.NotContains(t, fields, "script_order_data_source_count")
|
|
require.NotContains(t, fields, "script_order_rule_count")
|
|
}
|
|
|
|
func TestTelemetryScriptOrderSummaryError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
|
|
// Boundary counters are reset during collection, so their snapshot
|
|
// must still be delivered when script-order collection fails.
|
|
tracker := boundaryusage.NewTracker()
|
|
tracker.Track(uuid.New(), uuid.New(), 10, 2)
|
|
require.NoError(t, tracker.FlushToDB(ctx, db, uuid.New()))
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, &scriptOrderSummaryErrorStore{Store: db}, nil)
|
|
require.Nil(t, snapshot.ScriptOrderSummary)
|
|
require.NotNil(t, snapshot.BoundaryUsageSummary)
|
|
require.Equal(t, int64(1), snapshot.BoundaryUsageSummary.UniqueWorkspaces)
|
|
require.Equal(t, int64(1), snapshot.BoundaryUsageSummary.UniqueUsers)
|
|
require.Equal(t, int64(10), snapshot.BoundaryUsageSummary.AllowedRequests)
|
|
require.Equal(t, int64(2), snapshot.BoundaryUsageSummary.DeniedRequests)
|
|
}
|
|
|
|
type scriptOrderSummaryErrorStore struct {
|
|
database.Store
|
|
}
|
|
|
|
func (*scriptOrderSummaryErrorStore) GetScriptOrderTelemetrySummary(context.Context) (database.GetScriptOrderTelemetrySummaryRow, error) {
|
|
return database.GetScriptOrderTelemetrySummaryRow{}, sql.ErrConnDone
|
|
}
|
|
|
|
func TestTelemetryScriptOrderSummaryEmpty(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
|
|
// Verify an empty deployment reports an explicit all-zero
|
|
// summary, rather than a missing summary.
|
|
require.Equal(t, &telemetry.ScriptOrderSummary{}, snapshot.ScriptOrderSummary)
|
|
data, err := json.Marshal(snapshot)
|
|
require.NoError(t, err)
|
|
var fields map[string]json.RawMessage
|
|
require.NoError(t, json.Unmarshal(data, &fields))
|
|
require.JSONEq(t, `{
|
|
"templates_with_script_order":0,"data_source_count":0,"rule_count":0
|
|
}`, string(fields["script_order_summary"]))
|
|
}
|
|
|
|
func TestTelemetryScriptOrderSummaryOneReplicaPerPeriod(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db, _ := dbtestutil.NewDB(t)
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
const snapshotFrequency = 5 * time.Minute
|
|
clock := quartz.NewMock(t)
|
|
clock.Set(dbtime.Now().Truncate(snapshotFrequency))
|
|
deploymentID := uuid.NewString()
|
|
options := func(opts telemetry.Options) telemetry.Options {
|
|
opts.Clock = clock
|
|
opts.SnapshotFrequency = snapshotFrequency
|
|
opts.DeploymentID = deploymentID
|
|
return opts
|
|
}
|
|
|
|
// The elected replica reports zero counts too, so adoption can be cleared.
|
|
_, first := collectSnapshot(ctx, t, db, options)
|
|
require.Equal(t, &telemetry.ScriptOrderSummary{}, first.ScriptOrderSummary)
|
|
|
|
// Another replica later in the same period must not collect a summary.
|
|
clock.Advance(time.Second)
|
|
_, second := collectSnapshot(ctx, t, db, options)
|
|
require.Nil(t, second.ScriptOrderSummary)
|
|
data, err := json.Marshal(second)
|
|
require.NoError(t, err)
|
|
var fields map[string]json.RawMessage
|
|
require.NoError(t, json.Unmarshal(data, &fields))
|
|
require.Equal(t, "null", string(fields["script_order_summary"]))
|
|
|
|
// The previous period's lock must not prevent the next collection.
|
|
clock.Advance(snapshotFrequency - time.Second)
|
|
_, next := collectSnapshot(ctx, t, db, options)
|
|
require.Equal(t, &telemetry.ScriptOrderSummary{}, next.ScriptOrderSummary)
|
|
}
|
|
|
|
func TestCollectAgentsVirtualDesktop(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsVirtualDesktopTelemetry {
|
|
t.Helper()
|
|
var payload telemetry.AgentsVirtualDesktopTelemetry
|
|
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsVirtualDesktop(context.Background(), opts), &payload))
|
|
return payload
|
|
}
|
|
|
|
expectOverrides := func(db *dbmock.MockStore, rows []database.GetChatOrganizationModelOverridesByContextRow, err error) {
|
|
db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), string(codersdk.ChatModelOverrideContextComputerUse)).Return(rows, err)
|
|
}
|
|
|
|
t.Run("NoOverrides", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
expectOverrides(db, nil, nil)
|
|
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.False(t, payload.Enabled)
|
|
require.NotNil(t, payload.ComputerUse.Overrides)
|
|
require.Empty(t, payload.ComputerUse.Overrides)
|
|
})
|
|
|
|
t.Run("Overrides", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
available, inactive, unsupported := uuid.New(), uuid.New(), uuid.New()
|
|
bedrockClaude, bedrockNonClaude := uuid.New(), uuid.New()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
expectOverrides(db, []database.GetChatOrganizationModelOverridesByContextRow{
|
|
{
|
|
OrganizationID: available,
|
|
ModelAvailable: true,
|
|
Model: "claude-opus-5",
|
|
ProviderType: string(database.AIProviderTypeAnthropic),
|
|
},
|
|
{OrganizationID: inactive, ModelAvailable: false},
|
|
// The model API can move a saved override's model to an AI
|
|
// provider type that computer use does not support.
|
|
{
|
|
OrganizationID: unsupported,
|
|
ModelAvailable: true,
|
|
Model: "openai/gpt-test",
|
|
ProviderType: string(database.AIProviderTypeOpenrouter),
|
|
},
|
|
{
|
|
OrganizationID: bedrockClaude,
|
|
ModelAvailable: true,
|
|
Model: "us.anthropic.claude-opus-5-5",
|
|
ProviderType: string(database.AIProviderTypeBedrock),
|
|
},
|
|
{
|
|
OrganizationID: bedrockNonClaude,
|
|
ModelAvailable: true,
|
|
Model: "openai.gpt-5.4",
|
|
ProviderType: string(database.AIProviderTypeBedrock),
|
|
},
|
|
}, nil)
|
|
|
|
payload := collect(t, telemetry.Options{
|
|
Database: db,
|
|
Logger: testutil.Logger(t),
|
|
Experiments: codersdk.Experiments{codersdk.ExperimentChatVirtualDesktop},
|
|
})
|
|
require.True(t, payload.Enabled)
|
|
require.Equal(t, []telemetry.AgentsComputerUseOverrideTelemetry{
|
|
{OrganizationID: available.String(), Provider: string(database.AIProviderTypeAnthropic), Model: "claude-opus-5"},
|
|
{
|
|
OrganizationID: inactive.String(),
|
|
Provider: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
Model: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
},
|
|
{
|
|
OrganizationID: unsupported.String(),
|
|
Provider: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
Model: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
},
|
|
{
|
|
OrganizationID: bedrockClaude.String(),
|
|
Provider: string(database.AIProviderTypeBedrock),
|
|
Model: "us.anthropic.claude-opus-5-5",
|
|
},
|
|
{
|
|
OrganizationID: bedrockNonClaude.String(),
|
|
Provider: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
Model: telemetry.AgentsExperimentComputerUseOrgDefaultModel,
|
|
},
|
|
}, payload.ComputerUse.Overrides)
|
|
})
|
|
|
|
t.Run("QueryError", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
expectOverrides(db, nil, sql.ErrConnDone)
|
|
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Nil(t, payload.ComputerUse.Overrides)
|
|
})
|
|
}
|
|
|
|
func TestCollectAgentsAdvisor(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
collect := func(t *testing.T, opts telemetry.Options) telemetry.AgentsAdvisorTelemetry {
|
|
t.Helper()
|
|
var payload telemetry.AgentsAdvisorTelemetry
|
|
require.NoError(t, json.Unmarshal(telemetry.CollectAgentsAdvisor(context.Background(), opts), &payload))
|
|
return payload
|
|
}
|
|
marshalConfig := func(t *testing.T, cfg codersdk.AdvisorConfig) string {
|
|
t.Helper()
|
|
raw, err := json.Marshal(cfg)
|
|
require.NoError(t, err)
|
|
return string(raw)
|
|
}
|
|
expectOverrides := func(db *dbmock.MockStore, rows ...database.GetChatOrganizationModelOverridesByContextRow) {
|
|
db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), string(codersdk.ChatModelOverrideContextAdvisor)).Return(rows, nil)
|
|
}
|
|
|
|
t.Run("Defaults", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{}), nil)
|
|
expectOverrides(db)
|
|
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.True(t, payload.Enabled)
|
|
require.Zero(t, payload.MaxUsesPerRun)
|
|
require.Zero(t, payload.MaxOutputTokens)
|
|
require.Empty(t, payload.Overrides)
|
|
})
|
|
|
|
t.Run("ModelOverrides", func(t *testing.T) {
|
|
t.Parallel()
|
|
org := database.Organization{ID: uuid.New()}
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(marshalConfig(t, codersdk.AdvisorConfig{MaxUsesPerRun: 7, MaxOutputTokens: 2048}), nil)
|
|
expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
|
|
OrganizationID: org.ID,
|
|
ModelAvailable: true,
|
|
Model: "gpt-6-preview",
|
|
ProviderType: string(database.AIProviderTypeOpenai),
|
|
})
|
|
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.True(t, payload.Enabled)
|
|
require.Equal(t, 7, payload.MaxUsesPerRun)
|
|
require.Equal(t, int64(2048), payload.MaxOutputTokens)
|
|
require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
|
|
OrganizationID: org.ID.String(), Provider: string(database.AIProviderTypeOpenai), Model: "gpt-6-preview",
|
|
}}, payload.Overrides)
|
|
})
|
|
|
|
t.Run("InactiveModel", func(t *testing.T) {
|
|
t.Parallel()
|
|
org := database.Organization{ID: uuid.New()}
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
|
|
expectOverrides(db, database.GetChatOrganizationModelOverridesByContextRow{
|
|
OrganizationID: org.ID,
|
|
ModelAvailable: false,
|
|
})
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Equal(t, []telemetry.AgentsAdvisorOverrideTelemetry{{
|
|
OrganizationID: org.ID.String(),
|
|
Provider: telemetry.AgentsExperimentAdvisorReuseChatModel,
|
|
Model: telemetry.AgentsExperimentAdvisorReuseChatModel,
|
|
}}, payload.Overrides)
|
|
})
|
|
|
|
t.Run("NoOverride", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
|
|
expectOverrides(db)
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Empty(t, payload.Overrides)
|
|
})
|
|
|
|
t.Run("MalformedConfig", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("not-json", nil)
|
|
expectOverrides(db)
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Zero(t, payload.MaxUsesPerRun)
|
|
require.Zero(t, payload.MaxOutputTokens)
|
|
})
|
|
|
|
t.Run("ClampsNegativeLimits", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return(`{"max_uses_per_run":-3,"max_output_tokens":-99}`, nil)
|
|
expectOverrides(db)
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Zero(t, payload.MaxUsesPerRun)
|
|
require.Zero(t, payload.MaxOutputTokens)
|
|
})
|
|
|
|
t.Run("ConfigFetchError", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("", sql.ErrConnDone)
|
|
expectOverrides(db)
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Empty(t, payload.Overrides)
|
|
})
|
|
|
|
t.Run("OverridesFetchError", func(t *testing.T) {
|
|
t.Parallel()
|
|
db := dbmock.NewMockStore(gomock.NewController(t))
|
|
db.EXPECT().GetChatAdvisorConfig(gomock.Any()).Return("{}", nil)
|
|
db.EXPECT().GetChatOrganizationModelOverridesByContext(gomock.Any(), gomock.Any()).Return(nil, sql.ErrConnDone)
|
|
payload := collect(t, telemetry.Options{Database: db, Logger: testutil.Logger(t)})
|
|
require.Empty(t, payload.Overrides)
|
|
})
|
|
}
|