Telemetry builds now report whether Agent Relay created them. Before this, relay workspaces looked like any other build. A start build counts as a relay build when it has the `agent_relay_session_id` parameter. Stop and delete builds never get the field. For those builds, the new `agent_relay` field on `WorkspaceBuild` reports: - `provider`: `claude_code`, `cursor`, `ambiguous`, or `unknown` - `client_platform`: Claude Code only. Lowercased, characters outside `[a-z0-9_]` become `_`, and cut to 15 characters - `attempt`: omitted when missing or not an integer, so `0` is sent - `session_id_hash` and `pool_hash`: SHA-256 hashes No other parameter values are sent. Other builds leave the field out. The snapshot fetches these parameters with one new batch query. The workspace-create event reads the stored parameters with one extra query, only when telemetry is enabled, so it reports the same data as the snapshot. `docs/admin/setup/telemetry.md` lists the parameters that are read. The collector side is [coder/coder-telemetry-server#56](https://github.com/coder/coder-telemetry-server/pull/56). Merge this PR first. Refs [RELAY-9](https://linear.app/codercom/issue/RELAY-9/telemetry-report-agent-relay-usage-from-codercoder) <details> <summary>Decision log</summary> - **Detect relay builds from parameters.** This needs no relay changes. Relay registration or [CORE-426](https://linear.app/codercom/issue/CORE-426) provenance metadata can come later. - **Builds only, not workspaces.** `GetWorkspaces` doesn't return the latest build's parameters, so a workspace field would cost another query. The collector can join on `workspace_id`. - **Provider comes from parameter names only.** `agent_relay_claude_code_*` means Claude Code and `agent_relay_cursor_*` means Cursor. A build with both reports `ambiguous`; a build with neither reports `unknown`. coderd stores every declared parameter on every build (ephemeral ones with their default), so a Claude Code start after a stop still reports `claude_code`. - **Start builds only.** Stop and delete builds of a relay workspace carry the same parameters but are not dispatches, so the snapshot skips them before querying parameters. - **Hash session ID and pool.** Pool names are user content, and sessions only need counting. - **Known gaps.** Cursor has no client platform. Relay version and health would need relay registration. - **Parameter names are shared with coder/agent-relay.** If agent-relay renames one, telemetry silently stops reading it. </details> <details> <summary>Manual test on <code>develop.sh</code></summary> Ran on `8ad7cd491f` with `CODER_TELEMETRY_ENABLE=true CODER_TELEMETRY_URL=http://127.0.0.1:8081` and `go run ./scripts/telemetry-server --port 8081` as the collector. Pushed three templates with no infrastructure, each declaring the same parameters as the real modules: - `relay-claude`: `agent_relay_session_id`, `_delivery_id`, `_pool`, `_client_platform`, plus ephemeral `agent_relay_attempt` and `agent_relay_claude_code_lock_to_account` - `relay-cursor`: `agent_relay_session_id`, `_delivery_id`, `_pool`, `agent_relay_cursor_pool_name`, `_idle_release_timeout`, `_repo_url` - `relay-plain`: `region` Created workspaces with `POST /api/v2/users/me/workspaces` and `rich_parameter_values`, the same call the relay makes. Then ran stop and start builds on the relay workspaces, and restarted coderd to get a periodic snapshot. | Workspace | Build 1 (create event and snapshot) | Builds 2 and 3 (stop, start) | |---|---|---| | Claude (`claude_in_slack`, attempt 1) | `claude_code`, platform, attempt 1 | `claude_code`, platform, no attempt | | Cursor | `cursor` | `cursor` | | Plain | field omitted | n/a | | Claude template, no session ID | field omitted | n/a | - Stop and start builds keep `agent_relay_claude_code_lock_to_account`, with an empty value, so Claude Code rebuilds still report `claude_code`. - No session ID, pool name, or `region` value appears in the captured telemetry. The hashes match `sha256` of the inputs. - Fed the captured snapshot through the conversion code in coder/coder-telemetry-server#56: all 8 builds converted, 6 of them with `agent_relay` set. - No telemetry errors or panics in the coderd logs. This run predates the start-only change: build 2 (stop) carried the field then. Stop and delete builds are now omitted, which `TestTelemetry_WorkspaceBuildAgentRelay` covers. </details> --- *Generated by Coder Agents on behalf of @Emyrk.*
218 lines
6.7 KiB
Go
218 lines
6.7 KiB
Go
package telemetry_test
|
|
|
|
import (
|
|
"crypto/sha256"
|
|
"fmt"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbfake"
|
|
"github.com/coder/coder/v2/coderd/database/dbgen"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
"github.com/coder/coder/v2/coderd/telemetry"
|
|
"github.com/coder/coder/v2/coderd/util/ptr"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
func TestConvertWorkspaceBuild_AgentRelay(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hash := func(s string) string {
|
|
return fmt.Sprintf("%x", sha256.Sum256([]byte(s)))
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
params map[string]string
|
|
want *telemetry.WorkspaceBuildAgentRelay
|
|
}{
|
|
{
|
|
name: "NoParameters",
|
|
params: nil,
|
|
want: nil,
|
|
},
|
|
{
|
|
name: "NotRelay",
|
|
params: map[string]string{
|
|
"region": "us-east",
|
|
"agent_relay_pool": "pool",
|
|
},
|
|
want: nil,
|
|
},
|
|
{
|
|
name: "ClaudeCode",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-1",
|
|
"agent_relay_pool": "claude-pool",
|
|
"agent_relay_attempt": "2",
|
|
"agent_relay_client_platform": "claude_in_slack",
|
|
"agent_relay_claude_code_lock_to_account": "true",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderClaudeCode,
|
|
ClientPlatform: "claude_in_slack",
|
|
SessionIDHash: hash("session-1"),
|
|
PoolHash: hash("claude-pool"),
|
|
Attempt: ptr.Ref(2),
|
|
},
|
|
},
|
|
{
|
|
// Attempt 0 is the first dispatch and must stay distinct from a
|
|
// missing attempt.
|
|
name: "AttemptZero",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-1",
|
|
"agent_relay_attempt": "0",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderUnknown,
|
|
SessionIDHash: hash("session-1"),
|
|
Attempt: ptr.Ref(0),
|
|
},
|
|
},
|
|
{
|
|
// A template default stores an empty session ID on builds a
|
|
// person creates by hand.
|
|
name: "EmptySessionID",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "",
|
|
"agent_relay_pool": "pool",
|
|
},
|
|
want: nil,
|
|
},
|
|
{
|
|
name: "ClientPlatformNormalized",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-1",
|
|
"agent_relay_client_platform": "Claude-In-Slack <script>",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderUnknown,
|
|
ClientPlatform: "claude_in_slack",
|
|
SessionIDHash: hash("session-1"),
|
|
},
|
|
},
|
|
{
|
|
// Stop and restart builds keep the ephemeral parameter's row
|
|
// with its default value.
|
|
name: "ClaudeCodeRestart",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-1",
|
|
"agent_relay_attempt": "",
|
|
"agent_relay_claude_code_lock_to_account": "",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderClaudeCode,
|
|
SessionIDHash: hash("session-1"),
|
|
},
|
|
},
|
|
{
|
|
// The client platform alone does not identify a provider.
|
|
name: "UnknownWithClientPlatform",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-1",
|
|
"agent_relay_client_platform": "web_claude_ai",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderUnknown,
|
|
ClientPlatform: "web_claude_ai",
|
|
SessionIDHash: hash("session-1"),
|
|
},
|
|
},
|
|
{
|
|
name: "Ambiguous",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-4",
|
|
"agent_relay_claude_code_lock_to_account": "true",
|
|
"agent_relay_cursor_pool_name": "pool",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderAmbiguous,
|
|
SessionIDHash: hash("session-4"),
|
|
},
|
|
},
|
|
{
|
|
name: "Cursor",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-2",
|
|
"agent_relay_pool": "cursor-pool",
|
|
"agent_relay_cursor_pool_name": "cursor-pool",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderCursor,
|
|
SessionIDHash: hash("session-2"),
|
|
PoolHash: hash("cursor-pool"),
|
|
},
|
|
},
|
|
{
|
|
name: "UnknownProviderInvalidAttempt",
|
|
params: map[string]string{
|
|
"agent_relay_session_id": "session-3",
|
|
"agent_relay_attempt": "not-a-number",
|
|
},
|
|
want: &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderUnknown,
|
|
SessionIDHash: hash("session-3"),
|
|
},
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
got := telemetry.ConvertWorkspaceBuild(database.WorkspaceBuild{}, tt.params)
|
|
require.Equal(t, tt.want, got.AgentRelay)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestTelemetry_WorkspaceBuildAgentRelay(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
org := dbgen.Organization(t, db, database.Organization{})
|
|
user := dbgen.User(t, db, database.User{})
|
|
|
|
relayBuild := func(transition database.WorkspaceTransition) uuid.UUID {
|
|
return dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
}).Seed(database.WorkspaceBuild{Transition: transition}).Params(
|
|
database.WorkspaceBuildParameter{Name: "agent_relay_session_id", Value: "session-1"},
|
|
database.WorkspaceBuildParameter{Name: "agent_relay_pool", Value: "pool"},
|
|
database.WorkspaceBuildParameter{Name: "agent_relay_cursor_pool_name", Value: "pool"},
|
|
database.WorkspaceBuildParameter{Name: "region", Value: "us-east"},
|
|
).Do().Build.ID
|
|
}
|
|
startBuild := relayBuild(database.WorkspaceTransitionStart)
|
|
// Stop and delete builds keep the relay parameters but are not dispatches.
|
|
stopBuild := relayBuild(database.WorkspaceTransitionStop)
|
|
deleteBuild := relayBuild(database.WorkspaceTransitionDelete)
|
|
plainBuild := dbfake.WorkspaceBuild(t, db, database.WorkspaceTable{
|
|
OrganizationID: org.ID,
|
|
OwnerID: user.ID,
|
|
}).Params(
|
|
database.WorkspaceBuildParameter{Name: "region", Value: "us-east"},
|
|
).Do().Build.ID
|
|
|
|
_, snapshot := collectSnapshot(ctx, t, db, nil)
|
|
|
|
builds := make(map[uuid.UUID]telemetry.WorkspaceBuild, len(snapshot.WorkspaceBuilds))
|
|
for _, build := range snapshot.WorkspaceBuilds {
|
|
builds[build.ID] = build
|
|
}
|
|
require.Contains(t, builds, startBuild)
|
|
require.Equal(t, &telemetry.WorkspaceBuildAgentRelay{
|
|
Provider: telemetry.AgentRelayProviderCursor,
|
|
SessionIDHash: fmt.Sprintf("%x", sha256.Sum256([]byte("session-1"))),
|
|
PoolHash: fmt.Sprintf("%x", sha256.Sum256([]byte("pool"))),
|
|
}, builds[startBuild].AgentRelay)
|
|
for _, id := range []uuid.UUID{stopBuild, deleteBuild, plainBuild} {
|
|
require.Contains(t, builds, id)
|
|
require.Nil(t, builds[id].AgentRelay)
|
|
}
|
|
}
|