Files
coder/coderd/experiments_test.go
Thomas Kosiewski e307f1d5cf feat: add the user-scoped chat-automations experiment (#32064)
## Why

The Automations RFC controls availability with one user-scoped
experiment. Its text: "The user-scoped `chat-automations` experiment
controls availability. Operators configure defaults with `--experiments`
and can change per-user rules while Coder is running. Each surface
checks the relevant caller or automation's owner, and experiment
evaluation fails closed." The later backend PRs (management routes, the
`manage_automations` tool, `Publish`) need that check first. This PR
registers the experiment and proves the per-user, fail-closed check.
Nothing consumes it yet.

Part of CODAGT-1135

## What changed

- Registers `chat-automations` in `ExperimentsKnown` and
`ExperimentsUserScoped`. An earlier experiment with this name was
removed in #30412, and this PR restores none of its consumers.
- Keeps it out of `ExperimentsSafe`, so `--experiments=*` does not turn
it on. Automations act with their owner's authority, and a test pins the
choice.
- Adds the name to the `userScopedExperimentStaticCheck` ruleguard rule,
so server code cannot decide it from the static list.
`TestExperimentsUserScopedRuleguard` keeps the rule in step with
`ExperimentsUserScoped`.
- Regenerates the swagger files, `docs/reference/api/schemas.md` and
`typesGenerated.ts`.
- Adds tests for the real experiment name. The evaluator tests cover the
startup default, stored `on`, `off` and condition rules for two users,
and an off decision on a rules read error, a malformed rule, an unknown
mode, an attribute read error, a condition error and a nil subject. A
coderd test with a real database checks the same per-user behavior
through `api.ExperimentEvaluator` and `GET /api/v2/experiments`.

## Scope

No routes, tool, UI or chatd change. I added no helper:
`Evaluator.Enabled(ctx, userID, ex)` already decides per user and fails
closed, and API handlers reach it through `api.ExperimentEvaluator`.
chatd has no evaluator handle on main, because #31327 removed the unused
field. The first chatd consumer adds it together with its read.

## Blast radius

With the experiment off, which is the default, behavior on main does not
change. Two things become visible. `GET
/api/experimental/experiments/rules` and `coder exp experiment-rules
list` show a `chat-automations` row, and an administrator can store a
rule for it. A deployment that already passes
`--experiments=chat-automations` sees three small differences: the
startup log reports an opted-in hidden experiment where it reported an
unknown one, the deployment Overview page stops flagging the name as
invalid, and the rules row shows the static default as on. Nothing reads
the experiment yet, so no feature turns on.

## Live check

A local `./scripts/develop.sh` deployment built from this head ran with
no flag, with `--experiments=*` and with
`--experiments=chat-automations`. `GET /api/v2/experiments` lists
`chat-automations` for the owner and for a member only when the startup
list or a rule turns it on, and `--experiments=*` does not turn it on. A
condition rule `user.username == "member"` turned it on for the member
and off for the owner without a restart, and `reset` restored the
startup default.

---
_Generated with [`xum`](https://github.com/coder/xum) • Model:
`anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_
2026-11-11 23:54:05 +01:00

248 lines
8.3 KiB
Go

package coderd_test
import (
"context"
"fmt"
"net/http"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/coder/coder/v2/coderd/coderdtest"
"github.com/coder/coder/v2/coderd/database/dbauthz"
experimentrules "github.com/coder/coder/v2/coderd/experiments"
"github.com/coder/coder/v2/coderd/httpmw"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/testutil"
)
func Test_Experiments(t *testing.T) {
t.Parallel()
t.Run("empty", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
_ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
experiments, err := client.Experiments(ctx)
require.NoError(t, err)
require.NotNil(t, experiments)
require.Empty(t, experiments)
require.False(t, experiments.Enabled("foo"))
})
t.Run("multiple features", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"foo", "BAR"}
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
_ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
experiments, err := client.Experiments(ctx)
require.NoError(t, err)
require.NotNil(t, experiments)
// Should be lower-cased.
require.ElementsMatch(t, []codersdk.Experiment{"foo", "bar"}, experiments)
require.True(t, experiments.Enabled("foo"))
require.True(t, experiments.Enabled("bar"))
require.False(t, experiments.Enabled("baz"))
})
t.Run("wildcard", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"*"}
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
_ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
experiments, err := client.Experiments(ctx)
require.NoError(t, err)
require.NotNil(t, experiments)
require.ElementsMatch(t, codersdk.ExperimentsSafe, experiments)
for _, ex := range codersdk.ExperimentsSafe {
require.True(t, experiments.Enabled(ex))
}
require.False(t, experiments.Enabled("danger"))
})
t.Run("alternate wildcard with manual opt-in", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"*", "dAnGeR"}
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
_ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
experiments, err := client.Experiments(ctx)
require.NoError(t, err)
require.NotNil(t, experiments)
require.ElementsMatch(t, append(codersdk.ExperimentsSafe, "danger"), experiments)
for _, ex := range codersdk.ExperimentsSafe {
require.True(t, experiments.Enabled(ex))
}
require.True(t, experiments.Enabled("danger"))
require.False(t, experiments.Enabled("herebedragons"))
})
t.Run("Unauthorized", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"*"}
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
// Explicitly omit creating a user so we're unauthorized.
// _ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
_, err := client.Experiments(ctx)
require.Error(t, err)
require.ErrorContains(t, err, httpmw.SignedOutErrorMessage)
})
t.Run("rules per user", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{"foo", string(codersdk.ExperimentExample)}
ownerClient, db := coderdtest.NewWithDatabase(t, &coderdtest.Options{
DeploymentValues: cfg,
})
owner := coderdtest.CreateFirstUser(t, ownerClient)
memberClient, member := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
revisions := map[codersdk.Experiment]int64{}
writeRule := func(ex codersdk.Experiment, rule experimentrules.Rule) {
t.Helper()
_, stored, _, err := experimentrules.WriteRule(dbauthz.AsSystemRestricted(ctx), db, owner.UserID, ex, rule, revisions[ex])
require.NoError(t, err)
revisions[ex] = stored.Revision
}
requireExperiments := func(client *codersdk.Client, want ...codersdk.Experiment) {
t.Helper()
got, err := client.Experiments(ctx)
require.NoError(t, err)
require.ElementsMatch(t, want, got)
}
// A condition enables a user-scoped experiment only for matching
// users; a kill switch disables a statically enabled one.
writeRule(codersdk.ExperimentExample, experimentrules.Rule{
Mode: experimentrules.ModeCondition,
Condition: fmt.Sprintf("user.username == %q", member.Username),
})
requireExperiments(memberClient, "foo", codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo")
writeRule(codersdk.ExperimentExample, experimentrules.Rule{Mode: experimentrules.ModeOff})
requireExperiments(memberClient, "foo")
// Inherit restores the startup default.
writeRule(codersdk.ExperimentExample, experimentrules.Rule{Mode: experimentrules.ModeInherit})
requireExperiments(memberClient, "foo", codersdk.ExperimentExample)
requireExperiments(ownerClient, "foo", codersdk.ExperimentExample)
// The personalized result must not be cached.
res, err := memberClient.Request(ctx, http.MethodGet, "/api/v2/experiments", nil)
require.NoError(t, err)
defer res.Body.Close()
require.Equal(t, http.StatusOK, res.StatusCode)
require.Equal(t, "no-store", res.Header.Get("Cache-Control"))
})
// Later automation code asks api.ExperimentEvaluator about one user ID at
// a time: the caller, a chat's owner, or an automation's owner.
t.Run("chat automations per user", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
cfg.Experiments = []string{string(codersdk.ExperimentChatAutomations)}
ownerClient, _, api := coderdtest.NewWithAPI(t, &coderdtest.Options{
DeploymentValues: cfg,
})
owner := coderdtest.CreateFirstUser(t, ownerClient)
memberClient, member := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
ctx := testutil.Context(t, testutil.WaitLong)
var revision int64
writeRule := func(rule experimentrules.Rule) {
t.Helper()
_, stored, _, err := experimentrules.WriteRule(dbauthz.AsSystemRestricted(ctx), api.Database, owner.UserID, codersdk.ExperimentChatAutomations, rule, revision)
require.NoError(t, err)
revision = stored.Revision
}
requireEnabled := func(wantOwner, wantMember bool) {
t.Helper()
for _, tc := range []struct {
name string
client *codersdk.Client
userID uuid.UUID
want bool
}{
{"owner", ownerClient, owner.UserID, wantOwner},
{"member", memberClient, member.ID, wantMember},
} {
require.Equal(t, tc.want, api.ExperimentEvaluator.Enabled(ctx, tc.userID, codersdk.ExperimentChatAutomations), tc.name)
listed, err := tc.client.Experiments(ctx)
require.NoError(t, err)
require.Equal(t, tc.want, listed.Enabled(codersdk.ExperimentChatAutomations), tc.name)
}
}
requireEnabled(true, true)
require.False(t, api.ExperimentEvaluator.Enabled(ctx, uuid.Nil, codersdk.ExperimentChatAutomations), "nil subject")
writeRule(experimentrules.Rule{
Mode: experimentrules.ModeCondition,
Condition: fmt.Sprintf("user.username == %q", member.Username),
})
requireEnabled(false, true)
writeRule(experimentrules.Rule{Mode: experimentrules.ModeOff})
requireEnabled(false, false)
writeRule(experimentrules.Rule{Mode: experimentrules.ModeInherit})
requireEnabled(true, true)
})
t.Run("available experiments", func(t *testing.T) {
t.Parallel()
cfg := coderdtest.DeploymentValues(t)
client := coderdtest.New(t, &coderdtest.Options{
DeploymentValues: cfg,
})
_ = coderdtest.CreateFirstUser(t, client)
ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitLong)
defer cancel()
experiments, err := client.SafeExperiments(ctx)
require.NoError(t, err)
require.NotNil(t, experiments)
require.ElementsMatch(t, codersdk.ExperimentsSafe, experiments.Safe)
})
}