## Why The Automations RFC controls availability with one user-scoped experiment. Its text: "The user-scoped `chat-automations` experiment controls availability. Operators configure defaults with `--experiments` and can change per-user rules while Coder is running. Each surface checks the relevant caller or automation's owner, and experiment evaluation fails closed." The later backend PRs (management routes, the `manage_automations` tool, `Publish`) need that check first. This PR registers the experiment and proves the per-user, fail-closed check. Nothing consumes it yet. Part of CODAGT-1135 ## What changed - Registers `chat-automations` in `ExperimentsKnown` and `ExperimentsUserScoped`. An earlier experiment with this name was removed in #30412, and this PR restores none of its consumers. - Keeps it out of `ExperimentsSafe`, so `--experiments=*` does not turn it on. Automations act with their owner's authority, and a test pins the choice. - Adds the name to the `userScopedExperimentStaticCheck` ruleguard rule, so server code cannot decide it from the static list. `TestExperimentsUserScopedRuleguard` keeps the rule in step with `ExperimentsUserScoped`. - Regenerates the swagger files, `docs/reference/api/schemas.md` and `typesGenerated.ts`. - Adds tests for the real experiment name. The evaluator tests cover the startup default, stored `on`, `off` and condition rules for two users, and an off decision on a rules read error, a malformed rule, an unknown mode, an attribute read error, a condition error and a nil subject. A coderd test with a real database checks the same per-user behavior through `api.ExperimentEvaluator` and `GET /api/v2/experiments`. ## Scope No routes, tool, UI or chatd change. I added no helper: `Evaluator.Enabled(ctx, userID, ex)` already decides per user and fails closed, and API handlers reach it through `api.ExperimentEvaluator`. chatd has no evaluator handle on main, because #31327 removed the unused field. The first chatd consumer adds it together with its read. ## Blast radius With the experiment off, which is the default, behavior on main does not change. Two things become visible. `GET /api/experimental/experiments/rules` and `coder exp experiment-rules list` show a `chat-automations` row, and an administrator can store a rule for it. A deployment that already passes `--experiments=chat-automations` sees three small differences: the startup log reports an opted-in hidden experiment where it reported an unknown one, the deployment Overview page stops flagging the name as invalid, and the rules row shows the static default as on. Nothing reads the experiment yet, so no feature turns on. ## Live check A local `./scripts/develop.sh` deployment built from this head ran with no flag, with `--experiments=*` and with `--experiments=chat-automations`. `GET /api/v2/experiments` lists `chat-automations` for the owner and for a member only when the startup list or a rule turns it on, and `--experiments=*` does not turn it on. A condition rule `user.username == "member"` turned it on for the member and off for the owner without a restart, and `reset` restored the startup default. --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-sonnet-5-5` • Thinking: `xhigh`_
296 lines
13 KiB
Go
296 lines
13 KiB
Go
package coderd_test
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"net/http"
|
|
"testing"
|
|
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/coder/coder/v2/coderd/coderdtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbauthz"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
"github.com/coder/coder/v2/coderd/rbac"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
// ruleWriter writes experiment rules through the experimental API and
|
|
// tracks the revision of each experiment so that every write expects the
|
|
// current revision.
|
|
type ruleWriter struct {
|
|
t *testing.T
|
|
client *codersdk.ExperimentalClient
|
|
revisions map[codersdk.Experiment]int64
|
|
}
|
|
|
|
func newRuleWriter(t *testing.T, client *codersdk.Client) *ruleWriter {
|
|
return &ruleWriter{t: t, client: codersdk.NewExperimentalClient(client), revisions: map[codersdk.Experiment]int64{}}
|
|
}
|
|
|
|
func (w *ruleWriter) put(ctx context.Context, ex codersdk.Experiment, mode codersdk.ExperimentRuleMode, condition string) codersdk.ExperimentRule {
|
|
w.t.Helper()
|
|
rule, err := w.client.PutExperimentRule(ctx, ex, codersdk.PutExperimentRuleRequest{
|
|
Mode: mode,
|
|
Condition: condition,
|
|
ExpectedRevision: w.revisions[ex],
|
|
})
|
|
require.NoError(w.t, err)
|
|
require.Equal(w.t, w.revisions[ex]+1, rule.Revision)
|
|
w.revisions[ex] = rule.Revision
|
|
return rule
|
|
}
|
|
|
|
func requireEnabled(ctx context.Context, t *testing.T, client *codersdk.Client, ex codersdk.Experiment, want bool) {
|
|
t.Helper()
|
|
got, err := client.Experiments(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, want, got.Enabled(ex), "experiment %q for the caller", ex)
|
|
}
|
|
|
|
func requireStatus(t *testing.T, err error, status int) *codersdk.Error {
|
|
t.Helper()
|
|
var sdkErr *codersdk.Error
|
|
require.ErrorAs(t, err, &sdkErr)
|
|
require.Equal(t, status, sdkErr.StatusCode(), sdkErr.Error())
|
|
return sdkErr
|
|
}
|
|
|
|
func ruleEntry(ctx context.Context, t *testing.T, client *codersdk.ExperimentalClient, ex codersdk.Experiment) codersdk.ExperimentRuleEntry {
|
|
t.Helper()
|
|
entries, err := client.ExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
for _, entry := range entries {
|
|
if entry.Experiment == string(ex) {
|
|
return entry
|
|
}
|
|
}
|
|
t.Fatalf("no rules entry for experiment %q", ex)
|
|
return codersdk.ExperimentRuleEntry{}
|
|
}
|
|
|
|
func TestExperimentRules(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("WritesApplyToNextRequest", func(t *testing.T) {
|
|
t.Parallel()
|
|
ownerClient := coderdtest.New(t, &coderdtest.Options{
|
|
DeploymentValues: coderdtest.DeploymentValues(t, func(v *codersdk.DeploymentValues) {
|
|
v.Experiments = []string{string(codersdk.ExperimentExample)}
|
|
}),
|
|
})
|
|
owner := coderdtest.CreateFirstUser(t, ownerClient)
|
|
memberClient, member := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
w := newRuleWriter(t, ownerClient)
|
|
|
|
entries, err := w.client.ExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []codersdk.ExperimentRuleEntry{
|
|
{Experiment: string(codersdk.ExperimentExample), StaticDefault: true},
|
|
{Experiment: string(codersdk.ExperimentChatAutomations)},
|
|
}, entries)
|
|
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeCondition, fmt.Sprintf("user.username == %q", member.Username))
|
|
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)
|
|
requireEnabled(ctx, t, ownerClient, codersdk.ExperimentExample, false)
|
|
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOn, "")
|
|
requireEnabled(ctx, t, ownerClient, codersdk.ExperimentExample, true)
|
|
|
|
// off is a kill switch for a statically enabled experiment, and
|
|
// reset restores the startup default.
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOff, "")
|
|
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, false)
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeInherit, "")
|
|
requireEnabled(ctx, t, memberClient, codersdk.ExperimentExample, true)
|
|
|
|
entry := ruleEntry(ctx, t, w.client, codersdk.ExperimentExample)
|
|
require.True(t, entry.StaticDefault)
|
|
require.NotNil(t, entry.Rule)
|
|
require.Equal(t, string(codersdk.ExperimentRuleModeInherit), entry.Rule.Mode)
|
|
require.Equal(t, int64(4), entry.Rule.Revision)
|
|
require.Equal(t, owner.UserID, entry.Rule.UpdatedBy)
|
|
require.False(t, entry.Rule.UpdatedAt.IsZero())
|
|
|
|
// Responses carry condition text and must never be cached.
|
|
for _, req := range []struct {
|
|
method, path string
|
|
body any
|
|
}{
|
|
{http.MethodGet, "/api/experimental/experiments/rules", nil},
|
|
{http.MethodPut, "/api/experimental/experiments/rules/example", codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeInherit, ExpectedRevision: 4}},
|
|
} {
|
|
res, err := ownerClient.Request(ctx, req.method, req.path, req.body)
|
|
require.NoError(t, err)
|
|
_ = res.Body.Close()
|
|
require.Equal(t, http.StatusOK, res.StatusCode, req.method)
|
|
require.Equal(t, "no-store", res.Header.Get("Cache-Control"), req.method)
|
|
}
|
|
})
|
|
|
|
t.Run("RejectedWritesKeepRule", func(t *testing.T) {
|
|
t.Parallel()
|
|
ownerClient := coderdtest.New(t, nil)
|
|
_ = coderdtest.CreateFirstUser(t, ownerClient)
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
w := newRuleWriter(t, ownerClient)
|
|
stored := w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOn, "")
|
|
|
|
put := func(ex codersdk.Experiment, req codersdk.PutExperimentRuleRequest) error {
|
|
_, err := w.client.PutExperimentRule(ctx, ex, req)
|
|
return err
|
|
}
|
|
|
|
// The author of an invalid condition receives the diagnostics.
|
|
err := put(codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeCondition, Condition: `user.username == "a" &&`, ExpectedRevision: 1})
|
|
require.Contains(t, requireStatus(t, err, http.StatusBadRequest).Detail, "Syntax error")
|
|
|
|
for _, req := range []codersdk.PutExperimentRuleRequest{
|
|
{Mode: codersdk.ExperimentRuleModeOff, Condition: "true", ExpectedRevision: 1},
|
|
{Mode: codersdk.ExperimentRuleModeCondition, ExpectedRevision: 1},
|
|
{Mode: "percent", ExpectedRevision: 1},
|
|
{Mode: codersdk.ExperimentRuleModeOff, ExpectedRevision: -1},
|
|
} {
|
|
requireStatus(t, put(codersdk.ExperimentExample, req), http.StatusBadRequest)
|
|
}
|
|
for _, ex := range []codersdk.Experiment{codersdk.ExperimentWorkspaceUsage, "not-an-experiment"} {
|
|
requireStatus(t, put(ex, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn}), http.StatusBadRequest)
|
|
}
|
|
|
|
// A stale revision conflicts; the same rule at the current
|
|
// revision is an accepted no-op.
|
|
requireStatus(t, put(codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOff}), http.StatusConflict)
|
|
same, err := w.client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 1})
|
|
require.NoError(t, err)
|
|
require.Equal(t, stored, same)
|
|
|
|
entries, err := w.client.ExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []codersdk.ExperimentRuleEntry{
|
|
{Experiment: string(codersdk.ExperimentExample), Rule: &stored},
|
|
{Experiment: string(codersdk.ExperimentChatAutomations)},
|
|
}, entries)
|
|
})
|
|
|
|
// Authorization runs before the body is parsed or compiled: a caller
|
|
// without update access gets 403 even for an invalid condition.
|
|
t.Run("Authorization", func(t *testing.T) {
|
|
t.Parallel()
|
|
ownerClient := coderdtest.New(t, nil)
|
|
owner := coderdtest.CreateFirstUser(t, ownerClient)
|
|
memberClient, _ := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
|
|
auditorClient, _ := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID, rbac.RoleAuditor())
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
|
|
member := codersdk.NewExperimentalClient(memberClient)
|
|
_, err := member.ExperimentRules(ctx)
|
|
requireStatus(t, err, http.StatusForbidden)
|
|
|
|
auditor := codersdk.NewExperimentalClient(auditorClient)
|
|
_, err = auditor.ExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
|
|
for _, client := range []*codersdk.ExperimentalClient{member, auditor} {
|
|
for _, req := range []codersdk.PutExperimentRuleRequest{
|
|
{Mode: codersdk.ExperimentRuleModeOn},
|
|
{Mode: codersdk.ExperimentRuleModeCondition, Condition: "user."},
|
|
} {
|
|
_, err := client.PutExperimentRule(ctx, codersdk.ExperimentExample, req)
|
|
requireStatus(t, err, http.StatusForbidden)
|
|
}
|
|
}
|
|
require.Nil(t, ruleEntry(ctx, t, codersdk.NewExperimentalClient(ownerClient), codersdk.ExperimentExample).Rule)
|
|
})
|
|
|
|
// Stored rules the API cannot write are listed but ignored, and a
|
|
// malformed rule of a user-scoped experiment stays replaceable, even
|
|
// when its revision is unreadable.
|
|
t.Run("StoredRulesOutsideTheAPI", func(t *testing.T) {
|
|
t.Parallel()
|
|
ownerClient, db := coderdtest.NewWithDatabase(t, nil)
|
|
owner := coderdtest.CreateFirstUser(t, ownerClient)
|
|
memberClient, _ := coderdtest.CreateAnotherUser(t, ownerClient, owner.OrganizationID)
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
sysCtx := dbauthz.AsSystemRestricted(ctx)
|
|
store := func(ex codersdk.Experiment, value string) {
|
|
t.Helper()
|
|
require.NoError(t, db.UpsertExperimentRule(sysCtx, database.UpsertExperimentRuleParams{Experiment: string(ex), Value: value}))
|
|
}
|
|
for ex, value := range map[codersdk.Experiment]string{
|
|
"not-an-experiment": `{"mode":"on","revision":1}`,
|
|
codersdk.ExperimentWorkspaceUsage: `{"mode":"off","revision":4}`,
|
|
codersdk.ExperimentExample: `{"mode":"percent","revision":2}`,
|
|
} {
|
|
store(ex, value)
|
|
}
|
|
client := codersdk.NewExperimentalClient(ownerClient)
|
|
|
|
entries, err := client.ExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
require.Equal(t, []codersdk.ExperimentRuleEntry{
|
|
{Experiment: string(codersdk.ExperimentExample), Rule: &codersdk.ExperimentRule{Revision: 2}},
|
|
{Experiment: string(codersdk.ExperimentChatAutomations)},
|
|
{Experiment: "not-an-experiment", Rule: &codersdk.ExperimentRule{Mode: string(codersdk.ExperimentRuleModeOn), Revision: 1}, Ignored: true},
|
|
{Experiment: string(codersdk.ExperimentWorkspaceUsage), Rule: &codersdk.ExperimentRule{Mode: string(codersdk.ExperimentRuleModeOff), Revision: 4}, Ignored: true},
|
|
}, entries)
|
|
|
|
rule, err := client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 2})
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(3), rule.Revision)
|
|
|
|
store(codersdk.ExperimentExample, `{not json`)
|
|
require.Equal(t, &codersdk.ExperimentRule{}, ruleEntry(ctx, t, client, codersdk.ExperimentExample).Rule)
|
|
_, err = client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 5})
|
|
requireStatus(t, err, http.StatusConflict)
|
|
rule, err = client.PutExperimentRule(ctx, codersdk.ExperimentExample, codersdk.PutExperimentRuleRequest{Mode: codersdk.ExperimentRuleModeOn, ExpectedRevision: 0})
|
|
require.NoError(t, err)
|
|
require.Equal(t, int64(1), rule.Revision)
|
|
for _, c := range []*codersdk.Client{ownerClient, memberClient} {
|
|
requireEnabled(ctx, t, c, codersdk.ExperimentExample, true)
|
|
}
|
|
|
|
// An on rule without a positive revision is malformed: it is
|
|
// listed at revision 0 and decides off for everyone.
|
|
for _, value := range []string{`{"mode":"on","revision":0}`, `{"mode":"on"}`} {
|
|
store(codersdk.ExperimentExample, value)
|
|
require.Equal(t, &codersdk.ExperimentRule{}, ruleEntry(ctx, t, client, codersdk.ExperimentExample).Rule, value)
|
|
for _, c := range []*codersdk.Client{ownerClient, memberClient} {
|
|
requireEnabled(ctx, t, c, codersdk.ExperimentExample, false)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
|
|
// TestExperimentRulesReplicas checks that a rule written through one
|
|
// replica decides the next evaluation on another replica sharing the
|
|
// database, both for the API and for server checks through the evaluator.
|
|
func TestExperimentRulesReplicas(t *testing.T) {
|
|
t.Parallel()
|
|
db, ps := dbtestutil.NewDB(t)
|
|
options := func() *coderdtest.Options {
|
|
return &coderdtest.Options{Database: db, Pubsub: ps}
|
|
}
|
|
clientA := coderdtest.New(t, options())
|
|
clientB, _, apiB := coderdtest.NewWithAPI(t, options())
|
|
owner := coderdtest.CreateFirstUser(t, clientA)
|
|
memberA, member := coderdtest.CreateAnotherUser(t, clientA, owner.OrganizationID)
|
|
memberB := codersdk.New(clientB.URL)
|
|
memberB.SetSessionToken(memberA.SessionToken())
|
|
ctx := testutil.Context(t, testutil.WaitLong)
|
|
w := newRuleWriter(t, clientA)
|
|
|
|
requireB := func(want bool) {
|
|
t.Helper()
|
|
requireEnabled(ctx, t, memberB, codersdk.ExperimentExample, want)
|
|
require.Equal(t, want, apiB.ExperimentEvaluator.Enabled(ctx, member.ID, codersdk.ExperimentExample))
|
|
}
|
|
requireB(false)
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeCondition, fmt.Sprintf("user.id == %q", member.ID))
|
|
requireB(true)
|
|
w.put(ctx, codersdk.ExperimentExample, codersdk.ExperimentRuleModeOff, "")
|
|
requireB(false)
|
|
}
|