Slice 2 of 4 for runtime experiment rules: persist one rule per experiment in `site_configs`, with revisions that only increase, and add an audit record type. Nothing calls this code yet, so behavior does not change. ## What's in it - **Queries** in `siteconfig.sql`, all checked by dbauthz against `ResourceDeploymentConfig`: - `GetExperimentRules` (read) lists `experiment_rule:` keys with `starts_with`, not `LIKE`, because `_` is a `LIKE` wildcard. - `GetExperimentRule` (read) reads one row. - `UpsertExperimentRule` (update) writes one row. - **`experiments.WriteRule(ctx, db, actor, ex, rule, expectedRevision)`** validates the rule first, then does everything in one `READ COMMITTED` transaction: - It takes an advisory lock for that experiment, waiting at most 5 s. - It reads the current row; a missing row counts as revision 0. - It returns `*RevisionConflictError` when the expected revision is stale. - An identical mode and condition writes nothing and returns `changed=false`. The first `inherit` on a missing row is a change and stores revision 1. - Otherwise it writes `revision+1` with the actor and time, and returns the old and new rule as read and written inside the transaction. - **`experiments.NewDBStore(db)`** implements the evaluator's `Store`. It reads rules by prefix, plus the user's id, username, email, explicit site roles, member org names, and `<org>/<group>` group names (including `<org>/Everyone`). It runs as the system. - **Audit:** - A new `resource_type` enum value `experiment_rule` (migration `000602`; the down migration is a comment because PostgreSQL cannot drop enum values). - A new struct `database.ExperimentRule`. `experiments.AuditRecord` derives its ID from the experiment name, so audit history groups by experiment. - In the audit table, `experiment`, `mode` `revision` and `condition` are tracked (`ActionTrack`), so the audit diff shows the old and new condition. In Enterprise, the slog audit backend also writes that diff to the server log. Admins and auditors can already read the condition through the rules API. - The audit page filter includes the new resource type. ## Testing - `WriteRule` and `NewDBStore` tests run against real PostgreSQL. They cover stale revisions after a reset and a recreate, two concurrent writers with the same expected revision (exactly one wins), no-ops, and rollback after a forced error. - dbauthz method tests and an audit diff test that shows both condition texts. Remote dogfood UAT through Coder Agents passed the upgrade from slice 1 (existing state kept, enum value added), a fresh install, unchanged `/api/v2/experiments`, and the `WriteRule` tests under `-race`. The licensed audit-page check could not run: the test workspace has no premium license. That round tested `015979ca96`. The commit after it only adds `AuditRecord`, and a follow-up UAT round will cover the final head. > [!NOTE] > Migration `000604` was the next free number when this was written. Recheck it before merging. ## Stack 1. #29869: evaluator library 2. **This PR:** storage and audit types 3. Enforcement and discovery (`/api/v2/experiments`, meta tag, chatd `mcp-tool-search`, ruleguard) 4. Control plane (experimental API, `coder exp experiment-rules` CLI, docs) > [!NOTE] > This PR was written by an AI agent (Xum). --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `high`_ --------- Signed-off-by: Thomas Kosiewski <tk@coder.com>
315 lines
11 KiB
Go
315 lines
11 KiB
Go
package experiments_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync"
|
|
"testing"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"cdr.dev/slog/v3/sloggers/slogtest"
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbgen"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
"github.com/coder/coder/v2/coderd/experiments"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/testutil"
|
|
)
|
|
|
|
var (
|
|
ruleOn = experiments.Rule{Mode: experiments.ModeOn}
|
|
ruleOff = experiments.Rule{Mode: experiments.ModeOff}
|
|
// ruleInherit is what reset writes.
|
|
ruleInherit = experiments.Rule{Mode: experiments.ModeInherit}
|
|
)
|
|
|
|
// writeRule calls WriteRule and requires success.
|
|
func writeRule(ctx context.Context, t *testing.T, db database.Store, actor uuid.UUID, rule experiments.Rule, expected int64) (oldRule, newRule experiments.Rule, changed bool) {
|
|
t.Helper()
|
|
oldRule, newRule, changed, err := experiments.WriteRule(ctx, db, actor, scoped, rule, expected)
|
|
require.NoError(t, err)
|
|
return oldRule, newRule, changed
|
|
}
|
|
|
|
// storedValue returns the raw stored value for the scoped experiment.
|
|
func storedValue(ctx context.Context, t *testing.T, db database.Store) string {
|
|
t.Helper()
|
|
value, err := db.GetExperimentRule(ctx, string(scoped))
|
|
require.NoError(t, err)
|
|
return value
|
|
}
|
|
|
|
func requireConflict(t *testing.T, err error, expected, current int64) {
|
|
t.Helper()
|
|
var conflict *experiments.RevisionConflictError
|
|
require.ErrorAs(t, err, &conflict)
|
|
require.Equal(t, expected, conflict.Expected)
|
|
require.Equal(t, current, conflict.Current)
|
|
}
|
|
|
|
func TestWriteRule(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
t.Run("RevisionsIncrease", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
actor := uuid.New()
|
|
|
|
condition := experiments.Rule{Mode: experiments.ModeCondition, Condition: `user.username == "alice"`}
|
|
previous := experiments.Rule{}
|
|
for i, rule := range []experiments.Rule{ruleOn, ruleOff, condition, ruleInherit} {
|
|
oldRule, newRule, changed := writeRule(ctx, t, db, actor, rule, int64(i))
|
|
require.True(t, changed)
|
|
require.Equal(t, previous, oldRule)
|
|
require.Equal(t, rule.Mode, newRule.Mode)
|
|
require.Equal(t, rule.Condition, newRule.Condition)
|
|
require.Equal(t, int64(i+1), newRule.Revision)
|
|
require.Equal(t, actor, newRule.UpdatedBy)
|
|
require.False(t, newRule.UpdatedAt.IsZero())
|
|
previous = newRule
|
|
}
|
|
})
|
|
|
|
t.Run("IdenticalRuleIsNoOp", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
|
|
// The first inherit creates the row, so it is a change.
|
|
oldRule, created, changed := writeRule(ctx, t, db, uuid.New(), ruleInherit, 0)
|
|
require.True(t, changed)
|
|
require.Equal(t, experiments.Rule{}, oldRule)
|
|
require.Equal(t, int64(1), created.Revision)
|
|
before := storedValue(ctx, t, db)
|
|
|
|
// Another actor writing the same rule changes nothing.
|
|
oldRule, newRule, changed := writeRule(ctx, t, db, uuid.New(), ruleInherit, 1)
|
|
require.False(t, changed)
|
|
require.Equal(t, created, oldRule)
|
|
require.Equal(t, created, newRule)
|
|
require.Equal(t, before, storedValue(ctx, t, db))
|
|
})
|
|
|
|
t.Run("StaleRevisionAfterResetAndRecreate", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
actor := uuid.New()
|
|
|
|
writeRule(ctx, t, db, actor, ruleOn, 0)
|
|
writeRule(ctx, t, db, actor, ruleInherit, 1) // reset
|
|
writeRule(ctx, t, db, actor, ruleOn, 2) // recreate
|
|
before := storedValue(ctx, t, db)
|
|
|
|
// A client that read revision 1 sees the same content again but
|
|
// must still be rejected.
|
|
_, _, _, err := experiments.WriteRule(ctx, db, actor, scoped, ruleOff, 1)
|
|
requireConflict(t, err, 1, 3)
|
|
_, _, _, err = experiments.WriteRule(ctx, db, actor, scoped, ruleOn, 0)
|
|
requireConflict(t, err, 0, 3)
|
|
require.Equal(t, before, storedValue(ctx, t, db))
|
|
})
|
|
|
|
t.Run("ConcurrentWritersOneWins", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _, sqlDB := dbtestutil.NewDBWithSQLDB(t)
|
|
actor := uuid.New()
|
|
|
|
// Hold the write lock so that both writers are queued behind it
|
|
// before either reads the current revision.
|
|
holding := make(chan struct{})
|
|
release := make(chan struct{})
|
|
releaseLock := sync.OnceFunc(func() { close(release) })
|
|
t.Cleanup(releaseLock)
|
|
holderErr := make(chan error, 1)
|
|
go func() {
|
|
holderErr <- db.InTx(func(tx database.Store) error {
|
|
if err := tx.AcquireLock(ctx, database.GenLockID("experiment_rule:"+string(scoped))); err != nil {
|
|
return err
|
|
}
|
|
close(holding)
|
|
<-release
|
|
return nil
|
|
}, nil)
|
|
}()
|
|
testutil.TryReceive(ctx, t, holding)
|
|
|
|
results := make(chan error, 2)
|
|
for range 2 {
|
|
go func() {
|
|
_, _, _, err := experiments.WriteRule(ctx, db, actor, scoped, ruleOn, 0)
|
|
results <- err
|
|
}()
|
|
}
|
|
testutil.Eventually(ctx, t, func(ctx context.Context) bool {
|
|
var waiting int
|
|
err := sqlDB.QueryRowContext(ctx, `SELECT count(*) FROM pg_locks
|
|
WHERE locktype = 'advisory' AND NOT granted
|
|
AND database = (SELECT oid FROM pg_database WHERE datname = current_database())`).Scan(&waiting)
|
|
return err == nil && waiting == 2
|
|
}, testutil.IntervalFast, "both writers wait for the lock")
|
|
releaseLock()
|
|
require.NoError(t, testutil.TryReceive(ctx, t, holderErr))
|
|
|
|
var wins int
|
|
for range 2 {
|
|
err := testutil.TryReceive(ctx, t, results)
|
|
if err == nil {
|
|
wins++
|
|
continue
|
|
}
|
|
requireConflict(t, err, 0, 1)
|
|
}
|
|
require.Equal(t, 1, wins)
|
|
|
|
var stored experiments.Rule
|
|
require.NoError(t, json.Unmarshal([]byte(storedValue(ctx, t, db)), &stored))
|
|
require.Equal(t, int64(1), stored.Revision)
|
|
})
|
|
|
|
t.Run("ErrorAfterWriteRollsBack", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
actor := uuid.New()
|
|
|
|
writeRule(ctx, t, db, actor, ruleOn, 0)
|
|
before := storedValue(ctx, t, db)
|
|
|
|
_, _, _, err := experiments.WriteRule(ctx, failAfterUpsert{db}, actor, scoped, ruleOff, 1)
|
|
require.ErrorIs(t, err, errInjected)
|
|
require.Equal(t, before, storedValue(ctx, t, db))
|
|
})
|
|
|
|
t.Run("InvalidRuleNotStored", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
actor := uuid.New()
|
|
|
|
_, _, _, err := experiments.WriteRule(ctx, db, actor, unscoped, ruleOn, 0)
|
|
require.Error(t, err)
|
|
_, _, _, err = experiments.WriteRule(ctx, db, actor, scoped, experiments.Rule{Mode: experiments.ModeCondition, Condition: "user."}, 0)
|
|
require.Error(t, err)
|
|
|
|
rows, err := db.GetExperimentRules(ctx)
|
|
require.NoError(t, err)
|
|
require.Empty(t, rows)
|
|
})
|
|
|
|
// A stored value the Evaluator treats as malformed must stay
|
|
// replaceable through WriteRule as long as its revision is readable.
|
|
t.Run("MalformedRuleReplaceable", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
actor := uuid.New()
|
|
|
|
for i, value := range []string{
|
|
`{"revision":2,"updated_by":7}`,
|
|
`{"revision":3,"mode":"percent"}`,
|
|
`{"revision":4,"mode":["on"],"condition":{}}`,
|
|
} {
|
|
require.NoError(t, db.UpsertExperimentRule(ctx, database.UpsertExperimentRuleParams{Experiment: string(scoped), Value: value}))
|
|
revision := int64(i + 2)
|
|
|
|
_, _, _, err := experiments.WriteRule(ctx, db, actor, scoped, ruleOn, revision-1)
|
|
requireConflict(t, err, revision-1, revision)
|
|
|
|
oldRule, newRule, changed := writeRule(ctx, t, db, actor, ruleOn, revision)
|
|
require.True(t, changed, value)
|
|
require.Equal(t, revision, oldRule.Revision, value)
|
|
require.Equal(t, revision+1, newRule.Revision, value)
|
|
require.Equal(t, experiments.ModeOn, newRule.Mode, value)
|
|
}
|
|
|
|
// A value without a readable positive revision is at revision 0:
|
|
// only expected revision 0 replaces it, and it restarts at 1.
|
|
for _, value := range []string{
|
|
`{not json`,
|
|
`{"mode":"on"}`,
|
|
`{"revision":0,"mode":"on"}`,
|
|
`{"revision":-3,"mode":"on"}`,
|
|
} {
|
|
require.NoError(t, db.UpsertExperimentRule(ctx, database.UpsertExperimentRuleParams{Experiment: string(scoped), Value: value}))
|
|
|
|
_, _, _, err := experiments.WriteRule(ctx, db, actor, scoped, ruleOff, 5)
|
|
requireConflict(t, err, 5, 0)
|
|
require.Equal(t, value, storedValue(ctx, t, db))
|
|
|
|
oldRule, newRule, changed := writeRule(ctx, t, db, actor, ruleOff, 0)
|
|
require.True(t, changed, value)
|
|
require.Equal(t, experiments.Rule{}, oldRule, value)
|
|
require.Equal(t, int64(1), newRule.Revision, value)
|
|
require.Equal(t, experiments.ModeOff, newRule.Mode, value)
|
|
}
|
|
})
|
|
|
|
// The Evaluator reads what WriteRule stores, through NewDBStore.
|
|
t.Run("EvaluatorReadsWrittenRule", func(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := testutil.Context(t, testutil.WaitMedium)
|
|
db, _ := dbtestutil.NewDB(t)
|
|
alice := dbgen.User(t, db, database.User{Username: "alice"})
|
|
bob := dbgen.User(t, db, database.User{Username: "bob"})
|
|
evaluator, err := experiments.New(slogtest.Make(t, nil), experiments.NewDBStore(db), codersdk.Experiments{scoped})
|
|
require.NoError(t, err)
|
|
|
|
writeRule(ctx, t, db, alice.ID, experiments.Rule{Mode: experiments.ModeCondition, Condition: `user.username == "alice"`}, 0)
|
|
require.True(t, evaluator.Enabled(ctx, alice.ID, scoped))
|
|
require.False(t, evaluator.Enabled(ctx, bob.ID, scoped))
|
|
|
|
// A kill switch applies to the next decision.
|
|
writeRule(ctx, t, db, alice.ID, ruleOff, 1)
|
|
require.False(t, evaluator.Enabled(ctx, alice.ID, scoped))
|
|
})
|
|
}
|
|
|
|
var errInjected = xerrors.New("injected failure")
|
|
|
|
// failAfterUpsert fails every rule upsert after it runs on the transaction.
|
|
type failAfterUpsert struct {
|
|
database.Store
|
|
}
|
|
|
|
func (s failAfterUpsert) InTx(fn func(database.Store) error, opts *database.TxOptions) error {
|
|
return s.Store.InTx(func(tx database.Store) error {
|
|
return fn(failAfterUpsert{tx})
|
|
}, opts)
|
|
}
|
|
|
|
func (s failAfterUpsert) UpsertExperimentRule(ctx context.Context, arg database.UpsertExperimentRuleParams) error {
|
|
if err := s.Store.UpsertExperimentRule(ctx, arg); err != nil {
|
|
return err
|
|
}
|
|
return errInjected
|
|
}
|
|
|
|
func TestAuditRecord(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
rule := experiments.Rule{Mode: experiments.ModeCondition, Condition: `"owner" in user.roles`, Revision: 4}
|
|
got := experiments.AuditRecord(codersdk.ExperimentExample, rule)
|
|
require.Equal(t, database.ExperimentRule{
|
|
ID: got.ID,
|
|
Experiment: string(codersdk.ExperimentExample),
|
|
Mode: "condition",
|
|
Condition: `"owner" in user.roles`,
|
|
Revision: 4,
|
|
}, got)
|
|
|
|
// The ID identifies the experiment, not the rule: audit history for one
|
|
// experiment groups under one ID, and experiments never share one.
|
|
require.NotEqual(t, uuid.Nil, got.ID)
|
|
require.Equal(t, got.ID, experiments.AuditRecord(codersdk.ExperimentExample, experiments.Rule{}).ID)
|
|
require.NotEqual(t, got.ID, experiments.AuditRecord(codersdk.ExperimentAutoFillParameters, rule).ID)
|
|
// Pin the derivation so a namespace change, which would split existing
|
|
// audit history, fails here.
|
|
require.Equal(t, uuid.NewSHA1(uuid.MustParse("65b7b16d-c22d-4972-adca-014b0a4b77f0"), []byte("example")), got.ID)
|
|
}
|