Files
coder/coderd/experiments/evaluator.go
T
Thomas Kosiewski 84fe2dda92 feat: add per-user experiment rule evaluator (#29869)
Slice 1 of 4 for runtime experiment rules: adds an internal evaluator
that decides whether an experiment is enabled for a given user, based on
a stored rule and that user's attributes. The package has no storage,
API, or callers yet, so behavior does not change.

## What's in it

- `coderd/experiments`: `Evaluator` with `New(logger, store, static)`,
`Enabled(ctx, userID, ex)` and `EnabledExperiments(ctx, userID)`. It
also adds the `Store` interface, the `Rule`/`StoredRule`/`User` models
and `ValidateRule`.
- Rule modes: `inherit` (use the startup `--experiments` default), `on`,
`off` (a kill switch that wins over a static on), and `condition` (a CEL
expression over `user.id`, `user.username`, `user.email`, `user.roles`,
`user.organizations`, `user.groups`).
- **Fail closed:** the decision is off for a rules-read error, a
malformed rule (including a stored revision below 1) or unknown mode, a
failed attribute load, a compile or eval error, a cost overrun, a
cancelled context, or a `uuid.Nil` subject. This holds even when the
experiment is enabled statically. `New` rejects a nil store, so there is
no fallback.
- **Diagnostics:** only errors returned to the caller carry full CEL
diagnostics. Logs record the experiment, revision, error category and
line:column, never condition text or CEL message text.
- Limits: 4096 code points, cost limit 10,000, and evaluation stops when
the context is cancelled. Compiled programs go in a bounded,
concurrency-safe cache keyed by exact source text. Decisions are never
cached.
- `codersdk.ExperimentsUserScoped = {example}`. Only experiments listed
there accept rules.
- `coderd/experiments/experimentstest`: a fake `Store` for tests.

## Dependency

Adds `cel.dev/cel-go v0.32.0` (plus indirect `antlr4-go/antlr/v4
v4.13.1`). `cel.dev/cel-go` is the module path that
`github.com/google/cel-go` now declares. GHSA-gcjh-h69q-9w9g
(GO-2026-6094) is fixed in 0.29.0. `govulncheck
./coderd/experiments/...` reports nothing against either new module. Its
remaining findings are go1.26.5 standard-library issues, fixed in
go1.26.6, plus existing dependencies, and they show up on unrelated
packages too.

## Stack

1. **This PR:** evaluator library
2. Storage and audit types (`site_configs` rows, revisions, `WriteRule`)
3. Enforcement and discovery (`/api/v2/experiments`, meta tag, chatd
`mcp-tool-search`, ruleguard)
4. Control plane (experimental API, `coder exp experiment-rules` CLI,
docs)

Remote dogfood UAT through Coder Agents passed. `/api/v2/experiments` is
unchanged for admin and member with and without `--experiments=example`,
and the package tests pass with and without `-race`.

> [!NOTE]
> This PR was written by an AI agent (Xum).

---
_Generated with [`xum`](https://github.com/coder/xum) • Model:
`anthropic:claude-opus-5-5` • Thinking: `high`_

---------

Signed-off-by: Thomas Kosiewski <tk@coder.com>
2026-09-29 12:22:59 +00:00

237 lines
7.3 KiB
Go

package experiments
import (
"context"
"errors"
"slices"
"github.com/google/uuid"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/codersdk"
)
// Store supplies stored rules and subject attributes to the Evaluator.
type Store interface {
// Rules returns the stored rule for every configured experiment.
// Experiments without a rule are absent from the map.
Rules(ctx context.Context) (map[codersdk.Experiment]StoredRule, error)
// UserAttributes loads the attributes conditions can read for userID.
UserAttributes(ctx context.Context, userID uuid.UUID) (User, error)
}
// Evaluator decides whether experiments are enabled for a subject user.
// It is safe for concurrent use. It does not cache decisions: every call
// reads the rules once, so rule changes apply to the next call.
type Evaluator struct {
logger slog.Logger
store Store
static codersdk.Experiments
programs *programCache
}
// New returns an Evaluator. static is the startup --experiments list; it
// is the default for experiments whose rule is absent or inherits.
func New(logger slog.Logger, store Store, static codersdk.Experiments) (*Evaluator, error) {
if store == nil {
return nil, xerrors.New("experiments: store is required")
}
// Build the CEL environment now so a broken environment fails startup
// instead of every decision.
if _, err := celEnv(); err != nil {
return nil, xerrors.Errorf("experiments: create CEL environment: %w", err)
}
return &Evaluator{
logger: logger,
store: store,
static: slices.Clone(static),
programs: newProgramCache(),
}, nil
}
// Enabled reports whether ex is enabled for userID.
func (e *Evaluator) Enabled(ctx context.Context, userID uuid.UUID, ex codersdk.Experiment) bool {
if !IsUserScoped(ex) {
return e.static.Enabled(ex)
}
return e.newDecision(ctx, userID).enabled(ex)
}
// EnabledExperiments returns the experiments enabled for userID. Static
// entries keep their order, including unknown strings. User-scoped
// entries decided off are removed, and user-scoped entries decided on that
// are not in the static list are appended in ExperimentsKnown order. Rules
// and attributes are each loaded at most once.
func (e *Evaluator) EnabledExperiments(ctx context.Context, userID uuid.UUID) codersdk.Experiments {
d := e.newDecision(ctx, userID)
result := make(codersdk.Experiments, 0, len(e.static))
for _, ex := range e.static {
if IsUserScoped(ex) && !d.enabled(ex) {
continue
}
result = append(result, ex)
}
for _, ex := range codersdk.ExperimentsKnown {
if !IsUserScoped(ex) || e.static.Enabled(ex) {
continue
}
if d.enabled(ex) {
result = append(result, ex)
}
}
return result
}
// decision holds the state of one Enabled or EnabledExperiments call. It
// is not safe for concurrent use.
type decision struct {
e *Evaluator
ctx context.Context
userID uuid.UUID
// failed is set when every user-scoped decision in this call must be
// off: a nil subject or a rules read error.
failed bool
rules map[codersdk.Experiment]StoredRule
userLoaded bool
user User
userErr error
// decided memoizes per-experiment results within this call.
decided map[codersdk.Experiment]bool
}
func (e *Evaluator) newDecision(ctx context.Context, userID uuid.UUID) *decision {
d := &decision{e: e, ctx: ctx, userID: userID, decided: make(map[codersdk.Experiment]bool)}
if userID == uuid.Nil {
e.logger.Error(ctx, "experiment decision without a subject user; user-scoped experiments are off")
d.failed = true
return d
}
rules, err := e.store.Rules(ctx)
if err != nil {
e.logReadError(ctx, "read experiment rules failed; user-scoped experiments are off", err)
d.failed = true
return d
}
d.rules = rules
return d
}
func (d *decision) enabled(ex codersdk.Experiment) bool {
if result, ok := d.decided[ex]; ok {
return result
}
result := d.decide(ex)
d.decided[ex] = result
return result
}
// decide applies the rule order: failed call, canceled context, no rule or
// inherit, malformed rule, on or off, then condition.
func (d *decision) decide(ex codersdk.Experiment) bool {
if d.failed {
return false
}
// A canceled caller must not get an enabled decision, even from an on
// rule or a static default that needs no further reads.
if d.ctx.Err() != nil {
d.e.logger.Debug(d.ctx, "experiment decision canceled; experiment is off",
slog.F("experiment", ex),
slog.F("category", categoryCancelled))
return false
}
stored, ok := d.rules[ex]
if !ok {
return d.e.static.Enabled(ex)
}
rule, err := decodeRule(stored)
if err != nil {
// Never log err: JSON errors can quote stored content.
d.e.logger.Error(d.ctx, "experiment rule is malformed; experiment is off",
slog.F("experiment", ex),
slog.F("revision", rule.Revision),
slog.F("category", categoryMalformed))
return false
}
switch rule.Mode {
case ModeInherit:
return d.e.static.Enabled(ex)
case ModeOn:
return true
case ModeOff:
return false
case ModeCondition:
return d.evaluate(ex, rule)
default:
// decodeRule rejects unknown modes.
panic("developer error: unhandled experiment rule mode " + string(rule.Mode))
}
}
func (d *decision) evaluate(ex codersdk.Experiment, rule Rule) bool {
user, err := d.loadUser()
if err != nil {
d.e.logReadError(d.ctx, "load experiment subject attributes failed; experiment is off", err,
slog.F("experiment", ex),
slog.F("revision", rule.Revision))
return false
}
prg, err := d.e.programs.get(rule.Condition)
if err == nil {
var result bool
result, err = evalCondition(d.ctx, prg, user)
if err == nil {
return result
}
}
d.logConditionError(ex, rule, err)
return false
}
// logReadError logs a failed store read. A read that failed because the
// context was canceled or expired is expected when a caller goes away, so
// it logs at Debug with category canceled instead of as an error.
func (e *Evaluator) logReadError(ctx context.Context, msg string, err error, fields ...slog.Field) {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
e.logger.Debug(ctx, msg, append(fields, slog.F("category", categoryCancelled), slog.Error(err))...)
return
}
e.logger.Error(ctx, msg, append(fields, slog.F("category", categoryRead), slog.Error(err))...)
}
func (d *decision) loadUser() (User, error) {
if !d.userLoaded {
d.userLoaded = true
d.user, d.userErr = d.e.store.UserAttributes(d.ctx, d.userID)
}
return d.user, d.userErr
}
// logConditionError logs only the category and location. CEL messages can
// quote the condition, so err's text is never logged.
func (d *decision) logConditionError(ex codersdk.Experiment, rule Rule, err error) {
fields := []slog.Field{
slog.F("experiment", ex),
slog.F("revision", rule.Revision),
}
var cerr *conditionError
if !errors.As(err, &cerr) {
// Only environment setup errors reach here; they contain no
// condition text.
d.e.logger.Error(d.ctx, "experiment condition failed; experiment is off", append(fields, slog.Error(err))...)
return
}
fields = append(fields,
slog.F("category", cerr.category),
slog.F("line", cerr.line),
slog.F("column", cerr.column))
if cerr.category == categoryCancelled {
d.e.logger.Debug(d.ctx, "experiment condition canceled; experiment is off", fields...)
return
}
d.e.logger.Error(d.ctx, "experiment condition failed; experiment is off", fields...)
}