Files
coder/coderd/notifications/reports/generator.go
T
Susana Ferreira 8c5921db5e fix: count unpriced AI model usage by when it was recorded (#31751)
## Description

Updates the unpriced AI models query behind the weekly owner
notification to count token usage by when it was recorded
(`token_usages.created_at`) in a `[period_start, period_end)` window,
consistent with the other AI spend queries. It also returns one row per
provider name and model instead of per provider type, so the unpriced
models endpoint later in this stack can reuse it. The notification
payload keeps the same format.

## Changes

- Rename `GetUnpricedAIModelsSince` to `GetUnpricedAIModels`. It used
`interceptions.started_at` with no end.
- Treat usage as unpriced only when no price was found (`cost_micros IS
NULL AND priced_model IS NULL`), so usage whose price was found but
whose cost could not be computed is not reported.
- Return one row per live provider name and model, with the provider's
display name, icon and type.
- The notification passes the last 7 days and merges rows by provider
type, since site-wide prices are keyed by type, so its payload keeps the
same shape.
- Table-driven query tests and a notification test for providers sharing
a type.

Related to AIGOV-730

> [!NOTE]
> Generated by Coder Agents on behalf of @ssncferreira.
2026-11-13 13:21:32 +00:00

477 lines
16 KiB
Go

package reports
import (
"cmp"
"context"
"database/sql"
"io"
"slices"
"sort"
"strings"
"time"
"github.com/google/uuid"
"golang.org/x/xerrors"
"cdr.dev/slog/v3"
"github.com/coder/coder/v2/coderd/aibridge/prices/providers"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbauthz"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/notifications"
"github.com/coder/coder/v2/coderd/util/slice"
"github.com/coder/coder/v2/codersdk"
"github.com/coder/quartz"
)
const (
delay = 15 * time.Minute
)
// runReport executes one report in its own transaction, guarded by its own
// advisory lock so that only one replica generates it. A replica that cannot
// take the lock skips this tick and tries again on the next one; how often a
// report is actually sent is enforced by the report itself, against the
// timestamp it persists.
func runReport(ctx context.Context, logger slog.Logger, db database.Store, lockID int64, name string, report func(tx database.Store) error) {
err := db.InTx(func(tx database.Store) error {
ok, err := tx.TryAcquireLock(ctx, lockID)
if err != nil {
return xerrors.Errorf("failed to acquire report lock: %w", err)
}
if !ok {
logger.Debug(ctx, "unable to acquire lock for generating periodic report, skipping", slog.F("report", name))
return nil
}
return report(tx)
}, nil)
if err != nil {
logger.Error(ctx, "failed to generate report", slog.F("report", name), slog.Error(err))
}
}
// NewReportGenerator periodically generates failed workspace build and
// unpriced AI model reports.
func NewReportGenerator(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) io.Closer {
closed := make(chan struct{})
ctx, cancelFunc := context.WithCancel(ctx)
//nolint:gocritic // The system generates periodic reports without direct user input.
ctx = dbauthz.AsSystemRestricted(ctx)
// Start the ticker with the initial delay.
ticker := clk.NewTicker(delay)
ticker.Stop()
doTick := func(_ time.Time) {
defer ticker.Reset(delay)
// Reports are independent, so each runs in its own transaction under
// its own advisory lock. Sharing either would couple them.
runReport(ctx, logger, db, database.LockIDNotificationsReportGenerator, "failed workspace builds",
func(tx database.Store) error {
return reportFailedWorkspaceBuilds(ctx, logger, tx, enqueuer, clk)
})
runReport(ctx, logger, db, database.LockIDNotifyUnpricedAIModels, "unpriced AI models",
func(tx database.Store) error {
return reportUnpricedAIModels(ctx, logger, tx, enqueuer, clk)
})
}
go func() {
defer close(closed)
defer ticker.Stop()
// Force an initial tick.
doTick(dbtime.Time(clk.Now()).UTC())
for {
select {
case <-ctx.Done():
logger.Debug(ctx, "closing report generator")
return
case tick := <-ticker.C:
ticker.Stop()
doTick(dbtime.Time(tick).UTC())
}
}
}()
return &reportGenerator{
cancel: cancelFunc,
closed: closed,
}
}
type reportGenerator struct {
cancel context.CancelFunc
closed chan struct{}
}
func (i *reportGenerator) Close() error {
i.cancel()
<-i.closed
return nil
}
const (
failedWorkspaceBuildsReportFrequency = 7 * 24 * time.Hour
failedWorkspaceBuildsReportFrequencyLabel = "week"
)
type adminReport struct {
stats database.GetWorkspaceBuildStatsByTemplatesRow
failedBuilds []database.GetFailedWorkspaceBuildsByTemplateIDRow
}
func reportFailedWorkspaceBuilds(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) error {
now := clk.Now()
since := now.Add(-failedWorkspaceBuildsReportFrequency)
// Firstly, check if this is the first run of the job ever
reportLog, err := db.GetNotificationReportGeneratorLogByTemplate(ctx, notifications.TemplateWorkspaceBuildsFailedReport)
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
return xerrors.Errorf("unable to read report generator log: %w", err)
}
if xerrors.Is(err, sql.ErrNoRows) {
// First run? Check-in the job, and get back after one week.
logger.Info(ctx, "report generator is executing the job for the first time", slog.F("notification_template_id", notifications.TemplateWorkspaceBuildsFailedReport))
err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
LastGeneratedAt: dbtime.Time(now).UTC(),
})
if err != nil {
return xerrors.Errorf("unable to update report generator logs (first time execution): %w", err)
}
return nil
}
// Secondly, check if the job has not been running recently
if !reportLog.LastGeneratedAt.IsZero() && reportLog.LastGeneratedAt.Add(failedWorkspaceBuildsReportFrequency).After(now) {
return nil // reports sent recently, no need to send them now
}
// Thirdly, fetch workspace build stats by templates
templateStatsRows, err := db.GetWorkspaceBuildStatsByTemplates(ctx, dbtime.Time(since).UTC())
if err != nil {
return xerrors.Errorf("unable to fetch failed workspace builds: %w", err)
}
reports := make(map[uuid.UUID][]adminReport)
for _, stats := range templateStatsRows {
select {
case <-ctx.Done():
logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
break
default:
}
if stats.FailedBuilds == 0 {
logger.Info(ctx, "no failed workspace builds found for template", slog.F("template_id", stats.TemplateID), slog.Error(err))
continue
}
// Fetch template admins with org access to the templates
templateAdmins, err := findTemplateAdmins(ctx, db, stats)
if err != nil {
logger.Error(ctx, "unable to find template admins for template", slog.F("template_id", stats.TemplateID), slog.Error(err))
continue
}
// Fetch failed builds by the template
failedBuilds, err := db.GetFailedWorkspaceBuildsByTemplateID(ctx, database.GetFailedWorkspaceBuildsByTemplateIDParams{
TemplateID: stats.TemplateID,
Since: dbtime.Time(since).UTC(),
})
if err != nil {
logger.Error(ctx, "unable to fetch failed workspace builds", slog.F("template_id", stats.TemplateID), slog.Error(err))
continue
}
for _, templateAdmin := range templateAdmins {
adminReports := reports[templateAdmin.ID]
adminReports = append(adminReports, adminReport{
failedBuilds: failedBuilds,
stats: stats,
})
reports[templateAdmin.ID] = adminReports
}
}
for templateAdmin, reports := range reports {
select {
case <-ctx.Done():
logger.Debug(ctx, "context is canceled, quitting", slog.Error(ctx.Err()))
break
default:
}
reportData := buildDataForReportFailedWorkspaceBuilds(reports)
targets := []uuid.UUID{}
for _, report := range reports {
targets = append(targets, report.stats.TemplateID, report.stats.TemplateOrganizationID)
}
if _, err := enqueuer.EnqueueWithData(ctx, templateAdmin, notifications.TemplateWorkspaceBuildsFailedReport,
map[string]string{},
reportData,
"report_generator",
slice.Unique(targets)...,
); err != nil {
logger.Warn(ctx, "failed to send a report with failed workspace builds", slog.Error(err))
}
}
if xerrors.Is(ctx.Err(), context.Canceled) {
logger.Error(ctx, "report generator job is canceled")
return ctx.Err()
}
// Lastly, update the timestamp in the generator log.
err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
NotificationTemplateID: notifications.TemplateWorkspaceBuildsFailedReport,
LastGeneratedAt: dbtime.Time(now).UTC(),
})
if err != nil {
return xerrors.Errorf("unable to update report generator logs: %w", err)
}
return nil
}
const workspaceBuildsLimitPerTemplateVersion = 10
func buildDataForReportFailedWorkspaceBuilds(reports []adminReport) map[string]any {
templates := []map[string]any{}
for _, report := range reports {
// Build notification model for template versions and failed workspace builds.
//
// Failed builds are sorted by template version ascending, workspace build number descending.
// Review builds, group them by template versions, and assign to builds to template versions.
// The map requires `[]map[string]any{}` to be compatible with data passed to `NotificationEnqueuer`.
templateVersions := []map[string]any{}
for _, failedBuild := range report.failedBuilds {
c := len(templateVersions)
if c == 0 || templateVersions[c-1]["template_version_name"] != failedBuild.TemplateVersionName {
templateVersions = append(templateVersions, map[string]any{
"template_version_name": failedBuild.TemplateVersionName,
"failed_count": 1,
"failed_builds": []map[string]any{
{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"workspace_id": failedBuild.WorkspaceID,
"build_number": failedBuild.WorkspaceBuildNumber,
},
},
})
continue
}
tv := templateVersions[c-1]
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
tv["failed_count"] = tv["failed_count"].(int) + 1
//nolint:errorlint,forcetypeassert // only this function prepares the notification model
builds := tv["failed_builds"].([]map[string]any)
if len(builds) < workspaceBuildsLimitPerTemplateVersion {
// return N last builds to prevent long email reports
builds = append(builds, map[string]any{
"workspace_owner_username": failedBuild.WorkspaceOwnerUsername,
"workspace_name": failedBuild.WorkspaceName,
"workspace_id": failedBuild.WorkspaceID,
"build_number": failedBuild.WorkspaceBuildNumber,
})
tv["failed_builds"] = builds
}
templateVersions[c-1] = tv
}
templateDisplayName := report.stats.TemplateDisplayName
if templateDisplayName == "" {
templateDisplayName = report.stats.TemplateName
}
templates = append(templates, map[string]any{
"failed_builds": report.stats.FailedBuilds,
"total_builds": report.stats.TotalBuilds,
"versions": templateVersions,
"name": report.stats.TemplateName,
"display_name": templateDisplayName,
})
}
return map[string]any{
"report_frequency": failedWorkspaceBuildsReportFrequencyLabel,
"templates": templates,
}
}
func findTemplateAdmins(ctx context.Context, db database.Store, stats database.GetWorkspaceBuildStatsByTemplatesRow) ([]database.GetUsersRow, error) {
users, err := db.GetUsers(ctx, database.GetUsersParams{
RbacRole: []string{codersdk.RoleTemplateAdmin},
})
if err != nil {
return nil, xerrors.Errorf("unable to fetch template admins: %w", err)
}
var templateAdmins []database.GetUsersRow
if len(users) == 0 {
return templateAdmins, nil
}
usersByIDs := map[uuid.UUID]database.GetUsersRow{}
var userIDs []uuid.UUID
for _, user := range users {
usersByIDs[user.ID] = user
userIDs = append(userIDs, user.ID)
}
orgIDsByMemberIDs, err := db.GetOrganizationIDsByMemberIDs(ctx, userIDs)
if err != nil {
return nil, xerrors.Errorf("unable to fetch organization IDs by member IDs: %w", err)
}
for _, entry := range orgIDsByMemberIDs {
if slices.Contains(entry.OrganizationIDs, stats.TemplateOrganizationID) {
templateAdmins = append(templateAdmins, usersByIDs[entry.UserID])
}
}
sort.Slice(templateAdmins, func(i, j int) bool {
return templateAdmins[i].Username < templateAdmins[j].Username
})
return templateAdmins, nil
}
const (
unpricedAIModelsReportFrequency = 7 * 24 * time.Hour
unpricedAIModelsReportFrequencyLabel = "week"
// unpricedAIModelsLimit caps how many models a single report lists.
// A deployment can accumulate more unpriced models than we want to display
// in a single notification, so the remaining models are reported as a count.
unpricedAIModelsLimit = 100
)
// reportUnpricedAIModels notifies owners about models used without a price
// in the preceding week. Unpriced usage is recorded but contributes nothing to
// spend, so it is neither reported nor enforced against a budget.
//
// The set of unpriced models is derived at report time from interceptions and
// the price table, so setting a price removes the model from the next report.
func reportUnpricedAIModels(ctx context.Context, logger slog.Logger, db database.Store, enqueuer notifications.Enqueuer, clk quartz.Clock) error {
now := clk.Now()
since := now.Add(-unpricedAIModelsReportFrequency)
reportLog, err := db.GetNotificationReportGeneratorLogByTemplate(ctx, notifications.TemplateAIModelsUnpricedReport)
if err != nil && !xerrors.Is(err, sql.ErrNoRows) {
return xerrors.Errorf("unable to read report generator log: %w", err)
}
// Check if the job has not been running recently. The ticker alone cannot
// enforce the frequency: it restarts with the process and each replica
// runs on its own phase.
if !reportLog.LastGeneratedAt.IsZero() && reportLog.LastGeneratedAt.Add(unpricedAIModelsReportFrequency).After(now) {
return nil // reports sent recently, no need to send them now
}
// Fetch the models used without a price.
unpricedModels, err := db.GetUnpricedAIModels(ctx, database.GetUnpricedAIModelsParams{
PeriodStart: dbtime.Time(since).UTC(),
PeriodEnd: dbtime.Time(now).UTC(),
PriceableProviders: providers.SupportedStrings(),
})
if err != nil {
return xerrors.Errorf("unable to fetch unpriced AI models: %w", err)
}
if len(unpricedModels) > 0 {
owners, err := db.GetUsers(ctx, database.GetUsersParams{
RbacRole: []string{codersdk.RoleOwner},
})
if err != nil {
return xerrors.Errorf("unable to fetch owners: %w", err)
}
reportData := buildDataForReportUnpricedAIModels(unpricedAIModelsByProviderType(unpricedModels))
for _, owner := range owners {
if _, err := enqueuer.EnqueueWithData(ctx, owner.ID, notifications.TemplateAIModelsUnpricedReport,
map[string]string{},
reportData,
"report_generator",
); err != nil {
logger.Warn(ctx, "failed to send a report with unpriced AI models", slog.F("user_id", owner.ID), slog.Error(err))
}
}
}
// Update the timestamp in the generator log. This happens even
// when nothing was reported, so the next report covers one week rather
// than every week since usage was last seen.
err = db.UpsertNotificationReportGeneratorLog(ctx, database.UpsertNotificationReportGeneratorLogParams{
NotificationTemplateID: notifications.TemplateAIModelsUnpricedReport,
LastGeneratedAt: dbtime.Time(now).UTC(),
})
if err != nil {
return xerrors.Errorf("unable to update report generator logs: %w", err)
}
return nil
}
// unpricedAIModel is a model used without a price, across every provider of
// its type.
type unpricedAIModel struct {
providerType string
model string
tokenCount int64
}
// unpricedAIModelsByProviderType merges the models of providers that share a
// type, since site-wide prices are keyed by provider type, and sorts them by
// token usage, most first.
func unpricedAIModelsByProviderType(rows []database.GetUnpricedAIModelsRow) []unpricedAIModel {
type key struct{ providerType, model string }
modelIndex := make(map[key]int, len(rows))
models := make([]unpricedAIModel, 0, len(rows))
for _, row := range rows {
k := key{providerType: row.ProviderType, model: row.Model}
if i, ok := modelIndex[k]; ok {
models[i].tokenCount += row.TokenCount
continue
}
modelIndex[k] = len(models)
models = append(models, unpricedAIModel{providerType: row.ProviderType, model: row.Model, tokenCount: row.TokenCount})
}
slices.SortFunc(models, func(a, b unpricedAIModel) int {
return cmp.Or(
cmp.Compare(b.tokenCount, a.tokenCount),
cmp.Compare(a.providerType, b.providerType),
cmp.Compare(strings.ToLower(a.model), strings.ToLower(b.model)),
cmp.Compare(a.model, b.model),
)
})
return models
}
// buildDataForReportUnpricedAIModels renders the models most used first, so
// the models dropped by the limit are the ones with the least unreported
// usage.
func buildDataForReportUnpricedAIModels(unpricedModels []unpricedAIModel) map[string]any {
reportedCount := min(len(unpricedModels), unpricedAIModelsLimit)
reportedModels := make([]map[string]any, 0, reportedCount)
for _, m := range unpricedModels[:reportedCount] {
reportedModels = append(reportedModels, map[string]any{
"provider": m.providerType,
"model": m.model,
})
}
return map[string]any{
"report_frequency": unpricedAIModelsReportFrequencyLabel,
"models": reportedModels,
"total_count": len(unpricedModels),
"truncated": len(unpricedModels) > reportedCount,
}
}