Files
dylanhuff-at-coder 1e0a481a60 fix: retry workspace quota commits with jittered backoff (#31207)
Bursts of builds for the same user and org (usually autostarts) can hit
serialization failures in `CommitQuota`. `InTx` only retried those 3
times back-to-back, so once retries ran out the build failed outright.

`CommitQuota` stays `SERIALIZABLE` and now gets up to 10 retries with
full-jitter exponential backoff (50ms base, 2s cap, about 11s of backoff
at most). These builds aren't interactive, so a longer wait is a better
trade than a failed build. The wait respects the request context and
happens between transactions, so nothing holds a connection while it
sleeps. Each attempt also resets its results now, so a rolled-back
attempt can't leak `Ok: true` into the response.

The retry settings are opt-in fields on `database.TxOptions`
(`SerialMaxAttempts` and `BeforeSerialRetry`). Every other `InTx` caller
keeps the default 3 attempts with no delay. Since this touches
`coderd/database/db.go`, it'll need a look from a `coderd/database`
owner.

This replaces #30126 with the simpler approach Spike and I landed on. If
we still see failures after this ships, the next step is making
autostart replica-aware so a single replica handles each user, rather
than adding locking here.

Fixes [CORE-198](https://linear.app/codercom/issue/CORE-198)
2026-10-26 15:38:48 -07:00

201 lines
5.2 KiB
Go

package database_test
import (
"context"
"database/sql"
"testing"
"github.com/DATA-DOG/go-sqlmock"
"github.com/google/uuid"
"github.com/lib/pq"
"github.com/stretchr/testify/require"
"golang.org/x/xerrors"
"github.com/coder/coder/v2/coderd/database"
"github.com/coder/coder/v2/coderd/database/dbtestutil"
"github.com/coder/coder/v2/coderd/database/dbtime"
"github.com/coder/coder/v2/coderd/database/migrations"
)
func TestSerializedRetry(t *testing.T) {
t.Parallel()
if testing.Short() {
t.SkipNow()
}
sqlDB := testSQLDB(t)
db := database.New(sqlDB)
called := 0
txOpts := &database.TxOptions{Isolation: sql.LevelSerializable}
err := db.InTx(func(tx database.Store) error {
// Test nested error
return tx.InTx(func(tx database.Store) error {
// The easiest way to mock a serialization failure is to
// return a serialization failure error.
called++
return &pq.Error{
Code: "40001",
Message: "serialization_failure",
}
}, txOpts)
}, txOpts)
require.Error(t, err, "should fail")
// The double "execute transaction: execute transaction" is from the nested transactions.
// Just want to make sure we don't try 9 times.
require.Equal(t, err.Error(), "transaction failed after 3 attempts: execute transaction: execute transaction: pq: serialization_failure", "error message")
require.Equal(t, called, 3, "should retry 3 times")
}
func TestSerializedRetryOverride(t *testing.T) {
t.Parallel()
if testing.Short() {
t.SkipNow()
}
sqlDB := testSQLDB(t)
db := database.New(sqlDB)
serialErr := &pq.Error{Code: "40001", Message: "serialization_failure"}
t.Run("MaxAttempts", func(t *testing.T) {
t.Parallel()
called := 0
var retries []int
err := db.InTx(func(database.Store) error {
called++
return serialErr
}, &database.TxOptions{
Isolation: sql.LevelSerializable,
SerialMaxAttempts: 5,
BeforeSerialRetry: func(attempt int) error {
retries = append(retries, attempt)
return nil
},
})
require.True(t, database.IsSerializedError(err))
require.Equal(t, 5, called)
require.Equal(t, []int{1, 2, 3, 4}, retries)
})
t.Run("HookStopsRetry", func(t *testing.T) {
t.Parallel()
called := 0
stop := xerrors.New("stop")
err := db.InTx(func(database.Store) error {
called++
return serialErr
}, &database.TxOptions{
Isolation: sql.LevelSerializable,
SerialMaxAttempts: 5,
BeforeSerialRetry: func(attempt int) error {
if attempt == 2 {
return stop
}
return nil
},
})
require.ErrorIs(t, err, stop)
require.ErrorContains(t, err, serialErr.Error())
require.Equal(t, 2, called)
})
t.Run("HookNotCalledForOtherErrors", func(t *testing.T) {
t.Parallel()
called := 0
boom := xerrors.New("boom")
err := db.InTx(func(database.Store) error {
called++
return boom
}, &database.TxOptions{
Isolation: sql.LevelSerializable,
SerialMaxAttempts: 5,
BeforeSerialRetry: func(int) error {
t.Error("hook called for non-serialization error")
return nil
},
})
require.ErrorIs(t, err, boom)
require.Equal(t, 1, called)
})
}
func TestNestedInTx(t *testing.T) {
t.Parallel()
if testing.Short() {
t.SkipNow()
}
uid := uuid.New()
sqlDB := testSQLDB(t)
err := migrations.Up(sqlDB)
require.NoError(t, err, "migrations")
db := database.New(sqlDB)
err = db.InTx(func(outer database.Store) error {
return outer.InTx(func(inner database.Store) error {
//nolint:gocritic
require.Equal(t, outer, inner, "should be same transaction") // intxcheck:ignore // intentional: test asserts nested InTx returns same store
_, err := inner.InsertUser(context.Background(), database.InsertUserParams{
ID: uid,
Email: "coder@coder.com",
Username: "coder",
HashedPassword: []byte{},
CreatedAt: dbtime.Now(),
UpdatedAt: dbtime.Now(),
RBACRoles: []string{},
LoginType: database.LoginTypeGithub,
})
return err
}, nil)
}, nil)
require.NoError(t, err, "outer tx: %w", err)
user, err := db.GetUserByID(context.Background(), uid)
require.NoError(t, err, "user exists")
require.Equal(t, uid, user.ID, "user id expected")
}
func TestInTx_CapturesRollbackError(t *testing.T) {
t.Parallel()
sqlDB, mock, err := sqlmock.New()
require.NoError(t, err)
t.Cleanup(func() { _ = sqlDB.Close() })
db := database.New(sqlDB)
callbackErr := xerrors.New("callback failed")
rollbackErr := xerrors.New("rollback failed")
mock.ExpectBegin()
mock.ExpectRollback().WillReturnError(rollbackErr)
err = db.InTx(func(_ database.Store) error {
return callbackErr
}, nil)
require.EqualError(t, err, "defer (rollback failed): execute transaction: callback failed")
require.ErrorIs(t, err, callbackErr,
"returned error should still match the callback error when rollback fails")
require.NotErrorIs(t, err, rollbackErr,
"rollback failure should be reported in the message, not wrapped in the error chain")
require.NoError(t, mock.ExpectationsWereMet())
}
func testSQLDB(t testing.TB) *sql.DB {
t.Helper()
connection, err := dbtestutil.Open(t)
require.NoError(t, err)
db, err := sql.Open("postgres", connection)
require.NoError(t, err)
t.Cleanup(func() { _ = db.Close() })
return db
}