Bursts of builds for the same user and org (usually autostarts) can hit serialization failures in `CommitQuota`. `InTx` only retried those 3 times back-to-back, so once retries ran out the build failed outright. `CommitQuota` stays `SERIALIZABLE` and now gets up to 10 retries with full-jitter exponential backoff (50ms base, 2s cap, about 11s of backoff at most). These builds aren't interactive, so a longer wait is a better trade than a failed build. The wait respects the request context and happens between transactions, so nothing holds a connection while it sleeps. Each attempt also resets its results now, so a rolled-back attempt can't leak `Ok: true` into the response. The retry settings are opt-in fields on `database.TxOptions` (`SerialMaxAttempts` and `BeforeSerialRetry`). Every other `InTx` caller keeps the default 3 attempts with no delay. Since this touches `coderd/database/db.go`, it'll need a look from a `coderd/database` owner. This replaces #30126 with the simpler approach Spike and I landed on. If we still see failures after this ships, the next step is making autostart replica-aware so a single replica handles each user, rather than adding locking here. Fixes [CORE-198](https://linear.app/codercom/issue/CORE-198)
201 lines
5.2 KiB
Go
201 lines
5.2 KiB
Go
package database_test
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"testing"
|
|
|
|
"github.com/DATA-DOG/go-sqlmock"
|
|
"github.com/google/uuid"
|
|
"github.com/lib/pq"
|
|
"github.com/stretchr/testify/require"
|
|
"golang.org/x/xerrors"
|
|
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/database/dbtestutil"
|
|
"github.com/coder/coder/v2/coderd/database/dbtime"
|
|
"github.com/coder/coder/v2/coderd/database/migrations"
|
|
)
|
|
|
|
func TestSerializedRetry(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.SkipNow()
|
|
}
|
|
|
|
sqlDB := testSQLDB(t)
|
|
db := database.New(sqlDB)
|
|
|
|
called := 0
|
|
txOpts := &database.TxOptions{Isolation: sql.LevelSerializable}
|
|
err := db.InTx(func(tx database.Store) error {
|
|
// Test nested error
|
|
return tx.InTx(func(tx database.Store) error {
|
|
// The easiest way to mock a serialization failure is to
|
|
// return a serialization failure error.
|
|
called++
|
|
return &pq.Error{
|
|
Code: "40001",
|
|
Message: "serialization_failure",
|
|
}
|
|
}, txOpts)
|
|
}, txOpts)
|
|
require.Error(t, err, "should fail")
|
|
// The double "execute transaction: execute transaction" is from the nested transactions.
|
|
// Just want to make sure we don't try 9 times.
|
|
require.Equal(t, err.Error(), "transaction failed after 3 attempts: execute transaction: execute transaction: pq: serialization_failure", "error message")
|
|
require.Equal(t, called, 3, "should retry 3 times")
|
|
}
|
|
|
|
func TestSerializedRetryOverride(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.SkipNow()
|
|
}
|
|
|
|
sqlDB := testSQLDB(t)
|
|
db := database.New(sqlDB)
|
|
serialErr := &pq.Error{Code: "40001", Message: "serialization_failure"}
|
|
|
|
t.Run("MaxAttempts", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
called := 0
|
|
var retries []int
|
|
err := db.InTx(func(database.Store) error {
|
|
called++
|
|
return serialErr
|
|
}, &database.TxOptions{
|
|
Isolation: sql.LevelSerializable,
|
|
SerialMaxAttempts: 5,
|
|
BeforeSerialRetry: func(attempt int) error {
|
|
retries = append(retries, attempt)
|
|
return nil
|
|
},
|
|
})
|
|
require.True(t, database.IsSerializedError(err))
|
|
require.Equal(t, 5, called)
|
|
require.Equal(t, []int{1, 2, 3, 4}, retries)
|
|
})
|
|
|
|
t.Run("HookStopsRetry", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
called := 0
|
|
stop := xerrors.New("stop")
|
|
err := db.InTx(func(database.Store) error {
|
|
called++
|
|
return serialErr
|
|
}, &database.TxOptions{
|
|
Isolation: sql.LevelSerializable,
|
|
SerialMaxAttempts: 5,
|
|
BeforeSerialRetry: func(attempt int) error {
|
|
if attempt == 2 {
|
|
return stop
|
|
}
|
|
return nil
|
|
},
|
|
})
|
|
require.ErrorIs(t, err, stop)
|
|
require.ErrorContains(t, err, serialErr.Error())
|
|
require.Equal(t, 2, called)
|
|
})
|
|
|
|
t.Run("HookNotCalledForOtherErrors", func(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
called := 0
|
|
boom := xerrors.New("boom")
|
|
err := db.InTx(func(database.Store) error {
|
|
called++
|
|
return boom
|
|
}, &database.TxOptions{
|
|
Isolation: sql.LevelSerializable,
|
|
SerialMaxAttempts: 5,
|
|
BeforeSerialRetry: func(int) error {
|
|
t.Error("hook called for non-serialization error")
|
|
return nil
|
|
},
|
|
})
|
|
require.ErrorIs(t, err, boom)
|
|
require.Equal(t, 1, called)
|
|
})
|
|
}
|
|
|
|
func TestNestedInTx(t *testing.T) {
|
|
t.Parallel()
|
|
if testing.Short() {
|
|
t.SkipNow()
|
|
}
|
|
|
|
uid := uuid.New()
|
|
sqlDB := testSQLDB(t)
|
|
err := migrations.Up(sqlDB)
|
|
require.NoError(t, err, "migrations")
|
|
|
|
db := database.New(sqlDB)
|
|
err = db.InTx(func(outer database.Store) error {
|
|
return outer.InTx(func(inner database.Store) error {
|
|
//nolint:gocritic
|
|
require.Equal(t, outer, inner, "should be same transaction") // intxcheck:ignore // intentional: test asserts nested InTx returns same store
|
|
|
|
_, err := inner.InsertUser(context.Background(), database.InsertUserParams{
|
|
ID: uid,
|
|
Email: "coder@coder.com",
|
|
Username: "coder",
|
|
HashedPassword: []byte{},
|
|
CreatedAt: dbtime.Now(),
|
|
UpdatedAt: dbtime.Now(),
|
|
RBACRoles: []string{},
|
|
LoginType: database.LoginTypeGithub,
|
|
})
|
|
return err
|
|
}, nil)
|
|
}, nil)
|
|
require.NoError(t, err, "outer tx: %w", err)
|
|
|
|
user, err := db.GetUserByID(context.Background(), uid)
|
|
require.NoError(t, err, "user exists")
|
|
require.Equal(t, uid, user.ID, "user id expected")
|
|
}
|
|
|
|
func TestInTx_CapturesRollbackError(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
sqlDB, mock, err := sqlmock.New()
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = sqlDB.Close() })
|
|
|
|
db := database.New(sqlDB)
|
|
|
|
callbackErr := xerrors.New("callback failed")
|
|
rollbackErr := xerrors.New("rollback failed")
|
|
|
|
mock.ExpectBegin()
|
|
mock.ExpectRollback().WillReturnError(rollbackErr)
|
|
|
|
err = db.InTx(func(_ database.Store) error {
|
|
return callbackErr
|
|
}, nil)
|
|
require.EqualError(t, err, "defer (rollback failed): execute transaction: callback failed")
|
|
require.ErrorIs(t, err, callbackErr,
|
|
"returned error should still match the callback error when rollback fails")
|
|
require.NotErrorIs(t, err, rollbackErr,
|
|
"rollback failure should be reported in the message, not wrapped in the error chain")
|
|
|
|
require.NoError(t, mock.ExpectationsWereMet())
|
|
}
|
|
|
|
func testSQLDB(t testing.TB) *sql.DB {
|
|
t.Helper()
|
|
|
|
connection, err := dbtestutil.Open(t)
|
|
require.NoError(t, err)
|
|
|
|
db, err := sql.Open("postgres", connection)
|
|
require.NoError(t, err)
|
|
t.Cleanup(func() { _ = db.Close() })
|
|
|
|
return db
|
|
}
|