Closes CODAGT-878 When proxy.golang.org is having a bad day, a module download can stall for minutes before the proxy resets the stream with `INTERNAL_ERROR`. `go` has no read timeout of its own, so `retry.sh` only gets to retry once the proxy gives up. In [this `test-e2e` run](https://github.com/coder/coder/actions/runs/38567440844/job/115863803256) the second attempt sat there for 10 minutes, and the job hit its 20 minute timeout during Playwright, even though the third attempt passed straight away. To fix, `retry.sh` now takes a `--timeout`, and kills any attempt that runs longer than that. Killing `go mod download` is safe, and the next attempt picks up where the last one left off, as finished zips are already in the module cache. The timer is in bash as we can't rely on GNU `timeout` on the macOS runners, and a step `timeout-minutes` would kill the whole retry loop (it isn't supported in composite actions anyway). I've gone with 120s per attempt. Across the last 32 runs on main, a healthy download took at most 41s on any platform, with macOS the slowest at 26 to 41s. During the incident on 2026-10-14, attempts that failed on their own took between 66s and 10 minutes, and three passed slowly at 113s, 133s and 151s. Those last two would now get killed, but that only costs a retry, which just needs whatever's left. Three attempts at 120s plus backoff comes to about 6 minutes, which still fits in `test-e2e`, whose slowest green run takes about 11 of its 20 minutes. Separately, `make fmt` runs a bare `go mod tidy`, which needs modules that `go mod download` doesn't fetch (test dependencies of dependencies). So it was downloading 166 modules without any retries, and [failing here](https://github.com/coder/coder/actions/runs/38624596116/job/116064172735). The `fmt` job now runs `go mod tidy` through `retry.sh` instead of `go mod download`.
77 lines
1.4 KiB
Bash
Executable File
77 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Retry a command with exponential backoff.
|
|
#
|
|
# Example:
|
|
# retry.sh --max-attempts 3 -- go install gotest.tools/gotestsum@latest
|
|
#
|
|
# This will retry the command up to 3 times with exponential backoff
|
|
# (2s, 4s, 8s delays between attempts).
|
|
|
|
set -euo pipefail
|
|
|
|
# shellcheck source=scripts/lib.sh
|
|
source "$(dirname "${BASH_SOURCE[0]}")/../../scripts/lib.sh"
|
|
|
|
max_attempts=3
|
|
timeout=""
|
|
|
|
args="$(getopt -o "" -l max-attempts:,timeout: -- "$@")"
|
|
eval set -- "$args"
|
|
while true; do
|
|
case "$1" in
|
|
--max-attempts)
|
|
max_attempts="$2"
|
|
shift 2
|
|
;;
|
|
--timeout)
|
|
timeout="$2"
|
|
shift 2
|
|
;;
|
|
--)
|
|
shift
|
|
break
|
|
;;
|
|
*)
|
|
error "Unrecognized option: $1"
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ $# -lt 1 ]]; then
|
|
error "Usage: retry.sh [--max-attempts N] [--timeout SECONDS] -- <command...>"
|
|
fi
|
|
|
|
# Avoid a GNU timeout dependency for macOS runners.
|
|
run_attempt() {
|
|
if [[ -z "$timeout" ]]; then
|
|
"$@"
|
|
return
|
|
fi
|
|
|
|
set -m
|
|
"$@" &
|
|
local pid=$!
|
|
set +m
|
|
local deadline=$((SECONDS + timeout))
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
if ((SECONDS >= deadline)); then
|
|
log "Attempt $attempt/$max_attempts timed out after ${timeout}s"
|
|
kill -- -"$pid"
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
wait "$pid"
|
|
}
|
|
|
|
attempt=1
|
|
until run_attempt "$@"; do
|
|
if ((attempt >= max_attempts)); then
|
|
error "Command failed after $max_attempts attempts: $*"
|
|
fi
|
|
delay=$((2 ** attempt))
|
|
log "Attempt $attempt/$max_attempts failed, retrying in ${delay}s..."
|
|
sleep "$delay"
|
|
((attempt++))
|
|
done
|