Files
coder/.github/workflows/eval-coder-agents.yaml
T
Kyle CarberryandClaude Opus 5.5 555069ef80 ci: run eval-coder-agents in its own eval environment (#32363)
The `run` job in `eval-coder-agents.yaml` checks out the PR's head and
runs its code, including its own `scripts/harbor-coder`. It currently
uses the `main` environment, which holds main-only secrets for other
workflows. This moves it to a dedicated `eval` environment, so only eval
secrets are ever in scope.

- **Depends on** coder/github-org-terraform#124, which creates `eval`
(only `main` may deploy, no reviewers, no admin bypass). Merge that
first; otherwise GitHub creates `eval` unprotected on the first run.
- **`EVAL_CODER_SESSION_TOKEN`:** the job references it, but it isn't
set in `main` today. When it's added, put it in `eval`.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-18 01:51:27 +00:00

252 lines
11 KiB
YAML

# A/B Coder Agents on a PR branch vs main: each arm builds its own coderd,
# runs Terminal-Bench 2.0 through Harbor with the Coder environment provider
# (scripts/harbor-coder), and the comparison is posted on the PR.
#
# Trigger by commenting `/coder-agents-eval` on a PR (push access required,
# fork PRs refused since the candidate arm runs the PR's code with the gateway
# token in scope), or via workflow_dispatch to compare the current ref to main.
# Trials spend real model tokens, so this never runs automatically.
name: eval/coder-agents
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
pr:
description: PR number to post the comparison on (optional)
required: false
type: string
permissions: {}
# Non-eval events (every other PR comment) get a group of their own so they
# cannot replace a queued /coder-agents-eval run, which would cancel it.
concurrency:
group: >-
${{ (github.event_name == 'workflow_dispatch' || startsWith(github.event.comment.body, '/coder-agents-eval'))
&& format('eval-coder-agents-{0}', github.event.issue.number || github.ref)
|| format('eval-coder-agents-other-{0}', github.run_id) }}
cancel-in-progress: false
env:
HARBOR_DATASET: terminal-bench@2.0
# qemu-* and build-cython-ext are broken upstream (bit-rotted apt sources
# and a git clone in the verifier). The rest are ML training, emulation,
# and long compiles that each take 20 to 60 minutes and are not decided by
# the system prompt; dropping them cuts an arm from ~3h to ~80 minutes.
# crack-7z-hash, break-filter-js-from-html, and vulnerable-secret are
# always refused by Anthropic's content policy (cyber), so they score 0 in
# every arm and cannot move the comparison.
HARBOR_EXCLUDE: >-
-x qemu-* -x build-cython-ext
-x train-fasttext -x compile-compcert -x install-windows-3.11
-x winning-avg-corewars -x extract-moves-from-video -x mcmc-sampling-stan
-x regex-chess -x mteb-leaderboard -x caffe-cifar-10
-x crack-7z-hash -x break-filter-js-from-html -x vulnerable-secret
# Sonnet sits mid-range on Terminal-Bench, leaving headroom in both
# directions for a prompt change to move; Opus is near the ceiling.
HARBOR_CODER_MODEL: claude-sonnet-5-5
HARBOR_ATTEMPTS: "2"
# chatd admits at most 5 concurrent root chats without a license
# (coderd/x/chatd/agentadmission.go). Above that, trials queue on Harbor's
# timeout and score 0 depending on start order.
HARBOR_CONCURRENCY: "5"
jobs:
resolve:
if: >-
github.event_name == 'workflow_dispatch' ||
(github.event.issue.pull_request && startsWith(github.event.comment.body, '/coder-agents-eval'))
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
outputs:
candidate_ref: ${{ steps.pr.outputs.sha || github.sha }}
candidate_label: ${{ steps.pr.outputs.ref || github.ref_name }}
baseline_ref: ${{ steps.base.outputs.sha }}
pr_number: ${{ github.event.issue.number || inputs.pr }}
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- name: Resolve PR
id: pr
if: github.event_name == 'issue_comment'
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR: ${{ github.event.issue.number }}
COMMENTER: ${{ github.event.comment.user.login }}
COMMENT_ID: ${{ github.event.comment.id }}
run: |
set -euo pipefail
if [[ "$(gh api "/repos/${REPO}/collaborators/${COMMENTER}/permission" --jq .user.permissions.push)" != "true" ]]; then
echo "::error::${COMMENTER} does not have push access"; exit 1
fi
if [[ "$(gh pr view "${PR}" -R "${REPO}" --json isCrossRepository --jq .isCrossRepository)" == "true" ]]; then
echo "::error::refusing to run on a fork PR"; exit 1
fi
echo "sha=$(gh pr view "${PR}" -R "${REPO}" --json headRefOid --jq .headRefOid)" >> "${GITHUB_OUTPUT}"
echo "ref=$(gh pr view "${PR}" -R "${REPO}" --json headRefName --jq .headRefName)" >> "${GITHUB_OUTPUT}"
gh api -X POST "/repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -f content=eyes >/dev/null ||
echo "::warning::could not react to comment ${COMMENT_ID}"
# Compare against the commit the candidate branched from, so commits
# that landed on main since then are not attributed to the candidate.
- name: Resolve baseline
id: base
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
CANDIDATE: ${{ steps.pr.outputs.sha || github.sha }}
run: |
set -euo pipefail
sha="$(gh api "/repos/${REPO}/compare/main...${CANDIDATE}" --jq .merge_base_commit.sha)"
echo "baseline: ${sha}, candidate: ${CANDIDATE}"
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
run:
name: run (${{ matrix.arm }})
needs: resolve
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
# The candidate arm runs the PR's code, so only the eval secrets may be in scope; never use main.
environment: eval
timeout-minutes: 360
permissions:
contents: read
strategy:
fail-fast: false
matrix:
arm: [baseline, candidate]
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- name: Checkout arm ref
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ matrix.arm == 'baseline' && needs.resolve.outputs.baseline_ref || needs.resolve.outputs.candidate_ref }}
fetch-depth: 1
persist-credentials: false
# The harness always comes from the candidate so both arms run the same code.
- name: Checkout harness from candidate ref
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.candidate_ref }}
path: .eval-src
sparse-checkout: scripts/harbor-coder
persist-credentials: false
- run: rm -rf scripts/harbor-coder && cp -r .eval-src/scripts/harbor-coder scripts/harbor-coder
- uses: ./.github/actions/setup-go-paths
- uses: ./.github/actions/setup-mise
with:
install-args: "go"
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
- name: Install Harbor from the hash-pinned lock
run: |
./scripts/harbor-coder/install.sh "${RUNNER_TEMP}/harbor-venv"
echo "${RUNNER_TEMP}/harbor-venv/bin" >> "${GITHUB_PATH}"
- name: Build and serve coderd
env:
# A Coder API key for dogfood; the eval's coderd uses dogfood's AI
# gateway as its model upstream, so no provider key is involved.
HARBOR_CODER_UPSTREAM_TOKEN: ${{ secrets.EVAL_CODER_SESSION_TOKEN }}
run: ./scripts/harbor-coder/serve-coderd.sh "${RUNNER_TEMP}/coder.env"
- name: Run Harbor
env:
ARM: ${{ matrix.arm }}
run: |
set -euo pipefail
set -a; . "${RUNNER_TEMP}/coder.env"; set +a
export PATH="${HARBOR_CODER_BIN%/*}:${PATH}"
# shellcheck disable=SC2086
harbor run -d "${HARBOR_DATASET}" ${HARBOR_EXCLUDE} \
--agent harbor_coder.agent:CoderAgentsAgent \
--env harbor_coder.environment:CoderEnvironment \
-k "${HARBOR_ATTEMPTS}" -n "${HARBOR_CONCURRENCY}" -o jobs --job-name "${ARM}"
# Harbor exits 0 even when every trial raised.
if [[ "$(find "jobs/${ARM}" -name reward.txt -o -name reward.json | wc -l)" -eq 0 ]]; then
echo "::error::no trial produced a reward"; exit 1
fi
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: harbor-${{ matrix.arm }}
path: jobs/
# Only the log: the rest of .harbor-coder holds the owner password and
# session token. A separate artifact keeps the jobs/ layout compare expects.
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
if: always()
with:
name: coderd-log-${{ matrix.arm }}
path: .harbor-coder/coderd.log
include-hidden-files: true
if-no-files-found: ignore
compare:
needs: [resolve, run]
if: always() && needs.resolve.result == 'success'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: write
steps:
- name: Harden Runner
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ needs.resolve.outputs.candidate_ref }}
sparse-checkout: scripts/harbor-coder
persist-credentials: false
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
continue-on-error: true
with:
pattern: harbor-*
path: jobs
merge-multiple: true
- name: Compare
env:
LABEL: ${{ needs.resolve.outputs.candidate_label }}
RUN_RESULT: ${{ needs.run.result }}
BASELINE_SHA: ${{ needs.resolve.outputs.baseline_ref }}
CANDIDATE_SHA: ${{ needs.resolve.outputs.candidate_ref }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
set -euo pipefail
if [[ "${RUN_RESULT}" != "success" ]]; then
# A partial table would read as a verdict; the run log has the cause.
printf '## Coder Agents eval: %s vs main\n\n**The eval did not complete** (run: %s). See the [run](%s) for the failing arm.\n' \
"${LABEL}" "${RUN_RESULT}" "${RUN_URL}" > report.md
else
mkdir -p jobs/baseline jobs/candidate
PYTHONPATH=scripts/harbor-coder python3 -m harbor_coder.compare \
--baseline jobs/baseline --candidate jobs/candidate --labels main "${LABEL}" > report.md
printf '\n%s, %s, %s attempt(s) per task. Baseline %s (merge base with main), candidate %s. [Run](%s)\n' \
"${HARBOR_DATASET}" "${HARBOR_CODER_MODEL}" "${HARBOR_ATTEMPTS}" "${BASELINE_SHA:0:10}" "${CANDIDATE_SHA:0:10}" "${RUN_URL}" >> report.md
fi
cat report.md >> "${GITHUB_STEP_SUMMARY}"
- name: Comment on PR
if: needs.resolve.outputs.pr_number != ''
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR: ${{ needs.resolve.outputs.pr_number }}
run: gh pr comment "${PR}" -R "${REPO}" --body-file report.md