The `run` job in `eval-coder-agents.yaml` checks out the PR's head and runs its code, including its own `scripts/harbor-coder`. It currently uses the `main` environment, which holds main-only secrets for other workflows. This moves it to a dedicated `eval` environment, so only eval secrets are ever in scope. - **Depends on** coder/github-org-terraform#124, which creates `eval` (only `main` may deploy, no reviewers, no admin bypass). Merge that first; otherwise GitHub creates `eval` unprotected on the first run. - **`EVAL_CODER_SESSION_TOKEN`:** the job references it, but it isn't set in `main` today. When it's added, put it in `eval`. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
252 lines
11 KiB
YAML
252 lines
11 KiB
YAML
# A/B Coder Agents on a PR branch vs main: each arm builds its own coderd,
|
|
# runs Terminal-Bench 2.0 through Harbor with the Coder environment provider
|
|
# (scripts/harbor-coder), and the comparison is posted on the PR.
|
|
#
|
|
# Trigger by commenting `/coder-agents-eval` on a PR (push access required,
|
|
# fork PRs refused since the candidate arm runs the PR's code with the gateway
|
|
# token in scope), or via workflow_dispatch to compare the current ref to main.
|
|
# Trials spend real model tokens, so this never runs automatically.
|
|
|
|
name: eval/coder-agents
|
|
|
|
on:
|
|
issue_comment:
|
|
types: [created]
|
|
workflow_dispatch:
|
|
inputs:
|
|
pr:
|
|
description: PR number to post the comparison on (optional)
|
|
required: false
|
|
type: string
|
|
|
|
permissions: {}
|
|
|
|
# Non-eval events (every other PR comment) get a group of their own so they
|
|
# cannot replace a queued /coder-agents-eval run, which would cancel it.
|
|
concurrency:
|
|
group: >-
|
|
${{ (github.event_name == 'workflow_dispatch' || startsWith(github.event.comment.body, '/coder-agents-eval'))
|
|
&& format('eval-coder-agents-{0}', github.event.issue.number || github.ref)
|
|
|| format('eval-coder-agents-other-{0}', github.run_id) }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
HARBOR_DATASET: terminal-bench@2.0
|
|
# qemu-* and build-cython-ext are broken upstream (bit-rotted apt sources
|
|
# and a git clone in the verifier). The rest are ML training, emulation,
|
|
# and long compiles that each take 20 to 60 minutes and are not decided by
|
|
# the system prompt; dropping them cuts an arm from ~3h to ~80 minutes.
|
|
# crack-7z-hash, break-filter-js-from-html, and vulnerable-secret are
|
|
# always refused by Anthropic's content policy (cyber), so they score 0 in
|
|
# every arm and cannot move the comparison.
|
|
HARBOR_EXCLUDE: >-
|
|
-x qemu-* -x build-cython-ext
|
|
-x train-fasttext -x compile-compcert -x install-windows-3.11
|
|
-x winning-avg-corewars -x extract-moves-from-video -x mcmc-sampling-stan
|
|
-x regex-chess -x mteb-leaderboard -x caffe-cifar-10
|
|
-x crack-7z-hash -x break-filter-js-from-html -x vulnerable-secret
|
|
# Sonnet sits mid-range on Terminal-Bench, leaving headroom in both
|
|
# directions for a prompt change to move; Opus is near the ceiling.
|
|
HARBOR_CODER_MODEL: claude-sonnet-5-5
|
|
HARBOR_ATTEMPTS: "2"
|
|
# chatd admits at most 5 concurrent root chats without a license
|
|
# (coderd/x/chatd/agentadmission.go). Above that, trials queue on Harbor's
|
|
# timeout and score 0 depending on start order.
|
|
HARBOR_CONCURRENCY: "5"
|
|
|
|
jobs:
|
|
resolve:
|
|
if: >-
|
|
github.event_name == 'workflow_dispatch' ||
|
|
(github.event.issue.pull_request && startsWith(github.event.comment.body, '/coder-agents-eval'))
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
outputs:
|
|
candidate_ref: ${{ steps.pr.outputs.sha || github.sha }}
|
|
candidate_label: ${{ steps.pr.outputs.ref || github.ref_name }}
|
|
baseline_ref: ${{ steps.base.outputs.sha }}
|
|
pr_number: ${{ github.event.issue.number || inputs.pr }}
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Resolve PR
|
|
id: pr
|
|
if: github.event_name == 'issue_comment'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
PR: ${{ github.event.issue.number }}
|
|
COMMENTER: ${{ github.event.comment.user.login }}
|
|
COMMENT_ID: ${{ github.event.comment.id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "$(gh api "/repos/${REPO}/collaborators/${COMMENTER}/permission" --jq .user.permissions.push)" != "true" ]]; then
|
|
echo "::error::${COMMENTER} does not have push access"; exit 1
|
|
fi
|
|
if [[ "$(gh pr view "${PR}" -R "${REPO}" --json isCrossRepository --jq .isCrossRepository)" == "true" ]]; then
|
|
echo "::error::refusing to run on a fork PR"; exit 1
|
|
fi
|
|
echo "sha=$(gh pr view "${PR}" -R "${REPO}" --json headRefOid --jq .headRefOid)" >> "${GITHUB_OUTPUT}"
|
|
echo "ref=$(gh pr view "${PR}" -R "${REPO}" --json headRefName --jq .headRefName)" >> "${GITHUB_OUTPUT}"
|
|
gh api -X POST "/repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -f content=eyes >/dev/null ||
|
|
echo "::warning::could not react to comment ${COMMENT_ID}"
|
|
|
|
# Compare against the commit the candidate branched from, so commits
|
|
# that landed on main since then are not attributed to the candidate.
|
|
- name: Resolve baseline
|
|
id: base
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
CANDIDATE: ${{ steps.pr.outputs.sha || github.sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
sha="$(gh api "/repos/${REPO}/compare/main...${CANDIDATE}" --jq .merge_base_commit.sha)"
|
|
echo "baseline: ${sha}, candidate: ${CANDIDATE}"
|
|
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
|
|
|
run:
|
|
name: run (${{ matrix.arm }})
|
|
needs: resolve
|
|
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
|
|
# The candidate arm runs the PR's code, so only the eval secrets may be in scope; never use main.
|
|
environment: eval
|
|
timeout-minutes: 360
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
arm: [baseline, candidate]
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout arm ref
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ matrix.arm == 'baseline' && needs.resolve.outputs.baseline_ref || needs.resolve.outputs.candidate_ref }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
# The harness always comes from the candidate so both arms run the same code.
|
|
- name: Checkout harness from candidate ref
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ needs.resolve.outputs.candidate_ref }}
|
|
path: .eval-src
|
|
sparse-checkout: scripts/harbor-coder
|
|
persist-credentials: false
|
|
- run: rm -rf scripts/harbor-coder && cp -r .eval-src/scripts/harbor-coder scripts/harbor-coder
|
|
|
|
- uses: ./.github/actions/setup-go-paths
|
|
- uses: ./.github/actions/setup-mise
|
|
with:
|
|
install-args: "go"
|
|
- uses: astral-sh/setup-uv@1c37ad07a6a961277cf70c0d37d6f313000f5884 # v10.3.0
|
|
- name: Install Harbor from the hash-pinned lock
|
|
run: |
|
|
./scripts/harbor-coder/install.sh "${RUNNER_TEMP}/harbor-venv"
|
|
echo "${RUNNER_TEMP}/harbor-venv/bin" >> "${GITHUB_PATH}"
|
|
|
|
- name: Build and serve coderd
|
|
env:
|
|
# A Coder API key for dogfood; the eval's coderd uses dogfood's AI
|
|
# gateway as its model upstream, so no provider key is involved.
|
|
HARBOR_CODER_UPSTREAM_TOKEN: ${{ secrets.EVAL_CODER_SESSION_TOKEN }}
|
|
run: ./scripts/harbor-coder/serve-coderd.sh "${RUNNER_TEMP}/coder.env"
|
|
|
|
- name: Run Harbor
|
|
env:
|
|
ARM: ${{ matrix.arm }}
|
|
run: |
|
|
set -euo pipefail
|
|
set -a; . "${RUNNER_TEMP}/coder.env"; set +a
|
|
export PATH="${HARBOR_CODER_BIN%/*}:${PATH}"
|
|
# shellcheck disable=SC2086
|
|
harbor run -d "${HARBOR_DATASET}" ${HARBOR_EXCLUDE} \
|
|
--agent harbor_coder.agent:CoderAgentsAgent \
|
|
--env harbor_coder.environment:CoderEnvironment \
|
|
-k "${HARBOR_ATTEMPTS}" -n "${HARBOR_CONCURRENCY}" -o jobs --job-name "${ARM}"
|
|
# Harbor exits 0 even when every trial raised.
|
|
if [[ "$(find "jobs/${ARM}" -name reward.txt -o -name reward.json | wc -l)" -eq 0 ]]; then
|
|
echo "::error::no trial produced a reward"; exit 1
|
|
fi
|
|
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always()
|
|
with:
|
|
name: harbor-${{ matrix.arm }}
|
|
path: jobs/
|
|
|
|
# Only the log: the rest of .harbor-coder holds the owner password and
|
|
# session token. A separate artifact keeps the jobs/ layout compare expects.
|
|
- uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
|
if: always()
|
|
with:
|
|
name: coderd-log-${{ matrix.arm }}
|
|
path: .harbor-coder/coderd.log
|
|
include-hidden-files: true
|
|
if-no-files-found: ignore
|
|
|
|
compare:
|
|
needs: [resolve, run]
|
|
if: always() && needs.resolve.result == 'success'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
|
|
with:
|
|
egress-policy: audit
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ needs.resolve.outputs.candidate_ref }}
|
|
sparse-checkout: scripts/harbor-coder
|
|
persist-credentials: false
|
|
- uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
|
continue-on-error: true
|
|
with:
|
|
pattern: harbor-*
|
|
path: jobs
|
|
merge-multiple: true
|
|
- name: Compare
|
|
env:
|
|
LABEL: ${{ needs.resolve.outputs.candidate_label }}
|
|
RUN_RESULT: ${{ needs.run.result }}
|
|
BASELINE_SHA: ${{ needs.resolve.outputs.baseline_ref }}
|
|
CANDIDATE_SHA: ${{ needs.resolve.outputs.candidate_ref }}
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "${RUN_RESULT}" != "success" ]]; then
|
|
# A partial table would read as a verdict; the run log has the cause.
|
|
printf '## Coder Agents eval: %s vs main\n\n**The eval did not complete** (run: %s). See the [run](%s) for the failing arm.\n' \
|
|
"${LABEL}" "${RUN_RESULT}" "${RUN_URL}" > report.md
|
|
else
|
|
mkdir -p jobs/baseline jobs/candidate
|
|
PYTHONPATH=scripts/harbor-coder python3 -m harbor_coder.compare \
|
|
--baseline jobs/baseline --candidate jobs/candidate --labels main "${LABEL}" > report.md
|
|
printf '\n%s, %s, %s attempt(s) per task. Baseline %s (merge base with main), candidate %s. [Run](%s)\n' \
|
|
"${HARBOR_DATASET}" "${HARBOR_CODER_MODEL}" "${HARBOR_ATTEMPTS}" "${BASELINE_SHA:0:10}" "${CANDIDATE_SHA:0:10}" "${RUN_URL}" >> report.md
|
|
fi
|
|
cat report.md >> "${GITHUB_STEP_SUMMARY}"
|
|
- name: Comment on PR
|
|
if: needs.resolve.outputs.pr_number != ''
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
PR: ${{ needs.resolve.outputs.pr_number }}
|
|
run: gh pr comment "${PR}" -R "${REPO}" --body-file report.md
|