Bumps the github-actions group with 12 updates in the / directory: | Package | From | To | | --- | --- | --- | | [step-security/harden-runner](https://github.com/step-security/harden-runner) | `2.21.1` | `2.22.4` | | [actions/upload-artifact](https://github.com/actions/upload-artifact) | `7.0.1` | `7.0.2` | | [docker/login-action](https://github.com/docker/login-action) | `4.6.0` | `4.8.0` | | [pnpm/action-setup](https://github.com/pnpm/action-setup) | `6.1.0` | `6.1.1` | | [depot/build-push-action](https://github.com/depot/build-push-action) | `1.18.0` | `1.19.0` | | [actions/download-artifact](https://github.com/actions/download-artifact) | `7.0.0` | `8.0.2` | | [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) | `10.3.0` | `10.3.3` | | [linear/linear-release-action](https://github.com/linear/linear-release-action) | `0.18.1` | `0.18.2` | | [toshimaru/auto-author-assign](https://github.com/toshimaru/auto-author-assign) | `3.1.0` | `3.1.1` | | [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) | `4.39.0` | `4.39.3` | | [github/codeql-action/init](https://github.com/github/codeql-action) | `4.39.0` | `4.39.3` | | [github/codeql-action/analyze](https://github.com/github/codeql-action) | `4.39.0` | `4.39.3` | Updates `step-security/harden-runner` from 2.21.1 to 2.22.4 - [Release notes](https://github.com/step-security/harden-runner/releases) - [Commits](https://github.com/step-security/harden-runner/compare/v2.21.1...1d88384fd173af6ecb012dfd0264a306b711241a) Updates `actions/upload-artifact` from 7.0.1 to 7.0.2 - [Release notes](https://github.com/actions/upload-artifact/releases) - [Commits](https://github.com/actions/upload-artifact/compare/v7.0.1...cf430e030ddbb5b0abf93d22962f4752f3646cd9) Updates `docker/login-action` from 4.6.0 to 4.8.0 - [Release notes](https://github.com/docker/login-action/releases) - [Commits](https://github.com/docker/login-action/compare/dbcb813823bdd20940b903addbd779551569679f...8aa5831fb8d48d8f76594c7ddde61ae58019f550) Updates `pnpm/action-setup` from 6.1.0 to 6.1.1 - [Release notes](https://github.com/pnpm/action-setup/releases) - [Commits](https://github.com/pnpm/action-setup/compare/ea17c68df8912ef543352723c149a84f56e3d413...73e46992e05b19abd31f2f3c20e0e7370d762115) Updates `depot/build-push-action` from 1.18.0 to 1.19.0 - [Release notes](https://github.com/depot/build-push-action/releases) - [Commits](https://github.com/depot/build-push-action/compare/98e78adca7817480b8185f474a400b451d74e287...0f27be7c41d76663a98518dfe68f1c46582242c4) Updates `actions/download-artifact` from 7.0.0 to 8.0.2 - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v7...9000827ccba6bdab643e8b6fd33ac0654aef8333) Updates `astral-sh/setup-uv` from 10.3.0 to 10.3.3 - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/1c37ad07a6a961277cf70c0d37d6f313000f5884...68559e99cc807c1eb7e9238fd43a15bba460eec2) Updates `linear/linear-release-action` from 0.18.1 to 0.18.2 - [Release notes](https://github.com/linear/linear-release-action/releases) - [Commits](https://github.com/linear/linear-release-action/compare/30f9ae77461ec29f07fffe0c52edd1909bfbb6f5...dd4fc407f860b9460d2cf0b3fb1edd8985cf443f) Updates `toshimaru/auto-author-assign` from 3.1.0 to 3.1.1 - [Release notes](https://github.com/toshimaru/auto-author-assign/releases) - [Changelog](https://github.com/toshimaru/auto-author-assign/blob/main/CHANGELOG.md) - [Commits](https://github.com/toshimaru/auto-author-assign/compare/a78a94b219445cece8ccf0d45fec60af449f212a...415448809ea5ba0e171b251632720b009df3d8d6) Updates `github/codeql-action/upload-sarif` from 4.39.0 to 4.39.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...33d7b85da1a04171e211befe2c739303b9a4c066) Updates `github/codeql-action/init` from 4.39.0 to 4.39.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...33d7b85da1a04171e211befe2c739303b9a4c066) Updates `github/codeql-action/analyze` from 4.39.0 to 4.39.3 - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/6f2ac81f5cd81fe351a197ba0b00d91d03cd2aee...33d7b85da1a04171e211befe2c739303b9a4c066) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-version: 8.0.2 dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions - dependency-name: actions/upload-artifact dependency-version: 7.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: astral-sh/setup-uv dependency-version: 10.3.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: depot/build-push-action dependency-version: 1.19.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: docker/login-action dependency-version: 4.8.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: github/codeql-action/analyze dependency-version: 4.39.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: github/codeql-action/init dependency-version: 4.39.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.39.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: linear/linear-release-action dependency-version: 0.18.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: pnpm/action-setup dependency-version: 6.1.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: step-security/harden-runner dependency-version: 2.22.4 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: github-actions - dependency-name: toshimaru/auto-author-assign dependency-version: 3.1.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions ... Signed-off-by: dependabot[bot] <support@github.com>
252 lines
11 KiB
YAML
252 lines
11 KiB
YAML
# A/B Coder Agents on a PR branch vs main: each arm builds its own coderd,
|
|
# runs Terminal-Bench 2.0 through Harbor with the Coder environment provider
|
|
# (scripts/harbor-coder), and the comparison is posted on the PR.
|
|
#
|
|
# Trigger by commenting `/coder-agents-eval` on a PR (push access required,
|
|
# fork PRs refused since the candidate arm runs the PR's code with the gateway
|
|
# token in scope), or via workflow_dispatch to compare the current ref to main.
|
|
# Trials spend real model tokens, so this never runs automatically.
|
|
|
|
name: eval/coder-agents
|
|
|
|
on:
|
|
issue_comment:
|
|
types: [created]
|
|
workflow_dispatch:
|
|
inputs:
|
|
pr:
|
|
description: PR number to post the comparison on (optional)
|
|
required: false
|
|
type: string
|
|
|
|
permissions: {}
|
|
|
|
# Non-eval events (every other PR comment) get a group of their own so they
|
|
# cannot replace a queued /coder-agents-eval run, which would cancel it.
|
|
concurrency:
|
|
group: >-
|
|
${{ (github.event_name == 'workflow_dispatch' || startsWith(github.event.comment.body, '/coder-agents-eval'))
|
|
&& format('eval-coder-agents-{0}', github.event.issue.number || github.ref)
|
|
|| format('eval-coder-agents-other-{0}', github.run_id) }}
|
|
cancel-in-progress: false
|
|
|
|
env:
|
|
HARBOR_DATASET: terminal-bench@2.0
|
|
# qemu-* and build-cython-ext are broken upstream (bit-rotted apt sources
|
|
# and a git clone in the verifier). The rest are ML training, emulation,
|
|
# and long compiles that each take 20 to 60 minutes and are not decided by
|
|
# the system prompt; dropping them cuts an arm from ~3h to ~80 minutes.
|
|
# crack-7z-hash, break-filter-js-from-html, and vulnerable-secret are
|
|
# always refused by Anthropic's content policy (cyber), so they score 0 in
|
|
# every arm and cannot move the comparison.
|
|
HARBOR_EXCLUDE: >-
|
|
-x qemu-* -x build-cython-ext
|
|
-x train-fasttext -x compile-compcert -x install-windows-3.11
|
|
-x winning-avg-corewars -x extract-moves-from-video -x mcmc-sampling-stan
|
|
-x regex-chess -x mteb-leaderboard -x caffe-cifar-10
|
|
-x crack-7z-hash -x break-filter-js-from-html -x vulnerable-secret
|
|
# Sonnet sits mid-range on Terminal-Bench, leaving headroom in both
|
|
# directions for a prompt change to move; Opus is near the ceiling.
|
|
HARBOR_CODER_MODEL: claude-sonnet-5-5
|
|
HARBOR_ATTEMPTS: "2"
|
|
# chatd admits at most 5 concurrent root chats without a license
|
|
# (coderd/x/chatd/agentadmission.go). Above that, trials queue on Harbor's
|
|
# timeout and score 0 depending on start order.
|
|
HARBOR_CONCURRENCY: "5"
|
|
|
|
jobs:
|
|
resolve:
|
|
if: >-
|
|
github.event_name == 'workflow_dispatch' ||
|
|
(github.event.issue.pull_request && startsWith(github.event.comment.body, '/coder-agents-eval'))
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
outputs:
|
|
candidate_ref: ${{ steps.pr.outputs.sha || github.sha }}
|
|
candidate_label: ${{ steps.pr.outputs.ref || github.ref_name }}
|
|
baseline_ref: ${{ steps.base.outputs.sha }}
|
|
pr_number: ${{ github.event.issue.number || inputs.pr }}
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Resolve PR
|
|
id: pr
|
|
if: github.event_name == 'issue_comment'
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
PR: ${{ github.event.issue.number }}
|
|
COMMENTER: ${{ github.event.comment.user.login }}
|
|
COMMENT_ID: ${{ github.event.comment.id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "$(gh api "/repos/${REPO}/collaborators/${COMMENTER}/permission" --jq .user.permissions.push)" != "true" ]]; then
|
|
echo "::error::${COMMENTER} does not have push access"; exit 1
|
|
fi
|
|
if [[ "$(gh pr view "${PR}" -R "${REPO}" --json isCrossRepository --jq .isCrossRepository)" == "true" ]]; then
|
|
echo "::error::refusing to run on a fork PR"; exit 1
|
|
fi
|
|
echo "sha=$(gh pr view "${PR}" -R "${REPO}" --json headRefOid --jq .headRefOid)" >> "${GITHUB_OUTPUT}"
|
|
echo "ref=$(gh pr view "${PR}" -R "${REPO}" --json headRefName --jq .headRefName)" >> "${GITHUB_OUTPUT}"
|
|
gh api -X POST "/repos/${REPO}/issues/comments/${COMMENT_ID}/reactions" -f content=eyes >/dev/null ||
|
|
echo "::warning::could not react to comment ${COMMENT_ID}"
|
|
|
|
# Compare against the commit the candidate branched from, so commits
|
|
# that landed on main since then are not attributed to the candidate.
|
|
- name: Resolve baseline
|
|
id: base
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
CANDIDATE: ${{ steps.pr.outputs.sha || github.sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
sha="$(gh api "/repos/${REPO}/compare/main...${CANDIDATE}" --jq .merge_base_commit.sha)"
|
|
echo "baseline: ${sha}, candidate: ${CANDIDATE}"
|
|
echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
|
|
|
|
run:
|
|
name: run (${{ matrix.arm }})
|
|
needs: resolve
|
|
runs-on: ${{ github.repository_owner == 'coder' && 'depot-ubuntu-22.04-8' || 'ubuntu-latest' }}
|
|
# The candidate arm runs the PR's code, so only the eval secrets may be in scope; never use main.
|
|
environment: eval
|
|
timeout-minutes: 360
|
|
permissions:
|
|
contents: read
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
arm: [baseline, candidate]
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
|
|
with:
|
|
egress-policy: audit
|
|
|
|
- name: Checkout arm ref
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ matrix.arm == 'baseline' && needs.resolve.outputs.baseline_ref || needs.resolve.outputs.candidate_ref }}
|
|
fetch-depth: 1
|
|
persist-credentials: false
|
|
|
|
# The harness always comes from the candidate so both arms run the same code.
|
|
- name: Checkout harness from candidate ref
|
|
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ needs.resolve.outputs.candidate_ref }}
|
|
path: .eval-src
|
|
sparse-checkout: scripts/harbor-coder
|
|
persist-credentials: false
|
|
- run: rm -rf scripts/harbor-coder && cp -r .eval-src/scripts/harbor-coder scripts/harbor-coder
|
|
|
|
- uses: ./.github/actions/setup-go-paths
|
|
- uses: ./.github/actions/setup-mise
|
|
with:
|
|
install-args: "go"
|
|
- uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3
|
|
- name: Install Harbor from the hash-pinned lock
|
|
run: |
|
|
./scripts/harbor-coder/install.sh "${RUNNER_TEMP}/harbor-venv"
|
|
echo "${RUNNER_TEMP}/harbor-venv/bin" >> "${GITHUB_PATH}"
|
|
|
|
- name: Build and serve coderd
|
|
env:
|
|
# A Coder API key for dogfood; the eval's coderd uses dogfood's AI
|
|
# gateway as its model upstream, so no provider key is involved.
|
|
HARBOR_CODER_UPSTREAM_TOKEN: ${{ secrets.EVAL_CODER_SESSION_TOKEN }}
|
|
run: ./scripts/harbor-coder/serve-coderd.sh "${RUNNER_TEMP}/coder.env"
|
|
|
|
- name: Run Harbor
|
|
env:
|
|
ARM: ${{ matrix.arm }}
|
|
run: |
|
|
set -euo pipefail
|
|
set -a; . "${RUNNER_TEMP}/coder.env"; set +a
|
|
export PATH="${HARBOR_CODER_BIN%/*}:${PATH}"
|
|
# shellcheck disable=SC2086
|
|
harbor run -d "${HARBOR_DATASET}" ${HARBOR_EXCLUDE} \
|
|
--agent harbor_coder.agent:CoderAgentsAgent \
|
|
--env harbor_coder.environment:CoderEnvironment \
|
|
-k "${HARBOR_ATTEMPTS}" -n "${HARBOR_CONCURRENCY}" -o jobs --job-name "${ARM}"
|
|
# Harbor exits 0 even when every trial raised.
|
|
if [[ "$(find "jobs/${ARM}" -name reward.txt -o -name reward.json | wc -l)" -eq 0 ]]; then
|
|
echo "::error::no trial produced a reward"; exit 1
|
|
fi
|
|
|
|
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
if: always()
|
|
with:
|
|
name: harbor-${{ matrix.arm }}
|
|
path: jobs/
|
|
|
|
# Only the log: the rest of .harbor-coder holds the owner password and
|
|
# session token. A separate artifact keeps the jobs/ layout compare expects.
|
|
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
|
|
if: always()
|
|
with:
|
|
name: coderd-log-${{ matrix.arm }}
|
|
path: .harbor-coder/coderd.log
|
|
include-hidden-files: true
|
|
if-no-files-found: ignore
|
|
|
|
compare:
|
|
needs: [resolve, run]
|
|
if: always() && needs.resolve.result == 'success'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 10
|
|
permissions:
|
|
contents: read
|
|
pull-requests: write
|
|
steps:
|
|
- name: Harden Runner
|
|
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
|
|
with:
|
|
egress-policy: audit
|
|
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
|
|
with:
|
|
ref: ${{ needs.resolve.outputs.candidate_ref }}
|
|
sparse-checkout: scripts/harbor-coder
|
|
persist-credentials: false
|
|
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
|
|
continue-on-error: true
|
|
with:
|
|
pattern: harbor-*
|
|
path: jobs
|
|
merge-multiple: true
|
|
- name: Compare
|
|
env:
|
|
LABEL: ${{ needs.resolve.outputs.candidate_label }}
|
|
RUN_RESULT: ${{ needs.run.result }}
|
|
BASELINE_SHA: ${{ needs.resolve.outputs.baseline_ref }}
|
|
CANDIDATE_SHA: ${{ needs.resolve.outputs.candidate_ref }}
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
run: |
|
|
set -euo pipefail
|
|
if [[ "${RUN_RESULT}" != "success" ]]; then
|
|
# A partial table would read as a verdict; the run log has the cause.
|
|
printf '## Coder Agents eval: %s vs main\n\n**The eval did not complete** (run: %s). See the [run](%s) for the failing arm.\n' \
|
|
"${LABEL}" "${RUN_RESULT}" "${RUN_URL}" > report.md
|
|
else
|
|
mkdir -p jobs/baseline jobs/candidate
|
|
PYTHONPATH=scripts/harbor-coder python3 -m harbor_coder.compare \
|
|
--baseline jobs/baseline --candidate jobs/candidate --labels main "${LABEL}" > report.md
|
|
printf '\n%s, %s, %s attempt(s) per task. Baseline %s (merge base with main), candidate %s. [Run](%s)\n' \
|
|
"${HARBOR_DATASET}" "${HARBOR_CODER_MODEL}" "${HARBOR_ATTEMPTS}" "${BASELINE_SHA:0:10}" "${CANDIDATE_SHA:0:10}" "${RUN_URL}" >> report.md
|
|
fi
|
|
cat report.md >> "${GITHUB_STEP_SUMMARY}"
|
|
- name: Comment on PR
|
|
if: needs.resolve.outputs.pr_number != ''
|
|
env:
|
|
GH_TOKEN: ${{ github.token }}
|
|
REPO: ${{ github.repository }}
|
|
PR: ${{ needs.resolve.outputs.pr_number }}
|
|
run: gh pr comment "${PR}" -R "${REPO}" --body-file report.md
|