Files
LangBot/.github/workflows/plugins-runner-tests.yml
huanghuoguoguo 1f227f7aa8 ci: scope PR checks and run the complete matrix daily (#3056)
* ci: streamline pull request checks and run the full matrix daily

Refs #3055

* ci: run runtime contracts only on affected Core checks

* test(ci): distinguish SDK checks from SDK publication

* fix(ci): retain legacy regressions and scope production image builds

* docs(ci): clarify focused regressions and production image selection
2026-11-17 17:31:50 +08:00

119 lines
6.0 KiB
YAML

name: Plugins / Runner tests
on:
pull_request:
paths:
- "sdk/**"
- "src/langbot/pkg/api/application/**"
- "src/langbot/pkg/core/stages/**"
- "src/langbot/pkg/entity/**"
- "src/langbot/pkg/plugin/**"
- "src/langbot/pkg/workspace/**"
- "scripts/generate_plugin_workspace_api.py"
- "pyproject.toml"
- "uv.lock"
- "src/langbot/pkg/agent/**"
- "src/langbot/pkg/box/**"
- "src/langbot/pkg/pipeline/**"
- "src/langbot/pkg/storage/**"
- ".github/workflows/plugins-local-agent-tests.yml"
- "plugins/pytest.ini"
- "plugins/Runner/**"
- ".github/workflows/plugins-runner-tests.yml"
workflow_dispatch:
workflow_call:
permissions:
contents: read
concurrency:
group: plugin-runner-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:
runs-on: ubuntu-latest
timeout-minutes: 15
defaults:
run:
working-directory: plugins/Runner
steps:
- name: Checkout plugins
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: Set up uv
uses: astral-sh/setup-uv@v5
- name: Keep environment outside the checkout
run: |
printf 'UV_PROJECT_ENVIRONMENT=%s/runner-venv\n' "$RUNNER_TEMP" >> "$GITHUB_ENV"
printf 'RUNNER_EVIDENCE_DIR=%s/runner-evidence\n' "$RUNNER_TEMP" >> "$GITHUB_ENV"
- name: Install dependencies
run: uv sync --locked --dev
- name: Verify SDK snapshot
run: uv run --no-sync python -c 'from importlib.metadata import version; assert version("langbot-plugin") == "0.7.20"; print("langbot-plugin", version("langbot-plugin"))'
- name: Prepare isolated certified coding Runner SDK environment
run: |
uv venv "$RUNNER_TEMP/coding-sdk"
# Coding Runners call ctx.request_interaction, first released in SDK 0.7.20.
uv pip install --python "$RUNNER_TEMP/coding-sdk/bin/python" \
'langbot-plugin==0.7.20' \
-r acp-agent-runner/requirements.txt -r claude-code-agent/requirements.txt -r codex-agent/requirements.txt pytest pyyaml packaging
"$RUNNER_TEMP/coding-sdk/bin/python" -c 'from importlib.metadata import version; assert version("langbot-plugin") == "0.7.20"'
printf 'CODING_RUNNER_TEST_PYTHON=%s/coding-sdk/bin/python\n' "$RUNNER_TEMP" >> "$GITHUB_ENV"
- name: Prepare isolated LocalAgent environment with the latest stable SDK
run: |
UV_PROJECT_ENVIRONMENT="$RUNNER_TEMP/localagent-latest" uv sync --project LocalAgent --extra dev --upgrade-package langbot-plugin
"$RUNNER_TEMP/localagent-latest/bin/python" -c 'from importlib.metadata import version; print("langbot-plugin", version("langbot-plugin")); from inspect import signature; from langbot_plugin.api.entities.builtin.runner.context import RunnerContext; assert "paths" in signature(RunnerContext.export_box_files).parameters'
printf 'LOCALAGENT_TEST_PYTHON=%s/localagent-latest/bin/python\n' "$RUNNER_TEMP" >> "$GITHUB_ENV"
# Preserve the external Runner lint scope; LocalAgent has its own Ruff config.
# RunnerDemo did not have a Ruff gate before consolidation.
- name: Ruff check
run: uv run --no-sync ruff check tests http-certification-tests ./*-agent acp-agent-runner
- name: Ruff format check
run: uv run --no-sync ruff format --check tests http-certification-tests ./*-agent acp-agent-runner
- name: LocalAgent Ruff check
working-directory: plugins/Runner/LocalAgent
run: |
"$LOCALAGENT_TEST_PYTHON" -m ruff check .
- name: LocalAgent Ruff format check
working-directory: plugins/Runner/LocalAgent
run: |
"$LOCALAGENT_TEST_PYTHON" -m ruff format --check .
- name: All Runner tests, including isolated plugin suites (LocalAgent, RunnerDemo, shared agents)
run: uv run --no-sync python -m pytest --junitxml="${RUNNER_EVIDENCE_DIR}/runner.xml"
# Generic pkg/test module names require a fresh pytest interpreter.
- name: HTTP Runner certification regressions
run: uv run --no-sync python -m pytest http-certification-tests -q --junitxml="${RUNNER_EVIDENCE_DIR}/http-certification.xml"
- name: Certified RunnerDemo SDK admission and package tests
run: |
"$RUNNER_TEMP/coding-sdk/bin/python" certification-tests/test_runner_demo_dependency_admission.py -v
cd RunnerDemo
"$RUNNER_TEMP/coding-sdk/bin/python" -m unittest discover -s tests -v
- name: Test LocalAgent with the in-tree SDK
working-directory: plugins/Runner/LocalAgent
run: |
uv pip install --python "$LOCALAGENT_TEST_PYTHON" ../../../sdk
uv pip check --python "$LOCALAGENT_TEST_PYTHON"
"$LOCALAGENT_TEST_PYTHON" -c 'import json; from importlib.metadata import distribution; from pathlib import Path; sdk = distribution("langbot-plugin"); origin = json.loads(sdk.read_text("direct_url.json"))["url"]; assert origin == Path("../../../sdk").resolve().as_uri(), origin; print(sdk.version, origin)'
"$LOCALAGENT_TEST_PYTHON" -m pytest --junitxml="$RUNNER_EVIDENCE_DIR/localagent-in-tree.xml"
- name: Test Runner RPC consumers with the in-tree SDK
run: |
uv pip install --python "$UV_PROJECT_ENVIRONMENT/bin/python" ../../sdk
uv pip install --python "$RUNNER_TEMP/coding-sdk/bin/python" ../../sdk
RUNNER_EVIDENCE_DIR="$RUNNER_EVIDENCE_DIR/in-tree" uv run --no-sync python -m pytest \
tests/test_real_sdk_runtime.py -q --junitxml="${RUNNER_EVIDENCE_DIR}/in-tree.xml"
- name: Upload test evidence
if: always()
uses: actions/upload-artifact@v4
with:
name: runner-test-evidence
path: ${{ runner.temp }}/runner-evidence
if-no-files-found: warn