Files
coder/scripts/harbor-coder/harbor_coder/agent.py
T

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

138 lines
6.2 KiB
Python
Raw Normal View History

"""Harbor agent that drives Coder Agents (the coderd chat agent).
Runs in the Harbor host process and talks to coderd over REST. Requires
``CoderEnvironment`` so the chat can be bound to the trial's workspace; the
agent's tool calls then land in the container Harbor's verifier inspects.
"""
from __future__ import annotations
import asyncio
import json
from typing import Any
from harbor.agents.base import BaseAgent
from harbor.environments.base import BaseEnvironment
from harbor.models.agent.context import AgentContext
from harbor_coder.client import CoderAPIError, CoderClient
# Chat error kinds caused by the deployment or the gateway rather than by the
# agent's attempt at the task.
_INFRA_ERROR_KINDS = frozenset({"auth", "config", "usage_limit", "missing_key", "provider_disabled"})
class CoderInfraError(RuntimeError):
"""The chat failed for reasons outside the agent; ``compare`` excludes these trials."""
class CoderAgentsAgent(BaseAgent):
def __init__(self, *args: Any, **kwargs: Any):
super().__init__(*args, **kwargs)
self.client = CoderClient()
self._version: str | None = None
self._model: str | None = None
@staticmethod
def name() -> str:
return "coder-agents"
def version(self) -> str | None:
return self._version
async def setup(self, environment: BaseEnvironment) -> None:
self._version = (await self.client.request("GET", "/api/v2/buildinfo")).get("version")
if not getattr(environment, "workspace_id", None):
raise RuntimeError("coder-agents requires --env harbor_coder.environment:CoderEnvironment")
org = await self.client.default_organization()
self._model = await self.client.default_chat_model(org["id"])
if self._model is None:
raise RuntimeError("no default chat model configured on the coderd under test")
self.logger.info(f"coder-agents model: {self._model}")
async def run(self, instruction: str, environment: BaseEnvironment, context: AgentContext) -> None:
org = await self.client.default_organization()
chat = await self.client.create_chat(
{
"organization_id": org["id"],
"workspace_id": environment.workspace_id, # type: ignore[attr-defined]
"title": f"harbor {self.session_id}"[:80],
"client_type": "api",
# The task text verbatim; the working directory comes from the
# workspace agent's `dir`, as in a real workspace.
"content": [{"type": "text", "text": instruction}],
}
)
chat_id = chat["id"]
context.metadata = {"chat_id": chat_id, "model": self._model, "queued_for_capacity": False}
status = "running"
last_error: dict[str, Any] | None = None
try:
while True:
c = await self.client.chat(chat_id)
status = c.get("status")
# Queued time counts against Harbor's timeout; recording it
# makes a concurrency above chatd's cap visible in results.
if c.get("queued_for_capacity"):
context.metadata["queued_for_capacity"] = True
if status in ("waiting", "error"):
last_error = c.get("last_error")
break
await asyncio.sleep(3)
except asyncio.CancelledError:
# Harbor's timeout: stop the agent so it cannot keep mutating the
# workspace while the verifier runs, and wait for it to actually
# stop since interrupt is asynchronous server-side.
await asyncio.shield(self._interrupt_and_settle(chat_id))
status = "timeout"
raise
finally:
context.metadata["final_status"] = status
await asyncio.shield(self._collect(chat_id, context))
try:
await asyncio.shield(self.client.archive_chat(chat_id))
except CoderAPIError as e:
self.logger.warning(f"archive chat {chat_id}: {e}")
if status == "error":
err = last_error or {}
if err.get("retryable") or err.get("kind") in _INFRA_ERROR_KINDS:
raise CoderInfraError(f"chat ended in infrastructure error: {err}")
raise RuntimeError(f"chat ended in error: {err}")
async def _interrupt_and_settle(self, chat_id: str) -> None:
await self.client.interrupt_chat(chat_id)
status = None
for _ in range(20):
status = (await self.client.chat(chat_id)).get("status")
if status in ("waiting", "error"):
return
await asyncio.sleep(1)
# The verifier may now race a still-running tool call, and archive
# will fail because chatd only archives idle chats.
self.logger.warning(f"chat {chat_id} still {status!r} 20s after interrupt")
async def _collect(self, chat_id: str, context: AgentContext) -> None:
messages = await self.client.chat_messages(chat_id)
n_in = n_out = n_cache = 0
for m in messages:
u = m.get("usage") or {}
n_in += u.get("input_tokens") or 0
n_out += u.get("output_tokens") or 0
n_cache += (u.get("cache_read_tokens") or 0) + (u.get("cache_creation_tokens") or 0)
# Harbor defines n_input_tokens as including cached tokens.
context.n_input_tokens = n_in + n_cache
context.n_cache_tokens = n_cache
context.n_output_tokens = n_out
context.metadata["n_tool_calls"] = sum(
1 for m in messages for p in m.get("content") or [] if p.get("type") == "tool-call"
)
# Cost only counts finished gateway interceptions, so it may lag the
# status change by a moment.
for _ in range(5):
cost = await self.client.chat_cost(chat_id)
if cost.get("request_count", 0) > 0 or n_in == 0:
context.cost_usd = (cost.get("total_cost_micros") or 0) / 1_000_000
break
await asyncio.sleep(2)
self.logs_dir.mkdir(parents=True, exist_ok=True)
(self.logs_dir / "chat.json").write_text(json.dumps(messages, indent=1, default=str))