mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-11-22 14:27:26 +08:00
120 lines
4.1 KiB
JavaScript
120 lines
4.1 KiB
JavaScript
import fs from "node:fs";
|
|
import { describe, expect, it } from "vitest";
|
|
|
|
const manifest = JSON.parse(
|
|
fs.readFileSync("internal/test/deepswe-coding-slice-v2.json", "utf8"),
|
|
);
|
|
|
|
function languageCounts(taskIds) {
|
|
const tasksById = new Map(manifest.tasks.map((task) => [task.id, task]));
|
|
return taskIds.reduce((counts, id) => {
|
|
const task = tasksById.get(id);
|
|
expect(task, `missing task metadata for ${id}`).toBeDefined();
|
|
counts[task.language] = (counts[task.language] ?? 0) + 1;
|
|
return counts;
|
|
}, {});
|
|
}
|
|
|
|
describe("DeepSWE coding slice v2", () => {
|
|
it("pins the source and current-chain execution contract", () => {
|
|
expect(manifest.schemaVersion).toBe("lime-deepswe-coding-slice-v2");
|
|
expect(manifest.status).toBe("diagnostic_true_runs_blocked");
|
|
expect(manifest.source).toMatchObject({
|
|
commit: "435ee89ec2f2e2289f33b0da4f992f0b7b7266b9",
|
|
taskSchemaVersion: "1.3",
|
|
taskCount: 113,
|
|
runner: "datacurve-pier==0.3.1",
|
|
});
|
|
expect(manifest.executionContract.agentPath).toBe(
|
|
"Lime App Server JSON-RPC current chain",
|
|
);
|
|
expect(manifest.executionContract.adapterCommand).toBe(
|
|
"npm run harness:deepswe:run",
|
|
);
|
|
expect(manifest.executionContract.workspace).toContain(
|
|
"outside the Lime repository",
|
|
);
|
|
expect(manifest.executionContract).toMatchObject({
|
|
adapterVersion: "deepswe-current-chain-adapter-v7",
|
|
primaryModel:
|
|
"custom-637ea2d5-e430-43de-86de-39c5f1735438 / agnes-2.0-flash",
|
|
comparisonModel: "custom-1ae93b42-e57f-4a83-ac6e-3f5275a7b376 / gpt-5.5",
|
|
providerBudget: {
|
|
maxProviderSteps: 32,
|
|
tokenBudget: 500_000,
|
|
tokenFormula: "max(0,input_tokens-cached_input_tokens)+output_tokens",
|
|
enforcementOwner:
|
|
"agent-runtime reply loop before tool execution and next sampling",
|
|
},
|
|
requiredEvidence: {
|
|
agent: [
|
|
"run-context.json",
|
|
"trajectory.json",
|
|
"provider-steps.json",
|
|
"thread-turn-item.json",
|
|
"tool-lifecycle.json",
|
|
"patch.diff",
|
|
"failure-classification.json",
|
|
],
|
|
verifier: ["reward.json", "ctrf.json", "test-stdout.txt"],
|
|
blockedVerifier: [
|
|
"verifier-prerequisites.json",
|
|
"verifier-failure-classification.json",
|
|
],
|
|
},
|
|
});
|
|
expect(manifest.executionContract.invalidTrialConditions).toContain(
|
|
"task dependency resolution can reach Lime node_modules",
|
|
);
|
|
expect(manifest.executionContract.invalidTrialConditions).toContain(
|
|
"task git repository exposes refs or commits after the pinned base",
|
|
);
|
|
expect(manifest.executionContract.currentBlockers).toEqual([
|
|
"Agnes produced no candidate in TS/Go/Rust diagnostics, including runtime-capped runs with a complete provider tool catalog",
|
|
"No local Docker, Podman, nerdctl, or Colima container runtime is available",
|
|
]);
|
|
});
|
|
|
|
it("keeps Smoke 10 unique, balanced, and contained in Release 20", () => {
|
|
const smoke = manifest.slices["smoke-10"];
|
|
const release = manifest.slices["release-20"];
|
|
|
|
expect(smoke).toHaveLength(10);
|
|
expect(new Set(smoke).size).toBe(10);
|
|
expect(release).toHaveLength(20);
|
|
expect(new Set(release).size).toBe(20);
|
|
expect(smoke.every((id) => release.includes(id))).toBe(true);
|
|
expect(languageCounts(smoke)).toEqual({
|
|
typescript: 2,
|
|
go: 2,
|
|
python: 2,
|
|
rust: 2,
|
|
javascript: 2,
|
|
});
|
|
});
|
|
|
|
it("keeps Release 20 metadata complete and language-diverse", () => {
|
|
const release = manifest.slices["release-20"];
|
|
const metadataIds = manifest.tasks.map((task) => task.id);
|
|
|
|
expect(manifest.tasks).toHaveLength(20);
|
|
expect(new Set(metadataIds).size).toBe(20);
|
|
expect(new Set(metadataIds)).toEqual(new Set(release));
|
|
expect(
|
|
manifest.tasks.every(
|
|
(task) =>
|
|
task.repository.length > 0 &&
|
|
Array.isArray(task.focus) &&
|
|
task.focus.length >= 2,
|
|
),
|
|
).toBe(true);
|
|
expect(languageCounts(release)).toEqual({
|
|
typescript: 6,
|
|
go: 4,
|
|
python: 5,
|
|
rust: 3,
|
|
javascript: 2,
|
|
});
|
|
});
|
|
});
|