diff --git a/.gitea/workflows/harbor-ci.yaml b/.gitea/workflows/harbor-ci.yaml deleted file mode 100644 index cc07e865..00000000 --- a/.gitea/workflows/harbor-ci.yaml +++ /dev/null @@ -1,64 +0,0 @@ -name: Harbor CADIR benchmark - -on: - push: - branches: - - master - - dev - - ci/harbor-smoke - workflow_dispatch: - -concurrency: - group: harbor-cadir-${{ gitea.ref }} - cancel-in-progress: true - -jobs: - benchmark: - # Secrets and Docker access are limited to protected release branches, - # plus this exact disposable branch for the initial runner smoke test. - if: gitea.ref == 'refs/heads/master' || gitea.ref == 'refs/heads/dev' || (gitea.event_name == 'push' && gitea.ref == 'refs/heads/ci/harbor-smoke') - runs-on: ubuntu-latest - timeout-minutes: 150 - steps: - - name: Check out the CADIR commit - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - fetch-depth: 1 - - - name: Install uv and Python - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 - with: - python-version: "3.12" - enable-cache: true - - - name: Verify Docker runner - run: docker info && docker compose version - - - name: Install pinned Harbor CLI - run: uv tool install harbor==0.23.0 - - - name: Run public Harbor Hub benchmark - env: - HARBOR_CI_DATASET: ${{ vars.CADIR_CI_DATASET }} - HARBOR_CI_AGENT: ${{ vars.CADIR_CI_AGENT }} - HARBOR_CI_AGENT_VERSION: ${{ vars.CADIR_CI_AGENT_VERSION }} - HARBOR_CI_N_CONCURRENT: ${{ vars.CADIR_CI_N_CONCURRENT }} - HARBOR_CI_N_ATTEMPTS: ${{ vars.CADIR_CI_N_ATTEMPTS }} - HARBOR_CI_MAX_RETRIES: ${{ vars.CADIR_CI_MAX_RETRIES }} - BENCH_MODEL: ${{ vars.CADIR_CI_MODEL }} - OPENAI_API_KEY: ${{ secrets.CADIR_CI_OPENAI_API_KEY }} - OPENAI_BASE_URL: ${{ vars.CADIR_CI_OPENAI_BASE_URL }} - run: python -m tools.harbor_ci.run --cleanup-runtime - - - name: Upload Harbor jobs and summary - if: always() - # Use the v3 artifact protocol until the runner/reverse-proxy path is - # reliable with v4 CreateArtifact requests. The action code comes from - # Gitea's mirror; the runner still uploads to this Gitea instance. - uses: https://gitea.com/actions/upload-artifact@c6a3b2bd78b3985e4b2f15397fec357f0fd808de # v3.2.2-node20 - with: - name: cadir-harbor-ci-${{ gitea.sha }} - path: .harbor-ci/ - if-no-files-found: error - # The complete Harbor output tree lives below this hidden directory. - include-hidden-files: true diff --git a/.gitignore b/.gitignore index e62dcea9..8ac3622f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,6 @@ wheels/ .env output/ -.harbor-ci/ # Sandbox artifacts sandbox/ diff --git a/docs/harbor-ci.md b/docs/harbor-ci.md deleted file mode 100644 index c800e794..00000000 --- a/docs/harbor-ci.md +++ /dev/null @@ -1,108 +0,0 @@ -# Harbor Hub CADIR benchmark - -The Gitea workflow in `.gitea/workflows/harbor-ci.yaml` evaluates the checked-out CADIR commit against the public Harbor Hub dataset: - -```text -au12321ua/cadir-ci-benchmark -``` - -The runner performs five operations: - -1. checks out CADIR and installs `harbor==0.23.0`; -2. builds a content-addressed local runtime image from that checkout, with the - requested Codex CLI version already installed; -3. runs `harbor run -d au12321ua/cadir-ci-benchmark` on the runner; -4. writes and uploads `.harbor-ci/`, including native Harbor jobs and the summary report; -5. removes the per-run runtime image tag while retaining Docker/BuildKit caches. - -The artifact upload explicitly includes hidden files because the complete -benchmark output is rooted at `.harbor-ci/`. A missing output tree is treated as -an upload error instead of being silently ignored. The workflow currently uses -the pinned Node 20 build of the v3 artifact protocol from Gitea's action mirror; -the action source is external, but the runner-provided artifact endpoint keeps -the uploaded data on the CADIR Gitea instance. This avoids v4 `CreateArtifact` -connection resets seen on the self-hosted runner/reverse-proxy path. - -Codex is installed while the runtime image is built, rather than during every -Harbor trial. The complete pinned npm installation is copied into the runtime, -including Node.js, the supported `codex` launcher, its platform package, -`codex-code-mode-host`, and bundled resources. Copying only the main native -binary is insufficient: `codex exec --enable unified_exec` needs the matching -Code Mode host, and other Codex features use additional package resources. -Harbor's agent setup has a six-minute default timeout; installing NVM, Node.js, -and Codex separately in each fresh task container can consume that entire -window and produce `AgentSetupTimeoutError` before the task starts. The -launcher resolves the generic `--agent-version` setting into both the image -build and the Harbor agent configuration, so Harbor's setup step only verifies -the matching preinstalled version. The agent name and version are also part of -the runtime image digest. - -The runtime image is not pushed to Docker Hub. Harbor downloads the public task packages from Harbor Hub, then Docker executes the agent and separate verifier environments locally on the trusted runner. - -## Runner requirements - -- trusted Linux host runner labelled `cadir-ci`; -- Gitea Actions runner with the v3 artifact protocol enabled; -- Docker Engine with Compose v2; -- outbound access for uv, Harbor Hub, base images, Codex installation, and the selected model endpoint; -- enough disk for OCC/VTK and task images. - -Configure Gitea repository variables. Empty or missing variables fall back to -the values shown here: - -- `CADIR_CI_DATASET=au12321ua/cadir-ci-benchmark`; -- `CADIR_CI_AGENT=codex`; -- `CADIR_CI_AGENT_VERSION=0.155.1`; -- `CADIR_CI_N_CONCURRENT=1`: maximum number of trials running at once; -- `CADIR_CI_N_ATTEMPTS=1`: independent attempts per task; -- `CADIR_CI_MAX_RETRIES=0`: retries after a trial raises an exception. - -`N_CONCURRENT` controls parallelism, not the total number of runs. Increasing -`N_ATTEMPTS` multiplies the number of trials (and model usage). `MAX_RETRIES` -applies to exceptional failures and does not rerun a completed trial just -because its score was low. - -The external configuration uses generic agent names so another Harbor agent can -be added without changing the workflow contract. The runtime currently supports -only `codex`; any other `CADIR_CI_AGENT` value fails explicitly before a runtime -image is built. - -Configure the model and endpoint as Gitea repository variables: - -- `CADIR_CI_MODEL`; -- optional `CADIR_CI_OPENAI_BASE_URL`. - -Configure the credential as a Gitea secret: - -- `CADIR_CI_OPENAI_API_KEY`; - -The model credential is inherited by Harbor at runtime and is not written to the Job JSON or report. The selected dataset, agent, agent version, concurrency, attempts, and retries are written to the report for traceability. The tracked workflow only runs on the protected `master` and `dev` refs because the checked-out code receives model secrets and controls Docker; do not broaden the trigger to untrusted branches. - -If local use supplies `OPENAI_BASE_URL` only through `--env-file`, also pass its non-secret hostname with `--allow-agent-host `. Automatic allowlisting can only inspect variables exported to the launcher process. - -## Local run - -After the dataset has been published, install Harbor. A Codex run uses -`BENCH_MODEL` and `OPENAI_API_KEY` from the environment and accepts the same -generic configuration as CI: - -```bash -uv tool install harbor==0.23.0 -export BENCH_MODEL='openai/' -export OPENAI_API_KEY='...' -python -m tools.harbor_ci.run \ - --dataset au12321ua/cadir-ci-benchmark \ - --agent codex \ - --agent-version 0.155.1 \ - --n-concurrent 1 \ - --n-attempts 1 \ - --max-retries 0 -``` - -Result semantics: - -- `PASS`: every completed trial has reward 1; -- `FAIL`: valid scoring completed with at least one reward below 1; -- `ERROR`: missing/partial trials, verifier/agent infrastructure exceptions, or missing rewards. - -`FAIL` is report-only by default. Add `--require-pass` after the benchmark is stable if scored failure should fail the workflow. `ERROR` always exits 2. diff --git a/tests/test_harbor_ci.py b/tests/test_harbor_ci.py deleted file mode 100644 index 08ac0022..00000000 --- a/tests/test_harbor_ci.py +++ /dev/null @@ -1,269 +0,0 @@ -import importlib -import json -import subprocess -import sys -from pathlib import Path - -import pytest - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT)) -agents = importlib.import_module("tools.harbor_ci.agents") -results = importlib.import_module("tools.harbor_ci.results") -run = importlib.import_module("tools.harbor_ci.run") -runtime = importlib.import_module("tools.harbor_ci.runtime") - - -def _source_tree(root: Path) -> Path: - for directory in ("src/simplecadapi", "docs/skill"): - (root / directory).mkdir(parents=True) - for name in ( - "pyproject.toml", - "README.md", - "LICENSE", - "MANIFEST.in", - "skillproj.toml", - ): - (root / name).write_text(name) - (root / "src/simplecadapi/__init__.py").write_text("") - (root / "docs/skill/SKILL.md").write_text("skill") - return root - - -def test_runtime_snapshot_is_content_addressed_and_rejects_links(tmp_path): - source = _source_tree(tmp_path / "source") - first = runtime.snapshot_source(source, tmp_path / "first") - second = runtime.snapshot_source(source, tmp_path / "second") - assert first == second - - (source / "docs/skill/link").symlink_to(source / "README.md") - with pytest.raises(ValueError): - runtime.snapshot_source(source, tmp_path / "linked") - - -def test_runtime_build_identity_includes_agent_and_version(): - first = runtime.runtime_build_digest("source", b"dockerfile", "codex", "0.155.1") - second = runtime.runtime_build_digest("source", b"dockerfile", "codex", "0.156.0") - other_agent = runtime.runtime_build_digest( - "source", b"dockerfile", "future-agent", "0.155.1" - ) - assert first != second - assert first != other_agent - - -def test_agent_config_is_generic_but_only_codex_is_supported(): - agent = agents.resolve_agent("codex", "0.155.1") - assert agent.harbor_kwargs == ("version=0.155.1",) - assert agent.docker_build_args == (("CODEX_VERSION", "0.155.1"),) - - with pytest.raises(ValueError, match="unsupported Harbor agent"): - agents.resolve_agent("future-agent", "1.0.0") - - -def test_empty_actions_variables_use_defaults(monkeypatch): - for name in ( - "HARBOR_CI_DATASET", - "HARBOR_CI_AGENT", - "HARBOR_CI_AGENT_VERSION", - "HARBOR_CI_N_CONCURRENT", - "HARBOR_CI_N_ATTEMPTS", - "HARBOR_CI_MAX_RETRIES", - ): - monkeypatch.setenv(name, "") - monkeypatch.setattr(sys, "argv", ["harbor-ci"]) - - args = run.parse_args() - - assert args.dataset == run.DEFAULT_DATASET - assert args.agent == agents.DEFAULT_AGENT - assert args.agent_version == agents.DEFAULT_AGENT_VERSION - assert args.n_concurrent == run.DEFAULT_N_CONCURRENT - assert args.n_attempts == run.DEFAULT_N_ATTEMPTS - assert args.max_retries == run.DEFAULT_MAX_RETRIES - - -def test_harbor_job_settings_from_environment_and_cli(monkeypatch): - monkeypatch.setenv("HARBOR_CI_N_CONCURRENT", "2") - monkeypatch.setenv("HARBOR_CI_N_ATTEMPTS", "3") - monkeypatch.setenv("HARBOR_CI_MAX_RETRIES", "4") - monkeypatch.setattr(sys, "argv", ["harbor-ci", "--n-concurrent", "5"]) - - args = run.parse_args() - - assert (args.n_concurrent, args.n_attempts, args.max_retries) == (5, 3, 4) - - -@pytest.mark.parametrize( - ("flag", "value"), - [ - ("--n-concurrent", "0"), - ("--n-attempts", "0"), - ("--max-retries", "-1"), - ("--n-concurrent", "abc"), - ], -) -def test_harbor_job_settings_reject_invalid_values(monkeypatch, flag, value): - monkeypatch.setattr(sys, "argv", ["harbor-ci", flag, value]) - with pytest.raises(SystemExit, match="2"): - run.parse_args() - - -def test_invalid_actions_job_setting_fails_before_run(monkeypatch): - monkeypatch.setenv("HARBOR_CI_N_ATTEMPTS", "0") - monkeypatch.setattr(sys, "argv", ["harbor-ci"]) - with pytest.raises(SystemExit, match="2"): - run.parse_args() - - -def test_harbor_job_settings_reach_command_and_report(monkeypatch, tmp_path): - monkeypatch.setenv("BENCH_MODEL", "test-model") - monkeypatch.setattr( - sys, - "argv", - [ - "harbor-ci", - "--n-concurrent", - "2", - "--n-attempts", - "3", - "--max-retries", - "4", - "--output-dir", - str(tmp_path), - ], - ) - monkeypatch.setattr(run, "git_identity", lambda _: ("a" * 40, False)) - monkeypatch.setattr( - run, - "build_runtime", - lambda *_args, **_kwargs: { - "image": "test-image", - "image_id": "test-id", - "source_sha256": "source", - "build_sha256": "build", - }, - ) - commands = [] - - def fake_subprocess(command, **_kwargs): - commands.append(command) - return subprocess.CompletedProcess(command, 0) - - monkeypatch.setattr(run.subprocess, "run", fake_subprocess) - reports = [] - monkeypatch.setattr( - run, - "summarize", - lambda _path, metadata: {"overall": "PASS", "metadata": metadata}, - ) - - def fake_write_reports(report, report_dir): - reports.append(report) - (report_dir / "summary.md").write_text("PASS\n") - - monkeypatch.setattr(run, "write_reports", fake_write_reports) - - assert run.main() == 0 - command = commands[0] - assert command[command.index("--n-concurrent") + 1] == "2" - assert command[command.index("--n-attempts") + 1] == "3" - assert command[command.index("--max-retries") + 1] == "4" - assert reports[0]["metadata"]["n_concurrent"] == 2 - assert reports[0]["metadata"]["n_attempts"] == 3 - assert reports[0]["metadata"]["max_retries"] == 4 - - -def test_agent_version_cannot_be_overridden_by_agent_kwarg(): - agent = agents.resolve_agent("codex", "0.155.1") - assert run.agent_kwargs(["reasoning_effort=high"], agent) == [ - "reasoning_effort=high", - "version=0.155.1", - ] - with pytest.raises(ValueError, match="managed by --agent-version"): - run.agent_kwargs(["version=latest"], agent) - - -def test_result_summary_reads_native_trial_directories(tmp_path): - job = tmp_path / "job" - trial = job / "trial-one" - trial.mkdir(parents=True) - (job / "result.json").write_text(json.dumps({"n_total_trials": 1})) - (trial / "result.json").write_text( - json.dumps( - { - "task_name": "example/task", - "trial_name": "trial-one", - "exception_info": None, - "verifier_result": {"rewards": {"reward": 1.0}}, - } - ) - ) - report = results.summarize(job, {"dataset": "example@1.0"}) - assert report["overall"] == "PASS" - assert report["trials"][0]["status"] == "PASS" - - -def test_result_summary_distinguishes_failure_and_error(tmp_path): - job = tmp_path / "job" - trial = job / "trial-one" - trial.mkdir(parents=True) - (job / "result.json").write_text(json.dumps({"n_total_trials": 1})) - result = { - "task_name": "example/task", - "trial_name": "trial-one", - "exception_info": None, - "verifier_result": {"rewards": {"reward": 0.0}}, - } - (trial / "result.json").write_text(json.dumps(result)) - assert results.summarize(job, {})["overall"] == "FAIL" - - verifier = trial / "verifier" - verifier.mkdir() - (verifier / "status.json").write_text( - json.dumps( - { - "checks": [ - { - "id": "artifact.required", - "gating": True, - "passed": False, - "actual": {"reason": "missing model.step"}, - } - ] - } - ) - ) - report = results.summarize(job, {}) - assert report["trials"][0]["failed_checks"] == [ - { - "id": "artifact.required", - "gating": True, - "detail": "missing model.step", - } - ] - results.write_reports(report, tmp_path / "reports") - summary = (tmp_path / "reports" / "summary.md").read_text() - assert "failed `artifact.required` (gating): missing model.step" in summary - - result["exception_info"] = {"exception_type": "RuntimeError"} - (trial / "result.json").write_text(json.dumps(result)) - assert results.summarize(job, {})["overall"] == "ERROR" - - -@pytest.mark.parametrize("reward", [True, float("nan"), float("inf"), -0.1, 1.1]) -def test_result_summary_rejects_malformed_binary_rewards(tmp_path, reward): - job = tmp_path / "job" - trial = job / "trial-one" - trial.mkdir(parents=True) - (job / "result.json").write_text(json.dumps({"n_total_trials": 1})) - (trial / "result.json").write_text( - json.dumps( - { - "task_name": "example/task", - "trial_name": "trial-one", - "exception_info": None, - "verifier_result": {"rewards": {"reward": reward}}, - } - ) - ) - assert results.summarize(job, {})["overall"] == "ERROR" diff --git a/tools/__init__.py b/tools/__init__.py deleted file mode 100644 index 98720bff..00000000 --- a/tools/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Repository maintenance tools.""" diff --git a/tools/harbor_ci/__init__.py b/tools/harbor_ci/__init__.py deleted file mode 100644 index fc197607..00000000 --- a/tools/harbor_ci/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Thin Harbor Hub release-check launcher for CADIR CI.""" diff --git a/tools/harbor_ci/agents.py b/tools/harbor_ci/agents.py deleted file mode 100644 index 80179f9a..00000000 --- a/tools/harbor_ci/agents.py +++ /dev/null @@ -1,39 +0,0 @@ -"""Resolve Harbor agent settings to the runtime-specific implementation.""" - -from __future__ import annotations - -from dataclasses import dataclass - - -DEFAULT_AGENT = "codex" -DEFAULT_AGENT_VERSION = "0.155.1" - - -@dataclass(frozen=True) -class AgentConfig: - """Validated agent settings used by Harbor and the runtime image build.""" - - name: str - version: str - dockerfile: str - docker_build_args: tuple[tuple[str, str], ...] - harbor_kwargs: tuple[str, ...] - - -def resolve_agent(name: str, version: str) -> AgentConfig: - """Return the installed adapter or reject an unsupported agent explicitly.""" - name = name.strip() - version = version.strip() - if name != "codex": - raise ValueError( - f"unsupported Harbor agent {name!r}; currently supported: {DEFAULT_AGENT}" - ) - if not version: - raise ValueError("Codex requires --agent-version or HARBOR_CI_AGENT_VERSION") - return AgentConfig( - name=name, - version=version, - dockerfile="runtime.Dockerfile", - docker_build_args=(("CODEX_VERSION", version),), - harbor_kwargs=(f"version={version}",), - ) diff --git a/tools/harbor_ci/results.py b/tools/harbor_ci/results.py deleted file mode 100644 index 8971b575..00000000 --- a/tools/harbor_ci/results.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Interpret native Harbor job results without trusting CLI exit status alone.""" - -from __future__ import annotations - -import json -import math -from pathlib import Path - - -def _failed_verifier_checks(trial_dir: Path) -> list[dict]: - """Read compact failure evidence without copying large verifier payloads.""" - status_path = trial_dir / "verifier" / "status.json" - if not status_path.is_file(): - return [] - try: - status = json.loads(status_path.read_text()) - except (OSError, json.JSONDecodeError): - return [] - if not isinstance(status, dict): - return [] - - failed = [] - for check in status.get("checks") or []: - if not isinstance(check, dict) or check.get("passed") is not False: - continue - detail = check.get("reason") or check.get("message") or check.get("error") - actual = check.get("actual") - if detail is None and isinstance(actual, dict): - detail = ( - actual.get("reason") - or actual.get("message") - or actual.get("error") - ) - failed.append( - { - "id": check.get("id") or "unknown", - "gating": bool(check.get("gating")), - "detail": detail if isinstance(detail, str) else None, - } - ) - return failed - - -def summarize(job_dir: Path, metadata: dict) -> dict: - top_level = job_dir / "result.json" - if not top_level.is_file(): - return { - "overall": "ERROR", - "metadata": metadata, - "trials": [], - "error": "missing result.json", - } - - job = json.loads(top_level.read_text()) - trial_documents = [] - for child in sorted(job_dir.iterdir()): - result = child / "result.json" - if child.is_dir() and result.is_file(): - trial_documents.append( - (json.loads(result.read_text()), _failed_verifier_checks(child)) - ) - - trials = [] - for trial, failed_checks in trial_documents: - exception = trial.get("exception_info") - verifier = trial.get("verifier_result") or {} - rewards = verifier.get("rewards") or {} - reward = rewards.get("reward") - reward_is_valid = ( - not isinstance(reward, bool) - and isinstance(reward, (int, float)) - and math.isfinite(reward) - and 0 <= reward <= 1 - ) - if exception or not reward_is_valid: - status = "ERROR" - elif reward == 1: - status = "PASS" - else: - status = "FAIL" - trials.append( - { - "task": trial.get("task_name"), - "trial": trial.get("trial_name"), - "status": status, - "rewards": rewards, - "exception": exception, - "failed_checks": failed_checks, - } - ) - - expected = job.get("n_total_trials") - if ( - not isinstance(expected, int) - or expected < 1 - or len(trials) != expected - or any(trial["status"] == "ERROR" for trial in trials) - ): - overall = "ERROR" - elif any(trial["status"] == "FAIL" for trial in trials): - overall = "FAIL" - else: - overall = "PASS" - return {"overall": overall, "metadata": metadata, "trials": trials, "job": job} - - -def write_reports(report: dict, report_dir: Path) -> None: - report_dir.mkdir(parents=True, exist_ok=True) - (report_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n") - - lines = ["# CADIR Harbor CI", "", f"Overall: **{report['overall']}**", ""] - for trial in report.get("trials", []): - reward = trial.get("rewards", {}).get("reward", "missing") - lines.append( - f"- `{trial.get('task')}`: **{trial['status']}** (reward={reward})" - ) - if trial["status"] != "PASS" and trial.get("failed_checks"): - for check in trial["failed_checks"]: - qualifier = "gating" if check["gating"] else "non-gating" - detail = f": {check['detail']}" if check.get("detail") else "" - lines.append(f" - failed `{check['id']}` ({qualifier}){detail}") - if report.get("error"): - lines.extend(["", f"Error: {report['error']}"]) - (report_dir / "summary.md").write_text("\n".join(lines) + "\n") diff --git a/tools/harbor_ci/run.py b/tools/harbor_ci/run.py deleted file mode 100644 index f853f583..00000000 --- a/tools/harbor_ci/run.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Run the public CADIR benchmark from Harbor Hub on this runner.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -from datetime import datetime, timezone -from pathlib import Path -from urllib.parse import urlparse - -from .agents import DEFAULT_AGENT, DEFAULT_AGENT_VERSION, AgentConfig, resolve_agent -from .results import summarize, write_reports -from .runtime import build_runtime - -DEFAULT_DATASET = "au12321ua/cadir-ci-benchmark" -DEFAULT_N_CONCURRENT = 1 -DEFAULT_N_ATTEMPTS = 1 -DEFAULT_MAX_RETRIES = 0 - - -def environment_default(name: str, fallback: str) -> str: - """Treat an unset or empty Actions variable as absent.""" - return os.environ.get(name) or fallback - - -def positive_int(value: str) -> int: - number = int(value) - if number < 1: - raise argparse.ArgumentTypeError("must be at least 1") - return number - - -def nonnegative_int(value: str) -> int: - number = int(value) - if number < 0: - raise argparse.ArgumentTypeError("must be at least 0") - return number - - -def git_identity(repo_root: Path) -> tuple[str, bool]: - commit = subprocess.run( - ["git", "rev-parse", "HEAD"], - cwd=repo_root, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - dirty = bool( - subprocess.run( - ["git", "status", "--porcelain", "--untracked-files=normal"], - cwd=repo_root, - check=True, - capture_output=True, - text=True, - ).stdout.strip() - ) - return commit, dirty - - -def endpoint_host(environment: dict[str, str]) -> str | None: - endpoint = environment.get("OPENAI_BASE_URL") or environment.get("OPENAI_API_BASE") - if not endpoint: - return None - return urlparse(endpoint if "://" in endpoint else f"https://{endpoint}").hostname - - -def parse_args() -> argparse.Namespace: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "--dataset", default=environment_default("HARBOR_CI_DATASET", DEFAULT_DATASET) - ) - parser.add_argument( - "--agent", default=environment_default("HARBOR_CI_AGENT", DEFAULT_AGENT) - ) - parser.add_argument( - "--agent-version", - default=environment_default("HARBOR_CI_AGENT_VERSION", DEFAULT_AGENT_VERSION), - ) - parser.add_argument("--model", default=os.environ.get("BENCH_MODEL")) - parser.add_argument( - "--n-concurrent", - type=positive_int, - default=environment_default( - "HARBOR_CI_N_CONCURRENT", str(DEFAULT_N_CONCURRENT) - ), - help="Maximum number of trials running at once (default: 1)", - ) - parser.add_argument( - "--n-attempts", - type=positive_int, - default=environment_default("HARBOR_CI_N_ATTEMPTS", str(DEFAULT_N_ATTEMPTS)), - help="Number of independent attempts per task (default: 1)", - ) - parser.add_argument( - "--max-retries", - type=nonnegative_int, - default=environment_default("HARBOR_CI_MAX_RETRIES", str(DEFAULT_MAX_RETRIES)), - help="Maximum retries after trial exceptions (default: 0)", - ) - parser.add_argument( - "--agent-kwarg", action="append", default=[], metavar="KEY=VALUE" - ) - parser.add_argument("--allow-agent-host", action="append", default=[]) - parser.add_argument("--env-file", type=Path) - parser.add_argument("--job-name") - parser.add_argument("--output-dir", type=Path, default=Path(".harbor-ci")) - parser.add_argument("--require-pass", action="store_true") - parser.add_argument("--cleanup-runtime", action="store_true") - return parser.parse_args() - - -def agent_kwargs(items: list[str], agent: AgentConfig) -> list[str]: - """Validate extra arguments and reserve adapter-owned argument names.""" - validated: list[str] = [] - reserved = {item.partition("=")[0] for item in agent.harbor_kwargs} - for item in items: - key, separator, _ = item.partition("=") - if not separator or not key: - raise ValueError(f"--agent-kwarg must be KEY=VALUE: {item}") - if key in reserved: - raise ValueError(f"--agent-kwarg {key!r} is managed by --agent-version") - validated.append(item) - return [*validated, *agent.harbor_kwargs] - - -def main() -> int: - args = parse_args() - repo_root = Path(__file__).resolve().parents[2] - agent = resolve_agent(args.agent, args.agent_version) - kwargs = agent_kwargs(args.agent_kwarg, agent) - if not args.model: - raise ValueError(f"{agent.name} requires --model or BENCH_MODEL") - - commit, dirty = git_identity(repo_root) - runtime = build_runtime(repo_root, agent=agent) - timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") - short_commit = re.sub(r"[^A-Za-z0-9_.-]", "-", commit[:12]) - job_name = args.job_name or f"cadir-ci-{short_commit}-{timestamp}" - - output_dir = args.output_dir.resolve() - jobs_dir = output_dir / "jobs" - reports_dir = output_dir / "reports" - config_dir = output_dir / "configs" - for directory in (jobs_dir, reports_dir, config_dir): - directory.mkdir(parents=True, exist_ok=True) - - config_path = config_dir / f"{job_name}.json" - config_path.write_text( - json.dumps( - { - "environment": { - "env": { - "CAD_BENCH_AGENT_IMAGE": runtime["image"], - "CAD_BENCH_VERIFIER_IMAGE": runtime["image"], - } - } - }, - indent=2, - ) - + "\n" - ) - - command = [ - "harbor", - "run", - "-c", - str(config_path), - "-d", - args.dataset, - "-a", - agent.name, - "--n-concurrent", - str(args.n_concurrent), - "--n-attempts", - str(args.n_attempts), - "--max-retries", - str(args.max_retries), - "--job-name", - job_name, - "--jobs-dir", - str(jobs_dir), - ] - if args.model: - command.extend(["-m", args.model]) - - for item in kwargs: - command.extend(["--ak", item]) - - if args.env_file: - command.extend(["--env-file", str(args.env_file.resolve())]) - allowed_hosts = list(args.allow_agent_host) - host = endpoint_host(dict(os.environ)) - if host and host != "api.openai.com" and host not in allowed_hosts: - allowed_hosts.append(host) - for host in allowed_hosts: - if not re.fullmatch(r"[A-Za-z0-9.-]+", host): - raise ValueError(f"invalid --allow-agent-host: {host}") - command.extend(["--allow-agent-host", host]) - - harbor_exit = subprocess.run(command, cwd=repo_root, check=False).returncode - metadata = { - "dataset": args.dataset, - "cadir_commit": commit, - "cadir_dirty": dirty, - "runtime_image": runtime["image"], - "runtime_image_id": runtime["image_id"], - "cadir_source_sha256": runtime["source_sha256"], - "runtime_build_sha256": runtime["build_sha256"], - "agent": agent.name, - "agent_version": agent.version, - "model": args.model, - "n_concurrent": args.n_concurrent, - "n_attempts": args.n_attempts, - "max_retries": args.max_retries, - "harbor_command_exit": harbor_exit, - } - report = summarize(jobs_dir / job_name, metadata) - write_reports(report, reports_dir) - summary_path = reports_dir / "summary.md" - print(f"CADIR benchmark: {report['overall']}") - print(f"Summary: {summary_path}") - print(summary_path.read_text(), end="") - if args.cleanup_runtime: - subprocess.run(["docker", "image", "rm", runtime["image"]], check=False) - - if report["overall"] == "ERROR": - return 2 - if report["overall"] == "FAIL" and args.require_pass: - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/harbor_ci/runtime.Dockerfile b/tools/harbor_ci/runtime.Dockerfile deleted file mode 100644 index bf2bf887..00000000 --- a/tools/harbor_ci/runtime.Dockerfile +++ /dev/null @@ -1,36 +0,0 @@ -ARG NODE_IMAGE=node:22-bookworm-slim@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9 -ARG BASE_IMAGE=python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a -ARG CODEX_VERSION - -FROM ${NODE_IMAGE} AS codex-cli -ARG CODEX_VERSION -RUN npm install --global --omit=dev "@openai/codex@${CODEX_VERSION}" \ - && code_mode_host="$(find /usr/local/lib/node_modules/@openai/codex \ - -type f -path '*/vendor/*/bin/codex-code-mode-host' -print -quit)" \ - && test -n "${code_mode_host}" \ - && test "$(codex --version)" = "codex-cli ${CODEX_VERSION}" - -FROM ${BASE_IMAGE} - -ARG CODEX_VERSION -ENV DEBIAN_FRONTEND=noninteractive \ - PATH=/opt/cad-venv/bin:${PATH} \ - PYTHONUNBUFFERED=1 -RUN apt-get update && apt-get install -y --no-install-recommends \ - bash ca-certificates curl git ripgrep \ - libgl1 libglu1-mesa libx11-6 libxext6 libxrender1 libxinerama1 \ - libxrandr2 libxcursor1 libxi6 libsm6 libice6 libfontconfig1 libosmesa6 \ - && rm -rf /var/lib/apt/lists/* -COPY --from=codex-cli /usr/local /usr/local -RUN test "$(codex --version)" = "codex-cli ${CODEX_VERSION}" \ - && code_mode_host="$(find /usr/local/lib/node_modules/@openai/codex \ - -type f -path '*/vendor/*/bin/codex-code-mode-host' -print -quit)" \ - && test -x "${code_mode_host}" -RUN python -m venv /opt/cad-venv -COPY source /opt/simplecadapi -RUN --mount=type=cache,target=/root/.cache/pip \ - python -m pip install /opt/simplecadapi PyYAML==6.0.2 \ - && python -m pip freeze --all > /opt/cad-runtime-pip-freeze.txt -COPY source/docs/skill /skills/simplecadapi -RUN test -x /opt/cad-venv/bin/sca && test -r /skills/simplecadapi/SKILL.md -WORKDIR /workspace diff --git a/tools/harbor_ci/runtime.py b/tools/harbor_ci/runtime.py deleted file mode 100644 index 7dc3c5c3..00000000 --- a/tools/harbor_ci/runtime.py +++ /dev/null @@ -1,133 +0,0 @@ -"""Build a local Harbor runtime from the current CADIR checkout.""" - -from __future__ import annotations - -import hashlib -import os -import shutil -import subprocess -import tempfile -from pathlib import Path - -from .agents import AgentConfig - -SOURCE_ITEMS = ( - "pyproject.toml", - "README.md", - "LICENSE", - "MANIFEST.in", - "skillproj.toml", - "src", - "docs", -) -REQUIRED_SOURCE = ( - "pyproject.toml", - "README.md", - "LICENSE", - "MANIFEST.in", - "skillproj.toml", - "docs/skill/SKILL.md", -) - - -def _reject_links(path: Path) -> None: - for root, directories, files in os.walk(path, followlinks=False): - root_path = Path(root) - for name in [*directories, *files]: - candidate = root_path / name - if candidate.is_symlink(): - raise ValueError( - f"CADIR build input must not contain links: {candidate}" - ) - - -def _tree_digest(path: Path) -> str: - digest = hashlib.sha256() - for item in sorted(path.rglob("*")): - if item.is_file(): - digest.update(item.relative_to(path).as_posix().encode()) - digest.update(b"\0") - digest.update(item.read_bytes()) - digest.update(b"\0") - return digest.hexdigest() - - -def runtime_build_digest( - source_digest: str, - dockerfile: bytes, - agent_name: str, - agent_version: str, -) -> str: - """Identify every input that changes the benchmark runtime image.""" - return hashlib.sha256( - source_digest.encode() - + b"\0" - + dockerfile - + b"\0" - + agent_name.encode() - + b"\0" - + agent_version.encode() - ).hexdigest() - - -def snapshot_source(source: Path, destination: Path) -> str: - source = source.resolve(strict=True) - for relative in REQUIRED_SOURCE: - candidate = source / relative - if not candidate.exists() or candidate.is_symlink(): - raise ValueError(f"required CADIR source is missing or linked: {relative}") - - destination.mkdir(parents=True) - for relative in SOURCE_ITEMS: - candidate = source / relative - if not candidate.exists(): - continue - _reject_links(candidate) - target = destination / relative - if candidate.is_dir(): - shutil.copytree(candidate, target) - else: - target.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(candidate, target) - return _tree_digest(destination) - - -def build_runtime( - repo_root: Path, - *, - agent: AgentConfig, - image_prefix: str = "cadir-ci/runtime", -) -> dict[str, str]: - repo_root = repo_root.resolve(strict=True) - dockerfile = repo_root / "tools/harbor_ci" / agent.dockerfile - if not dockerfile.is_file(): - raise FileNotFoundError(dockerfile) - - with tempfile.TemporaryDirectory(prefix="cadir-harbor-ci-") as temporary: - context = Path(temporary) - source_digest = snapshot_source(repo_root, context / "source") - build_digest = runtime_build_digest( - source_digest, - dockerfile.read_bytes(), - agent.name, - agent.version, - ) - image = f"{image_prefix}:{build_digest[:20]}" - command = ["docker", "build", "-f", str(dockerfile), "-t", image] - for name, value in agent.docker_build_args: - command.extend(["--build-arg", f"{name}={value}"]) - command.append(str(context)) - subprocess.run(command, check=True) - - image_id = subprocess.run( - ["docker", "image", "inspect", image, "--format", "{{.Id}}"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - return { - "image": image, - "image_id": image_id, - "source_sha256": source_digest, - "build_sha256": build_digest, - }