Revert "Ci/harbor smoke (#12)"

This reverts commit d9d42a4d73.
This commit is contained in:
auua
2026-09-23 20:27:59 +08:00
parent d9d42a4d73
commit a3fa37e50b
11 changed files with 0 additions and 1013 deletions
-1
View File
@@ -1 +0,0 @@
"""Repository maintenance tools."""
-1
View File
@@ -1 +0,0 @@
"""Thin Harbor Hub release-check launcher for CADIR CI."""
-39
View File
@@ -1,39 +0,0 @@
"""Resolve Harbor agent settings to the runtime-specific implementation."""
from __future__ import annotations
from dataclasses import dataclass
DEFAULT_AGENT = "codex"
DEFAULT_AGENT_VERSION = "0.155.1"
@dataclass(frozen=True)
class AgentConfig:
"""Validated agent settings used by Harbor and the runtime image build."""
name: str
version: str
dockerfile: str
docker_build_args: tuple[tuple[str, str], ...]
harbor_kwargs: tuple[str, ...]
def resolve_agent(name: str, version: str) -> AgentConfig:
"""Return the installed adapter or reject an unsupported agent explicitly."""
name = name.strip()
version = version.strip()
if name != "codex":
raise ValueError(
f"unsupported Harbor agent {name!r}; currently supported: {DEFAULT_AGENT}"
)
if not version:
raise ValueError("Codex requires --agent-version or HARBOR_CI_AGENT_VERSION")
return AgentConfig(
name=name,
version=version,
dockerfile="runtime.Dockerfile",
docker_build_args=(("CODEX_VERSION", version),),
harbor_kwargs=(f"version={version}",),
)
-124
View File
@@ -1,124 +0,0 @@
"""Interpret native Harbor job results without trusting CLI exit status alone."""
from __future__ import annotations
import json
import math
from pathlib import Path
def _failed_verifier_checks(trial_dir: Path) -> list[dict]:
"""Read compact failure evidence without copying large verifier payloads."""
status_path = trial_dir / "verifier" / "status.json"
if not status_path.is_file():
return []
try:
status = json.loads(status_path.read_text())
except (OSError, json.JSONDecodeError):
return []
if not isinstance(status, dict):
return []
failed = []
for check in status.get("checks") or []:
if not isinstance(check, dict) or check.get("passed") is not False:
continue
detail = check.get("reason") or check.get("message") or check.get("error")
actual = check.get("actual")
if detail is None and isinstance(actual, dict):
detail = (
actual.get("reason")
or actual.get("message")
or actual.get("error")
)
failed.append(
{
"id": check.get("id") or "unknown",
"gating": bool(check.get("gating")),
"detail": detail if isinstance(detail, str) else None,
}
)
return failed
def summarize(job_dir: Path, metadata: dict) -> dict:
top_level = job_dir / "result.json"
if not top_level.is_file():
return {
"overall": "ERROR",
"metadata": metadata,
"trials": [],
"error": "missing result.json",
}
job = json.loads(top_level.read_text())
trial_documents = []
for child in sorted(job_dir.iterdir()):
result = child / "result.json"
if child.is_dir() and result.is_file():
trial_documents.append(
(json.loads(result.read_text()), _failed_verifier_checks(child))
)
trials = []
for trial, failed_checks in trial_documents:
exception = trial.get("exception_info")
verifier = trial.get("verifier_result") or {}
rewards = verifier.get("rewards") or {}
reward = rewards.get("reward")
reward_is_valid = (
not isinstance(reward, bool)
and isinstance(reward, (int, float))
and math.isfinite(reward)
and 0 <= reward <= 1
)
if exception or not reward_is_valid:
status = "ERROR"
elif reward == 1:
status = "PASS"
else:
status = "FAIL"
trials.append(
{
"task": trial.get("task_name"),
"trial": trial.get("trial_name"),
"status": status,
"rewards": rewards,
"exception": exception,
"failed_checks": failed_checks,
}
)
expected = job.get("n_total_trials")
if (
not isinstance(expected, int)
or expected < 1
or len(trials) != expected
or any(trial["status"] == "ERROR" for trial in trials)
):
overall = "ERROR"
elif any(trial["status"] == "FAIL" for trial in trials):
overall = "FAIL"
else:
overall = "PASS"
return {"overall": overall, "metadata": metadata, "trials": trials, "job": job}
def write_reports(report: dict, report_dir: Path) -> None:
report_dir.mkdir(parents=True, exist_ok=True)
(report_dir / "report.json").write_text(json.dumps(report, indent=2) + "\n")
lines = ["# CADIR Harbor CI", "", f"Overall: **{report['overall']}**", ""]
for trial in report.get("trials", []):
reward = trial.get("rewards", {}).get("reward", "missing")
lines.append(
f"- `{trial.get('task')}`: **{trial['status']}** (reward={reward})"
)
if trial["status"] != "PASS" and trial.get("failed_checks"):
for check in trial["failed_checks"]:
qualifier = "gating" if check["gating"] else "non-gating"
detail = f": {check['detail']}" if check.get("detail") else ""
lines.append(f" - failed `{check['id']}` ({qualifier}){detail}")
if report.get("error"):
lines.extend(["", f"Error: {report['error']}"])
(report_dir / "summary.md").write_text("\n".join(lines) + "\n")
-237
View File
@@ -1,237 +0,0 @@
"""Run the public CADIR benchmark from Harbor Hub on this runner."""
from __future__ import annotations
import argparse
import json
import os
import re
import subprocess
from datetime import datetime, timezone
from pathlib import Path
from urllib.parse import urlparse
from .agents import DEFAULT_AGENT, DEFAULT_AGENT_VERSION, AgentConfig, resolve_agent
from .results import summarize, write_reports
from .runtime import build_runtime
DEFAULT_DATASET = "au12321ua/cadir-ci-benchmark"
DEFAULT_N_CONCURRENT = 1
DEFAULT_N_ATTEMPTS = 1
DEFAULT_MAX_RETRIES = 0
def environment_default(name: str, fallback: str) -> str:
"""Treat an unset or empty Actions variable as absent."""
return os.environ.get(name) or fallback
def positive_int(value: str) -> int:
number = int(value)
if number < 1:
raise argparse.ArgumentTypeError("must be at least 1")
return number
def nonnegative_int(value: str) -> int:
number = int(value)
if number < 0:
raise argparse.ArgumentTypeError("must be at least 0")
return number
def git_identity(repo_root: Path) -> tuple[str, bool]:
commit = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
dirty = bool(
subprocess.run(
["git", "status", "--porcelain", "--untracked-files=normal"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout.strip()
)
return commit, dirty
def endpoint_host(environment: dict[str, str]) -> str | None:
endpoint = environment.get("OPENAI_BASE_URL") or environment.get("OPENAI_API_BASE")
if not endpoint:
return None
return urlparse(endpoint if "://" in endpoint else f"https://{endpoint}").hostname
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--dataset", default=environment_default("HARBOR_CI_DATASET", DEFAULT_DATASET)
)
parser.add_argument(
"--agent", default=environment_default("HARBOR_CI_AGENT", DEFAULT_AGENT)
)
parser.add_argument(
"--agent-version",
default=environment_default("HARBOR_CI_AGENT_VERSION", DEFAULT_AGENT_VERSION),
)
parser.add_argument("--model", default=os.environ.get("BENCH_MODEL"))
parser.add_argument(
"--n-concurrent",
type=positive_int,
default=environment_default(
"HARBOR_CI_N_CONCURRENT", str(DEFAULT_N_CONCURRENT)
),
help="Maximum number of trials running at once (default: 1)",
)
parser.add_argument(
"--n-attempts",
type=positive_int,
default=environment_default("HARBOR_CI_N_ATTEMPTS", str(DEFAULT_N_ATTEMPTS)),
help="Number of independent attempts per task (default: 1)",
)
parser.add_argument(
"--max-retries",
type=nonnegative_int,
default=environment_default("HARBOR_CI_MAX_RETRIES", str(DEFAULT_MAX_RETRIES)),
help="Maximum retries after trial exceptions (default: 0)",
)
parser.add_argument(
"--agent-kwarg", action="append", default=[], metavar="KEY=VALUE"
)
parser.add_argument("--allow-agent-host", action="append", default=[])
parser.add_argument("--env-file", type=Path)
parser.add_argument("--job-name")
parser.add_argument("--output-dir", type=Path, default=Path(".harbor-ci"))
parser.add_argument("--require-pass", action="store_true")
parser.add_argument("--cleanup-runtime", action="store_true")
return parser.parse_args()
def agent_kwargs(items: list[str], agent: AgentConfig) -> list[str]:
"""Validate extra arguments and reserve adapter-owned argument names."""
validated: list[str] = []
reserved = {item.partition("=")[0] for item in agent.harbor_kwargs}
for item in items:
key, separator, _ = item.partition("=")
if not separator or not key:
raise ValueError(f"--agent-kwarg must be KEY=VALUE: {item}")
if key in reserved:
raise ValueError(f"--agent-kwarg {key!r} is managed by --agent-version")
validated.append(item)
return [*validated, *agent.harbor_kwargs]
def main() -> int:
args = parse_args()
repo_root = Path(__file__).resolve().parents[2]
agent = resolve_agent(args.agent, args.agent_version)
kwargs = agent_kwargs(args.agent_kwarg, agent)
if not args.model:
raise ValueError(f"{agent.name} requires --model or BENCH_MODEL")
commit, dirty = git_identity(repo_root)
runtime = build_runtime(repo_root, agent=agent)
timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
short_commit = re.sub(r"[^A-Za-z0-9_.-]", "-", commit[:12])
job_name = args.job_name or f"cadir-ci-{short_commit}-{timestamp}"
output_dir = args.output_dir.resolve()
jobs_dir = output_dir / "jobs"
reports_dir = output_dir / "reports"
config_dir = output_dir / "configs"
for directory in (jobs_dir, reports_dir, config_dir):
directory.mkdir(parents=True, exist_ok=True)
config_path = config_dir / f"{job_name}.json"
config_path.write_text(
json.dumps(
{
"environment": {
"env": {
"CAD_BENCH_AGENT_IMAGE": runtime["image"],
"CAD_BENCH_VERIFIER_IMAGE": runtime["image"],
}
}
},
indent=2,
)
+ "\n"
)
command = [
"harbor",
"run",
"-c",
str(config_path),
"-d",
args.dataset,
"-a",
agent.name,
"--n-concurrent",
str(args.n_concurrent),
"--n-attempts",
str(args.n_attempts),
"--max-retries",
str(args.max_retries),
"--job-name",
job_name,
"--jobs-dir",
str(jobs_dir),
]
if args.model:
command.extend(["-m", args.model])
for item in kwargs:
command.extend(["--ak", item])
if args.env_file:
command.extend(["--env-file", str(args.env_file.resolve())])
allowed_hosts = list(args.allow_agent_host)
host = endpoint_host(dict(os.environ))
if host and host != "api.openai.com" and host not in allowed_hosts:
allowed_hosts.append(host)
for host in allowed_hosts:
if not re.fullmatch(r"[A-Za-z0-9.-]+", host):
raise ValueError(f"invalid --allow-agent-host: {host}")
command.extend(["--allow-agent-host", host])
harbor_exit = subprocess.run(command, cwd=repo_root, check=False).returncode
metadata = {
"dataset": args.dataset,
"cadir_commit": commit,
"cadir_dirty": dirty,
"runtime_image": runtime["image"],
"runtime_image_id": runtime["image_id"],
"cadir_source_sha256": runtime["source_sha256"],
"runtime_build_sha256": runtime["build_sha256"],
"agent": agent.name,
"agent_version": agent.version,
"model": args.model,
"n_concurrent": args.n_concurrent,
"n_attempts": args.n_attempts,
"max_retries": args.max_retries,
"harbor_command_exit": harbor_exit,
}
report = summarize(jobs_dir / job_name, metadata)
write_reports(report, reports_dir)
summary_path = reports_dir / "summary.md"
print(f"CADIR benchmark: {report['overall']}")
print(f"Summary: {summary_path}")
print(summary_path.read_text(), end="")
if args.cleanup_runtime:
subprocess.run(["docker", "image", "rm", runtime["image"]], check=False)
if report["overall"] == "ERROR":
return 2
if report["overall"] == "FAIL" and args.require_pass:
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
-36
View File
@@ -1,36 +0,0 @@
ARG NODE_IMAGE=node:22-bookworm-slim@sha256:48e4b67d85f87bd551df43704e24d252f56cc5f8e9718841aace50f19948f0f9
ARG BASE_IMAGE=python:3.12-slim@sha256:2c941e860699f878900b0edc2403613c234d4b32eda3cc9fa7036991a2a63c4a
ARG CODEX_VERSION
FROM ${NODE_IMAGE} AS codex-cli
ARG CODEX_VERSION
RUN npm install --global --omit=dev "@openai/codex@${CODEX_VERSION}" \
&& code_mode_host="$(find /usr/local/lib/node_modules/@openai/codex \
-type f -path '*/vendor/*/bin/codex-code-mode-host' -print -quit)" \
&& test -n "${code_mode_host}" \
&& test "$(codex --version)" = "codex-cli ${CODEX_VERSION}"
FROM ${BASE_IMAGE}
ARG CODEX_VERSION
ENV DEBIAN_FRONTEND=noninteractive \
PATH=/opt/cad-venv/bin:${PATH} \
PYTHONUNBUFFERED=1
RUN apt-get update && apt-get install -y --no-install-recommends \
bash ca-certificates curl git ripgrep \
libgl1 libglu1-mesa libx11-6 libxext6 libxrender1 libxinerama1 \
libxrandr2 libxcursor1 libxi6 libsm6 libice6 libfontconfig1 libosmesa6 \
&& rm -rf /var/lib/apt/lists/*
COPY --from=codex-cli /usr/local /usr/local
RUN test "$(codex --version)" = "codex-cli ${CODEX_VERSION}" \
&& code_mode_host="$(find /usr/local/lib/node_modules/@openai/codex \
-type f -path '*/vendor/*/bin/codex-code-mode-host' -print -quit)" \
&& test -x "${code_mode_host}"
RUN python -m venv /opt/cad-venv
COPY source /opt/simplecadapi
RUN --mount=type=cache,target=/root/.cache/pip \
python -m pip install /opt/simplecadapi PyYAML==6.0.2 \
&& python -m pip freeze --all > /opt/cad-runtime-pip-freeze.txt
COPY source/docs/skill /skills/simplecadapi
RUN test -x /opt/cad-venv/bin/sca && test -r /skills/simplecadapi/SKILL.md
WORKDIR /workspace
-133
View File
@@ -1,133 +0,0 @@
"""Build a local Harbor runtime from the current CADIR checkout."""
from __future__ import annotations
import hashlib
import os
import shutil
import subprocess
import tempfile
from pathlib import Path
from .agents import AgentConfig
SOURCE_ITEMS = (
"pyproject.toml",
"README.md",
"LICENSE",
"MANIFEST.in",
"skillproj.toml",
"src",
"docs",
)
REQUIRED_SOURCE = (
"pyproject.toml",
"README.md",
"LICENSE",
"MANIFEST.in",
"skillproj.toml",
"docs/skill/SKILL.md",
)
def _reject_links(path: Path) -> None:
for root, directories, files in os.walk(path, followlinks=False):
root_path = Path(root)
for name in [*directories, *files]:
candidate = root_path / name
if candidate.is_symlink():
raise ValueError(
f"CADIR build input must not contain links: {candidate}"
)
def _tree_digest(path: Path) -> str:
digest = hashlib.sha256()
for item in sorted(path.rglob("*")):
if item.is_file():
digest.update(item.relative_to(path).as_posix().encode())
digest.update(b"\0")
digest.update(item.read_bytes())
digest.update(b"\0")
return digest.hexdigest()
def runtime_build_digest(
source_digest: str,
dockerfile: bytes,
agent_name: str,
agent_version: str,
) -> str:
"""Identify every input that changes the benchmark runtime image."""
return hashlib.sha256(
source_digest.encode()
+ b"\0"
+ dockerfile
+ b"\0"
+ agent_name.encode()
+ b"\0"
+ agent_version.encode()
).hexdigest()
def snapshot_source(source: Path, destination: Path) -> str:
source = source.resolve(strict=True)
for relative in REQUIRED_SOURCE:
candidate = source / relative
if not candidate.exists() or candidate.is_symlink():
raise ValueError(f"required CADIR source is missing or linked: {relative}")
destination.mkdir(parents=True)
for relative in SOURCE_ITEMS:
candidate = source / relative
if not candidate.exists():
continue
_reject_links(candidate)
target = destination / relative
if candidate.is_dir():
shutil.copytree(candidate, target)
else:
target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(candidate, target)
return _tree_digest(destination)
def build_runtime(
repo_root: Path,
*,
agent: AgentConfig,
image_prefix: str = "cadir-ci/runtime",
) -> dict[str, str]:
repo_root = repo_root.resolve(strict=True)
dockerfile = repo_root / "tools/harbor_ci" / agent.dockerfile
if not dockerfile.is_file():
raise FileNotFoundError(dockerfile)
with tempfile.TemporaryDirectory(prefix="cadir-harbor-ci-") as temporary:
context = Path(temporary)
source_digest = snapshot_source(repo_root, context / "source")
build_digest = runtime_build_digest(
source_digest,
dockerfile.read_bytes(),
agent.name,
agent.version,
)
image = f"{image_prefix}:{build_digest[:20]}"
command = ["docker", "build", "-f", str(dockerfile), "-t", image]
for name, value in agent.docker_build_args:
command.extend(["--build-arg", f"{name}={value}"])
command.append(str(context))
subprocess.run(command, check=True)
image_id = subprocess.run(
["docker", "image", "inspect", image, "--format", "{{.Id}}"],
check=True,
capture_output=True,
text=True,
).stdout.strip()
return {
"image": image,
"image_id": image_id,
"source_sha256": source_digest,
"build_sha256": build_digest,
}