Files
cdsl-cad/backend/app/cad_agent/evals/resume_one_step.py
T

91 lines
3.4 KiB
Python

"""Advance one persisted event for a task in an isolated live-evaluation root.
Long real-provider evaluations can outlive a command host's execution window.
This utility takes exactly one event from ``WorkflowCoordinator.run`` and
closes the async generator after that event has been durably handled. Repeated
invocations therefore resume the same task without re-running already
persisted nodes.
"""
from __future__ import annotations
import argparse
import asyncio
from dataclasses import replace
import json
from pathlib import Path
import sys
from typing import Any
from app.cad_agent.application.workflow import ModelIdentity
from app.cad_agent.composition import compose_v3
from app.settings import get_settings
def _arguments() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Advance one event for an isolated live CAD evaluation task.")
parser.add_argument("--report-root", required=True, type=Path)
parser.add_argument("--task-id", required=True)
parser.add_argument("--author-provider")
parser.add_argument("--author-model")
parser.add_argument("--review-provider")
parser.add_argument("--review-model")
return parser.parse_args()
async def _advance(arguments: argparse.Namespace) -> dict[str, Any]:
settings = get_settings()
author_provider, author_model = settings.resolve_model(arguments.author_provider, arguments.author_model)
if arguments.review_provider or arguments.review_model:
review_provider, review_model = settings.resolve_model(arguments.review_provider, arguments.review_model)
else:
review_provider, review_model = settings.resolve_independent_review_model(author_provider, author_model)
report_root = arguments.report_root.resolve()
services = compose_v3(replace(
settings,
task_root=report_root / "artifacts",
conversation_root=report_root / "conversations",
))
before = services.repository.get_state(arguments.task_id)
if before is None:
raise ValueError(f"Unknown task {arguments.task_id!r} in {report_root}")
runner = services.workflow.run(
task_id=arguments.task_id,
author=ModelIdentity(author_provider.id, author_model.id),
reviewer=ModelIdentity(review_provider.id, review_model.id),
)
try:
name, payload = await anext(runner)
except StopAsyncIteration:
name, payload = "workflow_exhausted", {}
finally:
await runner.aclose()
after = services.repository.get_state(arguments.task_id)
return {
"task_id": arguments.task_id,
"event": {"name": name, "payload": payload},
"before": {"phase": before.phase.value, "version": before.version},
"after": {
"phase": after.phase.value if after is not None else "",
"version": after.version if after is not None else -1,
"active_revision": after.active_revision if after is not None else "",
"last_error": after.last_error.value if after is not None and after.last_error else "",
},
}
def main() -> int:
arguments = _arguments()
try:
result = asyncio.run(_advance(arguments))
except BaseException as error:
result = {"error": f"{type(error).__name__}: {str(error)[:1000]}"}
print(json.dumps(result, ensure_ascii=False))
return 2
print(json.dumps(result, ensure_ascii=False))
return 0
if __name__ == "__main__":
sys.exit(main())