This commit is contained in:
2026-09-04 11:16:23 +08:00
parent 9e27f11bb7
commit 921ba5eab3
36 changed files with 2665 additions and 29 deletions
+27 -3
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse, json
from pathlib import Path
from .describe import describe_samples
from .pipeline import load_samples, run_stage, scan, select_samples
from .reports import generate_markdown_report, generate_reports, read_json
@@ -12,11 +13,20 @@ DEFAULT_OUTPUT = Path("cadfs_to_cdsl/output")
def _parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="Convert CADFS FeatureScript to CDSL and validate against STEP")
commands = parser.add_subparsers(dest="command", required=True)
for name in ("scan", "convert", "rebuild", "compare", "report", "pipeline"):
for name in ("scan", "convert", "rebuild", "compare", "report", "pipeline", "describe"):
command = commands.add_parser(name)
command.add_argument("--input", type=Path, default=DEFAULT_INPUT)
command.add_argument("--output", type=Path, default=DEFAULT_OUTPUT)
if name != "scan":
if name == "describe":
command.add_argument("--shard", help="input shard directory to describe, for example 0005")
command.add_argument("--sample-id", action="append")
command.add_argument("--offset", type=int, default=0)
command.add_argument("--limit", type=int)
command.add_argument("--seed", type=int)
command.add_argument("--workers", type=int, default=1)
command.add_argument("--mode", choices=("local", "hybrid", "vision"), default="hybrid")
command.add_argument("--force", action="store_true")
elif name != "scan":
command.add_argument("--sample-id", action="append")
command.add_argument("--offset", type=int, default=0)
command.add_argument("--limit", type=int)
@@ -31,9 +41,23 @@ def _parser() -> argparse.ArgumentParser:
def main(argv: list[str] | None = None) -> int:
args = _parser().parse_args(argv); args.output.mkdir(parents=True, exist_ok=True)
args = _parser().parse_args(argv)
if args.command != "describe":
args.output.mkdir(parents=True, exist_ok=True)
if args.command == "scan":
records = scan(args.input, args.output); result = {"sample_count": len(records), "output": str(args.output / "dataset_index.json")}
elif args.command == "describe":
records, result = describe_samples(
args.input,
shard=args.shard,
sample_ids=args.sample_id,
mode=args.mode,
offset=args.offset,
limit=args.limit,
seed=args.seed,
force=args.force,
workers=args.workers,
)
elif args.command == "report":
manifest = args.output / "manifest.jsonl"; records = [json.loads(line) for line in manifest.read_text().splitlines() if line.strip()]
result = generate_reports(args.output, records)