优化流程
This commit is contained in:
@@ -0,0 +1,94 @@
|
||||
# Onshape to CDSL
|
||||
|
||||
This package produces an auditable CDSL dataset from official public Onshape
|
||||
Part Studio URL mappings. It never treats a feature tree as sufficient proof:
|
||||
each accepted record has its complete raw API evidence, original AP242 STEP,
|
||||
CDSL-only rebuilt STEP, and an independent strict geometry report.
|
||||
|
||||
```text
|
||||
scan -> select -> download -> convert -> validate -> dataset/<id>.cdsl.json
|
||||
```
|
||||
|
||||
Run from the repository root. The only runtime path setup needed is the engine
|
||||
and this source package; no new packaging tool is introduced.
|
||||
|
||||
```bash
|
||||
export ONSHAPE_ACCESS_KEY='...'
|
||||
export ONSHAPE_SECRET_KEY='...'
|
||||
export PYTHONPATH=backend/engine:onshape_to_cdsl/src
|
||||
|
||||
python -m onshape_to_cdsl scan --url-file /path/to/objects.yml --limit 500 --workers 1
|
||||
python -m onshape_to_cdsl select --count 100 --seed 0
|
||||
python -m onshape_to_cdsl download
|
||||
python -m onshape_to_cdsl convert
|
||||
python -m onshape_to_cdsl validate
|
||||
```
|
||||
|
||||
To find more than one URL range yields, scan non-overlapping batches and merge
|
||||
them before deterministic selection:
|
||||
|
||||
```bash
|
||||
python -m onshape_to_cdsl scan --url-file objects_0000.yml --offset 0 --limit 500 --out data/scan/00000.json
|
||||
python -m onshape_to_cdsl scan --url-file objects_0000.yml --offset 500 --limit 500 --out data/scan/00500.json
|
||||
python -m onshape_to_cdsl merge-scans --scan data/scan/00000.json --scan data/scan/00500.json --out data/scan/merged.json
|
||||
python -m onshape_to_cdsl select --scan data/scan/merged.json --count 100 --seed 0
|
||||
```
|
||||
|
||||
For hosts that limit foreground process duration, download the deterministic
|
||||
selection in resumable batches. Existing artifacts are reused by SHA-256:
|
||||
|
||||
```bash
|
||||
python -m onshape_to_cdsl download --offset 0 --limit 3
|
||||
python -m onshape_to_cdsl download --offset 3 --limit 3
|
||||
```
|
||||
|
||||
Or run all stages in sequence:
|
||||
|
||||
```bash
|
||||
python -m onshape_to_cdsl run --url-file /path/to/objects.yml --scan-limit 500 --scan-workers 1 --count 100 --seed 0
|
||||
```
|
||||
|
||||
`--url-file` accepts `id URL`, `id: URL`, and the same-line ID/URL records in
|
||||
ABC `objects_*.yml`. URLs are deliberately not hard-coded in the repository.
|
||||
Credentials are read from `ONSHAPE_ACCESS_KEY`/`ONSHAPE_SECRET_KEY` or hidden
|
||||
terminal prompts, and are never written to files or logs.
|
||||
|
||||
The official ABC mapping archive is published by NYU as
|
||||
`abc_objects_00-49.7z` and `abc_objects_50-99.7z`. Each contained
|
||||
`objects_*.yml` file has 10,000 URLs. Start with `--scan-limit 500`; the
|
||||
pipeline stratifies those API-inspected candidates and selects 100 complete
|
||||
downloads. The client honors `429 Retry-After` responses; use one scan worker
|
||||
unless the API quota explicitly permits higher concurrency.
|
||||
|
||||
All generated data lives in ignored `onshape_to_cdsl/data/`:
|
||||
|
||||
```text
|
||||
scan/manifest.json feature classifications for all source URLs
|
||||
selection/manifest.json deterministic, stratified selected candidates
|
||||
raw/<id>/ complete public API responses, meshes, STEP and hashes
|
||||
converted/<id>/ candidate CDSL, rebuild STEP, conversion/validation reports
|
||||
dataset/<id>.cdsl.json only strict-validation accepted CDSL
|
||||
rejected/<id>/ candidate/rebuild/report evidence for every failure
|
||||
reconstruction_issues.md generated rejection and engine capability work queue
|
||||
reconstruction_issues.json machine-readable version of the same register
|
||||
```
|
||||
|
||||
The v1 executable converter intentionally supports only solved `newSketch`
|
||||
geometry composed of lines, arcs and circles plus standard, one-direction,
|
||||
no-draft `BLIND` extrudes (`NEW`, `ADD`, `REMOVE`). The registry audits all
|
||||
other feature types. Unsupported standard operations are `deferred`; custom
|
||||
FeatureScript, imports and external/derived dependencies are
|
||||
`not_admissible`. Their raw feature trees, FeatureScript payload, B-rep and
|
||||
STEP remain in `raw/`, but they cannot enter `dataset/`.
|
||||
|
||||
Admission uses a separate comparator, not the engine's preview comparator.
|
||||
It requires bidirectional B-rep surface sample maximum and P99 distances at
|
||||
most `0.01 mm`, six-coordinate bounding-box error at most `0.01 mm`, volume
|
||||
and surface-area relative error at most `1e-5`, and equal solid counts.
|
||||
Topology counts/types are report-only diagnostics because kernel exports may
|
||||
split equivalent faces differently.
|
||||
|
||||
`validate` regenerates `reconstruction_issues.md` on every run. It classifies
|
||||
each failure as a source dependency limitation, converter gap, engine
|
||||
capability gap, engine execution failure, or strict geometric regression, and
|
||||
adds engine-specific blockers to an explicit implementation work queue.
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Download AP242 STEP files for public Onshape Part Studios only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import getpass
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import asdict, dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
API_VERSION = "v17"
|
||||
DEFAULT_OUTPUT = Path("onshape_to_cdsl/input/raw")
|
||||
DEFAULT_SAMPLES = (
|
||||
(
|
||||
"00000001",
|
||||
"https://cad.onshape.com/documents/1ffb81a71e5b402e966b9341/"
|
||||
"w/6e295017d1b34be684565c40/e/8df255ee6705423d8e85234e",
|
||||
),
|
||||
(
|
||||
"00000002",
|
||||
"https://cad.onshape.com/documents/1ffb81a71e5b402e966b9341/"
|
||||
"w/6e295017d1b34be684565c40/e/bb398e4615fe4025b34ea8f0",
|
||||
),
|
||||
)
|
||||
ONSHAPE_URL_RE = re.compile(
|
||||
r"^https://(?P<stack>[^/]+)/documents/(?P<did>[^/]+)/"
|
||||
r"(?P<wvm>w|v|m)/(?P<wvmid>[^/]+)/e/(?P<eid>[^/?#]+)"
|
||||
)
|
||||
JSON_ACCEPT = "application/json;charset=UTF-8; qs=0.09"
|
||||
STEP_ACCEPT = "application/step, application/octet-stream"
|
||||
USER_AGENT = "cdsl-cad-onshape-step-downloader/1.0"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PartStudioRef:
|
||||
sample_id: str
|
||||
source_url: str
|
||||
stack: str
|
||||
did: str
|
||||
wvm: str
|
||||
wvmid: str
|
||||
eid: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepResult:
|
||||
status: str
|
||||
path: str | None = None
|
||||
bytes: int | None = None
|
||||
sha256: str | None = None
|
||||
translation_id: str | None = None
|
||||
http_status: int | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def parse_part_studio_url(sample_id: str, url: str) -> PartStudioRef:
|
||||
match = ONSHAPE_URL_RE.match(url.strip())
|
||||
if match is None:
|
||||
raise ValueError(f"not an Onshape Part Studio URL: {url}")
|
||||
return PartStudioRef(sample_id=sample_id, source_url=url.strip(), **match.groupdict())
|
||||
|
||||
|
||||
def read_url_file(path: Path) -> list[tuple[str, str]]:
|
||||
"""Read ABC objects YAML lines or plain '<id> <url>' records."""
|
||||
records: list[tuple[str, str]] = []
|
||||
url_pattern = re.compile(r"https://cad\.onshape\.com/documents/[^'\"\s]+")
|
||||
id_pattern = re.compile(r"^\s*['\"]?([A-Za-z0-9_-]+)['\"]?\s*[:\s]")
|
||||
for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
url_match = url_pattern.search(line)
|
||||
if url_match is None:
|
||||
continue
|
||||
id_match = id_pattern.search(line)
|
||||
if id_match is None:
|
||||
raise ValueError(f"{path}:{line_number}: URL has no leading sample ID")
|
||||
records.append((id_match.group(1), url_match.group(0)))
|
||||
if not records:
|
||||
raise ValueError(f"no Onshape Part Studio URLs found in {path}")
|
||||
return records
|
||||
|
||||
|
||||
def credentials() -> tuple[str, str]:
|
||||
access_key = os.environ.get("ONSHAPE_ACCESS_KEY") or getpass.getpass("Onshape access key: ")
|
||||
secret_key = os.environ.get("ONSHAPE_SECRET_KEY") or getpass.getpass("Onshape secret key: ")
|
||||
if not access_key or not secret_key:
|
||||
raise ValueError("both Onshape access and secret keys are required")
|
||||
return access_key, secret_key
|
||||
|
||||
|
||||
def ca_bundle() -> str | None:
|
||||
try:
|
||||
import certifi
|
||||
except ImportError:
|
||||
return None
|
||||
return certifi.where()
|
||||
|
||||
|
||||
class OnshapeClient:
|
||||
def __init__(self, ref: PartStudioRef, authorization: str, timeout: float) -> None:
|
||||
self.ref = ref
|
||||
self.authorization = authorization
|
||||
self.timeout = timeout
|
||||
self.context = ssl.create_default_context(cafile=ca_bundle())
|
||||
|
||||
def url(self, path: str) -> str:
|
||||
return f"https://{self.ref.stack}/api/{API_VERSION}{path}"
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
path: str,
|
||||
*,
|
||||
body: dict[str, Any] | None = None,
|
||||
accept: str = JSON_ACCEPT,
|
||||
) -> bytes:
|
||||
payload = None if body is None else json.dumps(body).encode("utf-8")
|
||||
request = urllib.request.Request(
|
||||
self.url(path),
|
||||
data=payload,
|
||||
method=method,
|
||||
headers={
|
||||
"Accept": accept,
|
||||
"Authorization": self.authorization,
|
||||
"Content-Type": JSON_ACCEPT,
|
||||
"User-Agent": USER_AGENT,
|
||||
},
|
||||
)
|
||||
with urllib.request.urlopen(request, timeout=self.timeout, context=self.context) as response:
|
||||
return response.read()
|
||||
|
||||
def get_json(self, path: str) -> Any:
|
||||
return json.loads(self.request("GET", path).decode("utf-8"))
|
||||
|
||||
|
||||
def sha256(payload: bytes) -> str:
|
||||
return hashlib.sha256(payload).hexdigest()
|
||||
|
||||
|
||||
def export_step(
|
||||
client: OnshapeClient,
|
||||
root: Path,
|
||||
*,
|
||||
poll_seconds: float,
|
||||
poll_limit: int,
|
||||
) -> StepResult:
|
||||
ref = client.ref
|
||||
export_path = f"/partstudios/d/{ref.did}/{ref.wvm}/{ref.wvmid}/e/{ref.eid}/export/step"
|
||||
body = {
|
||||
"destinationName": f"{ref.sample_id}.step",
|
||||
"grouping": True,
|
||||
"notifyUser": False,
|
||||
"storeInDocument": False,
|
||||
"triggerAutoDownload": False,
|
||||
"stepUnit": "METER",
|
||||
"stepVersionString": "AP242",
|
||||
}
|
||||
try:
|
||||
translation = json.loads(client.request("POST", export_path, body=body).decode("utf-8"))
|
||||
translation_id = translation.get("id")
|
||||
if not isinstance(translation_id, str) or not translation_id:
|
||||
return StepResult("error", message="STEP export response has no translation id")
|
||||
translation_path = f"/translations/{urllib.parse.quote(translation_id, safe='')}"
|
||||
for _ in range(poll_limit):
|
||||
state = client.get_json(translation_path)
|
||||
request_state = state.get("requestState") if isinstance(state, dict) else None
|
||||
if request_state == "DONE":
|
||||
external_ids = state.get("resultExternalDataIds", [])
|
||||
if not isinstance(external_ids, list) or not external_ids or not isinstance(external_ids[0], str):
|
||||
return StepResult("error", translation_id=translation_id, message="completed STEP export has no external data id")
|
||||
payload = client.request(
|
||||
"GET",
|
||||
f"/documents/d/{ref.did}/externaldata/{urllib.parse.quote(external_ids[0], safe='')}",
|
||||
accept=STEP_ACCEPT,
|
||||
)
|
||||
if not payload.startswith(b"ISO-10303-21"):
|
||||
return StepResult("invalid_step", translation_id=translation_id, message="download is empty or lacks the ISO-10303-21 STEP header")
|
||||
destination = root / "model.step"
|
||||
destination.write_bytes(payload)
|
||||
return StepResult(
|
||||
"downloaded",
|
||||
path=destination.name,
|
||||
bytes=len(payload),
|
||||
sha256=sha256(payload),
|
||||
translation_id=translation_id,
|
||||
)
|
||||
if request_state == "FAILED":
|
||||
return StepResult("translation_failed", translation_id=translation_id, message=str(state.get("failureReason") or "translation failed"))
|
||||
time.sleep(poll_seconds)
|
||||
return StepResult("translation_timeout", translation_id=translation_id, message=f"STEP translation did not finish after {poll_limit} polls")
|
||||
except urllib.error.HTTPError as exc:
|
||||
return StepResult("http_error", http_status=exc.code, message=exc.read().decode("utf-8", "replace")[:1000])
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError, json.JSONDecodeError) as exc:
|
||||
return StepResult("error", message=str(exc))
|
||||
|
||||
|
||||
def download_one(ref: PartStudioRef, authorization: str, output: Path, *, timeout: float, poll_seconds: float, poll_limit: int) -> dict[str, Any]:
|
||||
root = output / ref.sample_id
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
result = export_step(OnshapeClient(ref, authorization, timeout), root, poll_seconds=poll_seconds, poll_limit=poll_limit)
|
||||
manifest = {
|
||||
"schema": "onshape.step_sample.v1",
|
||||
"source": asdict(ref),
|
||||
"artifact": {"name": "step", **asdict(result)},
|
||||
}
|
||||
(root / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
|
||||
return {"sample_id": ref.sample_id, **asdict(result)}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--url", action="append", default=[], metavar="ID=URL", help="Onshape Part Studio URL; repeatable")
|
||||
parser.add_argument("--url-file", type=Path, help="ABC objects YAML or text file containing ID and URL records")
|
||||
parser.add_argument(
|
||||
"--count",
|
||||
type=int,
|
||||
help="maximum samples to download (defaults to the two built-in samples; custom URLs are appended)",
|
||||
)
|
||||
parser.add_argument("--out", type=Path, default=DEFAULT_OUTPUT, help="output directory")
|
||||
parser.add_argument("--timeout", type=float, default=60.0, help="per-request timeout in seconds")
|
||||
parser.add_argument("--poll-seconds", type=float, default=3.0, help="STEP translation poll interval")
|
||||
parser.add_argument("--poll-limit", type=int, default=40, help="maximum STEP translation polls")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = build_parser().parse_args(argv)
|
||||
if (args.count is not None and args.count < 1) or args.poll_seconds <= 0 or args.poll_limit < 1:
|
||||
raise ValueError("count and poll limit must be positive; poll interval must be greater than zero")
|
||||
records: list[tuple[str, str]] = list(DEFAULT_SAMPLES)
|
||||
for value in args.url:
|
||||
if "=" not in value:
|
||||
raise ValueError("--url must use ID=URL syntax")
|
||||
records.append(tuple(value.split("=", 1)))
|
||||
if args.url_file:
|
||||
records.extend(read_url_file(args.url_file))
|
||||
limit = args.count if args.count is not None else len(records)
|
||||
if limit < 1:
|
||||
raise ValueError("count must be positive")
|
||||
refs = [parse_part_studio_url(sample_id, url) for sample_id, url in records[:limit]]
|
||||
access_key, secret_key = credentials()
|
||||
authorization = "Basic " + base64.b64encode(f"{access_key}:{secret_key}".encode("utf-8")).decode("ascii")
|
||||
results = [download_one(ref, authorization, args.out, timeout=args.timeout, poll_seconds=args.poll_seconds, poll_limit=args.poll_limit) for ref in refs]
|
||||
summary = {
|
||||
"schema": "onshape.step_download_summary.v1",
|
||||
"requested_count": len(refs),
|
||||
"downloaded_count": sum(item["status"] == "downloaded" for item in results),
|
||||
"failed_count": sum(item["status"] != "downloaded" for item in results),
|
||||
"results": results,
|
||||
}
|
||||
args.out.mkdir(parents=True, exist_ok=True)
|
||||
(args.out / "manifest.json").write_text(json.dumps(summary, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(summary, ensure_ascii=True, indent=2))
|
||||
return 0 if summary["failed_count"] == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except (OSError, ValueError) as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
raise SystemExit(2)
|
||||
@@ -0,0 +1,3 @@
|
||||
"""Reproducible Onshape Part Studio to CDSL dataset pipeline."""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,4 @@
|
||||
from .cli import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,161 @@
|
||||
"""Command line interface for the Onshape to CDSL dataset pipeline."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .convert import convert
|
||||
from .download import download
|
||||
from .manifests import output_is_current, read_json, sha256_file
|
||||
from .merge_scans import merge_scans
|
||||
from .scan import scan
|
||||
from .select import select
|
||||
from .validate import validate
|
||||
|
||||
|
||||
DEFAULT_DATA = Path("onshape_to_cdsl/data")
|
||||
|
||||
|
||||
def _data_path(data: Path, value: str) -> Path:
|
||||
return data / value
|
||||
|
||||
|
||||
def _cached_scan(path: Path, url_file: Path) -> dict | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
value = read_json(path)
|
||||
return value if value.get("url_file_sha256") == sha256_file(url_file) else None
|
||||
|
||||
|
||||
def _cached_selection(path: Path, scan_path: Path, count: int, seed: int) -> dict | None:
|
||||
if not path.exists():
|
||||
return None
|
||||
value = read_json(path)
|
||||
return value if value.get("scan_manifest_sha256") == sha256_file(scan_path) and value.get("requested_count") == count and value.get("seed") == seed else None
|
||||
|
||||
|
||||
def _cached_download(raw_root: Path, selection_path: Path) -> dict | None:
|
||||
path = raw_root / "manifest.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
value = read_json(path)
|
||||
if value.get("selection_sha256") != sha256_file(selection_path):
|
||||
return None
|
||||
for record in value.get("records", []):
|
||||
sample = raw_root / str(record.get("sample_id"))
|
||||
if not (sample / "model.step").is_file():
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _cached_conversion(converted_root: Path) -> dict | None:
|
||||
path = converted_root / "manifest.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
value = read_json(path)
|
||||
for record in value.get("records", []):
|
||||
if record.get("status") == "converted":
|
||||
candidate = converted_root / str(record.get("sample_id")) / "candidate.cdsl.json"
|
||||
if not output_is_current(candidate, record.get("cdsl_sha256")):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _cached_validation(converted_root: Path, dataset_root: Path) -> dict | None:
|
||||
path = converted_root / "validation-manifest.json"
|
||||
if not path.exists():
|
||||
return None
|
||||
value = read_json(path)
|
||||
for record in value.get("records", []):
|
||||
if record.get("status") == "accepted" and not output_is_current(Path(record["dataset_cdsl"]), record.get("dataset_cdsl_sha256")):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Resumable Onshape Part Studio to validated CDSL pipeline")
|
||||
parser.add_argument("--data-root", type=Path, default=DEFAULT_DATA, help="ignored local output root (default: onshape_to_cdsl/data)")
|
||||
commands = parser.add_subparsers(dest="command", required=True)
|
||||
scan_parser = commands.add_parser("scan", help="download feature trees only and classify candidates")
|
||||
scan_parser.add_argument("--url-file", required=True, type=Path)
|
||||
scan_parser.add_argument("--out", type=Path)
|
||||
scan_parser.add_argument("--timeout", type=float, default=60.0)
|
||||
scan_parser.add_argument("--limit", type=int, help="scan only the first N official URLs")
|
||||
scan_parser.add_argument("--offset", type=int, default=0, help="zero-based source URL offset")
|
||||
scan_parser.add_argument("--workers", type=int, default=1, help="bounded parallel API requests (default: 1; safest for API quota)")
|
||||
merge_parser = commands.add_parser("merge-scans", help="combine completed scan batches before selection")
|
||||
merge_parser.add_argument("--scan", action="append", required=True, type=Path, help="scan manifest; repeatable")
|
||||
merge_parser.add_argument("--out", required=True, type=Path)
|
||||
select_parser = commands.add_parser("select", help="deterministically stratify scan results")
|
||||
select_parser.add_argument("--scan", type=Path)
|
||||
select_parser.add_argument("--out", type=Path)
|
||||
select_parser.add_argument("--count", type=int, default=100)
|
||||
select_parser.add_argument("--seed", type=int, default=0)
|
||||
download_parser = commands.add_parser("download", help="download all public API evidence and original STEP")
|
||||
download_parser.add_argument("--selection", type=Path)
|
||||
download_parser.add_argument("--raw-root", type=Path)
|
||||
download_parser.add_argument("--timeout", type=float, default=60.0)
|
||||
download_parser.add_argument("--poll-seconds", type=float, default=3.0)
|
||||
download_parser.add_argument("--poll-limit", type=int, default=40)
|
||||
download_parser.add_argument("--offset", type=int, default=0, help="zero-based position in selection")
|
||||
download_parser.add_argument("--limit", type=int, help="download at most N selected samples")
|
||||
convert_parser = commands.add_parser("convert", help="convert locally executable histories to CDSL")
|
||||
convert_parser.add_argument("--raw-root", type=Path)
|
||||
convert_parser.add_argument("--converted-root", type=Path)
|
||||
validate_parser = commands.add_parser("validate", help="CDSL-only rebuild and strict STEP admission")
|
||||
validate_parser.add_argument("--converted-root", type=Path)
|
||||
validate_parser.add_argument("--dataset-root", type=Path)
|
||||
validate_parser.add_argument("--rejected-root", type=Path)
|
||||
run_parser = commands.add_parser("run", help="execute scan, select, download, convert, and validate")
|
||||
run_parser.add_argument("--url-file", required=True, type=Path)
|
||||
run_parser.add_argument("--count", type=int, default=100)
|
||||
run_parser.add_argument("--seed", type=int, default=0)
|
||||
run_parser.add_argument("--timeout", type=float, default=60.0)
|
||||
run_parser.add_argument("--poll-seconds", type=float, default=3.0)
|
||||
run_parser.add_argument("--poll-limit", type=int, default=40)
|
||||
run_parser.add_argument("--scan-limit", type=int, default=500, help="number of official URLs to scan before stratifying (default: 500)")
|
||||
run_parser.add_argument("--scan-workers", type=int, default=1, help="bounded parallel scan requests (default: 1; safest for API quota)")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
data = args.data_root
|
||||
scan_path, selection_path, raw_root, converted_root = _data_path(data, "scan/manifest.json"), _data_path(data, "selection/manifest.json"), _data_path(data, "raw"), _data_path(data, "converted")
|
||||
if args.command == "scan":
|
||||
result = scan(args.url_file, args.out or scan_path, timeout=args.timeout, limit=args.limit, offset=args.offset, workers=args.workers)
|
||||
elif args.command == "merge-scans":
|
||||
result = merge_scans(args.scan, args.out)
|
||||
elif args.command == "select":
|
||||
if args.count < 1:
|
||||
raise ValueError("--count must be positive")
|
||||
result = select(args.scan or scan_path, args.out or selection_path, count=args.count, seed=args.seed)
|
||||
elif args.command == "download":
|
||||
result = download(args.selection or selection_path, args.raw_root or raw_root, timeout=args.timeout, poll_seconds=args.poll_seconds, poll_limit=args.poll_limit, offset=args.offset, limit=args.limit)
|
||||
elif args.command == "convert":
|
||||
result = convert(args.raw_root or raw_root, args.converted_root or converted_root)
|
||||
elif args.command == "validate":
|
||||
result = validate(args.converted_root or converted_root, args.dataset_root or _data_path(data, "dataset"), args.rejected_root or _data_path(data, "rejected"))
|
||||
else:
|
||||
# A stage is reused only when its manifest proves that the material it
|
||||
# consumed/produced still has the recorded hash. This keeps `run`
|
||||
# restartable after an interrupted 100-sample network acquisition.
|
||||
cached_scan = _cached_scan(scan_path, args.url_file)
|
||||
if cached_scan and (cached_scan.get("scan_limit") != args.scan_limit or cached_scan.get("scan_workers") != args.scan_workers):
|
||||
cached_scan = None
|
||||
result = {"scan": cached_scan or scan(args.url_file, scan_path, timeout=args.timeout, limit=args.scan_limit, workers=args.scan_workers)}
|
||||
result["selection"] = _cached_selection(selection_path, scan_path, args.count, args.seed) or select(scan_path, selection_path, count=args.count, seed=args.seed)
|
||||
result["download"] = _cached_download(raw_root, selection_path) or download(selection_path, raw_root, timeout=args.timeout, poll_seconds=args.poll_seconds, poll_limit=args.poll_limit)
|
||||
result["convert"] = _cached_conversion(converted_root) or convert(raw_root, converted_root)
|
||||
dataset_root = _data_path(data, "dataset")
|
||||
result["validate"] = _cached_validation(converted_root, dataset_root) or validate(converted_root, dataset_root, _data_path(data, "rejected"))
|
||||
# Detailed reports remain on disk. Do not flood a terminal with runtime
|
||||
# topology evidence for every selected Part Studio.
|
||||
if args.command == "run":
|
||||
printable = {stage: {key: value for key, value in report.items() if key in {"schema", "sample_count", "selected_count", "converted_count", "accepted_count", "rejected_count"}} for stage, report in result.items()}
|
||||
else:
|
||||
printable = {key: value for key, value in result.items() if key in {"schema", "sample_count", "selected_count", "converted_count", "accepted_count", "rejected_count"}}
|
||||
print(json.dumps(printable, ensure_ascii=True, indent=2, sort_keys=True))
|
||||
return 0
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Independent strict STEP comparator used for dataset admission only."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
|
||||
TOLERANCE_MM = 0.01
|
||||
RELATIVE_TOLERANCE = 1e-5
|
||||
|
||||
|
||||
def _point(value: Any) -> tuple[float, float, float]:
|
||||
return (float(value.X), float(value.Y), float(value.Z))
|
||||
|
||||
|
||||
def _surface_points(shape: Any, tolerance_mm: float) -> list[tuple[float, float, float]]:
|
||||
"""Deterministically sample vertices and triangle centroids on every face."""
|
||||
points: dict[tuple[int, int, int], tuple[float, float, float]] = {}
|
||||
def add(point: tuple[float, float, float]) -> None:
|
||||
points.setdefault(tuple(round(value / 1e-8) for value in point), point)
|
||||
for face in shape.faces():
|
||||
try:
|
||||
vertices, triangles = face.tessellate(tolerance_mm)
|
||||
rendered = [_point(vertex) for vertex in vertices]
|
||||
for vertex in rendered:
|
||||
add(vertex)
|
||||
for triangle in triangles:
|
||||
indices = list(triangle)
|
||||
if len(indices) >= 3:
|
||||
first, second, third = (rendered[int(index)] for index in indices[:3])
|
||||
add(tuple((first[axis] + second[axis] + third[axis]) / 3.0 for axis in range(3)))
|
||||
except Exception:
|
||||
# A malformed individual face should not make the comparison pass.
|
||||
for vertex in face.vertices():
|
||||
add(_point(vertex))
|
||||
if not points:
|
||||
raise ValueError("STEP contains no sampleable faces")
|
||||
return list(points.values())
|
||||
|
||||
|
||||
def _distances(points: Iterable[tuple[float, float, float]], target: Any) -> list[float]:
|
||||
from OCP.BRepBuilderAPI import BRepBuilderAPI_MakeVertex
|
||||
from OCP.BRepExtrema import BRepExtrema_DistShapeShape
|
||||
from OCP.gp import gp_Pnt
|
||||
values: list[float] = []
|
||||
for x, y, z in points:
|
||||
vertex = BRepBuilderAPI_MakeVertex(gp_Pnt(x, y, z)).Vertex()
|
||||
distance = BRepExtrema_DistShapeShape()
|
||||
distance.LoadS1(vertex)
|
||||
distance.LoadS2(target.wrapped)
|
||||
distance.Perform()
|
||||
if not distance.IsDone() or distance.NbSolution() < 1:
|
||||
raise ValueError("OCC point-to-B-rep distance computation failed")
|
||||
values.append(float(distance.Value()))
|
||||
return values
|
||||
|
||||
|
||||
def _percentile(values: list[float], percentile: float) -> float:
|
||||
if not values:
|
||||
return math.inf
|
||||
ordered = sorted(values)
|
||||
index = min(len(ordered) - 1, math.ceil(percentile * len(ordered)) - 1)
|
||||
return ordered[index]
|
||||
|
||||
|
||||
def _bbox(shape: Any) -> list[float]:
|
||||
box = shape.bounding_box()
|
||||
return [float(box.min.X), float(box.min.Y), float(box.min.Z), float(box.max.X), float(box.max.Y), float(box.max.Z)]
|
||||
|
||||
|
||||
def strict_compare(gold_step: Path, rebuilt_step: Path, *, surface_tolerance_mm: float = 0.05) -> dict[str, Any]:
|
||||
"""Compare source and rebuilt B-reps without the engine's permissive rules."""
|
||||
from build123d import import_step
|
||||
gold, rebuilt = import_step(str(gold_step)), import_step(str(rebuilt_step))
|
||||
gold_samples = _surface_points(gold, surface_tolerance_mm)
|
||||
rebuilt_samples = _surface_points(rebuilt, surface_tolerance_mm)
|
||||
forward, reverse = _distances(gold_samples, rebuilt), _distances(rebuilt_samples, gold)
|
||||
gold_box, rebuilt_box = _bbox(gold), _bbox(rebuilt)
|
||||
bbox_delta = max(abs(a - b) for a, b in zip(gold_box, rebuilt_box))
|
||||
volume_error = abs(float(gold.volume) - float(rebuilt.volume)) / max(abs(float(gold.volume)), 1e-12)
|
||||
area_error = abs(float(gold.area) - float(rebuilt.area)) / max(abs(float(gold.area)), 1e-12)
|
||||
forward_max, reverse_max = max(forward), max(reverse)
|
||||
forward_p99, reverse_p99 = _percentile(forward, 0.99), _percentile(reverse, 0.99)
|
||||
reasons = []
|
||||
if max(forward_max, reverse_max) > TOLERANCE_MM:
|
||||
reasons.append("surface_max_exceeds_0.01mm")
|
||||
if max(forward_p99, reverse_p99) > TOLERANCE_MM:
|
||||
reasons.append("surface_p99_exceeds_0.01mm")
|
||||
if bbox_delta > TOLERANCE_MM:
|
||||
reasons.append("bbox_exceeds_0.01mm")
|
||||
if volume_error > RELATIVE_TOLERANCE:
|
||||
reasons.append("volume_relative_error_exceeds_1e-5")
|
||||
if area_error > RELATIVE_TOLERANCE:
|
||||
reasons.append("surface_area_relative_error_exceeds_1e-5")
|
||||
gold_solids, rebuilt_solids = len(gold.solids()), len(rebuilt.solids())
|
||||
if gold_solids != rebuilt_solids:
|
||||
reasons.append("solid_count_mismatch")
|
||||
return {
|
||||
"schema": "onshape_to_cdsl.strict_step_compare.v1", "tolerance_mm": TOLERANCE_MM, "surface_tessellation_tolerance_mm": surface_tolerance_mm,
|
||||
"gold_step": str(gold_step), "rebuilt_step": str(rebuilt_step), "passed": not reasons, "failure_reasons": reasons,
|
||||
"surface": {"gold_to_rebuilt": {"sample_count": len(forward), "max_mm": forward_max, "p95_mm": _percentile(forward, .95), "p99_mm": forward_p99, "over_tolerance_count": sum(value > TOLERANCE_MM for value in forward)}, "rebuilt_to_gold": {"sample_count": len(reverse), "max_mm": reverse_max, "p95_mm": _percentile(reverse, .95), "p99_mm": reverse_p99, "over_tolerance_count": sum(value > TOLERANCE_MM for value in reverse)}},
|
||||
"metrics": {"gold_bbox_mm": gold_box, "rebuilt_bbox_mm": rebuilt_box, "bbox_max_delta_mm": bbox_delta, "gold_volume_mm3": float(gold.volume), "rebuilt_volume_mm3": float(rebuilt.volume), "volume_relative_error": volume_error, "gold_surface_area_mm2": float(gold.area), "rebuilt_surface_area_mm2": float(rebuilt.area), "surface_area_relative_error": area_error, "gold_solid_count": gold_solids, "rebuilt_solid_count": rebuilt_solids},
|
||||
"diagnostic_topology": {"gold_faces": len(gold.faces()), "rebuilt_faces": len(rebuilt.faces()), "gold_edges": len(gold.edges()), "rebuilt_edges": len(rebuilt.edges()), "gold_surface_types": sorted(str(face.geom_type) for face in gold.faces()), "rebuilt_surface_types": sorted(str(face.geom_type) for face in rebuilt.faces())},
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Convert fully-audited standard feature histories into self-contained CDSL."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .manifests import read_json, sha256_file, write_json
|
||||
from .registry import rule_for
|
||||
from .sketches import sketch_to_cdsl
|
||||
from .units import length_mm
|
||||
|
||||
|
||||
_SKETCH_QUERY = re.compile(r'qSketchRegion\(id\+"(?P<id>[^"]+)"')
|
||||
|
||||
|
||||
def _parameters(feature: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||||
return {str(item.get("parameterId")): item for item in feature.get("parameters") or [] if isinstance(item, dict) and item.get("parameterId")}
|
||||
|
||||
|
||||
def _value(parameters: dict[str, dict[str, Any]], name: str) -> Any:
|
||||
return (parameters.get(name) or {}).get("value")
|
||||
|
||||
|
||||
def _sketch_id(parameters: dict[str, dict[str, Any]]) -> str:
|
||||
queries = (parameters.get("entities") or {}).get("queries") or []
|
||||
matches = [match.group("id") for query in queries if isinstance(query, dict) for match in _SKETCH_QUERY.finditer(str(query.get("queryString") or ""))]
|
||||
if len(set(matches)) != 1:
|
||||
raise ValueError("extrude profile query does not uniquely name one sketch")
|
||||
return matches[0]
|
||||
|
||||
|
||||
def _extrude(feature: dict[str, Any], previous: list[str]) -> dict[str, Any]:
|
||||
parameters = _parameters(feature)
|
||||
end_bound = _value(parameters, "endBound")
|
||||
operation = _value(parameters, "operationType")
|
||||
if end_bound != "BLIND":
|
||||
raise ValueError(f"only BLIND extrudes are executable, got {end_bound!r}")
|
||||
if _value(parameters, "hasDraft") is True or _value(parameters, "hasSecondDirection") is True:
|
||||
raise ValueError("draft and two-direction extrudes are not executable in v1")
|
||||
atomic = "extrude_cut_blind" if operation in {"REMOVE", "CUT"} else "extrude_add_blind" if operation in {"NEW", "ADD"} else None
|
||||
if atomic is None:
|
||||
raise ValueError(f"unsupported extrude operation {operation!r}")
|
||||
expression = (parameters.get("depth") or {}).get("expression")
|
||||
return {"id": "f_" + str(feature["featureId"]).replace("-", "_"), "name": str(feature.get("name") or feature["featureId"]), "atomic_id": atomic, "depends_on": list(previous[-1:]), "sketch_id": _sketch_id(parameters), "params": {"distance_mm": length_mm(expression), "reverse": bool(_value(parameters, "oppositeDirection"))}, "execution_status": "supported"}
|
||||
|
||||
|
||||
def convert_one(raw_dir: Path, converted_root: Path) -> dict[str, Any]:
|
||||
sample_id = raw_dir.name
|
||||
output = converted_root / sample_id
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
diagnostics: list[dict[str, Any]] = []
|
||||
try:
|
||||
features_payload = read_json(raw_dir / "features.json")
|
||||
sketches_payload = read_json(raw_dir / "sketches.json")
|
||||
features = features_payload.get("features") or []
|
||||
solved = {str(item.get("featureId")): item for item in sketches_payload.get("sketches") or [] if isinstance(item, dict) and item.get("featureId")}
|
||||
cdsl_sketches: dict[str, dict[str, Any]] = {}
|
||||
cdsl_features: list[dict[str, Any]] = []
|
||||
history: list[str] = []
|
||||
admissible = True
|
||||
for index, feature in enumerate(features):
|
||||
if not isinstance(feature, dict):
|
||||
continue
|
||||
feature_type = str(feature.get("featureType") or "unknown")
|
||||
rule = rule_for(feature_type)
|
||||
feature_id = str(feature.get("featureId") or f"index_{index}")
|
||||
if feature_type == "newSketch":
|
||||
if feature_id not in solved:
|
||||
diagnostics.append({"feature_id": feature_id, "feature_type": feature_type, "status": "deferred", "reason": "solved sketch payload missing"})
|
||||
admissible = False
|
||||
continue
|
||||
try:
|
||||
cdsl_sketches[feature_id] = sketch_to_cdsl(solved[feature_id])
|
||||
except ValueError as exc:
|
||||
diagnostics.append({"feature_id": feature_id, "feature_type": feature_type, "status": "deferred", "reason": str(exc)})
|
||||
admissible = False
|
||||
continue
|
||||
if feature_type == "extrude":
|
||||
try:
|
||||
converted = _extrude(feature, history)
|
||||
if converted["sketch_id"] not in cdsl_sketches:
|
||||
raise ValueError("referenced sketch is missing or not executable")
|
||||
cdsl_features.append(converted)
|
||||
history.append(converted["id"])
|
||||
except ValueError as exc:
|
||||
diagnostics.append({"feature_id": feature_id, "feature_type": feature_type, "status": "deferred", "reason": str(exc)})
|
||||
admissible = False
|
||||
continue
|
||||
diagnostics.append({"feature_id": feature_id, "feature_type": feature_type, "status": rule.status, "reason": rule.reason})
|
||||
if rule.status != "supported":
|
||||
admissible = False
|
||||
provenance = {name: sha256_file(raw_dir / name) for name in ("features.json", "sketches.json", "model.step") if (raw_dir / name).exists()}
|
||||
result: dict[str, Any] = {"schema": "onshape_to_cdsl.conversion.v1", "sample_id": sample_id, "raw_dir": str(raw_dir), "source_sha256": provenance, "diagnostics": diagnostics, "status": "converted" if admissible and cdsl_features else "rejected"}
|
||||
if result["status"] == "converted":
|
||||
cdsl = {"schema": "cad.cdsl.llm.v1", "schema_version": "1.0.0", "kind": "part", "part_id": sample_id, "meta": {"source": "onshape", "provenance": provenance}, "geometry": {"sketches": list(cdsl_sketches.values())}, "features": cdsl_features}
|
||||
write_json(output / "candidate.cdsl.json", cdsl)
|
||||
result["cdsl_sha256"] = sha256_file(output / "candidate.cdsl.json")
|
||||
write_json(output / "conversion.json", result)
|
||||
return result
|
||||
except Exception as exc:
|
||||
result = {"schema": "onshape_to_cdsl.conversion.v1", "sample_id": sample_id, "raw_dir": str(raw_dir), "status": "failed", "failure_reason": f"{type(exc).__name__}: {exc}"}
|
||||
write_json(output / "conversion.json", result)
|
||||
return result
|
||||
|
||||
|
||||
def convert(raw_root: Path, converted_root: Path) -> dict[str, Any]:
|
||||
records = [convert_one(path, converted_root) for path in sorted(raw_root.iterdir()) if path.is_dir() and (path / "manifest.json").exists()]
|
||||
manifest = {"schema": "onshape_to_cdsl.convert.v1", "raw_root": str(raw_root), "sample_count": len(records), "converted_count": sum(record["status"] == "converted" for record in records), "records": records}
|
||||
write_json(converted_root / "manifest.json", manifest)
|
||||
return manifest
|
||||
@@ -0,0 +1,167 @@
|
||||
"""Complete Part Studio acquisition with per-artifact hashes and safe resume."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import asdict, dataclass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
from typing import Any
|
||||
|
||||
from .manifests import output_is_current, sha256_file, write_json
|
||||
from .onshape_api import JSON_ACCEPT, OnshapeClient, authorization_from_environment, part_studio_path
|
||||
from .source_urls import PartStudioRef
|
||||
|
||||
|
||||
@dataclass
|
||||
class Artifact:
|
||||
name: str
|
||||
status: str
|
||||
path: str | None = None
|
||||
sha256: str | None = None
|
||||
bytes: int | None = None
|
||||
url: str | None = None
|
||||
http_status: int | None = None
|
||||
message: str | None = None
|
||||
|
||||
|
||||
def _save(client: OnshapeClient, root: Path, name: str, api_path: str, filename: str, query: dict[str, Any] | None = None, accept: str = JSON_ACCEPT) -> Artifact:
|
||||
target = root / filename
|
||||
if target.exists():
|
||||
return Artifact(name, "reused", filename, sha256_file(target), target.stat().st_size)
|
||||
try:
|
||||
payload, url = client.request("GET", api_path, query=query, accept=accept)
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_bytes(payload)
|
||||
return Artifact(name, "downloaded", filename, sha256_file(target), len(payload), url)
|
||||
except urllib.error.HTTPError as exc:
|
||||
return Artifact(name, "http_error", url=client.url(api_path, query), http_status=exc.code, message=exc.read().decode("utf-8", "replace")[:1000])
|
||||
except Exception as exc:
|
||||
return Artifact(name, "error", url=client.url(api_path, query), message=f"{type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
def _export_step(client: OnshapeClient, root: Path, sample_id: str, poll_seconds: float, poll_limit: int) -> Artifact:
|
||||
target = root / "model.step"
|
||||
if target.exists() and target.read_bytes()[:12] == b"ISO-10303-21":
|
||||
return Artifact("step", "reused", "model.step", sha256_file(target), target.stat().st_size)
|
||||
path = part_studio_path(client.ref, "/export/step")
|
||||
body = {"destinationName": f"{sample_id}.step", "grouping": True, "notifyUser": False, "storeInDocument": False, "triggerAutoDownload": False, "stepUnit": "METER", "stepVersionString": "AP242"}
|
||||
try:
|
||||
# STEP generation is asynchronous and can outlive a short-lived CLI
|
||||
# process. Keep only the opaque server job id locally so a resumed
|
||||
# download polls the existing export instead of creating duplicates.
|
||||
state_file = root / ".step-translation.json"
|
||||
try:
|
||||
translation_id = json.loads(state_file.read_text(encoding="utf-8")).get("translation_id")
|
||||
except (OSError, json.JSONDecodeError):
|
||||
translation_id = None
|
||||
if not isinstance(translation_id, str):
|
||||
response, _ = client.request("POST", path, body=body)
|
||||
translation_id = json.loads(response.decode("utf-8")).get("id")
|
||||
if isinstance(translation_id, str):
|
||||
write_json(state_file, {"translation_id": translation_id})
|
||||
if not isinstance(translation_id, str):
|
||||
return Artifact("step", "error", message="STEP export response has no translation id")
|
||||
translation_path = "/translations/" + urllib.parse.quote(translation_id, safe="")
|
||||
for _ in range(poll_limit):
|
||||
state = client.get_json(translation_path)
|
||||
if state.get("requestState") == "DONE":
|
||||
ids = state.get("resultExternalDataIds") or []
|
||||
if not ids or not isinstance(ids[0], str):
|
||||
return Artifact("step", "error", message="STEP translation has no external data id")
|
||||
artifact = _save(client, root, "step", f"/documents/d/{client.ref.did}/externaldata/{urllib.parse.quote(ids[0], safe='')}", "model.step", accept="application/step, application/octet-stream")
|
||||
if artifact.status in {"downloaded", "reused"}:
|
||||
state_file.unlink(missing_ok=True)
|
||||
return artifact
|
||||
if state.get("requestState") == "FAILED":
|
||||
state_file.unlink(missing_ok=True)
|
||||
return Artifact("step", "translation_failed", message=str(state.get("failureReason") or "translation failed"))
|
||||
time.sleep(poll_seconds)
|
||||
return Artifact("step", "translation_timeout", message=f"not complete after {poll_limit} polls")
|
||||
except Exception as exc:
|
||||
return Artifact("step", "error", message=f"{type(exc).__name__}: {exc}")
|
||||
|
||||
|
||||
def _sketch_artifacts(client: OnshapeClient, root: Path) -> list[Artifact]:
|
||||
sketches_path = root / "sketches.json"
|
||||
if not sketches_path.exists():
|
||||
return []
|
||||
try:
|
||||
sketches = json.loads(sketches_path.read_text(encoding="utf-8")).get("sketches", [])
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return []
|
||||
results: list[Artifact] = []
|
||||
for sketch in sketches if isinstance(sketches, list) else []:
|
||||
feature_id = sketch.get("featureId") if isinstance(sketch, dict) else None
|
||||
if not isinstance(feature_id, str):
|
||||
continue
|
||||
base = part_studio_path(client.ref, f"/sketches/{urllib.parse.quote(feature_id, safe='')}")
|
||||
results.append(_save(client, root, f"sketch/{feature_id}/bounding_box", base + "/boundingboxes", f"sketches/{feature_id}/bounding_box.json"))
|
||||
results.append(_save(client, root, f"sketch/{feature_id}/tessellation", base + "/tessellatedentities", f"sketches/{feature_id}/tessellated_entities.json"))
|
||||
return results
|
||||
|
||||
|
||||
def download_one(ref: PartStudioRef, authorization: str, raw_root: Path, *, timeout: float, poll_seconds: float, poll_limit: int) -> dict[str, Any]:
|
||||
root = raw_root / ref.sample_id
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
client = OnshapeClient(ref, authorization, timeout)
|
||||
common = {"rollbackBarIndex": -1}
|
||||
artifacts: list[Artifact] = []
|
||||
resources = [
|
||||
("document", f"/documents/{ref.did}", "document.json", None), ("workspaces", f"/documents/d/{ref.did}/workspaces", "workspaces.json", None),
|
||||
("elements", f"/documents/d/{ref.did}/{ref.wvm}/{ref.wvmid}/elements", "elements.json", None), ("unit_info", f"/documents/d/{ref.did}/{ref.wvm}/{ref.wvmid}/unitinfo", "unit_info.json", None),
|
||||
("configuration", f"/elements/d/{ref.did}/{ref.wvm}/{ref.wvmid}/e/{ref.eid}/configuration", "configuration.json", None),
|
||||
("parts", f"/parts/d/{ref.did}/{ref.wvm}/{ref.wvmid}/e/{ref.eid}", "parts.json", {"withThumbnails": "true", "includeFlatParts": "true"}),
|
||||
("features", part_studio_path(ref, "/features"), "features.json", {**common, "includeGeometryIds": "true", "noSketchGeometry": "false"}),
|
||||
("featurescript_representation", part_studio_path(ref, "/featurescriptrepresentation"), "featurescript_representation.json", common),
|
||||
("feature_specs", part_studio_path(ref, "/featurespecs"), "feature_specs.json", None),
|
||||
("body_details", part_studio_path(ref, "/bodydetails"), "body_details.json", {**common, "includeSurfaces": "true", "includeCompositeParts": "true", "includeGeometricData": "true"}),
|
||||
("bounding_boxes", part_studio_path(ref, "/boundingboxes"), "bounding_boxes.json", {"includeHidden": "true", "includeWireBodies": "true"}),
|
||||
("mass_properties", part_studio_path(ref, "/massproperties"), "mass_properties.json", {**common, "massAsGroup": "true"}),
|
||||
("sketches", part_studio_path(ref, "/sketches"), "sketches.json", {"includeGeometry": "true", "output3D": "true", "curvePoints": "true"}),
|
||||
("named_views", f"/partstudios/d/{ref.did}/e/{ref.eid}/namedViews", "named_views.json", None),
|
||||
("tessellated_faces", part_studio_path(ref, "/tessellatedfaces"), "tessellated_faces.json", {**common, "outputVertexNormals": "true", "outputFacetNormals": "true", "outputIndexTable": "true", "outputErrorFaces": "true"}),
|
||||
("tessellated_edges", part_studio_path(ref, "/tessellatededges"), "tessellated_edges.json", common),
|
||||
("shaded_views", part_studio_path(ref, "/shadedviews"), "shaded_views.json", {"viewMatrix": "front", "outputWidth": 512, "outputHeight": 512, "edges": "show", "showAllParts": "true"}),
|
||||
]
|
||||
# These document and Part Studio evidence endpoints are independent. A
|
||||
# bounded fan-out keeps an interactive acquisition from being cut off
|
||||
# before it can reach the asynchronous STEP export, while preserving the
|
||||
# same per-artifact resume and error records.
|
||||
worker_count = max(1, min(16, int(os.environ.get("ONSHAPE_DOWNLOAD_WORKERS", "4"))))
|
||||
with ThreadPoolExecutor(max_workers=worker_count, thread_name_prefix="onshape-download") as executor:
|
||||
artifacts.extend(executor.map(lambda item: _save(client, root, *item), resources))
|
||||
artifacts.extend(_sketch_artifacts(client, root))
|
||||
artifacts.extend([
|
||||
_save(client, root, "parasolid", part_studio_path(ref, "/parasolid"), "model.x_t", {"version": "0", "includeExportIds": "true", "binaryExport": "false"}, "text/plain, application/octet-stream"),
|
||||
_save(client, root, "gltf", part_studio_path(ref, "/gltf"), "model.gltf", {**common, "outputSeparateFaceNodes": "true"}, "model/gltf+json, application/octet-stream"),
|
||||
_export_step(client, root, ref.sample_id, poll_seconds, poll_limit),
|
||||
])
|
||||
manifest = {"schema": "onshape_to_cdsl.raw_sample.v1", "source": ref.as_dict(), "resource_count": len(artifacts), "downloaded_count": sum(item.status in {"downloaded", "reused"} for item in artifacts), "unavailable_count": sum(item.status not in {"downloaded", "reused"} for item in artifacts), "resources": [asdict(item) for item in artifacts]}
|
||||
write_json(root / "manifest.json", manifest)
|
||||
return manifest
|
||||
|
||||
|
||||
def download(selection: Path, raw_root: Path, *, timeout: float = 60.0, poll_seconds: float = 3.0, poll_limit: int = 40, offset: int = 0, limit: int | None = None) -> dict[str, Any]:
|
||||
from .manifests import read_json
|
||||
if offset < 0 or limit is not None and limit < 1:
|
||||
raise ValueError("download offset must be non-negative and limit must be positive")
|
||||
authorization = authorization_from_environment()
|
||||
records = []
|
||||
selected = list(read_json(selection).get("records", []))[offset: None if limit is None else offset + limit]
|
||||
for item in selected:
|
||||
ref = PartStudioRef(**item["source"])
|
||||
manifest = download_one(ref, authorization, raw_root, timeout=timeout, poll_seconds=poll_seconds, poll_limit=poll_limit)
|
||||
records.append({"sample_id": ref.sample_id, "status": "downloaded" if (raw_root / ref.sample_id / "model.step").exists() else "incomplete", "manifest": str(raw_root / ref.sample_id / "manifest.json"), "resource_count": manifest["resource_count"]})
|
||||
summary = {"schema": "onshape_to_cdsl.download.v1", "selection": str(selection), "selection_sha256": sha256_file(selection), "offset": offset, "limit": limit, "sample_count": len(records), "records": records}
|
||||
# A batch manifest remains useful evidence even when an interactive host
|
||||
# enforces a short process lifetime. The next batch reuses all artifacts.
|
||||
write_json(raw_root / f"manifest-{offset:05d}.json", summary)
|
||||
if offset == 0 and limit is None:
|
||||
write_json(raw_root / "manifest.json", summary)
|
||||
return summary
|
||||
@@ -0,0 +1,112 @@
|
||||
"""Generate a durable reconstruction issue register from pipeline evidence."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .manifests import read_json, write_json
|
||||
|
||||
|
||||
_ENGINE_FEATURES = {
|
||||
"revolve", "hole", "fillet", "chamfer", "linearPattern", "circularPattern", "mirror", "draft",
|
||||
"loft", "sweep", "shell", "thicken", "split", "moveFace", "replaceFace", "deleteFace", "rib",
|
||||
"booleanBodies", "sheetMetal",
|
||||
}
|
||||
|
||||
|
||||
def _issue(sample_id: str, category: str, subsystem: str, detail: str, *, feature_id: str | None = None, feature_type: str | None = None) -> dict[str, str]:
|
||||
result = {"sample_id": sample_id, "category": category, "subsystem": subsystem, "detail": detail}
|
||||
if feature_id:
|
||||
result["feature_id"] = feature_id
|
||||
if feature_type:
|
||||
result["feature_type"] = feature_type
|
||||
return result
|
||||
|
||||
|
||||
def _conversion_issues(sample_id: str, conversion: dict[str, Any]) -> list[dict[str, str]]:
|
||||
issues: list[dict[str, str]] = []
|
||||
for diagnostic in conversion.get("diagnostics") or []:
|
||||
if not isinstance(diagnostic, dict):
|
||||
continue
|
||||
feature_type = str(diagnostic.get("feature_type") or "unknown")
|
||||
detail = str(diagnostic.get("reason") or "no diagnostic reason")
|
||||
status = diagnostic.get("status")
|
||||
if status == "not_admissible":
|
||||
category, subsystem = "source_not_admissible", "source_dependency"
|
||||
elif feature_type in _ENGINE_FEATURES:
|
||||
category, subsystem = "engine_capability_gap", "cdsl_engine"
|
||||
else:
|
||||
category, subsystem = "converter_gap", "onshape_converter"
|
||||
issues.append(_issue(sample_id, category, subsystem, detail, feature_id=str(diagnostic.get("feature_id") or ""), feature_type=feature_type))
|
||||
if conversion.get("status") == "failed":
|
||||
issues.append(_issue(sample_id, "converter_failure", "onshape_converter", str(conversion.get("failure_reason") or "unknown conversion failure")))
|
||||
return issues
|
||||
|
||||
|
||||
def _validation_issues(sample_id: str, validation: dict[str, Any]) -> list[dict[str, str]]:
|
||||
if validation.get("status") != "rejected":
|
||||
return []
|
||||
comparison = validation.get("comparison")
|
||||
if isinstance(comparison, dict):
|
||||
reasons = comparison.get("failure_reasons") or ["strict_comparison_failed"]
|
||||
return [_issue(sample_id, "geometric_regression", "strict_step_comparator", str(reason)) for reason in reasons]
|
||||
reason = str(validation.get("failure_reason") or "unknown validation failure")
|
||||
if reason == "conversion_not_executable":
|
||||
return [] # The detailed conversion diagnostics above are the source of truth.
|
||||
return [_issue(sample_id, "engine_execution_failure", "cdsl_engine", reason)]
|
||||
|
||||
|
||||
def _markdown(issues: list[dict[str, str]], summary: dict[str, int]) -> str:
|
||||
lines = [
|
||||
"# Reconstruction Issue Register",
|
||||
"",
|
||||
"This file is generated by `onshape_to_cdsl validate`. It records rejected samples and separates source limitations, converter gaps, engine capability gaps, engine execution failures, and strict geometric regressions.",
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
"| Category | Count |",
|
||||
"| --- | ---: |",
|
||||
]
|
||||
lines.extend(f"| {category} | {count} |" for category, count in sorted(summary.items()))
|
||||
if not issues:
|
||||
lines.extend(["", "No reconstruction issues were recorded.", ""])
|
||||
return "\n".join(lines)
|
||||
lines.extend(["", "## Issues", "", "| Sample | Category | Subsystem | Feature | Detail |", "| --- | --- | --- | --- | --- |"])
|
||||
for issue in issues:
|
||||
feature = issue.get("feature_type") or issue.get("feature_id") or "-"
|
||||
detail = issue["detail"].replace("|", "\\|").replace("\n", " ")
|
||||
lines.append(f"| {issue['sample_id']} | {issue['category']} | {issue['subsystem']} | {feature} | {detail} |")
|
||||
lines.extend(["", "## Engine Work Queue", ""])
|
||||
grouped: dict[str, set[str]] = defaultdict(set)
|
||||
for issue in issues:
|
||||
if issue["category"] in {"engine_capability_gap", "engine_execution_failure"}:
|
||||
grouped[issue["category"]].add(issue["detail"])
|
||||
if not grouped:
|
||||
lines.append("No engine-specific blockers were observed in this run.")
|
||||
else:
|
||||
for category, details in sorted(grouped.items()):
|
||||
lines.append(f"### {category}")
|
||||
lines.append("")
|
||||
lines.extend(f"- {detail}" for detail in sorted(details))
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def write_issue_register(converted_root: Path, validation_records: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
"""Persist both machine-readable and Markdown summaries under `data/`."""
|
||||
issues: list[dict[str, str]] = []
|
||||
for validation in validation_records:
|
||||
sample_id = str(validation["sample_id"])
|
||||
conversion_path = converted_root / sample_id / "conversion.json"
|
||||
if conversion_path.exists():
|
||||
issues.extend(_conversion_issues(sample_id, read_json(conversion_path)))
|
||||
issues.extend(_validation_issues(sample_id, validation))
|
||||
issues.sort(key=lambda issue: (issue["category"], issue["sample_id"], issue.get("feature_type", ""), issue["detail"]))
|
||||
summary = dict(sorted(Counter(issue["category"] for issue in issues).items()))
|
||||
register = {"schema": "onshape_to_cdsl.reconstruction_issues.v1", "sample_count": len(validation_records), "issue_count": len(issues), "summary": summary, "issues": issues}
|
||||
data_root = converted_root.parent
|
||||
write_json(data_root / "reconstruction_issues.json", register)
|
||||
(data_root / "reconstruction_issues.md").write_text(_markdown(issues, summary), encoding="utf-8")
|
||||
return register
|
||||
@@ -0,0 +1,42 @@
|
||||
"""Small, atomic JSON helpers used by every resumable pipeline stage."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
||||
digest.update(block)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def write_json(path: Path, value: Any) -> None:
|
||||
"""Atomically write JSON so interrupted stages never leave a valid-looking file."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
temporary.write_text(json.dumps(value, ensure_ascii=True, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
os.replace(temporary, path)
|
||||
|
||||
|
||||
def read_json(path: Path) -> Any:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def output_is_current(path: Path, expected_sha256: str | None = None) -> bool:
|
||||
return path.is_file() and (expected_sha256 is None or sha256_file(path) == expected_sha256)
|
||||
|
||||
|
||||
def jsonl_write(path: Path, items: list[dict[str, Any]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".tmp")
|
||||
with temporary.open("w", encoding="utf-8") as stream:
|
||||
for item in items:
|
||||
stream.write(json.dumps(item, ensure_ascii=True, sort_keys=True) + "\n")
|
||||
os.replace(temporary, path)
|
||||
@@ -0,0 +1,31 @@
|
||||
"""Combine non-overlapping scan batches into one deterministic selection input."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .manifests import read_json, sha256_file, write_json
|
||||
|
||||
|
||||
def merge_scans(paths: list[Path], out: Path) -> dict[str, Any]:
|
||||
if not paths:
|
||||
raise ValueError("at least one scan manifest is required")
|
||||
records: list[dict[str, Any]] = []
|
||||
source_hashes: list[dict[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for path in paths:
|
||||
scan = read_json(path)
|
||||
for record in scan.get("records") or []:
|
||||
sample_id = str(record.get("sample_id") or "")
|
||||
if not sample_id:
|
||||
raise ValueError(f"{path}: scan record has no sample_id")
|
||||
if sample_id in seen:
|
||||
raise ValueError(f"{path}: duplicate sample_id {sample_id}")
|
||||
seen.add(sample_id)
|
||||
records.append(record)
|
||||
source_hashes.append({"path": str(path), "sha256": sha256_file(path)})
|
||||
records.sort(key=lambda item: str(item["sample_id"]))
|
||||
manifest = {"schema": "onshape_to_cdsl.merged_scan.v1", "source_scans": source_hashes, "sample_count": len(records), "records": records}
|
||||
write_json(out, manifest)
|
||||
return manifest
|
||||
@@ -0,0 +1,72 @@
|
||||
"""Minimal authenticated Onshape v17 client. Credentials never enter manifests."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import getpass
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Any
|
||||
|
||||
from .source_urls import PartStudioRef
|
||||
|
||||
|
||||
API_VERSION = "v17"
|
||||
JSON_ACCEPT = "application/json;charset=UTF-8; qs=0.09"
|
||||
|
||||
|
||||
def authorization_from_environment() -> str:
|
||||
access_key = os.environ.get("ONSHAPE_ACCESS_KEY") or getpass.getpass("Onshape access key: ")
|
||||
secret_key = os.environ.get("ONSHAPE_SECRET_KEY") or getpass.getpass("Onshape secret key: ")
|
||||
if not access_key or not secret_key:
|
||||
raise ValueError("both ONSHAPE_ACCESS_KEY and ONSHAPE_SECRET_KEY are required")
|
||||
return "Basic " + base64.b64encode(f"{access_key}:{secret_key}".encode("utf-8")).decode("ascii")
|
||||
|
||||
|
||||
class OnshapeClient:
|
||||
def __init__(self, ref: PartStudioRef, authorization: str, timeout: float = 60.0) -> None:
|
||||
self.ref, self.authorization, self.timeout = ref, authorization, timeout
|
||||
try:
|
||||
import certifi
|
||||
self.context = ssl.create_default_context(cafile=certifi.where())
|
||||
except ImportError:
|
||||
self.context = ssl.create_default_context()
|
||||
|
||||
def url(self, path: str, query: dict[str, Any] | None = None) -> str:
|
||||
encoded = urllib.parse.urlencode(query or {}, doseq=True)
|
||||
return f"https://{self.ref.stack}/api/{API_VERSION}{path}" + (f"?{encoded}" if encoded else "")
|
||||
|
||||
def request(self, method: str, path: str, *, query: dict[str, Any] | None = None, body: dict[str, Any] | None = None, accept: str = JSON_ACCEPT) -> tuple[bytes, str]:
|
||||
request = urllib.request.Request(
|
||||
self.url(path, query), data=json.dumps(body).encode("utf-8") if body is not None else None,
|
||||
method=method, headers={"Accept": accept, "Authorization": self.authorization, "Content-Type": JSON_ACCEPT, "User-Agent": "onshape-to-cdsl/0.1"},
|
||||
)
|
||||
# Public ABC URLs can be scanned in large batches. Respect Onshape's
|
||||
# rate limit rather than recording a transient 429 as a dead document.
|
||||
retry_limit = max(0, int(os.environ.get("ONSHAPE_RATE_LIMIT_RETRIES", "5")))
|
||||
for attempt in range(retry_limit + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(request, timeout=self.timeout, context=self.context) as response:
|
||||
return response.read(), response.geturl()
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 429 or attempt == retry_limit:
|
||||
raise
|
||||
retry_after = exc.headers.get("Retry-After")
|
||||
try:
|
||||
pause = float(retry_after) if retry_after is not None else min(60.0, 2.0 ** attempt)
|
||||
except ValueError:
|
||||
pause = min(60.0, 2.0 ** attempt)
|
||||
time.sleep(max(1.0, pause))
|
||||
raise RuntimeError("unreachable Onshape retry loop")
|
||||
|
||||
def get_json(self, path: str, query: dict[str, Any] | None = None) -> Any:
|
||||
return json.loads(self.request("GET", path, query=query)[0].decode("utf-8"))
|
||||
|
||||
|
||||
def part_studio_path(ref: PartStudioRef, suffix: str) -> str:
|
||||
return f"/partstudios/d/{ref.did}/{ref.wvm}/{ref.wvmid}/e/{ref.eid}{suffix}"
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Explicit feature support contract. Unknown does not mean executable."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FeatureRule:
|
||||
status: str
|
||||
atomic_id: str | None = None
|
||||
reason: str = ""
|
||||
|
||||
|
||||
RULES: dict[str, FeatureRule] = {
|
||||
"newSketch": FeatureRule("supported", reason="solved sketch is captured separately"),
|
||||
"extrude": FeatureRule("supported", reason="only BLIND without draft is executable in v1"),
|
||||
"revolve": FeatureRule("deferred", "revolve_add", "requires robust source axis/region resolution"),
|
||||
"hole": FeatureRule("deferred", "hole_blind", "requires host face topology binding"),
|
||||
"fillet": FeatureRule("deferred", "fillet", "requires stable source edge selector"),
|
||||
"chamfer": FeatureRule("deferred", "chamfer", "requires stable source edge selector"),
|
||||
"linearPattern": FeatureRule("deferred", "pattern_linear", "requires source feature references"),
|
||||
"circularPattern": FeatureRule("deferred", reason="circular pattern is not implemented by the engine"),
|
||||
"mirror": FeatureRule("deferred", "pattern_mirror", "requires source plane selector"),
|
||||
"draft": FeatureRule("deferred", reason="draft executor is not implemented by the engine"),
|
||||
"loft": FeatureRule("deferred", reason="loft executor is not implemented by the engine"),
|
||||
"sweep": FeatureRule("deferred", reason="sweep executor is not implemented by the engine"),
|
||||
"shell": FeatureRule("deferred", reason="shell executor is not implemented by the engine"),
|
||||
"thicken": FeatureRule("deferred", reason="thicken executor is not implemented by the engine"),
|
||||
"split": FeatureRule("deferred", reason="split executor is not implemented by the engine"),
|
||||
"moveFace": FeatureRule("deferred", reason="direct face editing is not generically replayable"),
|
||||
"replaceFace": FeatureRule("deferred", reason="direct face editing is not generically replayable"),
|
||||
"deleteFace": FeatureRule("deferred", reason="direct face editing is not generically replayable"),
|
||||
"derive": FeatureRule("not_admissible", reason="external document dependency"),
|
||||
"import": FeatureRule("not_admissible", reason="imported geometry has no local parametric source"),
|
||||
}
|
||||
|
||||
|
||||
def rule_for(feature_type: object) -> FeatureRule:
|
||||
name = str(feature_type or "unknown")
|
||||
if name in RULES:
|
||||
return RULES[name]
|
||||
if "custom" in name.lower() or "featurescript" in name.lower() or "derive" in name.lower() or "import" in name.lower():
|
||||
return FeatureRule("not_admissible", reason="custom FeatureScript, imported, or external feature")
|
||||
return FeatureRule("deferred", reason=f"no registered local executor for Onshape feature {name}")
|
||||
@@ -0,0 +1,69 @@
|
||||
"""Feature-only scan used for selection; it deliberately does not export STEP."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .manifests import sha256_file, write_json
|
||||
from .onshape_api import OnshapeClient, authorization_from_environment, part_studio_path
|
||||
from .registry import rule_for
|
||||
from .source_urls import read_url_file
|
||||
|
||||
|
||||
def _feature_summary(payload: Any) -> dict[str, Any]:
|
||||
features = payload.get("features", []) if isinstance(payload, dict) else []
|
||||
types: Counter[str] = Counter()
|
||||
states: Counter[str] = Counter()
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict):
|
||||
continue
|
||||
feature_type = str(feature.get("featureType") or "unknown")
|
||||
types[feature_type] += 1
|
||||
states[rule_for(feature_type).status] += 1
|
||||
return {"feature_count": sum(types.values()), "feature_types": dict(sorted(types.items())), "admissibility": dict(sorted(states.items()))}
|
||||
|
||||
|
||||
def _curve_types(payload: Any) -> Counter[str]:
|
||||
curves: Counter[str] = Counter()
|
||||
features = payload.get("features", []) if isinstance(payload, dict) else []
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or feature.get("featureType") != "newSketch":
|
||||
continue
|
||||
for entity in feature.get("entities") or []:
|
||||
geometry = entity.get("geometry") if isinstance(entity, dict) else None
|
||||
if isinstance(geometry, dict):
|
||||
curves[str(geometry.get("btType") or "unknown")] += 1
|
||||
return curves
|
||||
|
||||
|
||||
def scan(url_file: Path, out: Path, *, timeout: float = 60.0, limit: int | None = None, offset: int = 0, workers: int = 1) -> dict[str, Any]:
|
||||
authorization = authorization_from_environment()
|
||||
if offset < 0:
|
||||
raise ValueError("scan offset must be non-negative")
|
||||
if workers < 1 or workers > 32:
|
||||
raise ValueError("scan workers must be between 1 and 32")
|
||||
refs = read_url_file(url_file)
|
||||
if limit is not None:
|
||||
if limit < 1:
|
||||
raise ValueError("scan limit must be positive")
|
||||
refs = refs[offset:offset + limit]
|
||||
else:
|
||||
refs = refs[offset:]
|
||||
def inspect(ref: Any) -> dict[str, Any]:
|
||||
entry: dict[str, Any] = {"sample_id": ref.sample_id, "source": ref.as_dict()}
|
||||
try:
|
||||
payload = OnshapeClient(ref, authorization, timeout).get_json(part_studio_path(ref, "/features"), {"rollbackBarIndex": -1, "includeGeometryIds": "true", "noSketchGeometry": "false"})
|
||||
summary = _feature_summary(payload)
|
||||
summary["curve_types"] = dict(sorted(_curve_types(payload).items()))
|
||||
entry.update({"status": "scanned", **summary})
|
||||
except Exception as exc: # A failed public document must remain auditable too.
|
||||
entry.update({"status": "failed", "failure_reason": f"{type(exc).__name__}: {exc}"})
|
||||
return entry
|
||||
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix="onshape-scan") as executor:
|
||||
records = list(executor.map(inspect, refs))
|
||||
manifest = {"schema": "onshape_to_cdsl.scan.v1", "url_file": str(url_file), "url_file_sha256": sha256_file(url_file), "source_record_count": len(read_url_file(url_file)), "scan_offset": offset, "scan_limit": limit, "scan_workers": workers, "sample_count": len(records), "records": records}
|
||||
write_json(out, manifest)
|
||||
return manifest
|
||||
@@ -0,0 +1,40 @@
|
||||
"""Deterministic stratified candidate selection."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .manifests import read_json, sha256_file, write_json
|
||||
|
||||
|
||||
def _stratum(record: dict[str, Any]) -> str:
|
||||
feature_types = record.get("feature_types") or {}
|
||||
curves = record.get("curve_types") or {}
|
||||
feature_band = "1-3" if record.get("feature_count", 0) <= 3 else "4-8" if record.get("feature_count", 0) <= 8 else "9+"
|
||||
return "|".join((",".join(sorted(feature_types)) or "none", ",".join(sorted(curves)) or "none", feature_band))
|
||||
|
||||
|
||||
def select(scan_manifest: Path, out: Path, *, count: int, seed: int) -> dict[str, Any]:
|
||||
scan = read_json(scan_manifest)
|
||||
eligible = [record for record in scan.get("records", []) if record.get("status") == "scanned"]
|
||||
buckets: dict[str, list[dict[str, Any]]] = defaultdict(list)
|
||||
for record in eligible:
|
||||
buckets[_stratum(record)].append(record)
|
||||
rng = random.Random(seed)
|
||||
for values in buckets.values():
|
||||
rng.shuffle(values)
|
||||
chosen: list[dict[str, Any]] = []
|
||||
# Round robin preserves uncommon strata while keeping a fixed seed order.
|
||||
keys = sorted(buckets)
|
||||
while len(chosen) < min(count, len(eligible)) and any(buckets.values()):
|
||||
for key in keys:
|
||||
if buckets[key] and len(chosen) < count:
|
||||
item = dict(buckets[key].pop())
|
||||
item["stratum"] = key
|
||||
chosen.append(item)
|
||||
manifest = {"schema": "onshape_to_cdsl.selection.v1", "scan_manifest": str(scan_manifest), "scan_manifest_sha256": sha256_file(scan_manifest), "seed": seed, "requested_count": count, "selected_count": len(chosen), "records": chosen}
|
||||
write_json(out, manifest)
|
||||
return manifest
|
||||
@@ -0,0 +1,120 @@
|
||||
"""Solved Onshape sketch geometry to self-contained CDSL analytic contours."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
|
||||
MM_PER_M = 1000.0
|
||||
|
||||
|
||||
def _vector(value: Any) -> list[float]:
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError("missing Onshape 3D point")
|
||||
return [float(value[key]) * MM_PER_M for key in ("x", "y", "z")]
|
||||
|
||||
|
||||
def _dot(left: list[float], right: list[float]) -> float:
|
||||
return sum(a * b for a, b in zip(left, right))
|
||||
|
||||
|
||||
def _subtract(left: list[float], right: list[float]) -> list[float]:
|
||||
return [a - b for a, b in zip(left, right)]
|
||||
|
||||
|
||||
def _cross(left: list[float], right: list[float]) -> list[float]:
|
||||
return [left[1] * right[2] - left[2] * right[1], left[2] * right[0] - left[0] * right[2], left[0] * right[1] - left[1] * right[0]]
|
||||
|
||||
|
||||
def _unit(value: list[float]) -> list[float]:
|
||||
length = math.sqrt(_dot(value, value))
|
||||
if length < 1e-10:
|
||||
raise ValueError("zero sketch coordinate direction")
|
||||
return [component / length for component in value]
|
||||
|
||||
|
||||
def workplane_from_matrix(matrix: Any) -> dict[str, list[float]]:
|
||||
"""Decode Onshape's row-major 4x4 sketch transform into millimetres."""
|
||||
if not isinstance(matrix, list) or len(matrix) != 16:
|
||||
raise ValueError("sketchMatrix must contain 16 values")
|
||||
values = [float(value) for value in matrix]
|
||||
x_dir = _unit([values[0], values[1], values[2]])
|
||||
y_dir = _unit([values[4], values[5], values[6]])
|
||||
normal = _unit(_cross(x_dir, y_dir))
|
||||
# API sketchMatrix serializes a row-major transform; translation is 3,7,11.
|
||||
return {"origin_mm": [values[3] * MM_PER_M, values[7] * MM_PER_M, values[11] * MM_PER_M], "x_dir": x_dir, "y_dir": y_dir, "normal": normal}
|
||||
|
||||
|
||||
def _local(point: list[float], plane: dict[str, list[float]]) -> list[float]:
|
||||
displacement = _subtract(point, plane["origin_mm"])
|
||||
return [_dot(displacement, plane["x_dir"]), _dot(displacement, plane["y_dir"])]
|
||||
|
||||
|
||||
def _endpoint_key(point: list[float], tolerance_mm: float = 1e-5) -> tuple[int, int]:
|
||||
return (round(point[0] / tolerance_mm), round(point[1] / tolerance_mm))
|
||||
|
||||
|
||||
def _ordered_contours(segments: list[dict[str, Any]]) -> list[list[dict[str, Any]]]:
|
||||
"""Join non-circular edges by endpoints; circles are independent closed contours."""
|
||||
circles = [[segment] for segment in segments if segment["type"] == "circle"]
|
||||
edges = [segment for segment in segments if segment["type"] != "circle"]
|
||||
unused = set(range(len(edges)))
|
||||
contours: list[list[dict[str, Any]]] = []
|
||||
while unused:
|
||||
current_index = unused.pop()
|
||||
contour = [edges[current_index]]
|
||||
first = _endpoint_key(contour[0]["start_mm"])
|
||||
end = _endpoint_key(contour[0]["end_mm"])
|
||||
while end != first:
|
||||
match = next((index for index in unused if _endpoint_key(edges[index]["start_mm"]) == end or _endpoint_key(edges[index]["end_mm"]) == end), None)
|
||||
if match is None:
|
||||
raise ValueError("sketch has an open or disconnected profile contour")
|
||||
unused.remove(match)
|
||||
edge = dict(edges[match])
|
||||
if _endpoint_key(edge["end_mm"]) == end:
|
||||
edge["start_mm"], edge["end_mm"] = edge["end_mm"], edge["start_mm"]
|
||||
if edge["type"] == "arc":
|
||||
edge["clockwise"] = not bool(edge.get("clockwise", False))
|
||||
contour.append(edge)
|
||||
end = _endpoint_key(edge["end_mm"])
|
||||
contours.append(contour)
|
||||
return contours + circles
|
||||
|
||||
|
||||
def sketch_to_cdsl(sketch: dict[str, Any]) -> dict[str, Any]:
|
||||
plane = workplane_from_matrix(sketch.get("sketchMatrix"))
|
||||
segments: list[dict[str, Any]] = []
|
||||
unsupported: list[str] = []
|
||||
for entity in sketch.get("entities") or []:
|
||||
if not isinstance(entity, dict) or entity.get("isConstruction"):
|
||||
continue
|
||||
entity_type = entity.get("sketchEntityType")
|
||||
geometry = entity.get("geometry") or {}
|
||||
try:
|
||||
if entity_type == "skLineSegment":
|
||||
segments.append({"type": "line", "start_mm": _local(_vector(entity["startPosition3d"]), plane), "end_mm": _local(_vector(entity["endPosition3d"]), plane)})
|
||||
elif entity_type == "skArc":
|
||||
segments.append({"type": "arc", "start_mm": _local(_vector(entity["startPosition3d"]), plane), "end_mm": _local(_vector(entity["endPosition3d"]), plane), "center_mm": _local(_vector(geometry["center3d"]), plane), "radius_mm": float(geometry["radius"]) * MM_PER_M, "clockwise": bool(geometry.get("clockWise", False))})
|
||||
elif entity_type == "skCircle":
|
||||
segments.append({"type": "circle", "center_mm": _local(_vector(geometry["center3d"]), plane), "radius_mm": float(geometry["radius"]) * MM_PER_M})
|
||||
elif entity_type != "skPoint":
|
||||
unsupported.append(str(entity_type))
|
||||
except (KeyError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f"{entity.get('sketchEntityId', '?')}: {exc}") from exc
|
||||
if unsupported:
|
||||
raise ValueError("unsupported solved sketch entities: " + ", ".join(sorted(set(unsupported))))
|
||||
if not segments:
|
||||
raise ValueError("sketch has no non-construction executable geometry")
|
||||
contours = _ordered_contours(segments)
|
||||
|
||||
def profile_segment(segment: dict[str, Any]) -> dict[str, Any]:
|
||||
output = {key: value for key, value in segment.items() if key not in {"start_mm", "end_mm", "center_mm"}}
|
||||
if "start_mm" in segment:
|
||||
output["start"] = segment["start_mm"]
|
||||
output["end"] = segment["end_mm"]
|
||||
if "center_mm" in segment:
|
||||
output["center"] = segment["center_mm"]
|
||||
return output
|
||||
|
||||
return {"id": str(sketch["featureId"]), "name": str(sketch.get("name") or sketch["featureId"]), "workplane": plane, "profile": {"type": "analytic_contours", "contours": [{"role": "unknown", "closed": True, "segments": [profile_segment(segment) for segment in contour]} for contour in contours]}}
|
||||
@@ -0,0 +1,66 @@
|
||||
"""Parse official ABC URL mapping files without depending on PyYAML."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ONSHAPE_URL_RE = re.compile(
|
||||
r"^https://(?P<stack>[^/]+)/documents/(?P<did>[^/]+)/(?P<wvm>w|v|m)/(?P<wvmid>[^/]+)/e/(?P<eid>[^/?#]+)"
|
||||
)
|
||||
URL_RE = re.compile(r"https://[^'\"\s]+/documents/[^'\"\s]+")
|
||||
# ABC's source files are Python/YAML-style mapping literals: the first record
|
||||
# starts with `{`, all following records are indented, and the final record
|
||||
# ends in `}`. Accept the optional opening delimiter without parsing arbitrary
|
||||
# YAML or evaluating source text.
|
||||
LEADING_ID_RE = re.compile(r"^\s*(?:\{\s*)?(?:-\s*)?['\"]?(?P<id>[A-Za-z0-9_-]+)['\"]?\s*(?::|\s)")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PartStudioRef:
|
||||
sample_id: str
|
||||
source_url: str
|
||||
stack: str
|
||||
did: str
|
||||
wvm: str
|
||||
wvmid: str
|
||||
eid: str
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
def parse_part_studio_url(sample_id: str, url: str) -> PartStudioRef:
|
||||
source_url = url.strip().rstrip("',\"]")
|
||||
match = ONSHAPE_URL_RE.match(source_url)
|
||||
if match is None:
|
||||
raise ValueError(f"{sample_id}: not an Onshape Part Studio URL: {url}")
|
||||
return PartStudioRef(sample_id=str(sample_id), source_url=source_url, **match.groupdict())
|
||||
|
||||
|
||||
def read_url_file(path: Path) -> list[PartStudioRef]:
|
||||
"""Read `id URL`, `id: URL`, or ABC objects YAML records.
|
||||
|
||||
ABC mappings can contain unrelated fields. A URL line is accepted only
|
||||
when its object ID appears on the same line, which prevents silently
|
||||
associating a URL with an adjacent record.
|
||||
"""
|
||||
refs: list[PartStudioRef] = []
|
||||
seen: set[str] = set()
|
||||
for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
||||
url = URL_RE.search(line)
|
||||
if not url:
|
||||
continue
|
||||
identifier = LEADING_ID_RE.match(line)
|
||||
if not identifier:
|
||||
raise ValueError(f"{path}:{number}: Onshape URL requires a leading sample ID")
|
||||
sample_id = identifier.group("id")
|
||||
if sample_id in seen:
|
||||
raise ValueError(f"{path}:{number}: duplicate sample ID {sample_id!r}")
|
||||
seen.add(sample_id)
|
||||
refs.append(parse_part_studio_url(sample_id, url.group(0)))
|
||||
if not refs:
|
||||
raise ValueError(f"{path}: no Onshape Part Studio URLs found")
|
||||
return refs
|
||||
@@ -0,0 +1,18 @@
|
||||
"""Conservative parser for Onshape length expressions used by standard features."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
|
||||
_LENGTH_RE = re.compile(r"^\s*(?P<value>[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?)\s*(?:\*\s*)?(?P<unit>mm|millimeter(?:s)?|cm|m|meter(?:s)?|in|inch|inches|ft|foot|feet)\s*$", re.I)
|
||||
_SCALE = {"mm": 1.0, "millimeter": 1.0, "millimeters": 1.0, "cm": 10.0, "m": 1000.0, "meter": 1000.0, "meters": 1000.0, "in": 25.4, "inch": 25.4, "inches": 25.4, "ft": 304.8, "foot": 304.8, "feet": 304.8}
|
||||
|
||||
|
||||
def length_mm(expression: object) -> float:
|
||||
if not isinstance(expression, str):
|
||||
raise ValueError("missing length expression")
|
||||
match = _LENGTH_RE.match(expression)
|
||||
if not match:
|
||||
raise ValueError(f"unsupported length expression {expression!r}")
|
||||
return float(match.group("value")) * _SCALE[match.group("unit").lower()]
|
||||
@@ -0,0 +1,59 @@
|
||||
"""CDSL-only rebuild, strict comparison, and accepted/rejected dataset routing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .compare import strict_compare
|
||||
from .issues import write_issue_register
|
||||
from .manifests import read_json, sha256_file, write_json
|
||||
|
||||
|
||||
def validate_one(converted_dir: Path, dataset_root: Path, rejected_root: Path) -> dict[str, Any]:
|
||||
conversion = read_json(converted_dir / "conversion.json")
|
||||
sample_id = str(conversion["sample_id"])
|
||||
output = converted_dir / "rebuild.step"
|
||||
report: dict[str, Any] = {"schema": "onshape_to_cdsl.validation.v1", "sample_id": sample_id, "conversion_status": conversion.get("status")}
|
||||
if conversion.get("status") != "converted":
|
||||
report.update({"status": "rejected", "failure_reason": "conversion_not_executable"})
|
||||
else:
|
||||
try:
|
||||
from cdsl_engine.runtime import rebuild_cdsl
|
||||
cdsl = read_json(converted_dir / "candidate.cdsl.json")
|
||||
report["rebuild"] = rebuild_cdsl(cdsl, output, strict=True)
|
||||
report["rebuild"]["engine"] = "cdsl_session_runtime"
|
||||
gold = Path(str(conversion["raw_dir"])) / "model.step"
|
||||
report["comparison"] = strict_compare(gold, output)
|
||||
report["status"] = "accepted" if report["comparison"]["passed"] else "rejected"
|
||||
if report["status"] == "rejected":
|
||||
report["failure_reason"] = ",".join(report["comparison"]["failure_reasons"])
|
||||
except Exception as exc:
|
||||
report.update({"status": "rejected", "failure_reason": f"{type(exc).__name__}: {exc}"})
|
||||
if report["status"] == "accepted":
|
||||
dataset_root.mkdir(parents=True, exist_ok=True)
|
||||
destination = dataset_root / f"{sample_id}.cdsl.json"
|
||||
shutil.copy2(converted_dir / "candidate.cdsl.json", destination)
|
||||
report["dataset_cdsl"] = str(destination)
|
||||
report["dataset_cdsl_sha256"] = sha256_file(destination)
|
||||
else:
|
||||
destination = rejected_root / sample_id
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("candidate.cdsl.json", "rebuild.step", "conversion.json"):
|
||||
source = converted_dir / name
|
||||
if source.exists():
|
||||
shutil.copy2(source, destination / name)
|
||||
report["rejection_dir"] = str(destination)
|
||||
write_json(converted_dir / "validation.json", report)
|
||||
if report["status"] == "rejected":
|
||||
shutil.copy2(converted_dir / "validation.json", Path(report["rejection_dir"]) / "validation.json")
|
||||
return report
|
||||
|
||||
|
||||
def validate(converted_root: Path, dataset_root: Path, rejected_root: Path) -> dict[str, Any]:
|
||||
records = [validate_one(path, dataset_root, rejected_root) for path in sorted(converted_root.iterdir()) if path.is_dir() and (path / "conversion.json").exists()]
|
||||
issues = write_issue_register(converted_root, records)
|
||||
manifest = {"schema": "onshape_to_cdsl.validation_summary.v1", "converted_root": str(converted_root), "sample_count": len(records), "accepted_count": sum(item["status"] == "accepted" for item in records), "rejected_count": sum(item["status"] == "rejected" for item in records), "reconstruction_issues": {"path": str(converted_root.parent / "reconstruction_issues.md"), "issue_count": issues["issue_count"], "summary": issues["summary"]}, "records": records}
|
||||
write_json(converted_root / "validation-manifest.json", manifest)
|
||||
return manifest
|
||||
@@ -0,0 +1,126 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from onshape_to_cdsl.compare import strict_compare
|
||||
from onshape_to_cdsl.convert import convert_one
|
||||
from onshape_to_cdsl.issues import write_issue_register
|
||||
from onshape_to_cdsl.merge_scans import merge_scans
|
||||
from onshape_to_cdsl.select import select
|
||||
from onshape_to_cdsl.sketches import sketch_to_cdsl, workplane_from_matrix
|
||||
from onshape_to_cdsl.source_urls import read_url_file
|
||||
from onshape_to_cdsl.units import length_mm
|
||||
|
||||
|
||||
class PipelineTests(unittest.TestCase):
|
||||
def test_url_file_and_units(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
mapping = Path(temporary) / "objects.yml"
|
||||
mapping.write_text("{'00000352': 'https://cad.onshape.com/documents/doc/w/work/e/element'}\n", encoding="utf-8")
|
||||
reference = read_url_file(mapping)[0]
|
||||
self.assertEqual(reference.sample_id, "00000352")
|
||||
self.assertEqual(reference.did, "doc")
|
||||
self.assertAlmostEqual(length_mm(".063 in"), 1.6002)
|
||||
self.assertAlmostEqual(length_mm("1.2*cm"), 12.0)
|
||||
with self.assertRaises(ValueError):
|
||||
length_mm("width")
|
||||
|
||||
def test_solved_rectangle_becomes_closed_analytic_contour(self) -> None:
|
||||
matrix = [1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, .002, 0, 0, 0, 1]
|
||||
self.assertEqual(workplane_from_matrix(matrix)["origin_mm"], [0.0, 0.0, 2.0])
|
||||
point = lambda x, y: {"x": x / 1000, "y": y / 1000, "z": .002}
|
||||
entities = []
|
||||
for start, end in [((0, 0), (10, 0)), ((10, 0), (10, 5)), ((10, 5), (0, 5)), ((0, 5), (0, 0))]:
|
||||
entities.append({"sketchEntityType": "skLineSegment", "isConstruction": False, "startPosition3d": point(*start), "endPosition3d": point(*end)})
|
||||
cdsl = sketch_to_cdsl({"featureId": "sketch_one", "sketchMatrix": matrix, "entities": entities})
|
||||
contour = cdsl["profile"]["contours"][0]
|
||||
self.assertTrue(contour["closed"])
|
||||
self.assertEqual(len(contour["segments"]), 4)
|
||||
self.assertEqual(contour["segments"][0]["type"], "line")
|
||||
|
||||
def test_conversion_rejects_unsupported_history(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary) / "raw" / "sample_01"
|
||||
root.mkdir(parents=True)
|
||||
(root / "features.json").write_text(json.dumps({"features": [{"featureId": "x", "featureType": "loft", "parameters": []}]}), encoding="utf-8")
|
||||
(root / "sketches.json").write_text(json.dumps({"sketches": []}), encoding="utf-8")
|
||||
result = convert_one(root, Path(temporary) / "converted")
|
||||
self.assertEqual(result["status"], "rejected")
|
||||
self.assertEqual(result["diagnostics"][0]["status"], "deferred")
|
||||
|
||||
def test_issue_register_summarizes_engine_capability_gaps(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
converted = Path(temporary) / "converted"
|
||||
sample = converted / "sample_01"
|
||||
sample.mkdir(parents=True)
|
||||
(sample / "conversion.json").write_text(json.dumps({"sample_id": "sample_01", "status": "rejected", "diagnostics": [{"feature_id": "loft_1", "feature_type": "loft", "status": "deferred", "reason": "loft executor is not implemented by the engine"}]}), encoding="utf-8")
|
||||
register = write_issue_register(converted, [{"sample_id": "sample_01", "status": "rejected", "failure_reason": "conversion_not_executable"}])
|
||||
self.assertEqual(register["summary"], {"engine_capability_gap": 1})
|
||||
document = (Path(temporary) / "reconstruction_issues.md").read_text(encoding="utf-8")
|
||||
self.assertIn("Engine Work Queue", document)
|
||||
self.assertIn("loft executor", document)
|
||||
|
||||
def test_deterministic_stratified_selection(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
scan = Path(temporary) / "scan.json"
|
||||
records = [{"sample_id": str(index), "status": "scanned", "feature_count": index, "feature_types": {"extrude": 1}, "curve_types": {"line": 4}} for index in range(8)]
|
||||
scan.write_text(json.dumps({"records": records}), encoding="utf-8")
|
||||
one = select(scan, Path(temporary) / "one.json", count=4, seed=7)
|
||||
two = select(scan, Path(temporary) / "two.json", count=4, seed=7)
|
||||
self.assertEqual([item["sample_id"] for item in one["records"]], [item["sample_id"] for item in two["records"]])
|
||||
|
||||
def test_merge_scan_batches_rejects_duplicate_ids(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
first, second = root / "first.json", root / "second.json"
|
||||
first.write_text(json.dumps({"records": [{"sample_id": "0001", "status": "scanned"}]}), encoding="utf-8")
|
||||
second.write_text(json.dumps({"records": [{"sample_id": "0002", "status": "failed"}]}), encoding="utf-8")
|
||||
merged = merge_scans([first, second], root / "merged.json")
|
||||
self.assertEqual([item["sample_id"] for item in merged["records"]], ["0001", "0002"])
|
||||
second.write_text(json.dumps({"records": [{"sample_id": "0001", "status": "failed"}]}), encoding="utf-8")
|
||||
with self.assertRaises(ValueError):
|
||||
merge_scans([first, second], root / "bad.json")
|
||||
|
||||
def test_strict_comparator_accepts_identical_and_rejects_translation(self) -> None:
|
||||
from build123d import Box, export_step
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
source, same, shifted = root / "source.step", root / "same.step", root / "shifted.step"
|
||||
box = Box(10, 10, 10)
|
||||
export_step(box, source)
|
||||
export_step(box, same)
|
||||
moved = Box(10, 10, 10).translate((0.02, 0, 0))
|
||||
export_step(moved, shifted)
|
||||
self.assertTrue(strict_compare(source, same)["passed"])
|
||||
rejection = strict_compare(source, shifted)
|
||||
self.assertFalse(rejection["passed"])
|
||||
self.assertIn("bbox_exceeds_0.01mm", rejection["failure_reasons"])
|
||||
|
||||
def test_00000352_offline_end_to_end_when_fixture_is_available(self) -> None:
|
||||
fixture = Path.cwd() / "json_to_cdsl/input/onshape_complete/00000352"
|
||||
if not (fixture / "model.step").exists():
|
||||
self.skipTest("optional local 00000352 raw fixture is not installed")
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
raw = root / "raw" / "00000352"
|
||||
raw.parent.mkdir()
|
||||
shutil.copytree(fixture, raw)
|
||||
result = convert_one(raw, root / "converted")
|
||||
self.assertEqual(result["status"], "converted")
|
||||
from cdsl_engine.runtime import rebuild_cdsl
|
||||
from cdsl_engine.semantic_validation import validate_semantic_cdsl
|
||||
cdsl_path = root / "converted/00000352/candidate.cdsl.json"
|
||||
cdsl = json.loads(cdsl_path.read_text(encoding="utf-8"))
|
||||
validate_semantic_cdsl(cdsl)
|
||||
rebuilt = root / "rebuild.step"
|
||||
rebuild_cdsl(cdsl, rebuilt, strict=True)
|
||||
comparison = strict_compare(raw / "model.step", rebuilt)
|
||||
self.assertTrue(comparison["passed"], comparison["failure_reasons"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user