Files
make-cad/test/test_reconstruction_benchmark_evaluation.py
NiJingzhe 8971279494 feat: the geometry engine is C++ on OpenCASCADE, one package, prebuilt wheels
The modeling API (shapes, operators, sketches, products, queries, the reactive graph) is implemented in
C++20 on OpenCASCADE (src/cpp) and ships inside the wheel as simplecadapi._cpp.  The Python layers that stay
Python (runtime, build, artifacts, editor, inspect, exporter, translators, scene) run on it unchanged; the
previous Python test suite passes on the new engine.

Layout: src/cpp and src/simplecadapi side by side, one distribution built by scikit-build-core.
simplecadapi.core, .operators.*, .ql, .std.* and the other legacy modules are real forwarding modules; the
Python adapter layer is simplecadapi/_glue.  OCP is no longer a dependency and nothing imports it.

Release: .github/workflows/release.yml builds wheels for Linux x86_64 and aarch64, macOS arm64 and Windows
x86_64 (Python 3.10-3.13) against one static OCCT, runs the test suite on the Linux wheels and publishes to
GitHub and PyPI; ci.yml caches the static OCCT for the test jobs.

Errors: every message the library writes is English; the tag-validation and semantic-query errors render in
full with possible causes and fixes.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-24 02:55:35 +08:00

1461 lines
46 KiB
Python

from __future__ import annotations
import os
from pathlib import Path
import subprocess
import sys
import time
import pytest
from simplecadapi._cpp.occt.BRepPrimAPI import BRepPrimAPI_MakeBox
import simplecadapi as scad
from simplecadapi.inverse_engineer.brep import evaluation as benchmark_evaluation
from simplecadapi.kernel.ocp_export import export_step_shapes
_TRUSTED_TEST_PATH = Path(__file__).resolve()
def _config(
*,
target_kind: str = "solid",
sections: tuple[benchmark_evaluation.SectionEvaluationConfig, ...] = (),
) -> benchmark_evaluation.EvaluationConfig:
return benchmark_evaluation.EvaluationConfig(
target_kind=target_kind,
stage_timeout_seconds=30.0,
material_timeout_seconds=90.0,
strict_material_tolerance=1.0e-9,
boundary_linear_deflection=1.0,
boundary_max_samples=32,
sections=sections,
strict_geometric_tolerance=1.0e-7,
)
def _global_stage() -> dict:
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": {
"bounding_box": {"max_absolute_coordinate_delta": 0.0},
"centroid": {"distance": 0.0},
"surface_area": {"relative_delta": 0.0},
"material_body_count": {"delta": 0},
"volume": {
"target": 24.0,
"current": 24.0,
"absolute_delta": 0.0,
"relative_delta": 0.0,
},
},
"report_path": None,
"error": None,
}
def _strict_material_report(*, supported: bool = True) -> dict:
return {
"method": "bidirectional_cut" if supported else "common_volume",
"missing_material": {"volume": 0.0},
"excess_material": {"volume": 0.0},
"boolean_result_valid": True,
"strict_equality_supported": supported,
"volume_balance": {"valid": True},
}
def _inspection(
*,
valid: bool = True,
solid: int = 1,
shell: int = 1,
open_shell: int | None = None,
) -> dict:
payload = {
"status": "passed" if valid else "failed",
"gate_passed": valid,
"elapsed_seconds": 0.01,
"report": {
"source": "candidate.step",
"valid": valid,
"counts": {
"solid": solid,
"shell": shell,
"open_shell": (
open_shell if open_shell is not None else int(solid == 0 and shell > 0)
),
"closed_shell": shell - (
open_shell if open_shell is not None else int(solid == 0 and shell > 0)
),
"face_occurrences": 6,
"edge_occurrences": 24,
"vertex_occurrences": 48,
"unique_faces": 6,
"unique_edges": 12,
"unique_vertices": 8,
},
},
"report_path": "evaluation/inspection/report.json",
"error": None,
}
return benchmark_evaluation._seal_trusted_evidence(
payload,
evidence_kind="inspection",
context={
"candidate_path": str(_TRUSTED_TEST_PATH),
"candidate_sha256": benchmark_evaluation._file_sha256(
_TRUSTED_TEST_PATH
),
},
)
def _strict_report(**overrides) -> dict:
report = {
"target_minus_candidate_volume": 0.0,
"candidate_minus_target_volume": 0.0,
"same_geometric_point_set": True,
"geometry_labelled_incidence_graph_isomorphic": True,
"target_graph_nodes_edges": [26, 48],
"candidate_graph_nodes_edges": [26, 48],
"geometric_tolerance": 1.0e-7,
"boolean_volume_tolerance": 1.0e-9,
"hard_gate_passed": True,
"diagnostics": {
"step_validity": {"target": True, "candidate": True},
"topology_counts": {
"target": {"face": 6, "edge": 12, "vertex": 8},
"candidate": {"face": 6, "edge": 12, "vertex": 8},
},
"face_edge_topology": {
"target": [
{"edge_ids": [f"edge:{index}" for index in range(4)]}
for _ in range(6)
],
"candidate": [
{"edge_ids": [f"edge:{index}" for index in range(4)]}
for _ in range(6)
],
},
},
}
report.update(overrides)
return report
def _strict_stage(**report_overrides) -> dict:
return {
"status": "passed",
"gate_passed": True,
"checks": {"hard_gate": True},
"report": _strict_report(**report_overrides),
}
def _write_placeholder_steps(tmp_path: Path) -> tuple[Path, Path]:
target = tmp_path / "target.step"
candidate = tmp_path / "candidate.step"
target.write_bytes(b"target")
candidate.write_bytes(b"candidate")
return target, candidate
def _classify(**overrides):
inspection_was_supplied = "candidate_inspection" in overrides
arguments = {
"replay_succeeded": True,
"persistence_succeeded": True,
"baseline_integrity_passed": True,
"candidate_bytes_equal_target": False,
"target_kind": "solid",
"candidate_inspection": _inspection(),
"stages": {},
"strict_topology_requested": False,
"parameter_representation_required": False,
"parameter_representation_passed": None,
}
arguments.update(overrides)
inspection_context = benchmark_evaluation._trusted_evidence_context(
arguments["candidate_inspection"],
evidence_kind="inspection",
)
stages_context = benchmark_evaluation._trusted_evidence_context(
arguments["stages"],
evidence_kind="comparison_bundle",
)
if (
stages_context is not None
and inspection_context is not None
and not inspection_was_supplied
):
arguments["candidate_inspection"] = benchmark_evaluation._seal_trusted_evidence(
arguments["candidate_inspection"],
evidence_kind="inspection",
context={
"candidate_path": stages_context["candidate_path"],
"candidate_sha256": stages_context["candidate_sha256"],
},
)
if not isinstance(arguments["stages"], benchmark_evaluation._TrustedEvidence):
arguments["stages"] = benchmark_evaluation._seal_trusted_evidence(
arguments["stages"],
evidence_kind="comparison_bundle",
context={
"target_path": str(_TRUSTED_TEST_PATH),
"target_sha256": benchmark_evaluation._file_sha256(
_TRUSTED_TEST_PATH
),
"candidate_path": str(_TRUSTED_TEST_PATH),
"candidate_sha256": benchmark_evaluation._file_sha256(
_TRUSTED_TEST_PATH
),
"config": {"target_kind": arguments["target_kind"]},
"diagnostics": False,
"strict_topology": arguments["strict_topology_requested"],
},
)
return benchmark_evaluation.classify_benchmark_result(**arguments)
def test_comparison_bundle_requests_strict_bidirectional_material(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
requests = []
def fake_stage(**kwargs):
requests.append(kwargs["request"])
if kwargs["name"] == "global":
return _global_stage()
if kwargs["name"] == "material":
assert kwargs["timeout_seconds"] == 90.0
assert kwargs["request"]["options"] == {
"boolean_tolerance": None,
"include_components": True,
}
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": _strict_material_report(),
"report_path": None,
"error": None,
}
raise AssertionError(kwargs["name"])
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert [request["task"] for request in requests] == ["global", "material"]
assert stages["global"]["status"] == "completed"
assert stages["global"]["gate_passed"] is None
assert "checks" not in stages["global"]
assert stages["material"]["strict_point_set_equal"] is True
def test_large_global_differences_do_not_gate_or_skip_strict_material(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
requests = []
global_stage = _global_stage()
global_stage["report"].update(
{
"bounding_box": {"max_absolute_coordinate_delta": 1.0e9},
"centroid": {"distance": 1.0e9},
"surface_area": {"relative_delta": 1.0e9},
"material_body_count": {"delta": 99},
"volume": {
"target": 24.0,
"current": 2.4e10,
"absolute_delta": 2.4e10 - 24.0,
"relative_delta": 1.0e9 - 1.0,
},
}
)
def fake_stage(**kwargs):
requests.append(kwargs["request"])
if kwargs["name"] == "global":
return global_stage
if kwargs["name"] == "material":
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": _strict_material_report(),
"report_path": None,
"error": None,
}
raise AssertionError(kwargs["name"])
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert [request["task"] for request in requests] == ["global", "material"]
assert stages["global"]["status"] == "completed"
assert stages["global"]["gate_passed"] is None
assert "checks" not in stages["global"]
assert stages["material"]["status"] == "passed"
assert stages["material"]["gate_passed"] is True
assert stages["material"]["relative_total_difference"] == pytest.approx(0.0)
def test_sampled_mismatches_are_diagnostics_and_do_not_change_classification(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
section = benchmark_evaluation.SectionEvaluationConfig(
section_id="center",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
samples_per_edge=4,
)
def fake_stage(**kwargs):
if kwargs["name"] == "global":
return _global_stage()
if kwargs["name"] == "material":
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": _strict_material_report(),
"report_path": None,
"error": None,
}
if kwargs["name"] == "boundary":
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": {
"symmetric": {"hausdorff_approximation": 1.0e9},
"target_to_current": {"statistics": {"p95": 1.0e9}},
"current_to_target": {"statistics": {"p95": 1.0e9}},
},
"report_path": None,
"error": None,
}
if kwargs["name"] == "section-center":
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": {
"target": {"material_area": 100.0, "edge_count": 4},
"current": {"material_area": 0.0, "edge_count": 0},
"comparison": {
"empty_section_mismatch": True,
"hausdorff_approximation": 1.0e9,
},
},
"report_path": None,
"error": None,
}
raise AssertionError(kwargs["name"])
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(sections=(section,)),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
diagnostics=True,
)
boundary = stages["boundary"]
section_stage = stages["sections"]["reports"][0]
assert boundary["status"] == "completed"
assert boundary["gate_passed"] is None
assert "checks" not in boundary
assert section_stage["status"] == "completed"
assert section_stage["gate_passed"] is None
assert section_stage["relative_area_error"] == pytest.approx(1.0)
assert "checks" not in section_stage
assert stages["sections"]["status"] == "completed"
assert stages["sections"]["gate_passed"] is None
assert _classify(stages=stages) == {
"classification": "geometry_equivalent",
"reasons": ["strict_topology_not_requested"],
}
def test_evaluation_config_validates_closed_contract() -> None:
section = benchmark_evaluation.SectionEvaluationConfig(
section_id="center",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
samples_per_edge=4,
)
config = benchmark_evaluation.EvaluationConfig(sections=(section,))
assert config.sections == (section,)
assert tuple(config.__dataclass_fields__) == (
"target_kind",
"stage_timeout_seconds",
"material_timeout_seconds",
"global_max_bbox_delta",
"global_max_centroid_distance",
"global_max_relative_volume_error",
"global_max_relative_area_error",
"strict_material_tolerance",
"boundary_linear_deflection",
"boundary_max_samples",
"boundary_max_hausdorff",
"boundary_max_p95",
"sections",
"strict_geometric_tolerance",
)
assert tuple(section.__dataclass_fields__) == (
"section_id",
"origin",
"normal",
"tolerance",
"samples_per_edge",
"require_nonempty",
"max_hausdorff",
"max_relative_area_error",
)
with pytest.raises(ValueError, match="at least four"):
benchmark_evaluation.SectionEvaluationConfig(
section_id="invalid",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
samples_per_edge=3,
)
def test_deprecated_diagnostic_thresholds_are_accepted_but_ignored() -> None:
config = benchmark_evaluation.EvaluationConfig(
global_max_bbox_delta=0.0,
global_max_centroid_distance=0.0,
global_max_relative_volume_error=0.0,
global_max_relative_area_error=0.0,
boundary_max_hausdorff=0.0,
boundary_max_p95=0.0,
)
section = benchmark_evaluation.SectionEvaluationConfig(
section_id="legacy",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
require_nonempty=True,
max_hausdorff=0.0,
max_relative_area_error=0.0,
)
assert config.global_max_bbox_delta == 0.0
assert section.max_hausdorff == 0.0
@pytest.mark.parametrize("value", [True, 4.0, "4"])
def test_section_samples_per_edge_requires_an_integer(value) -> None:
with pytest.raises(TypeError, match="samples_per_edge must be an integer"):
benchmark_evaluation.SectionEvaluationConfig(
section_id="center",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
samples_per_edge=value,
)
@pytest.mark.parametrize("value", [True, 16.0, "16"])
def test_boundary_max_samples_requires_an_integer(value) -> None:
with pytest.raises(TypeError, match="boundary_max_samples must be an integer"):
benchmark_evaluation.EvaluationConfig(boundary_max_samples=value)
@pytest.mark.parametrize(
("config_type", "field"),
[
(benchmark_evaluation.EvaluationConfig, "stage_timeout_seconds"),
(benchmark_evaluation.EvaluationConfig, "strict_material_tolerance"),
(benchmark_evaluation.SectionEvaluationConfig, "tolerance"),
],
)
def test_numeric_config_fields_reject_booleans(config_type, field) -> None:
kwargs = {field: True}
if config_type is benchmark_evaluation.SectionEvaluationConfig:
kwargs.update(
section_id="center",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
)
with pytest.raises(ValueError, match=field):
config_type(**kwargs)
def test_evaluation_config_rejects_duplicate_section_ids() -> None:
section = benchmark_evaluation.SectionEvaluationConfig(
section_id="center",
origin=(0.0, 0.0, 0.0),
normal=(0.0, 0.0, 1.0),
)
with pytest.raises(ValueError, match="section IDs must be unique"):
benchmark_evaluation.EvaluationConfig(sections=(section, section))
def test_identical_solids_produce_real_strict_material_proof(
tmp_path: Path,
) -> None:
target = tmp_path / "target.step"
candidate = tmp_path / "candidate.step"
export_step_shapes(
[BRepPrimAPI_MakeBox(2.0, 3.0, 4.0).Shape()],
str(target),
)
export_step_shapes(
[BRepPrimAPI_MakeBox(2.0, 3.0, 4.0).Shape()],
str(candidate),
)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
material = stages["material"]
assert material["status"] == "passed"
assert material["strict_point_set_equal"] is True
assert material["report"]["method"] == "bidirectional_cut"
assert material["report"]["strict_equality_supported"] is True
assert material["report"]["boolean_result_valid"] is True
assert material["report"]["volume_balance"]["valid"] is True
def test_non_strict_material_estimate_cannot_prove_point_set_equality(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_stage(**kwargs):
if kwargs["name"] == "global":
return _global_stage()
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": _strict_material_report(supported=False),
"report_path": None,
"error": None,
}
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert stages["material"]["strict_point_set_equal"] is None
assert stages["material"]["status"] == "failed"
assert stages["material"]["gate_passed"] is False
assert stages["material"]["checks"]["strict_equality_supported"] is False
def test_valid_strict_material_residual_proves_point_sets_differ(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
report = _strict_material_report()
report["missing_material"]["volume"] = 0.01
def fake_stage(**kwargs):
if kwargs["name"] == "global":
return _global_stage()
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": report,
"report_path": None,
"error": None,
}
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert stages["material"]["strict_point_set_equal"] is False
assert stages["material"]["status"] == "failed"
assert stages["material"]["gate_passed"] is False
assert stages["material"]["checks"]["strict_missing_material"] is False
def test_invalid_material_boolean_remains_a_failed_gate(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
report = _strict_material_report()
report["boolean_result_valid"] = False
def fake_stage(**kwargs):
if kwargs["name"] == "global":
return _global_stage()
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": report,
"report_path": None,
"error": None,
}
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert stages["material"]["status"] == "failed"
assert stages["material"]["gate_passed"] is False
assert stages["material"]["strict_point_set_equal"] is None
assert stages["material"]["checks"]["boolean_result_valid"] is False
def test_material_proof_survives_unavailable_global_report(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fake_stage(**kwargs):
if kwargs["name"] == "global":
return {
"status": "error",
"gate_passed": False,
"elapsed_seconds": 0.01,
"report": None,
"report_path": None,
"error": "global worker failed",
}
return {
"status": "completed",
"gate_passed": None,
"elapsed_seconds": 0.01,
"report": _strict_material_report(),
"report_path": None,
"error": None,
}
monkeypatch.setattr(benchmark_evaluation, "_stage", fake_stage)
target, candidate = _write_placeholder_steps(tmp_path)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
)
assert stages["global"]["status"] == "error"
assert stages["material"]["status"] == "passed"
assert stages["material"]["strict_point_set_equal"] is True
assert stages["material"]["relative_total_difference"] is None
def test_material_timeout_leaves_valid_solid_as_approximation() -> None:
result = _classify(
stages={
"global": {"status": "passed", "gate_passed": True},
"material": {
"status": "error",
"gate_passed": False,
"error": "evaluation stage timed out",
},
},
)
assert result == {
"classification": "approximation",
"reasons": ["strict_material_equality_unproved"],
}
def test_failed_material_stage_cannot_claim_strict_equality() -> None:
result = _classify(
stages={
"material": {
"status": "error",
"gate_passed": False,
"strict_point_set_equal": True,
},
"strict": {"status": "passed", "gate_passed": True},
},
strict_topology_requested=True,
)
assert result == {
"classification": "approximation",
"reasons": ["strict_material_equality_unproved"],
}
def test_valid_open_shell_is_approximation_when_equivalence_is_unproved() -> None:
result = _classify(
target_kind="open_shell",
candidate_inspection=_inspection(solid=0, shell=1),
stages={
"global": {"status": "passed", "gate_passed": True},
"material": {"status": "not_applicable", "gate_passed": True},
"strict": {"status": "not_applicable", "gate_passed": False},
},
)
assert result == {
"classification": "approximation",
"reasons": ["open_shell_equivalence_unproved"],
}
def test_open_shell_still_requires_trusted_comparison_evidence() -> None:
result = benchmark_evaluation.classify_benchmark_result(
replay_succeeded=True,
persistence_succeeded=True,
baseline_integrity_passed=True,
candidate_bytes_equal_target=False,
target_kind="open_shell",
candidate_inspection=_inspection(solid=0, shell=1),
stages={},
strict_topology_requested=False,
parameter_representation_required=False,
parameter_representation_passed=None,
)
assert result == {
"classification": "approximation",
"reasons": ["comparison_evidence_untrusted"],
}
def test_real_open_shell_reloads_and_classifies_without_fabricating_material(
tmp_path: Path,
) -> None:
lower = scad.make_circle_rwire(center=(0.0, 0.0, 0.0), radius=2.0)
upper = scad.make_circle_rwire(center=(0.0, 0.0, 2.0), radius=1.0)
shell = scad.loft_rshell(sections=[lower, upper])
candidate = tmp_path / "candidate.step"
export_step_shapes([shell.wrapped], str(candidate))
inspection = benchmark_evaluation.inspect_benchmark_step(
candidate,
name="open-shell-inspection",
output_directory=tmp_path / "evaluation",
timeout_seconds=30.0,
)
stages = benchmark_evaluation.run_comparison_bundle(
_config(target_kind="open_shell"),
target_path=candidate,
candidate_path=candidate,
output_directory=tmp_path / "comparison",
)
report = inspection["report"]
result = _classify(
target_kind="open_shell",
candidate_inspection=inspection,
stages=stages,
)
assert report["counts"]["solid"] == 0
assert report["counts"]["shell"] == 1
assert result == {
"classification": "approximation",
"reasons": ["open_shell_equivalence_unproved"],
}
def test_inspection_accepts_valid_multi_root_step(tmp_path: Path) -> None:
candidate = tmp_path / "candidate.step"
export_step_shapes(
[
BRepPrimAPI_MakeBox(1.0, 1.0, 1.0).Shape(),
BRepPrimAPI_MakeBox(2.0, 2.0, 2.0).Shape(),
],
str(candidate),
)
inspection = benchmark_evaluation.inspect_benchmark_step(
candidate,
name="multi-root-inspection",
output_directory=tmp_path / "evaluation",
timeout_seconds=30.0,
)
assert inspection["status"] == "passed"
assert inspection["report"]["counts"]["solid"] == 2
def test_strict_comparison_accepts_identical_multi_root_steps(tmp_path: Path) -> None:
target = tmp_path / "target.step"
candidate = tmp_path / "candidate.step"
shapes = [
BRepPrimAPI_MakeBox(1.0, 1.0, 1.0).Shape(),
BRepPrimAPI_MakeBox(2.0, 2.0, 2.0).Shape(),
]
export_step_shapes(shapes, str(target))
export_step_shapes(shapes, str(candidate))
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "evaluation",
strict_topology=True,
)
assert stages["material"]["strict_point_set_equal"] is True
assert stages["strict"]["status"] == "passed"
assert stages["strict"]["report"]["hard_gate_passed"] is True
assert _classify(
stages=stages,
strict_topology_requested=True,
) == {"classification": "exact_brep", "reasons": []}
def test_open_shell_target_rejects_a_fabricated_solid() -> None:
result = _classify(
target_kind="open_shell",
candidate_inspection=_inspection(solid=1, shell=1),
stages={"global": {"status": "passed", "gate_passed": True}},
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["fabricated_solid_for_open_shell"],
}
def test_strict_topology_failure_does_not_erase_proven_geometry_equivalence() -> None:
result = _classify(
stages={
"global": {"status": "passed", "gate_passed": True},
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": {"status": "error", "gate_passed": False},
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_topology_unproved"],
}
def test_missing_parameter_representation_does_not_erase_geometry_equivalence() -> None:
result = _classify(
stages={
"global": {"status": "passed", "gate_passed": True},
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": _strict_stage(),
},
strict_topology_requested=True,
parameter_representation_required=True,
parameter_representation_passed=None,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["parameter_representation_unproved"],
}
def test_open_shell_target_requires_a_shell_candidate() -> None:
result = _classify(
target_kind="open_shell",
candidate_inspection=_inspection(solid=0, shell=0),
stages={"global": {"status": "passed", "gate_passed": True}},
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_kind_mismatch"],
}
def test_solid_target_requires_a_solid_candidate() -> None:
result = _classify(
candidate_inspection=_inspection(solid=0, shell=1),
stages={"global": {"status": "passed", "gate_passed": True}},
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_kind_mismatch"],
}
def test_open_shell_kind_is_derived_from_trusted_inspection_counts() -> None:
result = _classify(
target_kind="open_shell",
candidate_inspection=_inspection(solid=0, shell=0),
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_kind_mismatch"],
}
def test_open_shell_target_rejects_an_empty_shell_count_claim() -> None:
inspection = _inspection(solid=0, shell=1)
inspection["report"]["counts"]["unique_faces"] = 0
result = _classify(
target_kind="open_shell",
candidate_inspection=inspection,
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_inspection_untrusted"],
}
def test_strict_material_conflict_does_not_erase_prior_material_proof() -> None:
result = _classify(
stages={
"global": {"status": "passed", "gate_passed": True},
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": {
"status": "failed",
"gate_passed": False,
"report": {
"same_geometric_point_set": False,
"hard_gate_passed": False,
},
},
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_stage_conflicts_with_material_proof"],
}
@pytest.mark.parametrize(
("override", "reason"),
[
({"persistence_succeeded": False}, "step_persistence_failed"),
({"baseline_integrity_passed": False}, "baseline_integrity_failed"),
({"candidate_bytes_equal_target": True}, "candidate_bytes_equal_target"),
],
)
def test_missing_submission_evidence_blocks_classification(override, reason) -> None:
result = _classify(**override)
assert result["classification"] == "unsupported_or_incomplete"
assert reason in result["reasons"]
def test_required_parameter_representation_can_reach_exact_brep() -> None:
result = _classify(
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": _strict_stage(),
},
strict_topology_requested=True,
parameter_representation_required=True,
parameter_representation_passed=True,
)
assert result == {"classification": "exact_brep", "reasons": []}
def test_unsealed_reports_cannot_reach_exact_brep() -> None:
result = benchmark_evaluation.classify_benchmark_result(
replay_succeeded=True,
persistence_succeeded=True,
baseline_integrity_passed=True,
candidate_bytes_equal_target=False,
target_kind="solid",
candidate_inspection=dict(_inspection()),
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": _strict_stage(),
},
strict_topology_requested=True,
parameter_representation_required=False,
parameter_representation_passed=None,
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_inspection_untrusted"],
}
def test_mutated_strict_report_invalidates_trusted_bundle() -> None:
stages = benchmark_evaluation._seal_trusted_evidence(
{
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": _strict_stage(),
},
evidence_kind="comparison_bundle",
context={
"target_path": str(_TRUSTED_TEST_PATH),
"target_sha256": benchmark_evaluation._file_sha256(_TRUSTED_TEST_PATH),
"candidate_path": str(_TRUSTED_TEST_PATH),
"candidate_sha256": benchmark_evaluation._file_sha256(
_TRUSTED_TEST_PATH
),
"config": {"target_kind": "solid"},
"diagnostics": False,
"strict_topology": True,
},
)
stages["strict"]["report"]["target_minus_candidate_volume"] = 1.0
result = _classify(stages=stages, strict_topology_requested=True)
assert result == {
"classification": "approximation",
"reasons": ["comparison_evidence_untrusted"],
}
def test_comparison_bundle_must_match_inspected_candidate() -> None:
stages = benchmark_evaluation._seal_trusted_evidence(
{},
evidence_kind="comparison_bundle",
context={
"target_path": str(_TRUSTED_TEST_PATH),
"target_sha256": benchmark_evaluation._file_sha256(_TRUSTED_TEST_PATH),
"candidate_path": "another.step",
"candidate_sha256": "sha256:not-the-inspection-input",
"config": {"target_kind": "solid"},
"diagnostics": False,
"strict_topology": False,
},
)
result = _classify(stages=stages, candidate_inspection=_inspection())
assert result == {
"classification": "approximation",
"reasons": ["comparison_evidence_context_mismatch"],
}
def test_changed_candidate_invalidates_exact_evidence(tmp_path: Path) -> None:
target = tmp_path / "target.step"
candidate = tmp_path / "candidate.step"
export_step_shapes([BRepPrimAPI_MakeBox(1.0, 1.0, 1.0).Shape()], str(target))
export_step_shapes([BRepPrimAPI_MakeBox(1.0, 1.0, 1.0).Shape()], str(candidate))
inspection = benchmark_evaluation.inspect_benchmark_step(
candidate,
name="candidate-inspection",
output_directory=tmp_path / "inspection",
timeout_seconds=30.0,
)
stages = benchmark_evaluation.run_comparison_bundle(
_config(),
target_path=target,
candidate_path=candidate,
output_directory=tmp_path / "comparison",
strict_topology=True,
)
export_step_shapes([BRepPrimAPI_MakeBox(2.0, 1.0, 1.0).Shape()], str(candidate))
result = benchmark_evaluation.classify_benchmark_result(
replay_succeeded=True,
persistence_succeeded=True,
baseline_integrity_passed=True,
candidate_bytes_equal_target=False,
target_kind="solid",
candidate_inspection=inspection,
stages=stages,
strict_topology_requested=True,
parameter_representation_required=False,
parameter_representation_passed=None,
)
assert result == {
"classification": "approximation",
"reasons": ["comparison_evidence_inputs_changed"],
}
def test_claimed_strict_stage_status_cannot_reach_exact_brep() -> None:
result = _classify(
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": {"status": "passed", "gate_passed": True},
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_topology_unproved"],
}
def test_strict_report_without_trusted_checks_cannot_reach_exact_brep() -> None:
strict = _strict_stage()
strict.pop("checks")
result = _classify(
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": strict,
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_topology_unproved"],
}
def test_strict_report_must_contain_consistent_exact_brep_evidence() -> None:
result = _classify(
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": _strict_stage(
geometry_labelled_incidence_graph_isomorphic=False,
),
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_topology_unproved"],
}
def test_failed_strict_report_needs_trusted_evidence_to_disprove_topology() -> None:
result = _classify(
stages={
"material": {
"status": "passed",
"gate_passed": True,
"strict_point_set_equal": True,
},
"strict": {
"status": "failed",
"gate_passed": False,
"report": {
"same_geometric_point_set": True,
"hard_gate_passed": False,
},
},
},
strict_topology_requested=True,
)
assert result == {
"classification": "geometry_equivalent",
"reasons": ["strict_stage_conflicts_with_material_proof"],
}
def test_malformed_inspection_cannot_claim_candidate_validity_or_kind() -> None:
inspection = _inspection(solid=0, shell=1)
inspection["report"]["counts"]["shell"] = True
result = _classify(
target_kind="open_shell",
candidate_inspection=inspection,
)
assert result == {
"classification": "unsupported_or_incomplete",
"reasons": ["candidate_inspection_untrusted"],
}
def test_stage_resolves_relative_output_before_switching_worker_cwd(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.chdir(tmp_path)
output = Path("evaluation")
output.mkdir()
class FakeProcess:
returncode = 0
def wait(self, timeout=None):
return self.returncode
def fake_popen(command, **kwargs):
request_path = Path(command[-2])
report_path = Path(command[-1])
assert request_path.is_absolute()
assert report_path.is_absolute()
assert request_path.parent == report_path.parent
assert request_path.parent.parent == (tmp_path / "evaluation")
report_path.write_text("{}", encoding="utf-8")
return FakeProcess()
monkeypatch.setattr(benchmark_evaluation.subprocess, "Popen", fake_popen)
monkeypatch.setattr(benchmark_evaluation, "_create_windows_job", lambda: None)
result = benchmark_evaluation._stage(
name="global",
request={"task": "global"},
output_directory=output,
timeout_seconds=1.0,
python_executable="python",
)
assert result["status"] == "completed"
def test_stage_rejects_path_traversal_name(tmp_path: Path) -> None:
with pytest.raises(ValueError, match="filename-safe"):
benchmark_evaluation._stage(
name="../escaped",
request={"task": "global"},
output_directory=tmp_path,
timeout_seconds=1.0,
python_executable="python",
)
def test_stage_writes_inside_fresh_exclusive_subdirectories(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
output = tmp_path / "evaluation"
output.mkdir()
report_directories = []
class FakeProcess:
returncode = 0
def wait(self, timeout=None):
return self.returncode
def fake_popen(command, **kwargs):
report_path = Path(command[-1])
report_directories.append(report_path.parent)
report_path.write_text("{}", encoding="utf-8")
return FakeProcess()
monkeypatch.setattr(benchmark_evaluation.subprocess, "Popen", fake_popen)
monkeypatch.setattr(benchmark_evaluation, "_create_windows_job", lambda: None)
for _ in range(2):
benchmark_evaluation._stage(
name="global",
request={"task": "global"},
output_directory=output,
timeout_seconds=1.0,
python_executable="python",
)
assert len(set(report_directories)) == 2
assert all(path.parent == output for path in report_directories)
def test_stage_returns_structured_failure_when_process_cannot_start(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fail_start(*args, **kwargs):
raise OSError("executable not found")
monkeypatch.setattr(benchmark_evaluation.subprocess, "Popen", fail_start)
result = benchmark_evaluation._stage(
name="global",
request={"task": "global"},
output_directory=tmp_path,
timeout_seconds=1.0,
python_executable="missing-python",
)
assert result["status"] == "error"
assert result["gate_passed"] is False
assert result["report"] is None
assert "failed to start: executable not found" in result["error"]
def test_stage_returns_structured_failure_when_process_isolation_cannot_start(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
def fail_job_creation():
raise OSError("job unavailable")
monkeypatch.setattr(
benchmark_evaluation,
"_create_windows_job",
fail_job_creation,
)
result = benchmark_evaluation._stage(
name="global",
request={"task": "global"},
output_directory=tmp_path,
timeout_seconds=1.0,
python_executable="python",
)
assert result["status"] == "error"
assert result["gate_passed"] is False
assert result["report"] is None
assert "failed to start: job unavailable" in result["error"]
def test_stage_timeout_terminates_the_process_tree(
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
) -> None:
terminated = []
class TimedOutProcess:
pid = 123
returncode = None
def wait(self, timeout=None):
if timeout is not None:
raise subprocess.TimeoutExpired("worker", timeout)
self.returncode = -9
return self.returncode
process = TimedOutProcess()
monkeypatch.setattr(
benchmark_evaluation.subprocess,
"Popen",
lambda *args, **kwargs: process,
)
monkeypatch.setattr(benchmark_evaluation, "_create_windows_job", lambda: None)
monkeypatch.setattr(
benchmark_evaluation,
"_terminate_process_tree",
lambda current, windows_job=None: terminated.append(current),
)
result = benchmark_evaluation._stage(
name="global",
request={"task": "global"},
output_directory=tmp_path,
timeout_seconds=0.01,
python_executable="python",
)
assert terminated == [process]
assert result["status"] == "error"
assert "timed out" in result["error"]
@pytest.mark.skipif(os.name != "nt", reason="Windows Job Object contract")
def test_windows_worker_starts_suspended_before_job_assignment() -> None:
creationflags = benchmark_evaluation._process_group_options()["creationflags"]
assert creationflags & benchmark_evaluation._CREATE_SUSPENDED
assert creationflags & subprocess.CREATE_NEW_PROCESS_GROUP
def test_process_tree_termination_kills_a_spawned_descendant(tmp_path: Path) -> None:
ready_path = tmp_path / "ready.txt"
survivor_path = tmp_path / "survivor.txt"
child_code = (
"import pathlib,sys,time;"
"time.sleep(1.0);"
"pathlib.Path(sys.argv[1]).write_text('survived',encoding='utf-8')"
)
parent_code = (
"import os,pathlib,subprocess,sys,time;"
"child=subprocess.Popen([sys.executable,'-c',sys.argv[1],sys.argv[3]],"
"start_new_session=os.name!='nt');"
"pathlib.Path(sys.argv[2]).write_text(str(child.pid),encoding='utf-8');"
"time.sleep(30)"
)
process = subprocess.Popen(
[
sys.executable,
"-c",
parent_code,
child_code,
str(ready_path),
str(survivor_path),
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
**benchmark_evaluation._process_group_options(),
)
windows_job = benchmark_evaluation._create_windows_job()
try:
benchmark_evaluation._assign_process_to_windows_job(windows_job, process)
benchmark_evaluation._resume_windows_process(windows_job, process)
deadline = time.monotonic() + 5.0
while not ready_path.exists() and time.monotonic() < deadline:
time.sleep(0.02)
assert ready_path.exists(), "parent did not spawn its descendant"
benchmark_evaluation._terminate_process_tree(process, windows_job)
process.wait(timeout=10.0)
time.sleep(1.2)
assert not survivor_path.exists()
finally:
if process.poll() is None:
process.kill()
process.wait(timeout=10.0)
benchmark_evaluation._close_windows_job(windows_job)