Wave 30/509: kernel-cases: a regression rig that grades each kernel against the case's own answer

1 commits, squashed; the originals stay reachable from tag archive/merge-main-develop.

- 732fd2ace bherbruck: kernel-cases: a regression rig that grades each kernel against the case's own answer

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
This commit is contained in:
bherbruck
2026-10-21 05:23:10 -04:00
committed by jyviko
parent bdca59e7a0
commit 9f1b47d689
8 changed files with 677 additions and 16 deletions
+194
View File
@@ -0,0 +1,194 @@
//! The kernel regression rig: replay `tests/kernel-cases/<area>/<id>/recipe.json` (an oracle
//! recipe or a command script) and grade the result against the case's own `expected.json`:
//! closed-form numbers where the case has them, and always the invariants (the operation's
//! outcome, one closed valid solid per body, the expected counts). One kernel per run; kernels
//! are never compared with each other. See tests/kernel-cases/README.md.
use serde_json::{Value, json};
use crate::oracle::{ABS_FRAC, Checks, REL_TOL, totals, v3};
use crate::{read_json, recipe};
/// The kernel this process runs booleans with (`SOLIDCRAFT_BOOLEANS`, set by the caller).
fn kernel() -> String {
std::env::var("SOLIDCRAFT_BOOLEANS").ok().filter(|k| !k.is_empty()).unwrap_or_else(|| "default".into())
}
/// The case's commands: a script as is, or an oracle recipe translated; `$CASE` in a string is
/// the case directory (for input files such as `input.step`).
fn script(dir: &str, recipe: &Value) -> Result<Value, String> {
let s = if recipe.get("commands").is_some() || recipe.is_array() { recipe.clone() } else { recipe::to_script(recipe)? };
let text = serde_json::to_string(&s).map_err(|e| e.to_string())?;
let dir_json = serde_json::to_string(dir).map_err(|e| e.to_string())?;
let dir_inner = dir_json.get(1..dir_json.len().saturating_sub(1)).unwrap_or("");
serde_json::from_str(&text.replace("$CASE", dir_inner)).map_err(|e| e.to_string())
}
/// An error that means "this kernel can't do it yet", not a wrong answer.
fn unsupported(e: &str) -> bool {
e.contains("not supported yet")
}
/// A panic caught by the engine or the kernel guard.
fn crashed(e: &str) -> bool {
e.contains("internal error") || e.contains("panicked")
}
fn case_name(dir: &str) -> (String, String) {
let p = std::path::Path::new(dir);
let name = |q: Option<&std::path::Path>| q.and_then(|q| q.file_name()).map(|x| x.to_string_lossy().to_string()).unwrap_or_default();
(name(p.parent()), name(Some(p)))
}
fn check_case(dir: &str) -> Value {
let (area, name) = case_name(dir);
let mut out = json!({"case": name, "area": area, "kernel": kernel()});
let want = match read_json(&format!("{dir}/expected.json")) {
Ok(w) => w,
Err(e) => {
out["status"] = json!("fail");
out["error"] = json!(format!("expected.json: {e}"));
return out;
}
};
out["sources"] = want["sources"].clone();
let commands = match read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &r)) {
Ok(s) => s,
Err(e) => {
out["status"] = json!("fail");
out["error"] = json!(format!("recipe.json: {e}"));
return out;
}
};
if kernel() == "smlib" && !solidcraft_engine::kernel::smlib_built() {
out["status"] = json!("unsupported");
out["error"] = json!("this build has no SMLib: build solidcraft-cli with --features solidcraft-kernel/smlib");
return out;
}
let refused = want["outcome"].as_str() == Some("refused");
let mut s = solidcraft_engine::Session::default();
let ran = s.run_script(&commands);
match (ran, refused) {
(Err(e), _) if crashed(&e.to_string()) => {
out["status"] = json!("crash");
out["error"] = json!(e.to_string());
return out;
}
(Err(e), true) => {
// The right answer is a clean refusal.
out["status"] = json!("pass");
out["refused_with"] = json!(e.to_string());
return out;
}
(Err(e), false) => {
let e = e.to_string();
out["status"] = json!(if unsupported(&e) { "unsupported" } else { "fail" });
out["error"] = json!(e);
return out;
}
(Ok(_), true) => {
out["status"] = json!("fail");
out["error"] = json!("succeeded where the case expects a refusal");
return out;
}
(Ok(_), false) => {}
}
// Every body in the world, imported components included.
let world = s.world_state();
let mut bodies = Vec::new();
let mut c = Checks { list: Vec::new(), pass: true };
let want_valid = want["valid"].as_bool().unwrap_or(true);
for b in &world.bodies {
let m = match solidcraft_engine::kernel::measure(&b.body) {
Ok(m) => m,
Err(e) => {
c.push("measure", json!(format!("{}: {e}", b.name)), json!("measurable"), false);
continue;
}
};
bodies.push(json!({
"volume_mm3": m.volume, "area_mm2": m.area, "center_of_mass": m.centroid,
"bbox": {"min": m.bbox.min, "max": m.bbox.max},
"faces": m.merged.faces, "edges": m.merged.edges, "vertices": m.merged.vertices,
}));
if want_valid {
// One closed shell, and every validity check this build has.
c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1);
let v = solidcraft_engine::kernel::validity(&b.body);
c.push("record_valid", json!(v.record.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), v.record.is_ok());
if let Some(sm) = &v.smlib {
c.push("smlib_check", json!(sm.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), sm.is_ok());
}
}
}
let got = totals(&json!({"bodies": bodies}));
let (edges, vertices): (u64, u64) =
bodies.iter().fold((0, 0), |(e, v), b| (e + b["edges"].as_u64().unwrap_or(0), v + b["vertices"].as_u64().unwrap_or(0)));
let has = |k: &str| want.get(k).is_some_and(|v| !v.is_null());
let rel = want["tolerance"]["rel"].as_f64().unwrap_or(REL_TOL);
let diag = |b: ([f64; 3], [f64; 3])| ((b.1[0] - b.0[0]).powi(2) + (b.1[1] - b.0[1]).powi(2) + (b.1[2] - b.0[2]).powi(2)).sqrt();
let want_bbox = v3(&want["bbox"]["min"]).zip(v3(&want["bbox"]["max"]));
let abs = want["tolerance"]["abs_mm"].as_f64().unwrap_or_else(|| ABS_FRAC * want_bbox.map(diag).unwrap_or_else(|| diag(got.bbox)).max(1.0));
let count = |c: &mut Checks, k: &str, g: u64| {
if has(k) {
c.push(k, json!(g), want[k].clone(), want[k].as_u64() == Some(g));
}
};
count(&mut c, "body_count", got.body_count as u64);
count(&mut c, "faces", got.faces);
count(&mut c, "edges", edges);
count(&mut c, "vertices", vertices);
if has("volume_mm3") {
c.rel("volume_mm3", got.volume, want["volume_mm3"].as_f64(), rel);
}
if has("area_mm2") {
c.rel("area_mm2", got.area, want["area_mm2"].as_f64(), rel);
}
if has("center_of_mass") {
c.vec("center_of_mass", got.com, v3(&want["center_of_mass"]), abs);
}
if has("bbox") {
c.vec("bbox.min", got.bbox.0, want_bbox.map(|b| b.0), abs);
c.vec("bbox.max", got.bbox.1, want_bbox.map(|b| b.1), abs);
}
out["status"] = json!(if c.pass { "pass" } else { "fail" });
out["checks"] = json!(c.list);
out
}
fn report(args: &[String], results: &[Value]) {
if args.iter().any(|a| a == "--json") {
println!("{}", serde_json::to_string_pretty(&Value::Array(results.to_vec())).unwrap_or_default());
return;
}
for r in results {
let detail: Vec<String> = r["checks"]
.as_array()
.into_iter()
.flatten()
.filter(|c| !c["ok"].as_bool().unwrap_or(false))
.map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"]))
.collect();
let err = r["error"].as_str().map(|e| format!(" ({e})")).unwrap_or_default();
println!(
"{} {}/{} [{}]{err}{}",
r["status"].as_str().unwrap_or("?").to_uppercase(),
r["area"].as_str().unwrap_or(""),
r["case"].as_str().unwrap_or(""),
r["kernel"].as_str().unwrap_or(""),
if detail.is_empty() { String::new() } else { format!(": {}", detail.join("; ")) }
);
}
}
pub fn run(args: &[String]) -> Result<(), String> {
let dirs: Vec<&String> = args.iter().filter(|a| !a.starts_with("--")).collect();
if dirs.is_empty() {
return Err("kernel-cases: give one or more case directories (each with recipe.json and expected.json)".into());
}
let results: Vec<Value> = dirs.iter().map(|d| check_case(d)).collect();
report(args, &results);
let bad = results.iter().filter(|r| matches!(r["status"].as_str(), Some("fail" | "crash"))).count();
if bad > 0 { Err(format!("{bad} kernel case(s) failed")) } else { Ok(()) }
}
+6
View File
@@ -15,6 +15,7 @@
//! solidcraft-cli commands the command registry as JSON
//! solidcraft-cli recipe <recipe.json> translate an oracle recipe to a script
//! solidcraft-cli oracle <case-dir>... replay recipes and compare with expected.json
//! solidcraft-cli kernel-cases [--json] <case-dir>... the kernel regression rig (tests/kernel-cases)
//! solidcraft-cli mcp [--headless] [--in design|script] [--connect HOST:PORT [--token T]] [--allow-path DIR]... [--allow-any-path]
//! [--param-check off|warn|strict]
//! MCP server on stdio (docs/mcp.md)
@@ -31,6 +32,7 @@ mod bench_agent;
mod bench_assistant;
mod crashes;
mod fuzz;
mod kernel_cases;
mod logger;
mod oracle;
mod preview;
@@ -79,6 +81,8 @@ const USAGE_COMMANDS: &str = " solidcraft-cli commands\n";
const USAGE_RECIPE: &str = " solidcraft-cli recipe <recipe.json>\n";
/// `oracle`'s lines in the usage.
const USAGE_ORACLE: &str = " solidcraft-cli oracle <case-dir>... [--json]\n";
/// `kernel-cases`'s lines in the usage.
const USAGE_KERNEL_CASES: &str = " solidcraft-cli kernel-cases <case-dir>... [--json]\n (the kernel regression rig, tests/kernel-cases; SOLIDCRAFT_BOOLEANS picks the kernel)\n";
/// `fuzz`'s lines in the usage.
const USAGE_FUZZ: &str =
" solidcraft-cli fuzz [--from N] [--count N] [--jobs J] [--timeout S] [--steps N] [--minimise] [--out DIR] [--report FILE]\n";
@@ -116,6 +120,7 @@ const COMMANDS: &[Command] = &[
Command { name: "commands", usage: USAGE_COMMANDS, listed: true },
Command { name: "recipe", usage: USAGE_RECIPE, listed: true },
Command { name: "oracle", usage: USAGE_ORACLE, listed: true },
Command { name: "kernel-cases", usage: USAGE_KERNEL_CASES, listed: true },
Command { name: "fuzz", usage: USAGE_FUZZ, listed: true },
Command { name: "fuzz-one", usage: USAGE_FUZZ_ONE, listed: false },
Command { name: "mcp", usage: USAGE_MCP, listed: true },
@@ -164,6 +169,7 @@ fn main() -> ExitCode {
recipe::to_script(&v).map(|s| println!("{}", pretty(&s)))
}),
Some("oracle") => oracle::run(&args[1..]),
Some("kernel-cases") => kernel_cases::run(&args[1..]),
Some("fuzz") => fuzz::run(&args[1..]),
Some("fuzz-one") => fuzz::one(&args[1..]),
Some("mcp") => cmd_mcp(&args[1..]),
+16 -16
View File
@@ -13,28 +13,28 @@ use crate::{read_json, recipe};
/// Default relative tolerance for volume and area (our curved-face tessellation is within ~2e-4).
pub const REL_TOL: f64 = 1e-3;
/// Default tolerance for centre of mass and bounding box, as a fraction of the bbox diagonal.
const ABS_FRAC: f64 = 1e-3;
pub(crate) const ABS_FRAC: f64 = 1e-3;
/// Chord tolerance used to triangulate our part for the surface-distance check (mm).
const MESH_TOL: f64 = 0.01;
/// Relative tolerance on the principal second moments of the two triangulations.
const INERTIA_REL: f64 = 2e-3;
/// What SolidCraft built: totals over every body.
struct Got {
body_count: usize,
volume: f64,
area: f64,
com: [f64; 3],
bbox: ([f64; 3], [f64; 3]),
faces: u64,
pub(crate) struct Got {
pub(crate) body_count: usize,
pub(crate) volume: f64,
pub(crate) area: f64,
pub(crate) com: [f64; 3],
pub(crate) bbox: ([f64; 3], [f64; 3]),
pub(crate) faces: u64,
}
fn v3(v: &Value) -> Option<[f64; 3]> {
pub(crate) fn v3(v: &Value) -> Option<[f64; 3]> {
let a = v.as_array()?;
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
}
fn totals(m: &Value) -> Got {
pub(crate) fn totals(m: &Value) -> Got {
let bodies = m["bodies"].as_array().cloned().unwrap_or_default();
let (mut volume, mut area, mut faces) = (0.0, 0.0, 0);
let mut moment = [0.0; 3];
@@ -57,21 +57,21 @@ fn totals(m: &Value) -> Got {
Got { body_count: bodies.len(), volume, area, com, bbox: (lo, hi), faces }
}
struct Checks {
list: Vec<Value>,
pass: bool,
pub(crate) struct Checks {
pub(crate) list: Vec<Value>,
pub(crate) pass: bool,
}
impl Checks {
fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
pub(crate) fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
self.pass &= ok;
self.list.push(json!({"check": what, "got": got, "want": want, "ok": ok}));
}
fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
pub(crate) fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
let ok = w.is_some_and(|w| g.is_finite() && (g - w).abs() <= rel * w.abs().max(1.0));
self.push(what, json!(g), json!(w), ok);
}
fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
pub(crate) fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
let ok = w.is_some_and(|w| g.iter().zip(w).all(|(g, w)| g.is_finite() && (g - w).abs() <= tol));
self.push(what, json!(g), json!(w), ok);
}
+2
View File
@@ -152,6 +152,7 @@ mod touching_walls;
mod tspline;
#[cfg(test)]
mod untouched_faces_tests;
mod validity;
pub mod watch;
mod whole_face;
@@ -223,6 +224,7 @@ pub use sweep_path::{RailFit, SweepControl, SweepOrientation, sweep_path, sweep_
pub use topo::{CylinderFace, cylinder_end_at, cylinder_face, cylinder_face_at};
pub use topo::{TopoCounts, merged_topology, seam_flags};
pub use tspline::tspline_sheet;
pub use validity::{Validity, smlib_built, validity};
pub use whole_face::{WholeFace, cut_takes_only_the_face, cut_tool, face_prism, matching_face};
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
+49
View File
@@ -0,0 +1,49 @@
//! Is a body one valid solid? The kernel regression rig (`tests/kernel-cases`) asks this of every
//! result: the schema 4 record's structure and geometry (`Brep::validate`, `check_geometry`),
//! and, with the `smlib` feature, SMLib's own check of the same record.
use crate::{Body, brep_record};
/// What the validity checks found for one body. `Err` holds why a check failed.
#[derive(Clone, Debug)]
pub struct Validity {
/// The record: structure, then geometry against topology within tolerances.
pub record: Result<(), String>,
/// SMLib's `check(All)` of the record; `None` in a build without SMLib.
pub smlib: Option<Result<(), String>>,
}
impl Validity {
pub fn ok(&self) -> bool {
self.record.is_ok() && self.smlib.as_ref().is_none_or(Result::is_ok)
}
}
/// Runs every validity check this build has on `body`.
pub fn validity(body: &Body) -> Validity {
let record = brep_record(body).map_err(|e| e.to_string());
let checked = record.as_ref().map_err(Clone::clone).and_then(|r| {
r.validate().map_err(|e| format!("structure: {e}"))?;
r.check_geometry().map_err(|e| format!("geometry: {e}"))
});
Validity { record: checked, smlib: smlib_validity(body) }
}
#[cfg(feature = "smlib")]
fn smlib_validity(body: &Body) -> Option<Result<(), String>> {
let checked = crate::smlib_body(body).map_err(|e| e.to_string()).and_then(|b| {
let r = b.check(solidcraft_smlib::CheckLevel::All).map_err(|e| format!("SMLib check: {e}"))?;
if r.is_valid() { Ok(()) } else { Err(format!("SMLib check: {r}")) }
});
Some(checked)
}
#[cfg(not(feature = "smlib"))]
fn smlib_validity(_: &Body) -> Option<Result<(), String>> {
None
}
/// Whether this build links SMLib (the `smlib` feature).
pub fn smlib_built() -> bool {
cfg!(feature = "smlib")
}
+93
View File
@@ -0,0 +1,93 @@
# Kernel regression rig
Every geometry case the old kernel (truck) got wrong, recorded in an issue, a pull request or a
crash report, is a use case here with its own right answer. The rig holds every kernel to all of
them. `cargo xtask kernel-cases` runs them and writes [`docs/kernel-cases.md`](../../docs/kernel-cases.md).
```sh
cargo xtask kernel-cases # the default kernel
cargo xtask kernel-cases --kernel both --cli target/release/solidcraft-cli # and SMLib
cargo run -p solidcraft-cli -- kernel-cases tests/kernel-cases/booleans/<id> # one case
```
For the `smlib` column, build `solidcraft-cli` with `--features solidcraft-kernel/smlib` (after
`scripts/fetch-smlib.sh`). The rig sets `SOLIDCRAFT_BOOLEANS=smlib` for those runs.
## Rules
- **The answer never comes from a kernel.** Not from truck's output, nor any commercial
kernel's. It comes from analysis or a hand calculation, or else from the invariants below.
- **Kernels are graded against the case, never against each other.**
- **No user designs.** A crash report or a repro design becomes a small synthetic recipe that
shows the same geometry. Check for private content before committing anything.
- **No workaround on SolidCraft's side.** A case that fails on a kernel stays failing until that
kernel is fixed. SMLib defects go to tangency-labs/solidcore as issues (`priority: high`,
`bug`), and the fix goes into the fork.
## Layout
One directory per area, one per case: `tests/kernel-cases/<area>/<id>/`.
| Area | Owner |
|---|---|
| `booleans` (combine, cuts, holes, patterns, split, mirror, draft) | booleans/shell/import/crash rig |
| `shell` (shell, offset, thicken) | booleans/shell/import/crash rig |
| `import` (STEP and IGES geometry) | booleans/shell/import/crash rig |
| `crashes` (crashes and hangs inside kernel operations) | booleans/shell/import/crash rig |
| `fillets`, `sweeps`, `threads`, `sheet-metal`, `tessellation` | history rig |
`<id>` is `<source number>-<short-name>`, such as `1894-prism-cut-touching-walls`.
Each case has:
- `recipe.json`: an engine command script (`{"commands": [...]}`, as `solidcraft-cli run`
takes) or an oracle recipe (`tests/oracle/*/recipe.json`). `$CASE` in a string is the case
directory, for an input file such as `$CASE/input.step` (written by hand from the STEP or IGES
specification, never a user's file).
- `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`.
Each area also has `skipped.json`: every issue and PR looked at that did not become a case, with
one line each saying why.
```json
[{"source": "tangency-labs/solidcraft#123", "why": "UI only: the selection highlight, no geometry"}]
```
## expected.json
```json
{
"sources": ["tangency-labs/solidcraft#1894", "tangency-labs/solidcraft#1629"],
"what": "a prism cut whose walls only touch the target's walls",
"source": "analytic",
"method": "box 40x30x20 minus the prism 10x30x5 flush with the side: V = 24000 - 1500",
"body_count": 1,
"volume_mm3": 22500.0,
"area_mm2": 5200.0,
"faces": 8
}
```
| Field | Meaning |
|---|---|
| `sources` | The issues, PRs or crash reports the case comes from (`owner/repo#N`). |
| `what` | One line: the geometry that went wrong. |
| `source` | `analytic` (numbers worked out by hand; `method` gives the formulas) or `invariants` (no number is known: only the outcome, validity and counts are graded). |
| `method` | How the numbers were derived, or which invariants hold and why. |
| `outcome` | `solid` (default: the script succeeds) or `refused` (the right answer is a clean error, never a panic or a hang). |
| `valid` | Default `true`: every body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid. |
| `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). |
| `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. |
| `tolerance` | Optional overrides: `rel`, `abs_mm`. |
## Results
Per case and kernel: **pass**; **fail** with the numbers; **not supported yet** (the kernel says
so); **crash** (a panic caught by the guards, or the process died); **hang** (no result before
the timeout).
## Exporting to solidcore
solidcore#349 proposes a graded case format for the Rust port. The cases here are SolidCraft's
copy and the source; each one maps onto it field by field (`sources`, the derivation, the
graded quantities and invariants), so an exporter can write them out for solidcore unchanged.
+311
View File
@@ -0,0 +1,311 @@
//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases`, one process per case
//! (so a crash or a hang is that case's result, not the run's), with each kernel asked for, and
//! writes `docs/kernel-cases.md`: results per case and kernel, coverage per area and per source,
//! and the issues and PRs looked at and skipped. Each kernel is graded against the cases' own
//! answers, never against the other kernel.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use serde_json::{Value, json};
use crate::{cargo, root};
const DEFAULT_TIMEOUT_S: u64 = 180;
/// Per kernel, how many cases ended in each status.
type StatusCounts = BTreeMap<String, BTreeMap<String, usize>>;
struct Options {
kernels: Vec<String>,
cli: Option<String>,
area: Option<String>,
timeout: Duration,
jobs: usize,
}
fn options(rest: &[&str]) -> Result<Options, String> {
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
let kernels = match arg("--kernel").as_deref() {
None | Some("default") => vec!["default".to_string()],
Some("smlib") => vec!["smlib".to_string()],
Some("both") => vec!["default".to_string(), "smlib".to_string()],
Some(k) => return Err(format!("--kernel {k}: default, smlib or both")),
};
let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs })
}
fn read(path: &Path) -> Value {
std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null)
}
/// Every case directory, sorted by area then id.
fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
let mut out = Vec::new();
for area in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {
let area = area.path();
if !area.is_dir() || only.is_some_and(|a| area.file_name().is_none_or(|n| n != a)) {
continue;
}
for case in std::fs::read_dir(&area).map_err(|e| format!("{}: {e}", area.display()))?.flatten() {
let case = case.path();
if case.join("recipe.json").is_file() && case.join("expected.json").is_file() {
out.push(case);
}
}
}
out.sort();
Ok(out)
}
/// One case on one kernel, in its own process, killed after the timeout.
fn run_one(o: &Options, kernel: &str, case: &Path) -> Value {
let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
let mut cmd = match &o.cli {
Some(bin) => Command::new(bin),
None => {
let mut c = cargo();
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]);
c
}
};
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
if kernel == "default" {
cmd.env_remove("SOLIDCRAFT_BOOLEANS");
} else {
cmd.env("SOLIDCRAFT_BOOLEANS", kernel);
}
let base = |status: &str, error: String| {
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
json!({"case": id, "area": area, "kernel": kernel, "status": status, "error": error})
};
let mut child = match cmd.spawn() {
Ok(c) => c,
Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
};
let start = Instant::now();
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) if start.elapsed() > o.timeout => {
let _ = child.kill();
let _ = child.wait();
return base("hang", format!("no result after {} s", o.timeout.as_secs()));
}
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
Err(e) => return base("fail", format!("wait: {e}")),
}
}
let out = match child.wait_with_output() {
Ok(o) => o,
Err(e) => return base("fail", format!("output: {e}")),
};
match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
Some(mut r) => {
r["seconds"] = json!(start.elapsed().as_secs_f64());
r
}
// No JSON: the process died (a native crash, an abort).
None => {
let err = String::from_utf8_lossy(&out.stderr);
let tail: String = err.lines().rev().take(3).collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>().join(" / ");
base("crash", format!("process exited ({}) without a result: {tail}", out.status))
}
}
}
fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
let jobs: Vec<(String, PathBuf)> = o.kernels.iter().flat_map(|k| list.iter().map(move |c| (k.clone(), c.clone()))).collect();
let next = Arc::new(Mutex::new(0usize));
let results = Arc::new(Mutex::new(vec![Value::Null; jobs.len()]));
std::thread::scope(|s| {
for _ in 0..o.jobs.min(jobs.len().max(1)) {
let (next, results, jobs) = (Arc::clone(&next), Arc::clone(&results), &jobs);
s.spawn(move || {
loop {
let i = {
let Ok(mut n) = next.lock() else { return };
let i = *n;
*n += 1;
i
};
let Some((kernel, case)) = jobs.get(i) else { return };
let r = run_one(o, kernel, case);
println!(
"{:<11} {} [{kernel}]",
r["status"].as_str().unwrap_or("?").to_uppercase(),
case.strip_prefix(root()).unwrap_or(case).display()
);
if let Ok(mut all) = results.lock()
&& let Some(slot) = all.get_mut(i)
{
*slot = r;
}
}
});
}
});
Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
}
fn cell(r: Option<&Value>) -> String {
let Some(r) = r else { return String::new() };
let status = r["status"].as_str().unwrap_or("?");
let mut notes: Vec<String> = r["checks"]
.as_array()
.into_iter()
.flatten()
.filter(|c| !c["ok"].as_bool().unwrap_or(false))
.map(|c| format!("{} {} (want {})", c["check"].as_str().unwrap_or(""), short(&c["got"]), short(&c["want"])))
.collect();
if let Some(e) = r["error"].as_str() {
notes.push(e.chars().take(160).collect());
}
let s = match status {
"pass" => "pass".to_string(),
"unsupported" => "not supported yet".to_string(),
other => format!("**{}**", other.to_uppercase()),
};
if notes.is_empty() { s } else { format!("{s}: {}", notes.join("; ")).replace('|', "/").replace('\n', " ") }
}
fn short(v: &Value) -> String {
match v {
Value::Number(n) => n.as_f64().map(|x| format!("{x:.6}").trim_end_matches('0').trim_end_matches('.').to_string()).unwrap_or_default(),
Value::String(s) => s.chars().take(100).collect(),
Value::Array(a) => format!("[{}]", a.iter().map(short).collect::<Vec<_>>().join(", ")),
other => other.to_string(),
}
}
fn sources_of(want: &Value) -> Vec<String> {
want["sources"].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(str::to_string)).collect()
}
/// `tangency-labs/solidcraft#123` as a link.
fn link(source: &str) -> String {
match source.split_once('#') {
Some((repo, n)) if repo.contains('/') => format!("[{source}](https://github.com/{repo}/issues/{n})"),
_ => source.to_string(),
}
}
fn write_report(o: &Options, dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<String, String> {
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
let mut by_case: BTreeMap<(String, String), BTreeMap<String, Value>> = BTreeMap::new();
for r in results {
by_case.entry(key(r)).or_default().insert(r["kernel"].as_str().unwrap_or("").to_string(), r.clone());
}
// Coverage: per area, cases and their statuses per kernel; per source, its cases.
let mut areas: BTreeMap<String, (usize, StatusCounts)> = BTreeMap::new();
let mut per_source: BTreeMap<String, Vec<String>> = BTreeMap::new();
let mut kinds: BTreeMap<String, usize> = BTreeMap::new();
let mut rows = String::new();
for case in list {
let want = read(&case.join("expected.json"));
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let entry = areas.entry(area.clone()).or_default();
entry.0 += 1;
let runs = by_case.get(&(area.clone(), id.clone()));
for k in &o.kernels {
let status = runs.and_then(|r| r.get(k)).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
*entry.1.entry(k.clone()).or_default().entry(status).or_default() += 1;
}
let sources = sources_of(&want);
for s in &sources {
per_source.entry(s.clone()).or_default().push(format!("{area}/{id}"));
}
*kinds.entry(want["source"].as_str().unwrap_or("?").split_whitespace().next().unwrap_or("?").to_string()).or_default() += 1;
let cells: Vec<String> = o.kernels.iter().map(|k| cell(runs.and_then(|r| r.get(k)))).collect();
rows += &format!(
"| {area}/{id} | {} | {} | {} | {} |\n",
sources.iter().map(|s| link(s)).collect::<Vec<_>>().join(" "),
want["what"].as_str().unwrap_or("").replace('|', "/"),
want["source"].as_str().unwrap_or("?"),
cells.join(" | ")
);
}
let mut md = String::from(
"# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nEach kernel is graded against the case, never against the other kernel. **default** is the kernel SolidCraft ships today; **smlib** runs with `SOLIDCRAFT_BOOLEANS=smlib`. A failing case stays failing until its kernel is fixed; there is no workaround on SolidCraft's side. See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format.\n\n",
);
md += "## Coverage\n\n| Area | Cases |";
for k in &o.kernels {
md += &format!(" {k} |");
}
md += "\n|---|---|";
md += &"---|".repeat(o.kernels.len());
md += "\n";
for (area, (n, statuses)) in &areas {
md += &format!("| {area} | {n} |");
for k in &o.kernels {
let s = statuses.get(k).map(|m| m.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", ")).unwrap_or_default();
md += &format!(" {s} |");
}
md += "\n";
}
md += &format!(
"\n{} cases from {} sources; answers: {}.\n\n## Results\n\n| Case | Sources | What went wrong | Answer |",
list.len(),
per_source.len(),
kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::<Vec<_>>().join(", ")
);
for k in &o.kernels {
md += &format!(" {k} |");
}
md += "\n|---|---|---|---|";
md += &"---|".repeat(o.kernels.len());
md += "\n";
md += &rows;
md += "\n## Sources\n\n| Source | Cases |\n|---|---|\n";
for (s, cs) in &per_source {
md += &format!("| {} | {} |\n", link(s), cs.join(", "));
}
// What was looked at and skipped, area by area (`<area>/skipped.json`).
md += "\n## Looked at and skipped\n\n";
let mut skipped = 0;
let mut area_dirs: Vec<PathBuf> = std::fs::read_dir(dir).map_err(|e| e.to_string())?.flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect();
area_dirs.sort();
for a in area_dirs {
let list = read(&a.join("skipped.json"));
let Some(items) = list.as_array().filter(|l| !l.is_empty()) else { continue };
md += &format!("### {}\n\n| Source | Why skipped |\n|---|---|\n", a.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default());
for it in items {
skipped += 1;
md += &format!("| {} | {} |\n", link(it["source"].as_str().unwrap_or("?")), it["why"].as_str().unwrap_or("").replace('|', "/"));
}
md += "\n";
}
if skipped == 0 {
md += "None yet.\n";
}
std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?;
let summary: Vec<String> = o
.kernels
.iter()
.map(|k| {
let n = results.iter().filter(|r| r["kernel"] == k.as_str() && r["status"] == "pass").count();
format!("{k}: {n}/{} pass", list.len())
})
.collect();
Ok(format!("kernel-cases: {} → docs/kernel-cases.md ({skipped} skipped sources listed)", summary.join(", ")))
}
pub fn run(rest: &[&str]) -> Result<(), String> {
let o = options(rest)?;
let dir = root().join("tests/kernel-cases");
let list = cases(&dir, o.area.as_deref())?;
if list.is_empty() {
return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
}
let results = run_all(&o, &list);
println!("{}", write_report(&o, &dir, &list, &results)?);
Ok(())
}
+6
View File
@@ -10,6 +10,7 @@ mod external_corpus;
mod features;
mod i18n;
mod icons;
mod kernel_cases;
mod layers;
mod licences;
mod lists;
@@ -45,6 +46,10 @@ commands:
replay the reference tests (tests/oracle/*/recipe.json), compare with each
case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
already built, instead of cargo run --release)
kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N]
the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ (one
process each, killed after --timeout, default 180 s) with each kernel asked for,
grade it against the case's own answer and write docs/kernel-cases.md
action-matrix [--bless] [--results FILE] [--from DIR]
try every ordinary action on every face, edge, vertex and body of the design
corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;
@@ -102,6 +107,7 @@ fn main() -> ExitCode {
Some("app-icons") => app_icons::run(&root()),
Some("features") => features::run(&root()),
Some("i18n") => i18n::run(&root(), &rest),
Some("kernel-cases") => kernel_cases::run(&rest),
Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()),
Some("action-matrix") => {
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();