Wave 30/509: kernel-cases: a regression rig that grades each kernel against the case's own answer
1 commits, squashed; the originals stay reachable from tag archive/merge-main-develop. - 732fd2ace bherbruck: kernel-cases: a regression rig that grades each kernel against the case's own answer Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
This commit is contained in:
@@ -0,0 +1,194 @@
|
||||
//! The kernel regression rig: replay `tests/kernel-cases/<area>/<id>/recipe.json` (an oracle
|
||||
//! recipe or a command script) and grade the result against the case's own `expected.json`:
|
||||
//! closed-form numbers where the case has them, and always the invariants (the operation's
|
||||
//! outcome, one closed valid solid per body, the expected counts). One kernel per run; kernels
|
||||
//! are never compared with each other. See tests/kernel-cases/README.md.
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::oracle::{ABS_FRAC, Checks, REL_TOL, totals, v3};
|
||||
use crate::{read_json, recipe};
|
||||
|
||||
/// The kernel this process runs booleans with (`SOLIDCRAFT_BOOLEANS`, set by the caller).
|
||||
fn kernel() -> String {
|
||||
std::env::var("SOLIDCRAFT_BOOLEANS").ok().filter(|k| !k.is_empty()).unwrap_or_else(|| "default".into())
|
||||
}
|
||||
|
||||
/// The case's commands: a script as is, or an oracle recipe translated; `$CASE` in a string is
|
||||
/// the case directory (for input files such as `input.step`).
|
||||
fn script(dir: &str, recipe: &Value) -> Result<Value, String> {
|
||||
let s = if recipe.get("commands").is_some() || recipe.is_array() { recipe.clone() } else { recipe::to_script(recipe)? };
|
||||
let text = serde_json::to_string(&s).map_err(|e| e.to_string())?;
|
||||
let dir_json = serde_json::to_string(dir).map_err(|e| e.to_string())?;
|
||||
let dir_inner = dir_json.get(1..dir_json.len().saturating_sub(1)).unwrap_or("");
|
||||
serde_json::from_str(&text.replace("$CASE", dir_inner)).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
/// An error that means "this kernel can't do it yet", not a wrong answer.
|
||||
fn unsupported(e: &str) -> bool {
|
||||
e.contains("not supported yet")
|
||||
}
|
||||
|
||||
/// A panic caught by the engine or the kernel guard.
|
||||
fn crashed(e: &str) -> bool {
|
||||
e.contains("internal error") || e.contains("panicked")
|
||||
}
|
||||
|
||||
fn case_name(dir: &str) -> (String, String) {
|
||||
let p = std::path::Path::new(dir);
|
||||
let name = |q: Option<&std::path::Path>| q.and_then(|q| q.file_name()).map(|x| x.to_string_lossy().to_string()).unwrap_or_default();
|
||||
(name(p.parent()), name(Some(p)))
|
||||
}
|
||||
|
||||
fn check_case(dir: &str) -> Value {
|
||||
let (area, name) = case_name(dir);
|
||||
let mut out = json!({"case": name, "area": area, "kernel": kernel()});
|
||||
let want = match read_json(&format!("{dir}/expected.json")) {
|
||||
Ok(w) => w,
|
||||
Err(e) => {
|
||||
out["status"] = json!("fail");
|
||||
out["error"] = json!(format!("expected.json: {e}"));
|
||||
return out;
|
||||
}
|
||||
};
|
||||
out["sources"] = want["sources"].clone();
|
||||
let commands = match read_json(&format!("{dir}/recipe.json")).and_then(|r| script(dir, &r)) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
out["status"] = json!("fail");
|
||||
out["error"] = json!(format!("recipe.json: {e}"));
|
||||
return out;
|
||||
}
|
||||
};
|
||||
if kernel() == "smlib" && !solidcraft_engine::kernel::smlib_built() {
|
||||
out["status"] = json!("unsupported");
|
||||
out["error"] = json!("this build has no SMLib: build solidcraft-cli with --features solidcraft-kernel/smlib");
|
||||
return out;
|
||||
}
|
||||
let refused = want["outcome"].as_str() == Some("refused");
|
||||
let mut s = solidcraft_engine::Session::default();
|
||||
let ran = s.run_script(&commands);
|
||||
match (ran, refused) {
|
||||
(Err(e), _) if crashed(&e.to_string()) => {
|
||||
out["status"] = json!("crash");
|
||||
out["error"] = json!(e.to_string());
|
||||
return out;
|
||||
}
|
||||
(Err(e), true) => {
|
||||
// The right answer is a clean refusal.
|
||||
out["status"] = json!("pass");
|
||||
out["refused_with"] = json!(e.to_string());
|
||||
return out;
|
||||
}
|
||||
(Err(e), false) => {
|
||||
let e = e.to_string();
|
||||
out["status"] = json!(if unsupported(&e) { "unsupported" } else { "fail" });
|
||||
out["error"] = json!(e);
|
||||
return out;
|
||||
}
|
||||
(Ok(_), true) => {
|
||||
out["status"] = json!("fail");
|
||||
out["error"] = json!("succeeded where the case expects a refusal");
|
||||
return out;
|
||||
}
|
||||
(Ok(_), false) => {}
|
||||
}
|
||||
|
||||
// Every body in the world, imported components included.
|
||||
let world = s.world_state();
|
||||
let mut bodies = Vec::new();
|
||||
let mut c = Checks { list: Vec::new(), pass: true };
|
||||
let want_valid = want["valid"].as_bool().unwrap_or(true);
|
||||
for b in &world.bodies {
|
||||
let m = match solidcraft_engine::kernel::measure(&b.body) {
|
||||
Ok(m) => m,
|
||||
Err(e) => {
|
||||
c.push("measure", json!(format!("{}: {e}", b.name)), json!("measurable"), false);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
bodies.push(json!({
|
||||
"volume_mm3": m.volume, "area_mm2": m.area, "center_of_mass": m.centroid,
|
||||
"bbox": {"min": m.bbox.min, "max": m.bbox.max},
|
||||
"faces": m.merged.faces, "edges": m.merged.edges, "vertices": m.merged.vertices,
|
||||
}));
|
||||
if want_valid {
|
||||
// One closed shell, and every validity check this build has.
|
||||
c.push("shells", json!(format!("{}: {}", b.name, m.shells)), json!(1), m.shells == 1);
|
||||
let v = solidcraft_engine::kernel::validity(&b.body);
|
||||
c.push("record_valid", json!(v.record.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), v.record.is_ok());
|
||||
if let Some(sm) = &v.smlib {
|
||||
c.push("smlib_check", json!(sm.clone().err().unwrap_or_else(|| "ok".into())), json!("ok"), sm.is_ok());
|
||||
}
|
||||
}
|
||||
}
|
||||
let got = totals(&json!({"bodies": bodies}));
|
||||
let (edges, vertices): (u64, u64) =
|
||||
bodies.iter().fold((0, 0), |(e, v), b| (e + b["edges"].as_u64().unwrap_or(0), v + b["vertices"].as_u64().unwrap_or(0)));
|
||||
let has = |k: &str| want.get(k).is_some_and(|v| !v.is_null());
|
||||
let rel = want["tolerance"]["rel"].as_f64().unwrap_or(REL_TOL);
|
||||
let diag = |b: ([f64; 3], [f64; 3])| ((b.1[0] - b.0[0]).powi(2) + (b.1[1] - b.0[1]).powi(2) + (b.1[2] - b.0[2]).powi(2)).sqrt();
|
||||
let want_bbox = v3(&want["bbox"]["min"]).zip(v3(&want["bbox"]["max"]));
|
||||
let abs = want["tolerance"]["abs_mm"].as_f64().unwrap_or_else(|| ABS_FRAC * want_bbox.map(diag).unwrap_or_else(|| diag(got.bbox)).max(1.0));
|
||||
let count = |c: &mut Checks, k: &str, g: u64| {
|
||||
if has(k) {
|
||||
c.push(k, json!(g), want[k].clone(), want[k].as_u64() == Some(g));
|
||||
}
|
||||
};
|
||||
count(&mut c, "body_count", got.body_count as u64);
|
||||
count(&mut c, "faces", got.faces);
|
||||
count(&mut c, "edges", edges);
|
||||
count(&mut c, "vertices", vertices);
|
||||
if has("volume_mm3") {
|
||||
c.rel("volume_mm3", got.volume, want["volume_mm3"].as_f64(), rel);
|
||||
}
|
||||
if has("area_mm2") {
|
||||
c.rel("area_mm2", got.area, want["area_mm2"].as_f64(), rel);
|
||||
}
|
||||
if has("center_of_mass") {
|
||||
c.vec("center_of_mass", got.com, v3(&want["center_of_mass"]), abs);
|
||||
}
|
||||
if has("bbox") {
|
||||
c.vec("bbox.min", got.bbox.0, want_bbox.map(|b| b.0), abs);
|
||||
c.vec("bbox.max", got.bbox.1, want_bbox.map(|b| b.1), abs);
|
||||
}
|
||||
out["status"] = json!(if c.pass { "pass" } else { "fail" });
|
||||
out["checks"] = json!(c.list);
|
||||
out
|
||||
}
|
||||
|
||||
fn report(args: &[String], results: &[Value]) {
|
||||
if args.iter().any(|a| a == "--json") {
|
||||
println!("{}", serde_json::to_string_pretty(&Value::Array(results.to_vec())).unwrap_or_default());
|
||||
return;
|
||||
}
|
||||
for r in results {
|
||||
let detail: Vec<String> = r["checks"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|c| !c["ok"].as_bool().unwrap_or(false))
|
||||
.map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"]))
|
||||
.collect();
|
||||
let err = r["error"].as_str().map(|e| format!(" ({e})")).unwrap_or_default();
|
||||
println!(
|
||||
"{} {}/{} [{}]{err}{}",
|
||||
r["status"].as_str().unwrap_or("?").to_uppercase(),
|
||||
r["area"].as_str().unwrap_or(""),
|
||||
r["case"].as_str().unwrap_or(""),
|
||||
r["kernel"].as_str().unwrap_or(""),
|
||||
if detail.is_empty() { String::new() } else { format!(": {}", detail.join("; ")) }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn run(args: &[String]) -> Result<(), String> {
|
||||
let dirs: Vec<&String> = args.iter().filter(|a| !a.starts_with("--")).collect();
|
||||
if dirs.is_empty() {
|
||||
return Err("kernel-cases: give one or more case directories (each with recipe.json and expected.json)".into());
|
||||
}
|
||||
let results: Vec<Value> = dirs.iter().map(|d| check_case(d)).collect();
|
||||
report(args, &results);
|
||||
let bad = results.iter().filter(|r| matches!(r["status"].as_str(), Some("fail" | "crash"))).count();
|
||||
if bad > 0 { Err(format!("{bad} kernel case(s) failed")) } else { Ok(()) }
|
||||
}
|
||||
@@ -15,6 +15,7 @@
|
||||
//! solidcraft-cli commands the command registry as JSON
|
||||
//! solidcraft-cli recipe <recipe.json> translate an oracle recipe to a script
|
||||
//! solidcraft-cli oracle <case-dir>... replay recipes and compare with expected.json
|
||||
//! solidcraft-cli kernel-cases [--json] <case-dir>... the kernel regression rig (tests/kernel-cases)
|
||||
//! solidcraft-cli mcp [--headless] [--in design|script] [--connect HOST:PORT [--token T]] [--allow-path DIR]... [--allow-any-path]
|
||||
//! [--param-check off|warn|strict]
|
||||
//! MCP server on stdio (docs/mcp.md)
|
||||
@@ -31,6 +32,7 @@ mod bench_agent;
|
||||
mod bench_assistant;
|
||||
mod crashes;
|
||||
mod fuzz;
|
||||
mod kernel_cases;
|
||||
mod logger;
|
||||
mod oracle;
|
||||
mod preview;
|
||||
@@ -79,6 +81,8 @@ const USAGE_COMMANDS: &str = " solidcraft-cli commands\n";
|
||||
const USAGE_RECIPE: &str = " solidcraft-cli recipe <recipe.json>\n";
|
||||
/// `oracle`'s lines in the usage.
|
||||
const USAGE_ORACLE: &str = " solidcraft-cli oracle <case-dir>... [--json]\n";
|
||||
/// `kernel-cases`'s lines in the usage.
|
||||
const USAGE_KERNEL_CASES: &str = " solidcraft-cli kernel-cases <case-dir>... [--json]\n (the kernel regression rig, tests/kernel-cases; SOLIDCRAFT_BOOLEANS picks the kernel)\n";
|
||||
/// `fuzz`'s lines in the usage.
|
||||
const USAGE_FUZZ: &str =
|
||||
" solidcraft-cli fuzz [--from N] [--count N] [--jobs J] [--timeout S] [--steps N] [--minimise] [--out DIR] [--report FILE]\n";
|
||||
@@ -116,6 +120,7 @@ const COMMANDS: &[Command] = &[
|
||||
Command { name: "commands", usage: USAGE_COMMANDS, listed: true },
|
||||
Command { name: "recipe", usage: USAGE_RECIPE, listed: true },
|
||||
Command { name: "oracle", usage: USAGE_ORACLE, listed: true },
|
||||
Command { name: "kernel-cases", usage: USAGE_KERNEL_CASES, listed: true },
|
||||
Command { name: "fuzz", usage: USAGE_FUZZ, listed: true },
|
||||
Command { name: "fuzz-one", usage: USAGE_FUZZ_ONE, listed: false },
|
||||
Command { name: "mcp", usage: USAGE_MCP, listed: true },
|
||||
@@ -164,6 +169,7 @@ fn main() -> ExitCode {
|
||||
recipe::to_script(&v).map(|s| println!("{}", pretty(&s)))
|
||||
}),
|
||||
Some("oracle") => oracle::run(&args[1..]),
|
||||
Some("kernel-cases") => kernel_cases::run(&args[1..]),
|
||||
Some("fuzz") => fuzz::run(&args[1..]),
|
||||
Some("fuzz-one") => fuzz::one(&args[1..]),
|
||||
Some("mcp") => cmd_mcp(&args[1..]),
|
||||
|
||||
@@ -13,28 +13,28 @@ use crate::{read_json, recipe};
|
||||
/// Default relative tolerance for volume and area (our curved-face tessellation is within ~2e-4).
|
||||
pub const REL_TOL: f64 = 1e-3;
|
||||
/// Default tolerance for centre of mass and bounding box, as a fraction of the bbox diagonal.
|
||||
const ABS_FRAC: f64 = 1e-3;
|
||||
pub(crate) const ABS_FRAC: f64 = 1e-3;
|
||||
/// Chord tolerance used to triangulate our part for the surface-distance check (mm).
|
||||
const MESH_TOL: f64 = 0.01;
|
||||
/// Relative tolerance on the principal second moments of the two triangulations.
|
||||
const INERTIA_REL: f64 = 2e-3;
|
||||
|
||||
/// What SolidCraft built: totals over every body.
|
||||
struct Got {
|
||||
body_count: usize,
|
||||
volume: f64,
|
||||
area: f64,
|
||||
com: [f64; 3],
|
||||
bbox: ([f64; 3], [f64; 3]),
|
||||
faces: u64,
|
||||
pub(crate) struct Got {
|
||||
pub(crate) body_count: usize,
|
||||
pub(crate) volume: f64,
|
||||
pub(crate) area: f64,
|
||||
pub(crate) com: [f64; 3],
|
||||
pub(crate) bbox: ([f64; 3], [f64; 3]),
|
||||
pub(crate) faces: u64,
|
||||
}
|
||||
|
||||
fn v3(v: &Value) -> Option<[f64; 3]> {
|
||||
pub(crate) fn v3(v: &Value) -> Option<[f64; 3]> {
|
||||
let a = v.as_array()?;
|
||||
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
|
||||
}
|
||||
|
||||
fn totals(m: &Value) -> Got {
|
||||
pub(crate) fn totals(m: &Value) -> Got {
|
||||
let bodies = m["bodies"].as_array().cloned().unwrap_or_default();
|
||||
let (mut volume, mut area, mut faces) = (0.0, 0.0, 0);
|
||||
let mut moment = [0.0; 3];
|
||||
@@ -57,21 +57,21 @@ fn totals(m: &Value) -> Got {
|
||||
Got { body_count: bodies.len(), volume, area, com, bbox: (lo, hi), faces }
|
||||
}
|
||||
|
||||
struct Checks {
|
||||
list: Vec<Value>,
|
||||
pass: bool,
|
||||
pub(crate) struct Checks {
|
||||
pub(crate) list: Vec<Value>,
|
||||
pub(crate) pass: bool,
|
||||
}
|
||||
|
||||
impl Checks {
|
||||
fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
|
||||
pub(crate) fn push(&mut self, what: &str, got: Value, want: Value, ok: bool) {
|
||||
self.pass &= ok;
|
||||
self.list.push(json!({"check": what, "got": got, "want": want, "ok": ok}));
|
||||
}
|
||||
fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
|
||||
pub(crate) fn rel(&mut self, what: &str, g: f64, w: Option<f64>, rel: f64) {
|
||||
let ok = w.is_some_and(|w| g.is_finite() && (g - w).abs() <= rel * w.abs().max(1.0));
|
||||
self.push(what, json!(g), json!(w), ok);
|
||||
}
|
||||
fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
|
||||
pub(crate) fn vec(&mut self, what: &str, g: [f64; 3], w: Option<[f64; 3]>, tol: f64) {
|
||||
let ok = w.is_some_and(|w| g.iter().zip(w).all(|(g, w)| g.is_finite() && (g - w).abs() <= tol));
|
||||
self.push(what, json!(g), json!(w), ok);
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@ mod touching_walls;
|
||||
mod tspline;
|
||||
#[cfg(test)]
|
||||
mod untouched_faces_tests;
|
||||
mod validity;
|
||||
pub mod watch;
|
||||
mod whole_face;
|
||||
|
||||
@@ -223,6 +224,7 @@ pub use sweep_path::{RailFit, SweepControl, SweepOrientation, sweep_path, sweep_
|
||||
pub use topo::{CylinderFace, cylinder_end_at, cylinder_face, cylinder_face_at};
|
||||
pub use topo::{TopoCounts, merged_topology, seam_flags};
|
||||
pub use tspline::tspline_sheet;
|
||||
pub use validity::{Validity, smlib_built, validity};
|
||||
pub use whole_face::{WholeFace, cut_takes_only_the_face, cut_tool, face_prism, matching_face};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, thiserror::Error)]
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
//! Is a body one valid solid? The kernel regression rig (`tests/kernel-cases`) asks this of every
|
||||
//! result: the schema 4 record's structure and geometry (`Brep::validate`, `check_geometry`),
|
||||
//! and, with the `smlib` feature, SMLib's own check of the same record.
|
||||
|
||||
use crate::{Body, brep_record};
|
||||
|
||||
/// What the validity checks found for one body. `Err` holds why a check failed.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Validity {
|
||||
/// The record: structure, then geometry against topology within tolerances.
|
||||
pub record: Result<(), String>,
|
||||
/// SMLib's `check(All)` of the record; `None` in a build without SMLib.
|
||||
pub smlib: Option<Result<(), String>>,
|
||||
}
|
||||
|
||||
impl Validity {
|
||||
pub fn ok(&self) -> bool {
|
||||
self.record.is_ok() && self.smlib.as_ref().is_none_or(Result::is_ok)
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs every validity check this build has on `body`.
|
||||
pub fn validity(body: &Body) -> Validity {
|
||||
let record = brep_record(body).map_err(|e| e.to_string());
|
||||
let checked = record.as_ref().map_err(Clone::clone).and_then(|r| {
|
||||
r.validate().map_err(|e| format!("structure: {e}"))?;
|
||||
r.check_geometry().map_err(|e| format!("geometry: {e}"))
|
||||
});
|
||||
Validity { record: checked, smlib: smlib_validity(body) }
|
||||
}
|
||||
|
||||
#[cfg(feature = "smlib")]
|
||||
fn smlib_validity(body: &Body) -> Option<Result<(), String>> {
|
||||
let checked = crate::smlib_body(body).map_err(|e| e.to_string()).and_then(|b| {
|
||||
let r = b.check(solidcraft_smlib::CheckLevel::All).map_err(|e| format!("SMLib check: {e}"))?;
|
||||
if r.is_valid() { Ok(()) } else { Err(format!("SMLib check: {r}")) }
|
||||
});
|
||||
Some(checked)
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "smlib"))]
|
||||
fn smlib_validity(_: &Body) -> Option<Result<(), String>> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Whether this build links SMLib (the `smlib` feature).
|
||||
pub fn smlib_built() -> bool {
|
||||
cfg!(feature = "smlib")
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
# Kernel regression rig
|
||||
|
||||
Every geometry case the old kernel (truck) got wrong, recorded in an issue, a pull request or a
|
||||
crash report, is a use case here with its own right answer. The rig holds every kernel to all of
|
||||
them. `cargo xtask kernel-cases` runs them and writes [`docs/kernel-cases.md`](../../docs/kernel-cases.md).
|
||||
|
||||
```sh
|
||||
cargo xtask kernel-cases # the default kernel
|
||||
cargo xtask kernel-cases --kernel both --cli target/release/solidcraft-cli # and SMLib
|
||||
cargo run -p solidcraft-cli -- kernel-cases tests/kernel-cases/booleans/<id> # one case
|
||||
```
|
||||
|
||||
For the `smlib` column, build `solidcraft-cli` with `--features solidcraft-kernel/smlib` (after
|
||||
`scripts/fetch-smlib.sh`). The rig sets `SOLIDCRAFT_BOOLEANS=smlib` for those runs.
|
||||
|
||||
## Rules
|
||||
|
||||
- **The answer never comes from a kernel.** Not from truck's output, nor any commercial
|
||||
kernel's. It comes from analysis or a hand calculation, or else from the invariants below.
|
||||
- **Kernels are graded against the case, never against each other.**
|
||||
- **No user designs.** A crash report or a repro design becomes a small synthetic recipe that
|
||||
shows the same geometry. Check for private content before committing anything.
|
||||
- **No workaround on SolidCraft's side.** A case that fails on a kernel stays failing until that
|
||||
kernel is fixed. SMLib defects go to tangency-labs/solidcore as issues (`priority: high`,
|
||||
`bug`), and the fix goes into the fork.
|
||||
|
||||
## Layout
|
||||
|
||||
One directory per area, one per case: `tests/kernel-cases/<area>/<id>/`.
|
||||
|
||||
| Area | Owner |
|
||||
|---|---|
|
||||
| `booleans` (combine, cuts, holes, patterns, split, mirror, draft) | booleans/shell/import/crash rig |
|
||||
| `shell` (shell, offset, thicken) | booleans/shell/import/crash rig |
|
||||
| `import` (STEP and IGES geometry) | booleans/shell/import/crash rig |
|
||||
| `crashes` (crashes and hangs inside kernel operations) | booleans/shell/import/crash rig |
|
||||
| `fillets`, `sweeps`, `threads`, `sheet-metal`, `tessellation` | history rig |
|
||||
|
||||
`<id>` is `<source number>-<short-name>`, such as `1894-prism-cut-touching-walls`.
|
||||
|
||||
Each case has:
|
||||
|
||||
- `recipe.json`: an engine command script (`{"commands": [...]}`, as `solidcraft-cli run`
|
||||
takes) or an oracle recipe (`tests/oracle/*/recipe.json`). `$CASE` in a string is the case
|
||||
directory, for an input file such as `$CASE/input.step` (written by hand from the STEP or IGES
|
||||
specification, never a user's file).
|
||||
- `expected.json`: the right answer, graded by `solidcraft-cli kernel-cases`.
|
||||
|
||||
Each area also has `skipped.json`: every issue and PR looked at that did not become a case, with
|
||||
one line each saying why.
|
||||
|
||||
```json
|
||||
[{"source": "tangency-labs/solidcraft#123", "why": "UI only: the selection highlight, no geometry"}]
|
||||
```
|
||||
|
||||
## expected.json
|
||||
|
||||
```json
|
||||
{
|
||||
"sources": ["tangency-labs/solidcraft#1894", "tangency-labs/solidcraft#1629"],
|
||||
"what": "a prism cut whose walls only touch the target's walls",
|
||||
"source": "analytic",
|
||||
"method": "box 40x30x20 minus the prism 10x30x5 flush with the side: V = 24000 - 1500",
|
||||
"body_count": 1,
|
||||
"volume_mm3": 22500.0,
|
||||
"area_mm2": 5200.0,
|
||||
"faces": 8
|
||||
}
|
||||
```
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `sources` | The issues, PRs or crash reports the case comes from (`owner/repo#N`). |
|
||||
| `what` | One line: the geometry that went wrong. |
|
||||
| `source` | `analytic` (numbers worked out by hand; `method` gives the formulas) or `invariants` (no number is known: only the outcome, validity and counts are graded). |
|
||||
| `method` | How the numbers were derived, or which invariants hold and why. |
|
||||
| `outcome` | `solid` (default: the script succeeds) or `refused` (the right answer is a clean error, never a panic or a hang). |
|
||||
| `valid` | Default `true`: every body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's `check(All)`. `false` only where the result is legitimately not a solid. |
|
||||
| `body_count`, `faces`, `edges`, `vertices` | Exact counts, where the geometry fixes them. Faces and edges count whole analytic surfaces and curves once (a full cylinder side is one face). |
|
||||
| `volume_mm3`, `area_mm2`, `center_of_mass`, `bbox` | As in `tests/oracle`: totals over every body, relative tolerance 1e-3 for volume and area, 1e-3 of the box diagonal for positions. |
|
||||
| `tolerance` | Optional overrides: `rel`, `abs_mm`. |
|
||||
|
||||
## Results
|
||||
|
||||
Per case and kernel: **pass**; **fail** with the numbers; **not supported yet** (the kernel says
|
||||
so); **crash** (a panic caught by the guards, or the process died); **hang** (no result before
|
||||
the timeout).
|
||||
|
||||
## Exporting to solidcore
|
||||
|
||||
solidcore#349 proposes a graded case format for the Rust port. The cases here are SolidCraft's
|
||||
copy and the source; each one maps onto it field by field (`sources`, the derivation, the
|
||||
graded quantities and invariants), so an exporter can write them out for solidcore unchanged.
|
||||
@@ -0,0 +1,311 @@
|
||||
//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
|
||||
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases`, one process per case
|
||||
//! (so a crash or a hang is that case's result, not the run's), with each kernel asked for, and
|
||||
//! writes `docs/kernel-cases.md`: results per case and kernel, coverage per area and per source,
|
||||
//! and the issues and PRs looked at and skipped. Each kernel is graded against the cases' own
|
||||
//! answers, never against the other kernel.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use crate::{cargo, root};
|
||||
|
||||
const DEFAULT_TIMEOUT_S: u64 = 180;
|
||||
|
||||
/// Per kernel, how many cases ended in each status.
|
||||
type StatusCounts = BTreeMap<String, BTreeMap<String, usize>>;
|
||||
|
||||
struct Options {
|
||||
kernels: Vec<String>,
|
||||
cli: Option<String>,
|
||||
area: Option<String>,
|
||||
timeout: Duration,
|
||||
jobs: usize,
|
||||
}
|
||||
|
||||
fn options(rest: &[&str]) -> Result<Options, String> {
|
||||
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
|
||||
let kernels = match arg("--kernel").as_deref() {
|
||||
None | Some("default") => vec!["default".to_string()],
|
||||
Some("smlib") => vec!["smlib".to_string()],
|
||||
Some("both") => vec!["default".to_string(), "smlib".to_string()],
|
||||
Some(k) => return Err(format!("--kernel {k}: default, smlib or both")),
|
||||
};
|
||||
let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
|
||||
let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
|
||||
Ok(Options { kernels, cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs })
|
||||
}
|
||||
|
||||
fn read(path: &Path) -> Value {
|
||||
std::fs::read_to_string(path).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or(Value::Null)
|
||||
}
|
||||
|
||||
/// Every case directory, sorted by area then id.
|
||||
fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
|
||||
let mut out = Vec::new();
|
||||
for area in std::fs::read_dir(dir).map_err(|e| format!("{}: {e}", dir.display()))?.flatten() {
|
||||
let area = area.path();
|
||||
if !area.is_dir() || only.is_some_and(|a| area.file_name().is_none_or(|n| n != a)) {
|
||||
continue;
|
||||
}
|
||||
for case in std::fs::read_dir(&area).map_err(|e| format!("{}: {e}", area.display()))?.flatten() {
|
||||
let case = case.path();
|
||||
if case.join("recipe.json").is_file() && case.join("expected.json").is_file() {
|
||||
out.push(case);
|
||||
}
|
||||
}
|
||||
}
|
||||
out.sort();
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
/// One case on one kernel, in its own process, killed after the timeout.
|
||||
fn run_one(o: &Options, kernel: &str, case: &Path) -> Value {
|
||||
let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
|
||||
let mut cmd = match &o.cli {
|
||||
Some(bin) => Command::new(bin),
|
||||
None => {
|
||||
let mut c = cargo();
|
||||
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--"]);
|
||||
c
|
||||
}
|
||||
};
|
||||
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||
if kernel == "default" {
|
||||
cmd.env_remove("SOLIDCRAFT_BOOLEANS");
|
||||
} else {
|
||||
cmd.env("SOLIDCRAFT_BOOLEANS", kernel);
|
||||
}
|
||||
let base = |status: &str, error: String| {
|
||||
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
json!({"case": id, "area": area, "kernel": kernel, "status": status, "error": error})
|
||||
};
|
||||
let mut child = match cmd.spawn() {
|
||||
Ok(c) => c,
|
||||
Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
|
||||
};
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(_)) => break,
|
||||
Ok(None) if start.elapsed() > o.timeout => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return base("hang", format!("no result after {} s", o.timeout.as_secs()));
|
||||
}
|
||||
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
|
||||
Err(e) => return base("fail", format!("wait: {e}")),
|
||||
}
|
||||
}
|
||||
let out = match child.wait_with_output() {
|
||||
Ok(o) => o,
|
||||
Err(e) => return base("fail", format!("output: {e}")),
|
||||
};
|
||||
match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
|
||||
Some(mut r) => {
|
||||
r["seconds"] = json!(start.elapsed().as_secs_f64());
|
||||
r
|
||||
}
|
||||
// No JSON: the process died (a native crash, an abort).
|
||||
None => {
|
||||
let err = String::from_utf8_lossy(&out.stderr);
|
||||
let tail: String = err.lines().rev().take(3).collect::<Vec<_>>().into_iter().rev().collect::<Vec<_>>().join(" / ");
|
||||
base("crash", format!("process exited ({}) without a result: {tail}", out.status))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
|
||||
let jobs: Vec<(String, PathBuf)> = o.kernels.iter().flat_map(|k| list.iter().map(move |c| (k.clone(), c.clone()))).collect();
|
||||
let next = Arc::new(Mutex::new(0usize));
|
||||
let results = Arc::new(Mutex::new(vec![Value::Null; jobs.len()]));
|
||||
std::thread::scope(|s| {
|
||||
for _ in 0..o.jobs.min(jobs.len().max(1)) {
|
||||
let (next, results, jobs) = (Arc::clone(&next), Arc::clone(&results), &jobs);
|
||||
s.spawn(move || {
|
||||
loop {
|
||||
let i = {
|
||||
let Ok(mut n) = next.lock() else { return };
|
||||
let i = *n;
|
||||
*n += 1;
|
||||
i
|
||||
};
|
||||
let Some((kernel, case)) = jobs.get(i) else { return };
|
||||
let r = run_one(o, kernel, case);
|
||||
println!(
|
||||
"{:<11} {} [{kernel}]",
|
||||
r["status"].as_str().unwrap_or("?").to_uppercase(),
|
||||
case.strip_prefix(root()).unwrap_or(case).display()
|
||||
);
|
||||
if let Ok(mut all) = results.lock()
|
||||
&& let Some(slot) = all.get_mut(i)
|
||||
{
|
||||
*slot = r;
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
|
||||
}
|
||||
|
||||
fn cell(r: Option<&Value>) -> String {
|
||||
let Some(r) = r else { return String::new() };
|
||||
let status = r["status"].as_str().unwrap_or("?");
|
||||
let mut notes: Vec<String> = r["checks"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter(|c| !c["ok"].as_bool().unwrap_or(false))
|
||||
.map(|c| format!("{} {} (want {})", c["check"].as_str().unwrap_or(""), short(&c["got"]), short(&c["want"])))
|
||||
.collect();
|
||||
if let Some(e) = r["error"].as_str() {
|
||||
notes.push(e.chars().take(160).collect());
|
||||
}
|
||||
let s = match status {
|
||||
"pass" => "pass".to_string(),
|
||||
"unsupported" => "not supported yet".to_string(),
|
||||
other => format!("**{}**", other.to_uppercase()),
|
||||
};
|
||||
if notes.is_empty() { s } else { format!("{s}: {}", notes.join("; ")).replace('|', "/").replace('\n', " ") }
|
||||
}
|
||||
|
||||
fn short(v: &Value) -> String {
|
||||
match v {
|
||||
Value::Number(n) => n.as_f64().map(|x| format!("{x:.6}").trim_end_matches('0').trim_end_matches('.').to_string()).unwrap_or_default(),
|
||||
Value::String(s) => s.chars().take(100).collect(),
|
||||
Value::Array(a) => format!("[{}]", a.iter().map(short).collect::<Vec<_>>().join(", ")),
|
||||
other => other.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn sources_of(want: &Value) -> Vec<String> {
|
||||
want["sources"].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(str::to_string)).collect()
|
||||
}
|
||||
|
||||
/// `tangency-labs/solidcraft#123` as a link.
|
||||
fn link(source: &str) -> String {
|
||||
match source.split_once('#') {
|
||||
Some((repo, n)) if repo.contains('/') => format!("[{source}](https://github.com/{repo}/issues/{n})"),
|
||||
_ => source.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_report(o: &Options, dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<String, String> {
|
||||
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
|
||||
let mut by_case: BTreeMap<(String, String), BTreeMap<String, Value>> = BTreeMap::new();
|
||||
for r in results {
|
||||
by_case.entry(key(r)).or_default().insert(r["kernel"].as_str().unwrap_or("").to_string(), r.clone());
|
||||
}
|
||||
// Coverage: per area, cases and their statuses per kernel; per source, its cases.
|
||||
let mut areas: BTreeMap<String, (usize, StatusCounts)> = BTreeMap::new();
|
||||
let mut per_source: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
let mut kinds: BTreeMap<String, usize> = BTreeMap::new();
|
||||
let mut rows = String::new();
|
||||
for case in list {
|
||||
let want = read(&case.join("expected.json"));
|
||||
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
let entry = areas.entry(area.clone()).or_default();
|
||||
entry.0 += 1;
|
||||
let runs = by_case.get(&(area.clone(), id.clone()));
|
||||
for k in &o.kernels {
|
||||
let status = runs.and_then(|r| r.get(k)).and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
|
||||
*entry.1.entry(k.clone()).or_default().entry(status).or_default() += 1;
|
||||
}
|
||||
let sources = sources_of(&want);
|
||||
for s in &sources {
|
||||
per_source.entry(s.clone()).or_default().push(format!("{area}/{id}"));
|
||||
}
|
||||
*kinds.entry(want["source"].as_str().unwrap_or("?").split_whitespace().next().unwrap_or("?").to_string()).or_default() += 1;
|
||||
let cells: Vec<String> = o.kernels.iter().map(|k| cell(runs.and_then(|r| r.get(k)))).collect();
|
||||
rows += &format!(
|
||||
"| {area}/{id} | {} | {} | {} | {} |\n",
|
||||
sources.iter().map(|s| link(s)).collect::<Vec<_>>().join(" "),
|
||||
want["what"].as_str().unwrap_or("").replace('|', "/"),
|
||||
want["source"].as_str().unwrap_or("?"),
|
||||
cells.join(" | ")
|
||||
);
|
||||
}
|
||||
let mut md = String::from(
|
||||
"# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and, in builds with SMLib, SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nEach kernel is graded against the case, never against the other kernel. **default** is the kernel SolidCraft ships today; **smlib** runs with `SOLIDCRAFT_BOOLEANS=smlib`. A failing case stays failing until its kernel is fixed; there is no workaround on SolidCraft's side. See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format.\n\n",
|
||||
);
|
||||
md += "## Coverage\n\n| Area | Cases |";
|
||||
for k in &o.kernels {
|
||||
md += &format!(" {k} |");
|
||||
}
|
||||
md += "\n|---|---|";
|
||||
md += &"---|".repeat(o.kernels.len());
|
||||
md += "\n";
|
||||
for (area, (n, statuses)) in &areas {
|
||||
md += &format!("| {area} | {n} |");
|
||||
for k in &o.kernels {
|
||||
let s = statuses.get(k).map(|m| m.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", ")).unwrap_or_default();
|
||||
md += &format!(" {s} |");
|
||||
}
|
||||
md += "\n";
|
||||
}
|
||||
md += &format!(
|
||||
"\n{} cases from {} sources; answers: {}.\n\n## Results\n\n| Case | Sources | What went wrong | Answer |",
|
||||
list.len(),
|
||||
per_source.len(),
|
||||
kinds.iter().map(|(k, v)| format!("{v} {k}")).collect::<Vec<_>>().join(", ")
|
||||
);
|
||||
for k in &o.kernels {
|
||||
md += &format!(" {k} |");
|
||||
}
|
||||
md += "\n|---|---|---|---|";
|
||||
md += &"---|".repeat(o.kernels.len());
|
||||
md += "\n";
|
||||
md += &rows;
|
||||
md += "\n## Sources\n\n| Source | Cases |\n|---|---|\n";
|
||||
for (s, cs) in &per_source {
|
||||
md += &format!("| {} | {} |\n", link(s), cs.join(", "));
|
||||
}
|
||||
// What was looked at and skipped, area by area (`<area>/skipped.json`).
|
||||
md += "\n## Looked at and skipped\n\n";
|
||||
let mut skipped = 0;
|
||||
let mut area_dirs: Vec<PathBuf> = std::fs::read_dir(dir).map_err(|e| e.to_string())?.flatten().map(|e| e.path()).filter(|p| p.is_dir()).collect();
|
||||
area_dirs.sort();
|
||||
for a in area_dirs {
|
||||
let list = read(&a.join("skipped.json"));
|
||||
let Some(items) = list.as_array().filter(|l| !l.is_empty()) else { continue };
|
||||
md += &format!("### {}\n\n| Source | Why skipped |\n|---|---|\n", a.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default());
|
||||
for it in items {
|
||||
skipped += 1;
|
||||
md += &format!("| {} | {} |\n", link(it["source"].as_str().unwrap_or("?")), it["why"].as_str().unwrap_or("").replace('|', "/"));
|
||||
}
|
||||
md += "\n";
|
||||
}
|
||||
if skipped == 0 {
|
||||
md += "None yet.\n";
|
||||
}
|
||||
std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?;
|
||||
let summary: Vec<String> = o
|
||||
.kernels
|
||||
.iter()
|
||||
.map(|k| {
|
||||
let n = results.iter().filter(|r| r["kernel"] == k.as_str() && r["status"] == "pass").count();
|
||||
format!("{k}: {n}/{} pass", list.len())
|
||||
})
|
||||
.collect();
|
||||
Ok(format!("kernel-cases: {} → docs/kernel-cases.md ({skipped} skipped sources listed)", summary.join(", ")))
|
||||
}
|
||||
|
||||
pub fn run(rest: &[&str]) -> Result<(), String> {
|
||||
let o = options(rest)?;
|
||||
let dir = root().join("tests/kernel-cases");
|
||||
let list = cases(&dir, o.area.as_deref())?;
|
||||
if list.is_empty() {
|
||||
return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
|
||||
}
|
||||
let results = run_all(&o, &list);
|
||||
println!("{}", write_report(&o, &dir, &list, &results)?);
|
||||
Ok(())
|
||||
}
|
||||
@@ -10,6 +10,7 @@ mod external_corpus;
|
||||
mod features;
|
||||
mod i18n;
|
||||
mod icons;
|
||||
mod kernel_cases;
|
||||
mod layers;
|
||||
mod licences;
|
||||
mod lists;
|
||||
@@ -45,6 +46,10 @@ commands:
|
||||
replay the reference tests (tests/oracle/*/recipe.json), compare with each
|
||||
case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
|
||||
already built, instead of cargo run --release)
|
||||
kernel-cases [--kernel default|smlib|both] [--area AREA] [--cli BIN] [--timeout S] [--jobs N]
|
||||
the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ (one
|
||||
process each, killed after --timeout, default 180 s) with each kernel asked for,
|
||||
grade it against the case's own answer and write docs/kernel-cases.md
|
||||
action-matrix [--bless] [--results FILE] [--from DIR]
|
||||
try every ordinary action on every face, edge, vertex and body of the design
|
||||
corpus and compare with tests/action_matrix/baseline.json (--bless rewrites it;
|
||||
@@ -102,6 +107,7 @@ fn main() -> ExitCode {
|
||||
Some("app-icons") => app_icons::run(&root()),
|
||||
Some("features") => features::run(&root()),
|
||||
Some("i18n") => i18n::run(&root(), &rest),
|
||||
Some("kernel-cases") => kernel_cases::run(&rest),
|
||||
Some("oracle") => cmd_oracle(rest.iter().position(|a| *a == "--cli").and_then(|i| rest.get(i + 1)).copied()),
|
||||
Some("action-matrix") => {
|
||||
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).copied();
|
||||
|
||||
Reference in New Issue
Block a user