From bc68e14c3e8e80a3911f9cc42dbe00fb1c7adcc2 Mon Sep 17 00:00:00 2001 From: Jackson Oursland Date: Sun, 18 Oct 2026 15:10:36 -0700 Subject: [PATCH] Mutation testing: judge kernel mutants by the black-box suites (#2660) The kernel will move from truck to solidcore (#1597). The tests that must catch regressions across that switch are the ones that don't depend on the kernel: the reference tests and the robustness goldens. cargo-mutants checks whether they would notice a change in the kernel's behaviour. - apps/solidcraft-cli/tests/oracle_cases.rs: the reference tests as a cargo test (ignored; CI keeps replaying them through cargo xtask oracle), run over one process per core. - .cargo/mutants.toml: mutants are judged by that test and the pull-request sample of the robustness goldens, not by the kernel's own unit tests. - A mutants profile (ci with incremental builds): about 15 s to rebuild after a kernel change instead of about 3 minutes. - docs/mutation-testing.md: how to run it, its cost, two cargo-mutants 27.1 problems and the pilot's results (15% of sampled kernel mutants caught). Co-authored-by: Claude Opus 5.5 (1M context) --- .cargo/mutants.toml | 14 +++++ .gitignore | 3 + CONTRIBUTING.md | 2 + Cargo.toml | 6 ++ apps/solidcraft-cli/tests/oracle_cases.rs | 66 ++++++++++++++++++++++ docs/mutation-testing.md | 69 +++++++++++++++++++++++ 6 files changed, 160 insertions(+) create mode 100644 .cargo/mutants.toml create mode 100644 apps/solidcraft-cli/tests/oracle_cases.rs create mode 100644 docs/mutation-testing.md diff --git a/.cargo/mutants.toml b/.cargo/mutants.toml new file mode 100644 index 000000000..2cc2da3ea --- /dev/null +++ b/.cargo/mutants.toml @@ -0,0 +1,14 @@ +# cargo-mutants (docs/mutation-testing.md). Mutants are judged by the black-box suites that will +# outlive the kernel backend: the reference tests (tests/oracle, through the oracle_cases test) +# and the pull-request sample of the robustness goldens (every 8th seed, as ci.yml). The +# kernel's own unit tests are left out on purpose: a mutant only they catch is a behaviour the +# backend-neutral suites would miss when the kernel is swapped. +profile = "mutants" +additional_cargo_test_args = [ + "--test", "oracle_cases", "--lib", "--", "--include-ignored", + "oracle_suite", + "robustness_goldens::seed_00", "robustness_goldens::seed_08", "robustness_goldens::seed_16", + "robustness_goldens::seed_24", "robustness_goldens::seed_32", "robustness_goldens::seed_40", + "robustness_goldens::seed_48", "robustness_goldens::seed_56", "robustness_goldens::seed_64", + "robustness_goldens::seed_72", "robustness_goldens::seed_80", "robustness_goldens::seed_88", +] diff --git a/.gitignore b/.gitignore index 0a9913f05..8e0305251 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,6 @@ /.claude/worktrees/ /target-snapshot __pycache__/ + +# cargo-mutants output (docs/mutation-testing.md) +mutants.out*/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 971144e4e..b4567e198 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,6 +148,8 @@ the most complete description of the house style. A geometry change that changes the robustness goldens re-records them in the same pull request: `ROBUSTNESS_GOLDENS_BLESS="why it changed" cargo test --profile ci -p solidcraft-engine --lib robustness_goldens::` (see [docs/robustness.md](docs/robustness.md)); CI runs every seed on pull requests that touch geometry. +[docs/mutation-testing.md](docs/mutation-testing.md) shows how to find kernel behaviour these +black-box suites miss. **UI scenarios** drive the real app headless: start a tool, click a plane or a sketch point, press keys, then assert on the result. diff --git a/Cargo.toml b/Cargo.toml index ba9d53840..6369851fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -142,3 +142,9 @@ matext4cgmath = { path = "vendor/matext4cgmath" } # Keep OpenGL pipelines, resolve textures and CAD callbacks at the same safe sample count. egui-wgpu = { path = "vendor/egui-wgpu" } + +# Mutation testing (cargo-mutants, docs/mutation-testing.md): the ci profile with incremental +# builds, since every mutant rebuilds the crates above the one it changes. +[profile.mutants] +inherits = "ci" +incremental = true diff --git a/apps/solidcraft-cli/tests/oracle_cases.rs b/apps/solidcraft-cli/tests/oracle_cases.rs new file mode 100644 index 000000000..76b062763 --- /dev/null +++ b/apps/solidcraft-cli/tests/oracle_cases.rs @@ -0,0 +1,66 @@ +//! The reference tests (tests/oracle, docs/oracle.md) as a `cargo test`, so tools that drive +//! `cargo test` (cargo-mutants) can use them: every case with expected values and no +//! `known_failure` mark must pass, as `cargo xtask oracle` requires. + +use std::path::Path; +use std::process::Command; + +use serde_json::Value; + +#[test] +#[ignore = "run by cargo-mutants (.cargo/mutants.toml); CI replays the oracle through cargo xtask oracle"] +fn oracle_suite() { + let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); + let dir = root.join("tests/oracle"); + let mut cases: Vec = std::fs::read_dir(&dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.join("recipe.json").is_file()) + .map(|p| format!("tests/oracle/{}", p.file_name().unwrap().to_string_lossy())) + .collect(); + cases.sort(); + assert!(cases.len() >= 50, "oracle cases in {}", dir.display()); + // One process per core, cases dealt round-robin so the slow ones spread out. + let n = std::thread::available_parallelism().map_or(4, |n| n.get()).min(cases.len()); + let children: Vec<_> = (0..n) + .map(|i| { + let chunk: Vec<&String> = cases.iter().skip(i).step_by(n).collect(); + Command::new(env!("CARGO_BIN_EXE_solidcraft-cli")) + .current_dir(&root) + .arg("oracle") + .arg("--json") + .args(chunk) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .unwrap() + }) + .collect(); + let mut results: Vec = Vec::new(); + for child in children { + let out = child.wait_with_output().unwrap(); + let part: Vec = + serde_json::from_slice(&out.stdout).unwrap_or_else(|e| panic!("oracle JSON: {e}\n{}", String::from_utf8_lossy(&out.stderr))); + results.extend(part); + } + assert_eq!(results.len(), cases.len()); + let mut failed = Vec::new(); + for r in &results { + let case = r["case"].as_str().unwrap_or(""); + let want: Value = + std::fs::read_to_string(dir.join(case).join("expected.json")).ok().and_then(|t| serde_json::from_str(&t).ok()).unwrap_or_default(); + if want.is_null() || want["known_failure"].is_string() || r["pass"].as_bool().unwrap_or(false) { + continue; + } + let checks: Vec = r["checks"] + .as_array() + .into_iter() + .flatten() + .filter(|c| !c["ok"].as_bool().unwrap_or(false)) + .map(|c| format!("{} got {} want {}", c["check"].as_str().unwrap_or(""), c["got"], c["want"])) + .collect(); + failed.push(format!("{case}: {} {}", r["error"].as_str().unwrap_or(""), checks.join("; "))); + } + assert!(failed.is_empty(), "oracle cases failed:\n{}", failed.join("\n")); +} diff --git a/docs/mutation-testing.md b/docs/mutation-testing.md new file mode 100644 index 000000000..74c54f49f --- /dev/null +++ b/docs/mutation-testing.md @@ -0,0 +1,69 @@ +# Mutation testing + +SolidCraft will move from truck to solidcore (#1597). The tests that have to catch regressions +across that switch are the ones that don't depend on the kernel: the reference tests +([docs/oracle.md](oracle.md)), the robustness goldens ([docs/robustness.md](robustness.md)) and +the engine's command tests. Most of the kernel's own unit tests call truck-specific code and go +away with it. + +Mutation testing checks those suites. [cargo-mutants](https://mutants.rs) makes small changes to +the source, one at a time (`*` becomes `/`, a condition is negated, a function returns a default +value), rebuilds, and runs the tests. A mutant the tests don't catch is a change in behaviour +nobody would notice, and solidcore could get the same thing wrong without anyone noticing. + +## What judges a mutant + +`.cargo/mutants.toml` judges mutants by the black-box suites only: + +- `oracle_suite` (`apps/solidcraft-cli/tests/oracle_cases.rs`): every case in `tests/oracle` with + expected values and no `known_failure` must pass, as `cargo xtask oracle` requires. It is + `#[ignore]`d so CI doesn't replay the oracle twice. +- The pull-request sample of the robustness goldens: every 8th seed, as `ci.yml` runs it. + +The kernel's unit tests are left out on purpose. A mutant that only they catch marks a behaviour +the black-box suites would miss once the kernel is swapped. + +## Running it + +``` +cargo install cargo-mutants --locked +CARGO_TARGET_DIR=$PWD/target cargo mutants --in-place \ + -f crates/kernel/src/measure.rs \ + --test-package solidcraft-cli --test-package solidcraft-engine \ + --baseline skip --timeout 900 --build-timeout 900 +``` + +- Use a worktree you don't otherwise touch: `--in-place` writes each mutant into the source tree + and restores it afterwards. +- `--shard 0/200 --sharding round-robin` with `-d crates/kernel` takes an even sample (every 200th + mutant) of the whole crate. +- The `mutants` profile in `Cargo.toml` is the `ci` profile with incremental builds. It cuts the + rebuild after a kernel change from about 3 minutes to about 15 seconds. +- A mutant costs 1.5–5 minutes on a 12-core Mac, mostly the tests. The kernel has about 40,000 + mutants, so run a sample or one file at a time, never the whole crate. This is a tool for + finding gaps, not a CI gate. + +Two problems with cargo-mutants 27.1 are behind the flags above: + +- The baseline (the unmutated run) ignores `--test-package` and tests the mutated crate, which + fails. Skip it with `--baseline skip`, set `--timeout` yourself, and check the unmutated tree + passes first. +- `--re` and `--exclude-re` don't filter "delete field" mutants. Saved diffs in + `mutants.out/diff/` name the mutation, not the file, in their `+++` line: apply one with + `patch < `. + +## Pilot results (October 2026, `cb857f1f`) + +| Sample | Mutants that build | Caught by the black-box suites | +|---|---:|---:| +| `crates/kernel/src/measure.rs`, every mutant | 85 | 85% | +| Whole kernel, every 200th mutant | 173 | 15% (95% interval 10–21%) | + +Out of 49 of the kernel sample's survivors, the kernel's own unit tests caught 21 (43%) and +nothing caught 28 (57%). About 11% of survivors swap `>` and `>=` on floating-point values that +are never exactly equal, so they don't change behaviour. + +The black-box suites are weakest on tessellation and the display mesh (0 of 19 mutants caught), +offset, shell, emboss and lip (0 of 14), blends (15%) and STEP/IGES exchange (17%). For a +survivor that a kernel unit test catches, that test's design is the starting point for a new +oracle case or engine test.