Wave 39/45: diagnostics.kernel takes a typed contract, as main's other diagnostics commands do

1 commits, squashed; the originals stay reachable from branch jyviko/merge-main-develop-solidcore (#11360).

- 7ad1d3e12 jyviko: diagnostics.kernel takes a typed contract, as main's other diagnostics commands do

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
This commit is contained in:
jyviko
2026-11-02 16:58:17 -05:00
parent f582b1f39e
commit 28e983e191
115 changed files with 18703 additions and 600 deletions
+344 -56
View File
@@ -1,12 +1,18 @@
//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases` on SMLib, one process
//! per case (so a crash or a hang is that case's result, not the run's), and writes
//! `docs/kernel-cases.md`: results per case, coverage per area and per source, and the issues and
//! PRs looked at and skipped. Each case is graded against its own answer.
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases` on solidcore, or the
//! C++ SMLib kernel when asked for, one process per case (so a crash or a hang is that case's result, not the run's), and writes
//! its report: results per case, coverage per area and per source, and the issues and PRs looked
//! at and skipped. Each case is graded against its own answer.
//!
//! The run's output folder (`--out`, default `target/kernel-cases`) holds `results.json`, every
//! case's full result, and `dumps/<area>-<case>/`, the inputs of each SMLib call a case's run
//! failed in (`solidcraft-cli kernel-cases --dumps`), for replay.
//!
//! `--kernel solidcore|smlib|both` (default solidcore) picks the backend each case's process runs
//! on (`SOLIDCRAFT_KERNEL`). A solidcore run keeps its results, dumps and report in
//! `<out>/solidcore/`; an smlib run writes `docs/kernel-cases.md`; `both` runs every case on each,
//! keeps `docs/kernel-cases.md` as the SMLib report and writes the side by side to
//! `docs/kernel-cases-backends.md` and `<out>/backends.json`.
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
@@ -20,6 +26,9 @@ use crate::{cargo, root};
const DEFAULT_TIMEOUT_S: u64 = 180;
/// The backends `--kernel` may name, as `SOLIDCRAFT_KERNEL` takes them.
const BACKENDS: [&str; 2] = ["smlib", "solidcore"];
/// How many cases ended in each status.
type StatusCounts = BTreeMap<String, usize>;
@@ -30,6 +39,8 @@ pub(crate) struct Options {
pub(crate) jobs: usize,
/// The run's output folder: `results.json` and the SMLib dumps (`dumps/`).
out: PathBuf,
/// The backends each case runs on (`--kernel`, default solidcore alone).
pub(crate) kernels: Vec<&'static str>,
/// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]).
known_failing: Option<PathBuf>,
/// `--validation MODE`: when SMLib's full check runs on the cases' features (each_step,
@@ -37,28 +48,56 @@ pub(crate) struct Options {
validation: Option<String>,
}
impl Options {
/// A backend's output folder: the run's own for SMLib, `<out>/<backend>/` for any other.
fn out_of(&self, kernel: &str) -> PathBuf {
match kernel {
"smlib" => self.out.clone(),
other => self.out.join(other),
}
}
}
/// The run's options; `out` is its output folder unless `--out` gives one.
pub(crate) fn options(rest: &[&str], out: &str) -> Result<Options, String> {
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
let out = arg("--out").map_or_else(|| root().join(out), PathBuf::from);
let kernels = match arg("--kernel").as_deref() {
Some("smlib") => vec![BACKENDS[0]],
None | Some("solidcore") => vec![BACKENDS[1]],
Some("both") => BACKENDS.to_vec(),
Some(k) => return Err(format!("--kernel: unknown backend '{k}' (smlib, solidcore or both)")),
};
if rest.contains(&"--route") {
return Err("--route is gone: a run has one kernel, --kernel solidcore, smlib or both".into());
}
let known_failing = arg("--known-failing").map(|p| {
let p = PathBuf::from(p);
if p.is_absolute() { p } else { root().join(p) }
});
// A list is one kernel's: the gate grades the run's one kernel against it.
if known_failing.is_some() && kernels.len() != 1 {
return Err("--known-failing: a list is one kernel's, so not with --kernel both".into());
}
let validation = arg("--validation");
if let Some(v) = validation.as_deref().filter(|v| !["each_step", "each_feature", "off"].contains(v)) {
return Err(format!("--validation must be each_step, each_feature or off, not `{v}`"));
}
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing, validation })
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, kernels, known_failing, validation })
}
/// Every case's full result as JSON, into the run's output folder.
pub(crate) fn save_results(o: &Options, results: &[Value]) -> Result<(), String> {
std::fs::create_dir_all(&o.out).map_err(|e| format!("{}: {e}", o.out.display()))?;
let path = o.out.join("results.json");
let text = serde_json::to_string_pretty(results).map_err(|e| e.to_string())?;
save_json(&o.out, "results.json", results)
}
/// `value` as pretty JSON into `dir/name`.
fn save_json<T: serde::Serialize + ?Sized>(dir: &Path, name: &str, value: &T) -> Result<(), String> {
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
let path = dir.join(name);
let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?;
std::fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))
}
@@ -87,26 +126,66 @@ fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
/// One case, in its own process, killed after the timeout; `path` is the case's directory under
/// `tests/kernel-cases/`.
fn run_one(o: &Options, case: &Path) -> Value {
let mut r = run_case(o, case);
fn run_one(o: &Options, case: &Path, kernel: &str) -> Value {
let mut r = run_case(o, case, kernel);
let base = root().join("tests/kernel-cases");
r["path"] = json!(case.strip_prefix(&base).unwrap_or(case).to_string_lossy().replace('\\', "/"));
r
}
fn run_case(o: &Options, case: &Path) -> Value {
/// Waits for `child` for at most `timeout` while reading its stdout and stderr, so a child that
/// writes more than the pipe buffer (64 KiB on Linux) is not blocked in `write` and taken for a
/// hang. `None`: the child was still running at the timeout and has been killed.
fn wait_draining(mut child: std::process::Child, timeout: Duration) -> Result<Option<std::process::Output>, String> {
fn drain<R: std::io::Read + Send + 'static>(pipe: Option<R>) -> std::thread::JoinHandle<Vec<u8>> {
std::thread::spawn(move || {
let mut buf = Vec::new();
if let Some(mut p) = pipe {
let _ = p.read_to_end(&mut buf);
}
buf
})
}
let (stdout, stderr) = (drain(child.stdout.take()), drain(child.stderr.take()));
let start = Instant::now();
let status = loop {
match child.try_wait() {
Ok(Some(status)) => break status,
Ok(None) if start.elapsed() > timeout => {
let _ = child.kill();
let _ = child.wait();
// The readers are left to finish on their own: a grandchild (`cargo run`'s CLI)
// may still hold the pipes open.
return Ok(None);
}
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
Err(e) => return Err(format!("wait: {e}")),
}
};
let join = |h: std::thread::JoinHandle<Vec<u8>>| h.join().map_err(|_| "output: a pipe reader stopped".to_string());
Ok(Some(std::process::Output { status, stdout: join(stdout)?, stderr: join(stderr)? }))
}
/// The features `cargo run` builds the CLI with for a run on `kernels`: the C++ kernel is opt-in at
/// build time (`--features smlib`), so only a run that asks for it builds it. One build for the
/// whole run (`both` included), so the cases don't rebuild the CLI between kernels.
fn cli_features(kernels: &[&str]) -> &'static [&'static str] {
if kernels.contains(&"smlib") { &["--features", "smlib"] } else { &[] }
}
fn run_case(o: &Options, case: &Path, kernel: &str) -> Value {
let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
let mut cmd = match &o.cli {
Some(bin) => Command::new(bin),
None => {
let mut c = cargo();
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--features", "smlib", "--"]);
c.args(["run", "-q", "--release", "-p", "solidcraft-cli"]).args(cli_features(&o.kernels)).arg("--");
c
}
};
let mut dumps = std::ffi::OsString::from("--dumps=");
dumps.push(o.out.join("dumps"));
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
dumps.push(o.out_of(kernel).join("dumps"));
cmd.env("SOLIDCRAFT_KERNEL", kernel).current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
if let Some(v) = &o.validation {
cmd.arg(format!("--validation={v}"));
}
@@ -116,28 +195,17 @@ fn run_case(o: &Options, case: &Path) -> Value {
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
json!({"case": id, "area": area, "status": status, "error": error})
};
let mut child = match cmd.spawn() {
let child = match cmd.spawn() {
Ok(c) => c,
Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
};
// A case may need longer than the run's timeout (`timeout_s`, at most 30 min).
let timeout = read(&case.join("expected.json"))["timeout_s"].as_u64().map_or(o.timeout, |s| Duration::from_secs(s.min(1800)).max(o.timeout));
let start = Instant::now();
loop {
match child.try_wait() {
Ok(Some(_)) => break,
Ok(None) if start.elapsed() > timeout => {
let _ = child.kill();
let _ = child.wait();
return base("hang", format!("no result after {} s", timeout.as_secs()));
}
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
Err(e) => return base("fail", format!("wait: {e}")),
}
}
let out = match child.wait_with_output() {
Ok(o) => o,
Err(e) => return base("fail", format!("output: {e}")),
let out = match wait_draining(child, timeout) {
Ok(Some(out)) => out,
Ok(None) => return base("hang", format!("no result after {} s", timeout.as_secs())),
Err(e) => return base("fail", e),
};
match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
Some(mut r) => {
@@ -154,11 +222,27 @@ fn run_case(o: &Options, case: &Path) -> Value {
}
}
/// Every case on the run's one backend (the ladder's: it refuses `--kernel both`).
pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
run_on(o, list, &o.kernels).into_iter().next().unwrap_or_default()
}
/// Every case on each backend, one job pool for all the runs; the results per backend, in order.
fn run_on(o: &Options, list: &[PathBuf], kernels: &[&str]) -> Vec<Vec<Value>> {
// Without --cli each case is `cargo run`: build the CLI once first, so the first cases don't
// spend their timeout compiling it (the C++ kernel's build most of all).
if o.cli.is_none() {
let mut c = cargo();
c.args(["build", "-q", "--release", "-p", "solidcraft-cli"]).args(cli_features(kernels));
if let Err(e) = c.status().map_err(|e| e.to_string()).and_then(|s| if s.success() { Ok(()) } else { Err(s.to_string()) }) {
eprintln!("kernel-cases: building solidcraft-cli failed ({e}); each case will say why");
}
}
let jobs = list.len() * kernels.len();
let next = Arc::new(Mutex::new(0usize));
let results = Arc::new(Mutex::new(vec![Value::Null; list.len()]));
let results = Arc::new(Mutex::new(vec![Value::Null; jobs]));
std::thread::scope(|s| {
for _ in 0..o.jobs.min(list.len().max(1)) {
for _ in 0..o.jobs.min(jobs.max(1)) {
let (next, results) = (Arc::clone(&next), Arc::clone(&results));
s.spawn(move || {
loop {
@@ -168,9 +252,15 @@ pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
*n += 1;
i
};
let Some(case) = list.get(i) else { return };
let r = run_one(o, case);
println!("{:<11} {}", r["status"].as_str().unwrap_or("?").to_uppercase(), case.strip_prefix(root()).unwrap_or(case).display());
// Case by case, each on every backend in turn.
let (Some(case), Some(kernel)) = (list.get(i / kernels.len().max(1)), kernels.get(i % kernels.len().max(1))) else { return };
let r = run_one(o, case, kernel);
let tag = if kernels.len() > 1 { format!("{kernel:<10} ") } else { String::new() };
println!(
"{:<11} {tag}{}",
r["status"].as_str().unwrap_or("?").to_uppercase(),
case.strip_prefix(root()).unwrap_or(case).display()
);
if let Ok(mut all) = results.lock()
&& let Some(slot) = all.get_mut(i)
{
@@ -180,12 +270,12 @@ pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
});
}
});
Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
let all = Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default();
(0..kernels.len()).map(|k| all.iter().skip(k).step_by(kernels.len().max(1)).cloned().collect()).collect()
}
pub(crate) fn cell(r: Option<&Value>) -> String {
let Some(r) = r else { return String::new() };
let status = r["status"].as_str().unwrap_or("?");
/// What went wrong in a result: its failed checks, then its error.
fn notes(r: &Value) -> Vec<String> {
let mut notes: Vec<String> = r["checks"]
.as_array()
.into_iter()
@@ -196,6 +286,13 @@ pub(crate) fn cell(r: Option<&Value>) -> String {
if let Some(e) = r["error"].as_str() {
notes.push(e.chars().take(160).collect());
}
notes
}
pub(crate) fn cell(r: Option<&Value>) -> String {
let Some(r) = r else { return String::new() };
let status = r["status"].as_str().unwrap_or("?");
let notes = notes(r);
let s = match status {
"pass" => "pass".to_string(),
"unsupported" => "not supported yet".to_string(),
@@ -256,7 +353,9 @@ fn source_kinds(dir: &Path, cited: &BTreeMap<String, Vec<String>>) -> Result<Str
Ok(md)
}
fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<String, String> {
/// The report of one backend's run into `to` (relative to the repository root, or absolute);
/// `kernel` is the backend's label, `env` what its processes were given.
fn write_report(dir: &Path, list: &[PathBuf], results: &[Value], kernel: &str, env: &str, to: &Path) -> Result<String, String> {
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
let by_case: BTreeMap<(String, String), &Value> = results.iter().map(|r| (key(r), r)).collect();
// Coverage: per area, cases and their statuses; per source, its cases.
@@ -289,6 +388,9 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<Strin
let mut md = String::from(
"# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nThe cases run on SMLib, SolidCraft's kernel. A failing case stays failing until the kernel is fixed; there is no workaround on SolidCraft's side. A failed SMLib call leaves its inputs for replay in the run's output folder (`target/kernel-cases/dumps/<area>-<case>/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. Every area is also a regression gate: CI's Kernel rig workflow runs them on every pull request that touches the kernel and fails on a case that fails and its area's `KNOWN_FAILING` (such as [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING)) does not list, or a listed case that passes.\n\n",
);
if kernel != "smlib" {
md += &format!("**This run: `{env}`.**\n\n");
}
md += "## Coverage\n\n| Area | Cases | Results |\n|---|---|---|\n";
for (area, (n, statuses)) in &areas {
md += &format!("| {area} | {n} | {} |\n", statuses.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", "));
@@ -335,13 +437,96 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<Strin
if skipped == 0 {
md += "None yet.\n";
}
std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?;
let path = root().join(to);
let shown = path.strip_prefix(root()).unwrap_or(&path).display().to_string();
std::fs::write(&path, md).map_err(|e| format!("{shown}: {e}"))?;
let pass = results.iter().filter(|r| r["status"] == "pass").count();
Ok(format!("kernel-cases: {pass}/{} pass → docs/kernel-cases.md ({skipped} skipped sources listed)", list.len()))
let on = if kernel == "smlib" { String::new() } else { format!(" on {kernel}") };
Ok(format!("kernel-cases: {pass}/{} pass{on} → {shown} ({skipped} skipped sources listed)", list.len()))
}
/// pass, fail, or other (crash, hang, not supported yet...), for the summary.
fn bucket(status: &str) -> &'static str {
match status {
"pass" => "pass",
"fail" => "fail",
_ => "other",
}
}
/// A result's commands' time (or, without one, its process's).
fn seconds(r: &Value) -> Option<f64> {
r["seconds"].as_f64().or_else(|| r["process_seconds"].as_f64())
}
/// The two backends side by side: per case (area, id) each one's status, time and what went wrong,
/// as JSON, and the Markdown report: a summary, then per area the cases that differ first.
fn side_by_side(smlib: &[Value], solidcore: &[Value]) -> (Value, String) {
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
let mut pairs: BTreeMap<(String, String), [Option<&Value>; 2]> = BTreeMap::new();
for (k, rs) in [smlib, solidcore].iter().enumerate() {
for r in rs.iter().filter(|r| !r.is_null()) {
if let Some(slot) = pairs.entry(key(r)).or_default().get_mut(k) {
*slot = Some(r);
}
}
}
let status = |r: Option<&Value>| r.and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
let mut rows = Vec::new();
let (mut counts, mut times, mut agree) = ([StatusCounts::new(), StatusCounts::new()], [0.0f64; 2], 0);
for ((area, case), runs) in &pairs {
let same = status(runs[0]) == status(runs[1]);
agree += usize::from(same);
let mut row = json!({"case": case, "area": area, "agree": same});
for (k, name) in BACKENDS.iter().enumerate() {
let r = runs[k];
*counts[k].entry(bucket(&status(r)).to_string()).or_default() += 1;
times[k] += r.and_then(seconds).unwrap_or(0.0);
let error = r.map(|r| notes(r).join("; ")).filter(|e| !e.is_empty());
row[*name] = json!({"status": status(r), "seconds": r.and_then(seconds), "error": error});
}
rows.push(row);
}
let differ = pairs.len() - agree;
let mut md = String::from(
"# Kernel regression rig: SMLib and solidcore\n\nGenerated by `cargo xtask kernel-cases --kernel both`: every case in [`tests/kernel-cases/`](../tests/kernel-cases/) run on each backend (`SOLIDCRAFT_KERNEL`), graded against its own answer. Times are the commands' own. [`kernel-cases.md`](kernel-cases.md) is the SMLib run in full.\n\n",
);
md += "## Summary\n\n| Backend | Pass | Fail | Other | Time (s) |\n|---|---|---|---|---|\n";
for (k, name) in BACKENDS.iter().enumerate() {
let n = |b: &str| counts[k].get(b).copied().unwrap_or(0);
md += &format!("| {name} | {} | {} | {} | {:.1} |\n", n("pass"), n("fail"), n("other"), times[k]);
}
md += &format!("\n{} cases: {agree} agree, {differ} differ.\n", pairs.len());
let side = |r: &Value| match r["seconds"].as_f64() {
Some(s) => format!("{} ({s:.2} s)", r["status"].as_str().unwrap_or("?")),
None => r["status"].as_str().unwrap_or("?").to_string(),
};
let mut by_area: BTreeMap<&str, Vec<&Value>> = BTreeMap::new();
for row in &rows {
by_area.entry(row["area"].as_str().unwrap_or("")).or_default().push(row);
}
for (area, mut list) in by_area {
list.sort_by_key(|r| (r["agree"].as_bool().unwrap_or(false), r["case"].as_str().unwrap_or("").to_string()));
md += &format!("\n## {area}\n\n| Case | {} | {} | |\n|---|---|---|---|\n", BACKENDS[0], BACKENDS[1]);
for r in list {
let mark = if r["agree"].as_bool().unwrap_or(false) { "" } else { "**differ**" };
md += &format!("| {} | {} | {} | {mark} |\n", r["case"].as_str().unwrap_or(""), side(&r["smlib"]), side(&r["solidcore"]));
}
}
let summary: Value = BACKENDS
.iter()
.enumerate()
.map(|(k, name)| (name.to_string(), json!({"counts": counts[k], "seconds": times[k]})))
.collect::<serde_json::Map<_, _>>()
.into();
(json!({"summary": summary, "agree": agree, "differ": differ, "cases": rows}), md)
}
pub fn run(rest: &[&str]) -> Result<(), String> {
if rest.contains(&"--ladder") {
if options(rest, "target/kernel-ladder")?.kernels.len() != 1 {
return Err("--kernel: the ladder runs on one backend, smlib or solidcore".into());
}
return crate::kernel_ladder::run(rest);
}
let o = options(rest, "target/kernel-cases")?;
@@ -354,18 +539,39 @@ pub fn run(rest: &[&str]) -> Result<(), String> {
if list.is_empty() {
return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
}
let results = run_all(&o, &list);
save_results(&o, &results)?;
let runs = run_on(&o, &list, &o.kernels);
for (kernel, results) in o.kernels.iter().zip(&runs) {
save_json(&o.out_of(kernel), "results.json", results)?;
}
if let Some(file) = &o.known_failing {
// A gate run grades one set against its baseline; the report stays the full run's.
// A gate run grades the run's one kernel against its baseline, and writes no report; its
// results.json is still saved above.
let text = std::fs::read_to_string(file).map_err(|e| format!("{}: {e}", file.display()))?;
let base = file.parent().unwrap_or(&dir);
let paths: Vec<String> = list.iter().map(|c| c.strip_prefix(base).unwrap_or(c).to_string_lossy().replace('\\', "/")).collect();
let statuses: Vec<&str> = results.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect();
let graded = runs.first().ok_or("--known-failing: no run to grade")?;
let statuses: Vec<&str> = graded.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect();
println!("{}", gate(&text, &paths, &statuses).map_err(|e| format!("{}: {e}", file.display()))?);
return Ok(());
}
println!("{}", write_report(&dir, &list, &results)?);
for (kernel, results) in o.kernels.iter().zip(&runs) {
let out = o.out_of(kernel);
let to = if *kernel == "smlib" { PathBuf::from("docs/kernel-cases.md") } else { out.join("report.md") };
let env = format!("SOLIDCRAFT_KERNEL={kernel}");
println!("{}", write_report(&dir, &list, results, kernel, &env, &to)?);
}
if let [smlib, solidcore] = runs.as_slice() {
let (data, md) = side_by_side(smlib, solidcore);
let into = o.out.clone();
save_json(&into, "backends.json", &data)?;
std::fs::write(root().join("docs/kernel-cases-backends.md"), md).map_err(|e| format!("docs/kernel-cases-backends.md: {e}"))?;
println!(
"kernel-cases: {} agree, {} differ → docs/kernel-cases-backends.md, {}",
data["agree"],
data["differ"],
into.join("backends.json").display()
);
}
Ok(())
}
@@ -563,9 +769,91 @@ fn run_coverage(o: &Options, dir: &Path) -> Result<String, String> {
#[cfg(test)]
mod tests {
use super::{Exempt, coverage, exempt, gate, recipe_commands};
use super::*;
fn run(text: &str, cases: &[(&str, &str)]) -> Result<String, String> {
/// A child that writes more than the pipe buffer still finishes, with all its output.
#[cfg(unix)]
#[test]
fn a_child_writing_more_than_the_pipe_buffer_is_not_a_hang() {
let child = Command::new("sh")
.args(["-c", "head -c 200000 /dev/zero; head -c 100000 /dev/zero >&2"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let out = wait_draining(child, Duration::from_secs(60)).unwrap().unwrap();
assert!(out.status.success());
assert_eq!((out.stdout.len(), out.stderr.len()), (200_000, 100_000));
let slow = Command::new("sh").args(["-c", "sleep 30"]).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
let t = Instant::now();
assert!(wait_draining(slow, Duration::from_millis(200)).unwrap().is_none());
assert!(t.elapsed() < Duration::from_secs(20));
}
#[test]
fn side_by_side_counts_and_puts_differences_first() {
let r = |area: &str, case: &str, status: &str, s: f64| json!({"area": area, "case": case, "status": status, "seconds": s});
let smlib = [r("booleans", "a", "pass", 1.0), r("booleans", "b", "pass", 2.0), r("shell", "c", "crash", 0.5)];
let mut hang = json!({"area": "booleans", "case": "b", "status": "hang", "error": "no result after 180 s"});
hang["process_seconds"] = json!(180.0);
let solidcore = [r("booleans", "a", "pass", 0.5), hang, r("shell", "c", "fail", 1.5)];
let (data, md) = side_by_side(&smlib, &solidcore);
assert_eq!((data["agree"].as_u64(), data["differ"].as_u64()), (Some(1), Some(2)));
assert_eq!(data["summary"]["smlib"]["counts"], json!({"pass": 2, "other": 1}));
assert_eq!(data["summary"]["solidcore"]["counts"], json!({"pass": 1, "fail": 1, "other": 1}));
assert_eq!(data["summary"]["solidcore"]["seconds"].as_f64(), Some(182.0));
let b = data["cases"].as_array().and_then(|c| c.iter().find(|c| c["case"] == "b")).cloned().unwrap_or_default();
assert_eq!((b["solidcore"]["status"].as_str(), b["solidcore"]["error"].as_str()), (Some("hang"), Some("no result after 180 s")));
// In booleans, b (differs) comes before a (agrees).
let (pos_a, pos_b) = (md.find("| a |"), md.find("| b |"));
assert!(pos_b.is_some() && pos_a.is_some() && pos_b < pos_a, "{md}");
assert!(md.contains("| b | pass (2.00 s) | hang (180.00 s) | **differ** |"), "{md}");
assert!(md.contains("3 cases: 1 agree, 2 differ."), "{md}");
}
/// `--route`, the old per-family run, is no option: each case's process has one kernel. Passed
/// anyway, it is refused rather than run as something else.
#[test]
fn a_kernel_has_its_own_folder_and_route_is_refused() {
let o = options(&["--kernel", "both", "--out", "/r"], "x").unwrap();
assert_eq!((o.out_of("smlib"), o.out_of("solidcore")), (PathBuf::from("/r"), PathBuf::from("/r/solidcore")));
assert!(options(&["--route", "boolean=solidcore"], "x").is_err());
assert!(options(&["--kernel", "both", "--route"], "x").is_err());
}
/// `--known-failing` follows the kernel given, solidcore by default: a list is one kernel's.
#[test]
fn the_gate_grades_the_kernel_given() {
for (args, kernel) in [
(&["--known-failing", "/k"][..], "solidcore"),
(&["--kernel", "solidcore", "--known-failing", "/k"], "solidcore"),
(&["--kernel", "smlib", "--known-failing", "/k"], "smlib"),
] {
let o = options(args, "x").unwrap();
assert_eq!((o.known_failing, o.kernels), (Some(PathBuf::from("/k")), vec![kernel]), "{args:?}");
}
assert!(options(&["--kernel", "both", "--known-failing", "/k"], "x").is_err());
}
/// With no `--kernel`, a run is on solidcore; the C++ kernel is asked for.
#[test]
fn the_rig_defaults_to_solidcore() {
assert_eq!(options(&[], "x").unwrap().kernels, ["solidcore"]);
assert_eq!(options(&["--kernel", "smlib"], "x").unwrap().kernels, ["smlib"]);
assert_eq!(options(&["--kernel", "both"], "x").unwrap().kernels, ["smlib", "solidcore"]);
}
/// Only a run that asks for the C++ kernel builds the CLI with it.
#[test]
fn the_cli_is_built_with_the_cpp_kernel_only_when_a_run_asks_for_it() {
let features = |args: &[&str]| cli_features(&options(args, "x").unwrap().kernels).to_vec();
assert!(features(&[]).is_empty());
assert!(features(&["--kernel", "solidcore"]).is_empty());
assert_eq!(features(&["--kernel", "smlib"]), ["--features", "smlib"]);
assert_eq!(features(&["--kernel", "both"]), ["--features", "smlib"]);
}
fn gate_on(text: &str, cases: &[(&str, &str)]) -> Result<String, String> {
let names: Vec<String> = cases.iter().map(|(c, _)| c.to_string()).collect();
let statuses: Vec<&str> = cases.iter().map(|(_, s)| *s).collect();
gate(text, &names, &statuses)
@@ -574,20 +862,20 @@ mod tests {
#[test]
fn the_gate_fails_on_a_regression_and_on_a_fixed_case_still_listed() {
let list = "# today\na-case SMLib refuses the corner\nb-case G1 at the corner point\n";
assert!(run(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok());
let regression = run(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err();
assert!(gate_on(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok());
let regression = gate_on(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err();
assert!(regression.contains("REGRESSION (hang): c-case"), "{regression}");
let fixed = run(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err();
let fixed = gate_on(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err();
assert!(fixed.contains("FIXED, remove it from the known-failing list: a-case"), "{fixed}");
let stale = run(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err();
let stale = gate_on(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err();
assert!(stale.contains("not a case of this set: b-case"), "{stale}");
}
#[test]
fn the_known_failing_list_is_sorted_with_a_cause_each() {
assert!(run("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order"));
assert!(run("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of"));
assert!(run("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause"));
assert!(gate_on("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order"));
assert!(gate_on("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of"));
assert!(gate_on("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause"));
}
#[test]
+1 -1
View File
@@ -3,7 +3,7 @@
//! `tests/kernel-cases/ladder/ladder.json` lists tiers in order of complexity and, per tier, its
//! cases in order (paths under `tests/kernel-cases/`, so a tier may also list rig cases from other
//! areas). Each case is a test like any rig case, graded against its own `expected.json`, on
//! SMLib. On top, `tests/kernel-cases/ladder/golden.json` records per case the status, the
//! `--kernel`'s one backend. The golden is SMLib's, so a gate or a record says `--kernel smlib`. On top, `tests/kernel-cases/ladder/golden.json` records per case the status, the
//! commands' time and what was made (counts, volume, area): the run compares with it and flags
//! slowdowns and changes; cases run one at a time unless `--jobs` says otherwise, so the times are
//! not shared with other cases. `--record REASON` re-measures the golden and re-sorts each tier
+11 -5
View File
@@ -1,6 +1,7 @@
//! `cargo xtask known-failing --junit PATH... [--list FILE]`: grades a nextest run (its junit
//! report) against the tests known to fail today, `tests/KNOWN_FAILING_TESTS`. CI's test jobs
//! run it after nextest, in place of failing on any failed test. It fails on a test that failed
//! report) against the tests known to fail today on the default kernel,
//! `tests/KNOWN_FAILING_TESTS` (the C++ kernel's are `tests/KNOWN_FAILING_TESTS.smlib`, for a run
//! with `SOLIDCRAFT_KERNEL=smlib` and `--list`). CI's test jobs run it after nextest, in place of failing on any failed test. It fails on a test that failed
//! and is not listed, and on a listed test that passed (fixed: its line comes off in the same
//! pull request); a listed test that did not run (a pull request tests only the packages it
//! touches) is neither. The list has the kernel rig's format (`kernel_cases::known_failing`):
@@ -15,6 +16,8 @@ use quick_xml::Reader;
use quick_xml::events::{BytesStart, Event};
pub const LIST: &str = "tests/KNOWN_FAILING_TESTS";
/// The same for the C++ kernel (`SOLIDCRAFT_KERNEL=smlib`), graded with `--list`.
pub const SMLIB_LIST: &str = "tests/KNOWN_FAILING_TESTS.smlib";
pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> {
let mut reports = Vec::new();
@@ -60,9 +63,12 @@ fn add(all: &mut BTreeMap<String, bool>, xml: &str) -> Result<(), String> {
/// `cargo xtask lists`: the list is sorted, each test once, with a cause each.
pub fn check_list(root: &Path) -> Result<(), String> {
let file = root.join(LIST);
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
crate::kernel_cases::known_failing(&text).map(|_| ()).map_err(|e| format!("{LIST}: {e}"))
for list in [LIST, SMLIB_LIST] {
let file = root.join(list);
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
crate::kernel_cases::known_failing(&text).map_err(|e| format!("{list}: {e}"))?;
}
Ok(())
}
/// `path` itself, or every `.xml` file under it (sorted), into `out`.
+1 -1
View File
@@ -5,7 +5,7 @@
//!
//! - The message catalogs (`assets/locales/*.json`): one message per line, sorted by id, no
//! duplicate ids, as `serde_json` pretty-prints them.
//! - `tests/KNOWN_FAILING_TESTS`: one test a line with its cause, sorted, each test once
//! - `tests/KNOWN_FAILING_TESTS` and `.smlib`: one test a line with its cause, sorted, each test once
//! (checked only; `cargo xtask known-failing` reads it).
//! - Growing Rust lists: no written array lengths on more than ten lines of entries.
+17 -8
View File
@@ -49,21 +49,29 @@ commands:
replay the reference tests (tests/oracle/*/recipe.json), compare with each
case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
already built, instead of cargo run --release)
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--validation MODE] [--coverage]
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--kernel K] [--route R] [--known-failing FILE] [--validation MODE] [--coverage]
the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ on
SMLib (one process each, killed after --timeout, default 180 s), grade it against
the case's own answer and write docs/kernel-cases.md; DIR (default
target/kernel-cases) gets results.json and dumps/<area>-<case>/, the inputs of
each failing SMLib call (--cli: a solidcraft-cli built with --features smlib)
solidcore (one process each, killed after --timeout, default 180 s), grade it against
the case's own answer; DIR (default target/kernel-cases) gets results.json and
dumps/<area>-<case>/, the inputs of each failing SMLib call (--cli: a
solidcraft-cli already built);
--kernel solidcore|smlib|both (default solidcore) sets SOLIDCRAFT_KERNEL per case:
solidcore keeps results.json, dumps/ and report.md in DIR/solidcore/; smlib
writes docs/kernel-cases.md; both runs each case on each, and also writes the
side by side to docs/kernel-cases-backends.md and DIR/backends.json; smlib and
both need the C++ kernel, so without --cli they build the CLI with --features smlib
(a --cli built without it reports every case as unsupported)
--known-failing FILE: a regression gate instead of the report: fails on a case
that fails and FILE does not list, and on a listed case that passes
(tests/kernel-cases/<area>/KNOWN_FAILING, CI's Kernel rig workflow)
(tests/kernel-cases/<area>/KNOWN_FAILING, CI's Kernel rig workflow); it grades
the run's kernel, so a list is one kernel's (KNOWN_FAILING.solidcore,
KNOWN_FAILING for --kernel smlib) and --kernel both is refused
--validation MODE: when SMLib's full check runs on the cases' features:
each_step, each_feature (the default) or off (solidcraft-cli kernel-cases
--validation=MODE); with --ladder too
--ladder [--record REASON | --gate]: the complexity ladder instead
(tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared
with its timing golden (one case at a time unless --jobs), into
with its timing golden (one case at a time unless --jobs), on --kernel's one backend, into
docs/kernel-ladder.md (DIR default target/kernel-ladder); --record re-measures
the golden and re-sorts each tier by time; --gate fails on any case whose status
or counts differ from the golden, instead of the report
@@ -72,7 +80,8 @@ commands:
uses and tests/kernel-cases/EXEMPT does not list, and on a stale EXEMPT line
known-failing --junit PATH... [--list FILE]
grade a nextest run's junit report (a file, or every .xml under a directory)
against the tests known to fail today, tests/KNOWN_FAILING_TESTS (FILE): fails
against the tests known to fail today on the default kernel,
tests/KNOWN_FAILING_TESTS (FILE: .smlib is the C++ kernel's): fails
on a test that failed and is not listed, and on a listed test that passed
(fixed: take its line off); a listed test that did not run is neither. CI's
test jobs run it after nextest