Wave 39/45: diagnostics.kernel takes a typed contract, as main's other diagnostics commands do
1 commits, squashed; the originals stay reachable from branch jyviko/merge-main-develop-solidcore (#11360). - 7ad1d3e12 jyviko: diagnostics.kernel takes a typed contract, as main's other diagnostics commands do Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
This commit is contained in:
+344
-56
@@ -1,12 +1,18 @@
|
||||
//! `cargo xtask kernel-cases`: the kernel regression rig. Runs every case in
|
||||
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases` on SMLib, one process
|
||||
//! per case (so a crash or a hang is that case's result, not the run's), and writes
|
||||
//! `docs/kernel-cases.md`: results per case, coverage per area and per source, and the issues and
|
||||
//! PRs looked at and skipped. Each case is graded against its own answer.
|
||||
//! `tests/kernel-cases/<area>/<id>/` through `solidcraft-cli kernel-cases` on solidcore, or the
|
||||
//! C++ SMLib kernel when asked for, one process per case (so a crash or a hang is that case's result, not the run's), and writes
|
||||
//! its report: results per case, coverage per area and per source, and the issues and PRs looked
|
||||
//! at and skipped. Each case is graded against its own answer.
|
||||
//!
|
||||
//! The run's output folder (`--out`, default `target/kernel-cases`) holds `results.json`, every
|
||||
//! case's full result, and `dumps/<area>-<case>/`, the inputs of each SMLib call a case's run
|
||||
//! failed in (`solidcraft-cli kernel-cases --dumps`), for replay.
|
||||
//!
|
||||
//! `--kernel solidcore|smlib|both` (default solidcore) picks the backend each case's process runs
|
||||
//! on (`SOLIDCRAFT_KERNEL`). A solidcore run keeps its results, dumps and report in
|
||||
//! `<out>/solidcore/`; an smlib run writes `docs/kernel-cases.md`; `both` runs every case on each,
|
||||
//! keeps `docs/kernel-cases.md` as the SMLib report and writes the side by side to
|
||||
//! `docs/kernel-cases-backends.md` and `<out>/backends.json`.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -20,6 +26,9 @@ use crate::{cargo, root};
|
||||
|
||||
const DEFAULT_TIMEOUT_S: u64 = 180;
|
||||
|
||||
/// The backends `--kernel` may name, as `SOLIDCRAFT_KERNEL` takes them.
|
||||
const BACKENDS: [&str; 2] = ["smlib", "solidcore"];
|
||||
|
||||
/// How many cases ended in each status.
|
||||
type StatusCounts = BTreeMap<String, usize>;
|
||||
|
||||
@@ -30,6 +39,8 @@ pub(crate) struct Options {
|
||||
pub(crate) jobs: usize,
|
||||
/// The run's output folder: `results.json` and the SMLib dumps (`dumps/`).
|
||||
out: PathBuf,
|
||||
/// The backends each case runs on (`--kernel`, default solidcore alone).
|
||||
pub(crate) kernels: Vec<&'static str>,
|
||||
/// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]).
|
||||
known_failing: Option<PathBuf>,
|
||||
/// `--validation MODE`: when SMLib's full check runs on the cases' features (each_step,
|
||||
@@ -37,28 +48,56 @@ pub(crate) struct Options {
|
||||
validation: Option<String>,
|
||||
}
|
||||
|
||||
impl Options {
|
||||
/// A backend's output folder: the run's own for SMLib, `<out>/<backend>/` for any other.
|
||||
fn out_of(&self, kernel: &str) -> PathBuf {
|
||||
match kernel {
|
||||
"smlib" => self.out.clone(),
|
||||
other => self.out.join(other),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The run's options; `out` is its output folder unless `--out` gives one.
|
||||
pub(crate) fn options(rest: &[&str], out: &str) -> Result<Options, String> {
|
||||
let arg = |name: &str| rest.iter().position(|a| *a == name).and_then(|i| rest.get(i + 1)).map(|s| s.to_string());
|
||||
let secs = arg("--timeout").map(|t| t.parse::<u64>().map_err(|e| format!("--timeout: {e}"))).transpose()?.unwrap_or(DEFAULT_TIMEOUT_S);
|
||||
let jobs = arg("--jobs").map(|j| j.parse::<usize>().map_err(|e| format!("--jobs: {e}"))).transpose()?.unwrap_or(4).clamp(1, 32);
|
||||
let out = arg("--out").map_or_else(|| root().join(out), PathBuf::from);
|
||||
let kernels = match arg("--kernel").as_deref() {
|
||||
Some("smlib") => vec![BACKENDS[0]],
|
||||
None | Some("solidcore") => vec![BACKENDS[1]],
|
||||
Some("both") => BACKENDS.to_vec(),
|
||||
Some(k) => return Err(format!("--kernel: unknown backend '{k}' (smlib, solidcore or both)")),
|
||||
};
|
||||
if rest.contains(&"--route") {
|
||||
return Err("--route is gone: a run has one kernel, --kernel solidcore, smlib or both".into());
|
||||
}
|
||||
let known_failing = arg("--known-failing").map(|p| {
|
||||
let p = PathBuf::from(p);
|
||||
if p.is_absolute() { p } else { root().join(p) }
|
||||
});
|
||||
// A list is one kernel's: the gate grades the run's one kernel against it.
|
||||
if known_failing.is_some() && kernels.len() != 1 {
|
||||
return Err("--known-failing: a list is one kernel's, so not with --kernel both".into());
|
||||
}
|
||||
let validation = arg("--validation");
|
||||
if let Some(v) = validation.as_deref().filter(|v| !["each_step", "each_feature", "off"].contains(v)) {
|
||||
return Err(format!("--validation must be each_step, each_feature or off, not `{v}`"));
|
||||
}
|
||||
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing, validation })
|
||||
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, kernels, known_failing, validation })
|
||||
}
|
||||
|
||||
/// Every case's full result as JSON, into the run's output folder.
|
||||
pub(crate) fn save_results(o: &Options, results: &[Value]) -> Result<(), String> {
|
||||
std::fs::create_dir_all(&o.out).map_err(|e| format!("{}: {e}", o.out.display()))?;
|
||||
let path = o.out.join("results.json");
|
||||
let text = serde_json::to_string_pretty(results).map_err(|e| e.to_string())?;
|
||||
save_json(&o.out, "results.json", results)
|
||||
}
|
||||
|
||||
/// `value` as pretty JSON into `dir/name`.
|
||||
fn save_json<T: serde::Serialize + ?Sized>(dir: &Path, name: &str, value: &T) -> Result<(), String> {
|
||||
std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
|
||||
let path = dir.join(name);
|
||||
let text = serde_json::to_string_pretty(value).map_err(|e| e.to_string())?;
|
||||
std::fs::write(&path, text).map_err(|e| format!("{}: {e}", path.display()))
|
||||
}
|
||||
|
||||
@@ -87,26 +126,66 @@ fn cases(dir: &Path, only: Option<&str>) -> Result<Vec<PathBuf>, String> {
|
||||
|
||||
/// One case, in its own process, killed after the timeout; `path` is the case's directory under
|
||||
/// `tests/kernel-cases/`.
|
||||
fn run_one(o: &Options, case: &Path) -> Value {
|
||||
let mut r = run_case(o, case);
|
||||
fn run_one(o: &Options, case: &Path, kernel: &str) -> Value {
|
||||
let mut r = run_case(o, case, kernel);
|
||||
let base = root().join("tests/kernel-cases");
|
||||
r["path"] = json!(case.strip_prefix(&base).unwrap_or(case).to_string_lossy().replace('\\', "/"));
|
||||
r
|
||||
}
|
||||
|
||||
fn run_case(o: &Options, case: &Path) -> Value {
|
||||
/// Waits for `child` for at most `timeout` while reading its stdout and stderr, so a child that
|
||||
/// writes more than the pipe buffer (64 KiB on Linux) is not blocked in `write` and taken for a
|
||||
/// hang. `None`: the child was still running at the timeout and has been killed.
|
||||
fn wait_draining(mut child: std::process::Child, timeout: Duration) -> Result<Option<std::process::Output>, String> {
|
||||
fn drain<R: std::io::Read + Send + 'static>(pipe: Option<R>) -> std::thread::JoinHandle<Vec<u8>> {
|
||||
std::thread::spawn(move || {
|
||||
let mut buf = Vec::new();
|
||||
if let Some(mut p) = pipe {
|
||||
let _ = p.read_to_end(&mut buf);
|
||||
}
|
||||
buf
|
||||
})
|
||||
}
|
||||
let (stdout, stderr) = (drain(child.stdout.take()), drain(child.stderr.take()));
|
||||
let start = Instant::now();
|
||||
let status = loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => break status,
|
||||
Ok(None) if start.elapsed() > timeout => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
// The readers are left to finish on their own: a grandchild (`cargo run`'s CLI)
|
||||
// may still hold the pipes open.
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
|
||||
Err(e) => return Err(format!("wait: {e}")),
|
||||
}
|
||||
};
|
||||
let join = |h: std::thread::JoinHandle<Vec<u8>>| h.join().map_err(|_| "output: a pipe reader stopped".to_string());
|
||||
Ok(Some(std::process::Output { status, stdout: join(stdout)?, stderr: join(stderr)? }))
|
||||
}
|
||||
|
||||
/// The features `cargo run` builds the CLI with for a run on `kernels`: the C++ kernel is opt-in at
|
||||
/// build time (`--features smlib`), so only a run that asks for it builds it. One build for the
|
||||
/// whole run (`both` included), so the cases don't rebuild the CLI between kernels.
|
||||
fn cli_features(kernels: &[&str]) -> &'static [&'static str] {
|
||||
if kernels.contains(&"smlib") { &["--features", "smlib"] } else { &[] }
|
||||
}
|
||||
|
||||
fn run_case(o: &Options, case: &Path, kernel: &str) -> Value {
|
||||
let rel = case.strip_prefix(root()).unwrap_or(case).to_path_buf();
|
||||
let mut cmd = match &o.cli {
|
||||
Some(bin) => Command::new(bin),
|
||||
None => {
|
||||
let mut c = cargo();
|
||||
c.args(["run", "-q", "--release", "-p", "solidcraft-cli", "--features", "smlib", "--"]);
|
||||
c.args(["run", "-q", "--release", "-p", "solidcraft-cli"]).args(cli_features(&o.kernels)).arg("--");
|
||||
c
|
||||
}
|
||||
};
|
||||
let mut dumps = std::ffi::OsString::from("--dumps=");
|
||||
dumps.push(o.out.join("dumps"));
|
||||
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
|
||||
dumps.push(o.out_of(kernel).join("dumps"));
|
||||
cmd.env("SOLIDCRAFT_KERNEL", kernel).current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
|
||||
if let Some(v) = &o.validation {
|
||||
cmd.arg(format!("--validation={v}"));
|
||||
}
|
||||
@@ -116,28 +195,17 @@ fn run_case(o: &Options, case: &Path) -> Value {
|
||||
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
json!({"case": id, "area": area, "status": status, "error": error})
|
||||
};
|
||||
let mut child = match cmd.spawn() {
|
||||
let child = match cmd.spawn() {
|
||||
Ok(c) => c,
|
||||
Err(e) => return base("fail", format!("solidcraft-cli: {e}")),
|
||||
};
|
||||
// A case may need longer than the run's timeout (`timeout_s`, at most 30 min).
|
||||
let timeout = read(&case.join("expected.json"))["timeout_s"].as_u64().map_or(o.timeout, |s| Duration::from_secs(s.min(1800)).max(o.timeout));
|
||||
let start = Instant::now();
|
||||
loop {
|
||||
match child.try_wait() {
|
||||
Ok(Some(_)) => break,
|
||||
Ok(None) if start.elapsed() > timeout => {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
return base("hang", format!("no result after {} s", timeout.as_secs()));
|
||||
}
|
||||
Ok(None) => std::thread::sleep(Duration::from_millis(50)),
|
||||
Err(e) => return base("fail", format!("wait: {e}")),
|
||||
}
|
||||
}
|
||||
let out = match child.wait_with_output() {
|
||||
Ok(o) => o,
|
||||
Err(e) => return base("fail", format!("output: {e}")),
|
||||
let out = match wait_draining(child, timeout) {
|
||||
Ok(Some(out)) => out,
|
||||
Ok(None) => return base("hang", format!("no result after {} s", timeout.as_secs())),
|
||||
Err(e) => return base("fail", e),
|
||||
};
|
||||
match serde_json::from_slice::<Value>(&out.stdout).ok().and_then(|v| v.as_array().and_then(|a| a.first().cloned())) {
|
||||
Some(mut r) => {
|
||||
@@ -154,11 +222,27 @@ fn run_case(o: &Options, case: &Path) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
/// Every case on the run's one backend (the ladder's: it refuses `--kernel both`).
|
||||
pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
|
||||
run_on(o, list, &o.kernels).into_iter().next().unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Every case on each backend, one job pool for all the runs; the results per backend, in order.
|
||||
fn run_on(o: &Options, list: &[PathBuf], kernels: &[&str]) -> Vec<Vec<Value>> {
|
||||
// Without --cli each case is `cargo run`: build the CLI once first, so the first cases don't
|
||||
// spend their timeout compiling it (the C++ kernel's build most of all).
|
||||
if o.cli.is_none() {
|
||||
let mut c = cargo();
|
||||
c.args(["build", "-q", "--release", "-p", "solidcraft-cli"]).args(cli_features(kernels));
|
||||
if let Err(e) = c.status().map_err(|e| e.to_string()).and_then(|s| if s.success() { Ok(()) } else { Err(s.to_string()) }) {
|
||||
eprintln!("kernel-cases: building solidcraft-cli failed ({e}); each case will say why");
|
||||
}
|
||||
}
|
||||
let jobs = list.len() * kernels.len();
|
||||
let next = Arc::new(Mutex::new(0usize));
|
||||
let results = Arc::new(Mutex::new(vec![Value::Null; list.len()]));
|
||||
let results = Arc::new(Mutex::new(vec![Value::Null; jobs]));
|
||||
std::thread::scope(|s| {
|
||||
for _ in 0..o.jobs.min(list.len().max(1)) {
|
||||
for _ in 0..o.jobs.min(jobs.max(1)) {
|
||||
let (next, results) = (Arc::clone(&next), Arc::clone(&results));
|
||||
s.spawn(move || {
|
||||
loop {
|
||||
@@ -168,9 +252,15 @@ pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
|
||||
*n += 1;
|
||||
i
|
||||
};
|
||||
let Some(case) = list.get(i) else { return };
|
||||
let r = run_one(o, case);
|
||||
println!("{:<11} {}", r["status"].as_str().unwrap_or("?").to_uppercase(), case.strip_prefix(root()).unwrap_or(case).display());
|
||||
// Case by case, each on every backend in turn.
|
||||
let (Some(case), Some(kernel)) = (list.get(i / kernels.len().max(1)), kernels.get(i % kernels.len().max(1))) else { return };
|
||||
let r = run_one(o, case, kernel);
|
||||
let tag = if kernels.len() > 1 { format!("{kernel:<10} ") } else { String::new() };
|
||||
println!(
|
||||
"{:<11} {tag}{}",
|
||||
r["status"].as_str().unwrap_or("?").to_uppercase(),
|
||||
case.strip_prefix(root()).unwrap_or(case).display()
|
||||
);
|
||||
if let Ok(mut all) = results.lock()
|
||||
&& let Some(slot) = all.get_mut(i)
|
||||
{
|
||||
@@ -180,12 +270,12 @@ pub(crate) fn run_all(o: &Options, list: &[PathBuf]) -> Vec<Value> {
|
||||
});
|
||||
}
|
||||
});
|
||||
Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default()
|
||||
let all = Arc::try_unwrap(results).ok().and_then(|m| m.into_inner().ok()).unwrap_or_default();
|
||||
(0..kernels.len()).map(|k| all.iter().skip(k).step_by(kernels.len().max(1)).cloned().collect()).collect()
|
||||
}
|
||||
|
||||
pub(crate) fn cell(r: Option<&Value>) -> String {
|
||||
let Some(r) = r else { return String::new() };
|
||||
let status = r["status"].as_str().unwrap_or("?");
|
||||
/// What went wrong in a result: its failed checks, then its error.
|
||||
fn notes(r: &Value) -> Vec<String> {
|
||||
let mut notes: Vec<String> = r["checks"]
|
||||
.as_array()
|
||||
.into_iter()
|
||||
@@ -196,6 +286,13 @@ pub(crate) fn cell(r: Option<&Value>) -> String {
|
||||
if let Some(e) = r["error"].as_str() {
|
||||
notes.push(e.chars().take(160).collect());
|
||||
}
|
||||
notes
|
||||
}
|
||||
|
||||
pub(crate) fn cell(r: Option<&Value>) -> String {
|
||||
let Some(r) = r else { return String::new() };
|
||||
let status = r["status"].as_str().unwrap_or("?");
|
||||
let notes = notes(r);
|
||||
let s = match status {
|
||||
"pass" => "pass".to_string(),
|
||||
"unsupported" => "not supported yet".to_string(),
|
||||
@@ -256,7 +353,9 @@ fn source_kinds(dir: &Path, cited: &BTreeMap<String, Vec<String>>) -> Result<Str
|
||||
Ok(md)
|
||||
}
|
||||
|
||||
fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<String, String> {
|
||||
/// The report of one backend's run into `to` (relative to the repository root, or absolute);
|
||||
/// `kernel` is the backend's label, `env` what its processes were given.
|
||||
fn write_report(dir: &Path, list: &[PathBuf], results: &[Value], kernel: &str, env: &str, to: &Path) -> Result<String, String> {
|
||||
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
|
||||
let by_case: BTreeMap<(String, String), &Value> = results.iter().map(|r| (key(r), r)).collect();
|
||||
// Coverage: per area, cases and their statuses; per source, its cases.
|
||||
@@ -289,6 +388,9 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<Strin
|
||||
let mut md = String::from(
|
||||
"# Kernel regression rig\n\nGenerated by `cargo xtask kernel-cases`. Every geometry case the old kernel (truck) got wrong, as an issue, a PR or a crash report recorded it, is a use case in [`tests/kernel-cases/`](../tests/kernel-cases/) with its own right answer: numbers from analysis or a hand calculation (**analytic**) where the geometry has them, and otherwise the invariants (**invariants**): the operation succeeds, or refuses cleanly where that is the right answer; each body is one closed shell that passes the record's structure and geometry checks and SMLib's own check; the expected face, edge and vertex counts. Never a kernel's output, and never a commercial kernel's.\n\nThe cases run on SMLib, SolidCraft's kernel. A failing case stays failing until the kernel is fixed; there is no workaround on SolidCraft's side. A failed SMLib call leaves its inputs for replay in the run's output folder (`target/kernel-cases/dumps/<area>-<case>/`). See [`tests/kernel-cases/README.md`](../tests/kernel-cases/README.md) for the format. Every area is also a regression gate: CI's Kernel rig workflow runs them on every pull request that touches the kernel and fails on a case that fails and its area's `KNOWN_FAILING` (such as [`fillets/KNOWN_FAILING`](../tests/kernel-cases/fillets/KNOWN_FAILING)) does not list, or a listed case that passes.\n\n",
|
||||
);
|
||||
if kernel != "smlib" {
|
||||
md += &format!("**This run: `{env}`.**\n\n");
|
||||
}
|
||||
md += "## Coverage\n\n| Area | Cases | Results |\n|---|---|---|\n";
|
||||
for (area, (n, statuses)) in &areas {
|
||||
md += &format!("| {area} | {n} | {} |\n", statuses.iter().map(|(s, c)| format!("{c} {s}")).collect::<Vec<_>>().join(", "));
|
||||
@@ -335,13 +437,96 @@ fn write_report(dir: &Path, list: &[PathBuf], results: &[Value]) -> Result<Strin
|
||||
if skipped == 0 {
|
||||
md += "None yet.\n";
|
||||
}
|
||||
std::fs::write(root().join("docs/kernel-cases.md"), md).map_err(|e| format!("docs/kernel-cases.md: {e}"))?;
|
||||
let path = root().join(to);
|
||||
let shown = path.strip_prefix(root()).unwrap_or(&path).display().to_string();
|
||||
std::fs::write(&path, md).map_err(|e| format!("{shown}: {e}"))?;
|
||||
let pass = results.iter().filter(|r| r["status"] == "pass").count();
|
||||
Ok(format!("kernel-cases: {pass}/{} pass → docs/kernel-cases.md ({skipped} skipped sources listed)", list.len()))
|
||||
let on = if kernel == "smlib" { String::new() } else { format!(" on {kernel}") };
|
||||
Ok(format!("kernel-cases: {pass}/{} pass{on} → {shown} ({skipped} skipped sources listed)", list.len()))
|
||||
}
|
||||
|
||||
/// pass, fail, or other (crash, hang, not supported yet...), for the summary.
|
||||
fn bucket(status: &str) -> &'static str {
|
||||
match status {
|
||||
"pass" => "pass",
|
||||
"fail" => "fail",
|
||||
_ => "other",
|
||||
}
|
||||
}
|
||||
|
||||
/// A result's commands' time (or, without one, its process's).
|
||||
fn seconds(r: &Value) -> Option<f64> {
|
||||
r["seconds"].as_f64().or_else(|| r["process_seconds"].as_f64())
|
||||
}
|
||||
|
||||
/// The two backends side by side: per case (area, id) each one's status, time and what went wrong,
|
||||
/// as JSON, and the Markdown report: a summary, then per area the cases that differ first.
|
||||
fn side_by_side(smlib: &[Value], solidcore: &[Value]) -> (Value, String) {
|
||||
let key = |r: &Value| (r["area"].as_str().unwrap_or("").to_string(), r["case"].as_str().unwrap_or("").to_string());
|
||||
let mut pairs: BTreeMap<(String, String), [Option<&Value>; 2]> = BTreeMap::new();
|
||||
for (k, rs) in [smlib, solidcore].iter().enumerate() {
|
||||
for r in rs.iter().filter(|r| !r.is_null()) {
|
||||
if let Some(slot) = pairs.entry(key(r)).or_default().get_mut(k) {
|
||||
*slot = Some(r);
|
||||
}
|
||||
}
|
||||
}
|
||||
let status = |r: Option<&Value>| r.and_then(|r| r["status"].as_str()).unwrap_or("not run").to_string();
|
||||
let mut rows = Vec::new();
|
||||
let (mut counts, mut times, mut agree) = ([StatusCounts::new(), StatusCounts::new()], [0.0f64; 2], 0);
|
||||
for ((area, case), runs) in &pairs {
|
||||
let same = status(runs[0]) == status(runs[1]);
|
||||
agree += usize::from(same);
|
||||
let mut row = json!({"case": case, "area": area, "agree": same});
|
||||
for (k, name) in BACKENDS.iter().enumerate() {
|
||||
let r = runs[k];
|
||||
*counts[k].entry(bucket(&status(r)).to_string()).or_default() += 1;
|
||||
times[k] += r.and_then(seconds).unwrap_or(0.0);
|
||||
let error = r.map(|r| notes(r).join("; ")).filter(|e| !e.is_empty());
|
||||
row[*name] = json!({"status": status(r), "seconds": r.and_then(seconds), "error": error});
|
||||
}
|
||||
rows.push(row);
|
||||
}
|
||||
let differ = pairs.len() - agree;
|
||||
let mut md = String::from(
|
||||
"# Kernel regression rig: SMLib and solidcore\n\nGenerated by `cargo xtask kernel-cases --kernel both`: every case in [`tests/kernel-cases/`](../tests/kernel-cases/) run on each backend (`SOLIDCRAFT_KERNEL`), graded against its own answer. Times are the commands' own. [`kernel-cases.md`](kernel-cases.md) is the SMLib run in full.\n\n",
|
||||
);
|
||||
md += "## Summary\n\n| Backend | Pass | Fail | Other | Time (s) |\n|---|---|---|---|---|\n";
|
||||
for (k, name) in BACKENDS.iter().enumerate() {
|
||||
let n = |b: &str| counts[k].get(b).copied().unwrap_or(0);
|
||||
md += &format!("| {name} | {} | {} | {} | {:.1} |\n", n("pass"), n("fail"), n("other"), times[k]);
|
||||
}
|
||||
md += &format!("\n{} cases: {agree} agree, {differ} differ.\n", pairs.len());
|
||||
let side = |r: &Value| match r["seconds"].as_f64() {
|
||||
Some(s) => format!("{} ({s:.2} s)", r["status"].as_str().unwrap_or("?")),
|
||||
None => r["status"].as_str().unwrap_or("?").to_string(),
|
||||
};
|
||||
let mut by_area: BTreeMap<&str, Vec<&Value>> = BTreeMap::new();
|
||||
for row in &rows {
|
||||
by_area.entry(row["area"].as_str().unwrap_or("")).or_default().push(row);
|
||||
}
|
||||
for (area, mut list) in by_area {
|
||||
list.sort_by_key(|r| (r["agree"].as_bool().unwrap_or(false), r["case"].as_str().unwrap_or("").to_string()));
|
||||
md += &format!("\n## {area}\n\n| Case | {} | {} | |\n|---|---|---|---|\n", BACKENDS[0], BACKENDS[1]);
|
||||
for r in list {
|
||||
let mark = if r["agree"].as_bool().unwrap_or(false) { "" } else { "**differ**" };
|
||||
md += &format!("| {} | {} | {} | {mark} |\n", r["case"].as_str().unwrap_or(""), side(&r["smlib"]), side(&r["solidcore"]));
|
||||
}
|
||||
}
|
||||
let summary: Value = BACKENDS
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(k, name)| (name.to_string(), json!({"counts": counts[k], "seconds": times[k]})))
|
||||
.collect::<serde_json::Map<_, _>>()
|
||||
.into();
|
||||
(json!({"summary": summary, "agree": agree, "differ": differ, "cases": rows}), md)
|
||||
}
|
||||
|
||||
pub fn run(rest: &[&str]) -> Result<(), String> {
|
||||
if rest.contains(&"--ladder") {
|
||||
if options(rest, "target/kernel-ladder")?.kernels.len() != 1 {
|
||||
return Err("--kernel: the ladder runs on one backend, smlib or solidcore".into());
|
||||
}
|
||||
return crate::kernel_ladder::run(rest);
|
||||
}
|
||||
let o = options(rest, "target/kernel-cases")?;
|
||||
@@ -354,18 +539,39 @@ pub fn run(rest: &[&str]) -> Result<(), String> {
|
||||
if list.is_empty() {
|
||||
return Err(format!("no cases (recipe.json + expected.json) under {}", dir.display()));
|
||||
}
|
||||
let results = run_all(&o, &list);
|
||||
save_results(&o, &results)?;
|
||||
let runs = run_on(&o, &list, &o.kernels);
|
||||
for (kernel, results) in o.kernels.iter().zip(&runs) {
|
||||
save_json(&o.out_of(kernel), "results.json", results)?;
|
||||
}
|
||||
if let Some(file) = &o.known_failing {
|
||||
// A gate run grades one set against its baseline; the report stays the full run's.
|
||||
// A gate run grades the run's one kernel against its baseline, and writes no report; its
|
||||
// results.json is still saved above.
|
||||
let text = std::fs::read_to_string(file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||||
let base = file.parent().unwrap_or(&dir);
|
||||
let paths: Vec<String> = list.iter().map(|c| c.strip_prefix(base).unwrap_or(c).to_string_lossy().replace('\\', "/")).collect();
|
||||
let statuses: Vec<&str> = results.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect();
|
||||
let graded = runs.first().ok_or("--known-failing: no run to grade")?;
|
||||
let statuses: Vec<&str> = graded.iter().map(|r| r["status"].as_str().unwrap_or("?")).collect();
|
||||
println!("{}", gate(&text, &paths, &statuses).map_err(|e| format!("{}: {e}", file.display()))?);
|
||||
return Ok(());
|
||||
}
|
||||
println!("{}", write_report(&dir, &list, &results)?);
|
||||
for (kernel, results) in o.kernels.iter().zip(&runs) {
|
||||
let out = o.out_of(kernel);
|
||||
let to = if *kernel == "smlib" { PathBuf::from("docs/kernel-cases.md") } else { out.join("report.md") };
|
||||
let env = format!("SOLIDCRAFT_KERNEL={kernel}");
|
||||
println!("{}", write_report(&dir, &list, results, kernel, &env, &to)?);
|
||||
}
|
||||
if let [smlib, solidcore] = runs.as_slice() {
|
||||
let (data, md) = side_by_side(smlib, solidcore);
|
||||
let into = o.out.clone();
|
||||
save_json(&into, "backends.json", &data)?;
|
||||
std::fs::write(root().join("docs/kernel-cases-backends.md"), md).map_err(|e| format!("docs/kernel-cases-backends.md: {e}"))?;
|
||||
println!(
|
||||
"kernel-cases: {} agree, {} differ → docs/kernel-cases-backends.md, {}",
|
||||
data["agree"],
|
||||
data["differ"],
|
||||
into.join("backends.json").display()
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -563,9 +769,91 @@ fn run_coverage(o: &Options, dir: &Path) -> Result<String, String> {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{Exempt, coverage, exempt, gate, recipe_commands};
|
||||
use super::*;
|
||||
|
||||
fn run(text: &str, cases: &[(&str, &str)]) -> Result<String, String> {
|
||||
/// A child that writes more than the pipe buffer still finishes, with all its output.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn a_child_writing_more_than_the_pipe_buffer_is_not_a_hang() {
|
||||
let child = Command::new("sh")
|
||||
.args(["-c", "head -c 200000 /dev/zero; head -c 100000 /dev/zero >&2"])
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let out = wait_draining(child, Duration::from_secs(60)).unwrap().unwrap();
|
||||
assert!(out.status.success());
|
||||
assert_eq!((out.stdout.len(), out.stderr.len()), (200_000, 100_000));
|
||||
let slow = Command::new("sh").args(["-c", "sleep 30"]).stdout(Stdio::piped()).stderr(Stdio::piped()).spawn().unwrap();
|
||||
let t = Instant::now();
|
||||
assert!(wait_draining(slow, Duration::from_millis(200)).unwrap().is_none());
|
||||
assert!(t.elapsed() < Duration::from_secs(20));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn side_by_side_counts_and_puts_differences_first() {
|
||||
let r = |area: &str, case: &str, status: &str, s: f64| json!({"area": area, "case": case, "status": status, "seconds": s});
|
||||
let smlib = [r("booleans", "a", "pass", 1.0), r("booleans", "b", "pass", 2.0), r("shell", "c", "crash", 0.5)];
|
||||
let mut hang = json!({"area": "booleans", "case": "b", "status": "hang", "error": "no result after 180 s"});
|
||||
hang["process_seconds"] = json!(180.0);
|
||||
let solidcore = [r("booleans", "a", "pass", 0.5), hang, r("shell", "c", "fail", 1.5)];
|
||||
let (data, md) = side_by_side(&smlib, &solidcore);
|
||||
assert_eq!((data["agree"].as_u64(), data["differ"].as_u64()), (Some(1), Some(2)));
|
||||
assert_eq!(data["summary"]["smlib"]["counts"], json!({"pass": 2, "other": 1}));
|
||||
assert_eq!(data["summary"]["solidcore"]["counts"], json!({"pass": 1, "fail": 1, "other": 1}));
|
||||
assert_eq!(data["summary"]["solidcore"]["seconds"].as_f64(), Some(182.0));
|
||||
let b = data["cases"].as_array().and_then(|c| c.iter().find(|c| c["case"] == "b")).cloned().unwrap_or_default();
|
||||
assert_eq!((b["solidcore"]["status"].as_str(), b["solidcore"]["error"].as_str()), (Some("hang"), Some("no result after 180 s")));
|
||||
// In booleans, b (differs) comes before a (agrees).
|
||||
let (pos_a, pos_b) = (md.find("| a |"), md.find("| b |"));
|
||||
assert!(pos_b.is_some() && pos_a.is_some() && pos_b < pos_a, "{md}");
|
||||
assert!(md.contains("| b | pass (2.00 s) | hang (180.00 s) | **differ** |"), "{md}");
|
||||
assert!(md.contains("3 cases: 1 agree, 2 differ."), "{md}");
|
||||
}
|
||||
|
||||
/// `--route`, the old per-family run, is no option: each case's process has one kernel. Passed
|
||||
/// anyway, it is refused rather than run as something else.
|
||||
#[test]
|
||||
fn a_kernel_has_its_own_folder_and_route_is_refused() {
|
||||
let o = options(&["--kernel", "both", "--out", "/r"], "x").unwrap();
|
||||
assert_eq!((o.out_of("smlib"), o.out_of("solidcore")), (PathBuf::from("/r"), PathBuf::from("/r/solidcore")));
|
||||
assert!(options(&["--route", "boolean=solidcore"], "x").is_err());
|
||||
assert!(options(&["--kernel", "both", "--route"], "x").is_err());
|
||||
}
|
||||
|
||||
/// `--known-failing` follows the kernel given, solidcore by default: a list is one kernel's.
|
||||
#[test]
|
||||
fn the_gate_grades_the_kernel_given() {
|
||||
for (args, kernel) in [
|
||||
(&["--known-failing", "/k"][..], "solidcore"),
|
||||
(&["--kernel", "solidcore", "--known-failing", "/k"], "solidcore"),
|
||||
(&["--kernel", "smlib", "--known-failing", "/k"], "smlib"),
|
||||
] {
|
||||
let o = options(args, "x").unwrap();
|
||||
assert_eq!((o.known_failing, o.kernels), (Some(PathBuf::from("/k")), vec![kernel]), "{args:?}");
|
||||
}
|
||||
assert!(options(&["--kernel", "both", "--known-failing", "/k"], "x").is_err());
|
||||
}
|
||||
|
||||
/// With no `--kernel`, a run is on solidcore; the C++ kernel is asked for.
|
||||
#[test]
|
||||
fn the_rig_defaults_to_solidcore() {
|
||||
assert_eq!(options(&[], "x").unwrap().kernels, ["solidcore"]);
|
||||
assert_eq!(options(&["--kernel", "smlib"], "x").unwrap().kernels, ["smlib"]);
|
||||
assert_eq!(options(&["--kernel", "both"], "x").unwrap().kernels, ["smlib", "solidcore"]);
|
||||
}
|
||||
|
||||
/// Only a run that asks for the C++ kernel builds the CLI with it.
|
||||
#[test]
|
||||
fn the_cli_is_built_with_the_cpp_kernel_only_when_a_run_asks_for_it() {
|
||||
let features = |args: &[&str]| cli_features(&options(args, "x").unwrap().kernels).to_vec();
|
||||
assert!(features(&[]).is_empty());
|
||||
assert!(features(&["--kernel", "solidcore"]).is_empty());
|
||||
assert_eq!(features(&["--kernel", "smlib"]), ["--features", "smlib"]);
|
||||
assert_eq!(features(&["--kernel", "both"]), ["--features", "smlib"]);
|
||||
}
|
||||
|
||||
fn gate_on(text: &str, cases: &[(&str, &str)]) -> Result<String, String> {
|
||||
let names: Vec<String> = cases.iter().map(|(c, _)| c.to_string()).collect();
|
||||
let statuses: Vec<&str> = cases.iter().map(|(_, s)| *s).collect();
|
||||
gate(text, &names, &statuses)
|
||||
@@ -574,20 +862,20 @@ mod tests {
|
||||
#[test]
|
||||
fn the_gate_fails_on_a_regression_and_on_a_fixed_case_still_listed() {
|
||||
let list = "# today\na-case SMLib refuses the corner\nb-case G1 at the corner point\n";
|
||||
assert!(run(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok());
|
||||
let regression = run(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err();
|
||||
assert!(gate_on(list, &[("a-case", "fail"), ("b-case", "crash"), ("c-case", "pass")]).is_ok());
|
||||
let regression = gate_on(list, &[("a-case", "fail"), ("b-case", "fail"), ("c-case", "hang")]).unwrap_err();
|
||||
assert!(regression.contains("REGRESSION (hang): c-case"), "{regression}");
|
||||
let fixed = run(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err();
|
||||
let fixed = gate_on(list, &[("a-case", "pass"), ("b-case", "fail"), ("c-case", "pass")]).unwrap_err();
|
||||
assert!(fixed.contains("FIXED, remove it from the known-failing list: a-case"), "{fixed}");
|
||||
let stale = run(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err();
|
||||
let stale = gate_on(list, &[("a-case", "fail"), ("c-case", "pass")]).unwrap_err();
|
||||
assert!(stale.contains("not a case of this set: b-case"), "{stale}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_known_failing_list_is_sorted_with_a_cause_each() {
|
||||
assert!(run("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order"));
|
||||
assert!(run("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of"));
|
||||
assert!(run("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause"));
|
||||
assert!(gate_on("b x\na y\n", &[("a", "fail"), ("b", "fail")]).unwrap_err().contains("out of order"));
|
||||
assert!(gate_on("a x\na y\n", &[("a", "fail")]).unwrap_err().contains("a repeat of"));
|
||||
assert!(gate_on("a\n", &[("a", "fail")]).unwrap_err().contains("gives no cause"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
//! `tests/kernel-cases/ladder/ladder.json` lists tiers in order of complexity and, per tier, its
|
||||
//! cases in order (paths under `tests/kernel-cases/`, so a tier may also list rig cases from other
|
||||
//! areas). Each case is a test like any rig case, graded against its own `expected.json`, on
|
||||
//! SMLib. On top, `tests/kernel-cases/ladder/golden.json` records per case the status, the
|
||||
//! `--kernel`'s one backend. The golden is SMLib's, so a gate or a record says `--kernel smlib`. On top, `tests/kernel-cases/ladder/golden.json` records per case the status, the
|
||||
//! commands' time and what was made (counts, volume, area): the run compares with it and flags
|
||||
//! slowdowns and changes; cases run one at a time unless `--jobs` says otherwise, so the times are
|
||||
//! not shared with other cases. `--record REASON` re-measures the golden and re-sorts each tier
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! `cargo xtask known-failing --junit PATH... [--list FILE]`: grades a nextest run (its junit
|
||||
//! report) against the tests known to fail today, `tests/KNOWN_FAILING_TESTS`. CI's test jobs
|
||||
//! run it after nextest, in place of failing on any failed test. It fails on a test that failed
|
||||
//! report) against the tests known to fail today on the default kernel,
|
||||
//! `tests/KNOWN_FAILING_TESTS` (the C++ kernel's are `tests/KNOWN_FAILING_TESTS.smlib`, for a run
|
||||
//! with `SOLIDCRAFT_KERNEL=smlib` and `--list`). CI's test jobs run it after nextest, in place of failing on any failed test. It fails on a test that failed
|
||||
//! and is not listed, and on a listed test that passed (fixed: its line comes off in the same
|
||||
//! pull request); a listed test that did not run (a pull request tests only the packages it
|
||||
//! touches) is neither. The list has the kernel rig's format (`kernel_cases::known_failing`):
|
||||
@@ -15,6 +16,8 @@ use quick_xml::Reader;
|
||||
use quick_xml::events::{BytesStart, Event};
|
||||
|
||||
pub const LIST: &str = "tests/KNOWN_FAILING_TESTS";
|
||||
/// The same for the C++ kernel (`SOLIDCRAFT_KERNEL=smlib`), graded with `--list`.
|
||||
pub const SMLIB_LIST: &str = "tests/KNOWN_FAILING_TESTS.smlib";
|
||||
|
||||
pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> {
|
||||
let mut reports = Vec::new();
|
||||
@@ -60,9 +63,12 @@ fn add(all: &mut BTreeMap<String, bool>, xml: &str) -> Result<(), String> {
|
||||
|
||||
/// `cargo xtask lists`: the list is sorted, each test once, with a cause each.
|
||||
pub fn check_list(root: &Path) -> Result<(), String> {
|
||||
let file = root.join(LIST);
|
||||
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||||
crate::kernel_cases::known_failing(&text).map(|_| ()).map_err(|e| format!("{LIST}: {e}"))
|
||||
for list in [LIST, SMLIB_LIST] {
|
||||
let file = root.join(list);
|
||||
let text = std::fs::read_to_string(&file).map_err(|e| format!("{}: {e}", file.display()))?;
|
||||
crate::kernel_cases::known_failing(&text).map_err(|e| format!("{list}: {e}"))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `path` itself, or every `.xml` file under it (sorted), into `out`.
|
||||
|
||||
+1
-1
@@ -5,7 +5,7 @@
|
||||
//!
|
||||
//! - The message catalogs (`assets/locales/*.json`): one message per line, sorted by id, no
|
||||
//! duplicate ids, as `serde_json` pretty-prints them.
|
||||
//! - `tests/KNOWN_FAILING_TESTS`: one test a line with its cause, sorted, each test once
|
||||
//! - `tests/KNOWN_FAILING_TESTS` and `.smlib`: one test a line with its cause, sorted, each test once
|
||||
//! (checked only; `cargo xtask known-failing` reads it).
|
||||
//! - Growing Rust lists: no written array lengths on more than ten lines of entries.
|
||||
|
||||
|
||||
+17
-8
@@ -49,21 +49,29 @@ commands:
|
||||
replay the reference tests (tests/oracle/*/recipe.json), compare with each
|
||||
case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
|
||||
already built, instead of cargo run --release)
|
||||
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--validation MODE] [--coverage]
|
||||
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--kernel K] [--route R] [--known-failing FILE] [--validation MODE] [--coverage]
|
||||
the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ on
|
||||
SMLib (one process each, killed after --timeout, default 180 s), grade it against
|
||||
the case's own answer and write docs/kernel-cases.md; DIR (default
|
||||
target/kernel-cases) gets results.json and dumps/<area>-<case>/, the inputs of
|
||||
each failing SMLib call (--cli: a solidcraft-cli built with --features smlib)
|
||||
solidcore (one process each, killed after --timeout, default 180 s), grade it against
|
||||
the case's own answer; DIR (default target/kernel-cases) gets results.json and
|
||||
dumps/<area>-<case>/, the inputs of each failing SMLib call (--cli: a
|
||||
solidcraft-cli already built);
|
||||
--kernel solidcore|smlib|both (default solidcore) sets SOLIDCRAFT_KERNEL per case:
|
||||
solidcore keeps results.json, dumps/ and report.md in DIR/solidcore/; smlib
|
||||
writes docs/kernel-cases.md; both runs each case on each, and also writes the
|
||||
side by side to docs/kernel-cases-backends.md and DIR/backends.json; smlib and
|
||||
both need the C++ kernel, so without --cli they build the CLI with --features smlib
|
||||
(a --cli built without it reports every case as unsupported)
|
||||
--known-failing FILE: a regression gate instead of the report: fails on a case
|
||||
that fails and FILE does not list, and on a listed case that passes
|
||||
(tests/kernel-cases/<area>/KNOWN_FAILING, CI's Kernel rig workflow)
|
||||
(tests/kernel-cases/<area>/KNOWN_FAILING, CI's Kernel rig workflow); it grades
|
||||
the run's kernel, so a list is one kernel's (KNOWN_FAILING.solidcore,
|
||||
KNOWN_FAILING for --kernel smlib) and --kernel both is refused
|
||||
--validation MODE: when SMLib's full check runs on the cases' features:
|
||||
each_step, each_feature (the default) or off (solidcraft-cli kernel-cases
|
||||
--validation=MODE); with --ladder too
|
||||
--ladder [--record REASON | --gate]: the complexity ladder instead
|
||||
(tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared
|
||||
with its timing golden (one case at a time unless --jobs), into
|
||||
with its timing golden (one case at a time unless --jobs), on --kernel's one backend, into
|
||||
docs/kernel-ladder.md (DIR default target/kernel-ladder); --record re-measures
|
||||
the golden and re-sorts each tier by time; --gate fails on any case whose status
|
||||
or counts differ from the golden, instead of the report
|
||||
@@ -72,7 +80,8 @@ commands:
|
||||
uses and tests/kernel-cases/EXEMPT does not list, and on a stale EXEMPT line
|
||||
known-failing --junit PATH... [--list FILE]
|
||||
grade a nextest run's junit report (a file, or every .xml under a directory)
|
||||
against the tests known to fail today, tests/KNOWN_FAILING_TESTS (FILE): fails
|
||||
against the tests known to fail today on the default kernel,
|
||||
tests/KNOWN_FAILING_TESTS (FILE: .smlib is the C++ kernel's): fails
|
||||
on a test that failed and is not listed, and on a listed test that passed
|
||||
(fixed: take its line off); a listed test that did not run is neither. CI's
|
||||
test jobs run it after nextest
|
||||
|
||||
Reference in New Issue
Block a user