Wave 445/509: Engine: diagnostics.validation reads and sets the mode (and 3 more)

4 commits, squashed; the originals stay reachable from tag archive/merge-main-develop.

- 5d80fc571 jyviko: Engine: diagnostics.validation reads and sets the mode
- 1a93a3977 jyviko: CLI: run and kernel-cases take the validation mode
- 50501fe14 jyviko: Preferences: Body check, once per feature by default
- 6a133042c jyviko: Preferences: the body check is the kernel's, not named after SMLib

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01DtL5M41G8Nua7HvuFPZkmJ
This commit is contained in:
jyviko
2026-10-30 10:48:24 -04:00
parent 143a6cb987
commit 99cd20ea20
19 changed files with 408 additions and 33 deletions
+13 -2
View File
@@ -32,6 +32,9 @@ pub(crate) struct Options {
out: PathBuf,
/// `--known-failing FILE`: run as a regression gate against that baseline (see [`gate`]).
known_failing: Option<PathBuf>,
/// `--validation MODE`: when SMLib's full check runs on the cases' features (each_step,
/// each_feature, off; the CLI's default, each_feature, when not given).
validation: Option<String>,
}
/// The run's options; `out` is its output folder unless `--out` gives one.
@@ -44,7 +47,11 @@ pub(crate) fn options(rest: &[&str], out: &str) -> Result<Options, String> {
let p = PathBuf::from(p);
if p.is_absolute() { p } else { root().join(p) }
});
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing })
let validation = arg("--validation");
if let Some(v) = validation.as_deref().filter(|v| !["each_step", "each_feature", "off"].contains(v)) {
return Err(format!("--validation must be each_step, each_feature or off, not `{v}`"));
}
Ok(Options { cli: arg("--cli"), area: arg("--area"), timeout: Duration::from_secs(secs), jobs, out, known_failing, validation })
}
/// Every case's full result as JSON, into the run's output folder.
@@ -99,7 +106,11 @@ fn run_case(o: &Options, case: &Path) -> Value {
};
let mut dumps = std::ffi::OsString::from("--dumps=");
dumps.push(o.out.join("dumps"));
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps).arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
cmd.current_dir(root()).args(["kernel-cases", "--json"]).arg(dumps);
if let Some(v) = &o.validation {
cmd.arg(format!("--validation={v}"));
}
cmd.arg(&rel).stdout(Stdio::piped()).stderr(Stdio::piped());
let base = |status: &str, error: String| {
let area = case.parent().and_then(|p| p.file_name()).map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
let id = case.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
+4 -1
View File
@@ -47,7 +47,7 @@ commands:
replay the reference tests (tests/oracle/*/recipe.json), compare with each
case's expected.json and write docs/oracle.md (--cli: a solidcraft-cli
already built, instead of cargo run --release)
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--coverage]
kernel-cases [--area AREA] [--cli BIN] [--timeout S] [--jobs N] [--out DIR] [--known-failing FILE] [--validation MODE] [--coverage]
the kernel regression rig: run every case in tests/kernel-cases/<area>/<id>/ on
SMLib (one process each, killed after --timeout, default 180 s), grade it against
the case's own answer and write docs/kernel-cases.md; DIR (default
@@ -56,6 +56,9 @@ commands:
--known-failing FILE: a regression gate instead of the report: fails on a case
that fails and FILE does not list, and on a listed case that passes
(tests/kernel-cases/<area>/KNOWN_FAILING, CI's Kernel rig workflow)
--validation MODE: when SMLib's full check runs on the cases' features:
each_step, each_feature (the default) or off (solidcraft-cli kernel-cases
--validation=MODE); with --ladder too
--ladder [--record REASON | --gate]: the complexity ladder instead
(tests/kernel-cases/ladder/ladder.json, tiers in order), graded and compared
with its timing golden (one case at a time unless --jobs), into