diff --git a/apps/solidcraft-cli/src/kernel_cases.rs b/apps/solidcraft-cli/src/kernel_cases.rs index b07c44e9b..bb30f7990 100644 --- a/apps/solidcraft-cli/src/kernel_cases.rs +++ b/apps/solidcraft-cli/src/kernel_cases.rs @@ -745,6 +745,10 @@ fn check_case_in( // The trace ends with the commands: grading reads the bodies through the kernel too. if let Some(c) = capture.as_deref_mut() { c.events = solidcraft_engine::kernel::smlib_trace::finish(); + c.ran = Some(match &ran { + Ok(_) => s.world_state().map(|w| w.bodies.iter().map(|b| (b.name.clone(), b.body.record().clone())).collect()).map_err(|e| e.to_string()), + Err(e) => Err(e.to_string()), + }); } // The commands' own time, without process start or grading: the ladder's timing golden. out["seconds"] = json!(clock.elapsed().as_secs_f64()); @@ -796,9 +800,6 @@ fn check_case_in( return out; } }; - if let Some(c) = capture.as_deref_mut() { - c.world = world.bodies.iter().map(|b| (b.name.clone(), b.body.record().clone())).collect(); - } let mut bodies = Vec::new(); let mut c = Checks { list: Vec::new(), pass: true }; let want_valid = want["valid"].as_bool().unwrap_or(true); diff --git a/apps/solidcraft-cli/src/kernel_cases_export.rs b/apps/solidcraft-cli/src/kernel_cases_export.rs index 469497afd..e160abe17 100644 --- a/apps/solidcraft-cli/src/kernel_cases_export.rs +++ b/apps/solidcraft-cli/src/kernel_cases_export.rs @@ -3,18 +3,17 @@ //! (`smlib_trace`); the entry holds the operation (or the chain of operations) that made the //! bodies the case ends with, each operand that came in as a record given as that record (the //! B-rep record, solidcore-record's JSON), the fix set, the case's expected values and the rig's -//! own result. A case whose result no kernel operation alone accounts for is written too, marked -//! not expressible with the reasons, so nothing is dropped silently. +//! own result. A case whose result no kernel operation alone accounts for is written too, with +//! the reasons, so nothing is dropped silently. //! -//! An entry is expressible when: -//! - the run failed in a kernel operation (the engine's error carries the operation's), or every -//! body the case ends with is the record a traced operation's output was written as, or an -//! operand some traced operation took, as its record; -//! - its expected values are all the kernel's measures: none of `continuity`, `watertight`, -//! `files`, `results`, `construction` or `flat_pattern`, which SolidCraft computes; -//! - the counts it grades (faces, edges, vertices) are the records' own: the rig counts faces and -//! edges merged across one analytic surface or curve (`merged_topology`, from a mesh), so an -//! entry where those differ is not expressible. +//! An entry's operation is expressible (`operation_expressible`) when the run failed in a kernel +//! operation (the last one traced failed), or when every body the case ends with is the record a +//! traced operation's output was written as, or an operand some traced operation took, as its +//! record. The entry is wholly expressible (`expressible`) when, besides, every value it grades is +//! the kernel's: none of `continuity`, `watertight`, `files`, `results`, `construction` or +//! `flat_pattern` (SolidCraft computes them), no face, edge or vertex count that the rig's merging +//! across analytic surfaces (`merged_topology`) changes, and no refusal graded on SolidCraft's +//! wording of the kernel's error. Those are listed in `ungraded`. use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::{Path, PathBuf}; @@ -25,11 +24,12 @@ use solidcraft_engine::kernel::smlib_trace::{self, Event}; use crate::read_json; -/// What a traced case leaves for its entry: the trace of its commands and the bodies it ends with. +/// What a traced case leaves for its entry: the trace of its commands, and the bodies it ends +/// with by name or the engine's error. #[derive(Default)] pub struct Capture { pub events: Vec, - pub world: Vec<(String, Brep)>, + pub ran: Option, String>>, } /// The expected values SolidCraft computes, never the kernel. @@ -130,132 +130,144 @@ pub fn write(root: &Path, dir: &str, out: &Value, capture: Capture) -> Result<() let (area, id) = area_and_id(dir)?; let want = read_json(&format!("{dir}/expected.json"))?; let t = Trace::of(&capture.events); + // Why the operation is not the kernel's alone, and what the entry cannot grade without + // SolidCraft. let mut why_not: Vec = Vec::new(); - let status = out["status"].as_str().unwrap_or(""); + let mut ungraded: Vec = Vec::new(); + let has = |k: &str| want.get(k).is_some_and(|v| !v.is_null()); if let Some(missing) = want["missing"].as_str() { why_not.push(format!("SolidCraft has no command for it yet: {missing}")); } - let grader: Vec<&str> = GRADER_ONLY.iter().copied().filter(|k| want.get(*k).is_some_and(|v| !v.is_null())).collect(); - if !grader.is_empty() { - why_not.push(format!("graded on what SolidCraft computes: {}", grader.join(", "))); + for k in GRADER_ONLY.iter().filter(|k| has(k)) { + ungraded.push(format!("{k}: SolidCraft computes it")); } - // The bodies the entry ends with: what each is, by name. let mut result: Vec = Vec::new(); let mut ends: Vec = Vec::new(); let mut failed_call: Option = None; - let error = out["refused_with"].as_str().or_else(|| out["error"].as_str()).filter(|_| out.get("checks").is_none()); - if let Some(e) = error.filter(|_| want.get("missing").is_none()) { - // The run stopped: the kernel operation that failed with the message the engine gave. - failed_call = t.calls.iter().rposition(|c| matches!(c.3, Err(m) if !m.is_empty() && e.contains(m.as_str()))); - match failed_call { - Some(i) => ends.extend(t.calls.get(i).map(|c| c.1.to_vec()).unwrap_or_default()), - None if t.calls.is_empty() => why_not.push(format!("no kernel operation ran before the failure: {e}")), - None => why_not.push(format!("the failure is SolidCraft's, not a kernel operation's: {e}")), - } - } else if want.get("missing").is_none() { - let operands = t.operands(); - let mut made = 0; - for (name, record) in &capture.world { - let from_op = t.exports.iter().rev().find(|(_, r)| *r == record).map(|(id, _)| *id).filter(|id| t.producer.contains_key(id)); - let operand = t.imports.iter().find(|(i, (r, _, _))| *r == record && operands.contains(i)).map(|(i, _)| *i); - match (from_op, operand) { - (Some(id), _) => { - made += 1; - ends.push(id); - result.push(json!({"body": name, "made": format!("b{id}")})); + let mut outcome = json!({"ran": true}); + match (&capture.ran, has("missing")) { + (_, true) => {} + (None, false) => why_not.push("the case did not run".into()), + (Some(Err(e)), false) => { + // The run stopped. It stopped in the kernel when the last operation traced failed: the + // engine reports that failure in its own words. + outcome = json!({"ran": false, "engine_error": e}); + match t.calls.len().checked_sub(1).and_then(|i| t.calls.get(i).map(|c| (i, c))) { + Some((i, c)) if c.3.is_err() => { + failed_call = Some(i); + ends.extend(c.1.iter().copied()); + if let Err(k) = c.3 { + outcome["kernel_error"] = json!(k); + if let Some(why) = want["refusal"].as_str().filter(|w| !k.contains(*w)) { + ungraded.push(format!("refusal {why:?}: SolidCraft's wording of the kernel's error")); + } + } } - (None, Some(id)) => { - ends.push(id); - result.push(json!({"body": name, "operand": format!("b{id}")})); - } - (None, None) => why_not.push(format!( - "body {name:?} is no kernel operation's output nor an operand: SolidCraft built it or changed the kernel's record" - )), + Some(_) => why_not.push(format!("the failure is SolidCraft's, not a kernel operation's: {e}")), + None => why_not.push(format!("no kernel operation ran before the failure: {e}")), } } - if capture.world.is_empty() { - why_not.push("the case ends with no body".into()); - } else if made == 0 && why_not.is_empty() { - why_not.push("no kernel operation made a body the case ends with".into()); - } - // Counts graded as SolidCraft merges them, where that differs from the records'. - let raw = |k: &str| -> u64 { - capture - .world - .iter() - .map(|(_, r)| match k { - "faces" => r.faces.len(), - "edges" => r.edges.len(), - _ => r.vertices.len(), - } as u64) - .sum() - }; - for k in ["faces", "edges", "vertices"] { - let graded = want.get(k).is_some_and(|v| !v.is_null()); - let merged = out["measured"][k].as_u64(); - if graded && merged.is_some_and(|m| m != raw(k)) { - why_not.push(format!("{k} graded merged across analytic surfaces ({}), the records have {}", merged.unwrap_or(0), raw(k))); + (Some(Ok(world)), false) => { + let operands = t.operands(); + let mut made = 0; + for (name, record) in world { + let from_op = t.exports.iter().rev().find(|(_, r)| *r == record).map(|(id, _)| *id).filter(|id| t.producer.contains_key(id)); + let operand = t.imports.iter().filter(|(i, (r, _, _))| *r == record && operands.contains(i)).map(|(i, _)| *i).min(); + match (from_op, operand) { + (Some(id), _) => { + made += 1; + ends.push(id); + result.push(json!({"body": name, "made": format!("b{id}")})); + } + (None, Some(id)) => { + ends.push(id); + result.push(json!({"body": name, "operand": format!("b{id}")})); + } + (None, None) => why_not.push(format!( + "body {name:?} is no kernel operation's output nor an operand: SolidCraft built it or changed the kernel's record" + )), + } } + if world.is_empty() { + why_not.push("the case ends with no body".into()); + } else if made == 0 && why_not.is_empty() { + why_not.push("no kernel operation made a body the case ends with".into()); + } + // Counts graded as SolidCraft merges them, where that differs from the records'. + let raw = |k: &str| -> u64 { + world + .iter() + .map(|(_, r)| match k { + "faces" => r.faces.len(), + "edges" => r.edges.len(), + _ => r.vertices.len(), + } as u64) + .sum() + }; + for k in ["faces", "edges", "vertices"] { + if let Some(merged) = out["measured"][k].as_u64().filter(|&m| has(k) && m != raw(k)) { + ungraded.push(format!("{k}: graded merged across analytic surfaces ({merged}), the records have {}", raw(k))); + } + } + outcome["raw_counts"] = json!({"faces": raw("faces"), "edges": raw("edges"), "vertices": raw("vertices")}); } } let mut unknown = BTreeSet::new(); - let order = chain(&t, &ends, &mut unknown); + let mut order = chain(&t, &ends, &mut unknown); if !unknown.is_empty() { why_not.push(format!("operands made by nothing traced: {unknown:?}")); } - let order: Vec = match failed_call { - Some(i) if !order.contains(&i) => order.into_iter().chain([i]).collect(), - _ => order, - }; - // Operand records: every body the chain takes that came in as a record. + if let Some(i) = failed_call.filter(|i| !order.contains(i)) { + order.push(i); + } + // Operand records: every body the chain takes, or the case ends with, that came in as a record. let mut bodies = BTreeMap::new(); - let calls: Vec = order - .iter() - .filter_map(|&i| t.calls.get(i)) - .map(|(op, inputs, params, outputs)| { - for id in inputs.iter() { - if let Some((record, trusted, splines)) = t.imports.get(id) { - bodies.insert(format!("b{id}"), json!({"trusted": trusted, "keeping_splines": splines, "record": record})); - } - } - let mut c = json!({"op": op, "inputs": inputs.iter().map(|i| format!("b{i}")).collect::>(), "params": params}); - match outputs { - Ok(made) => c["outputs"] = json!(made.iter().map(|i| format!("b{i}")).collect::>()), - Err(e) => c["error"] = json!(e), - } - c - }) - .collect(); - for (_, record) in &capture.world { - // An operand the case ends with, never operated on in the chain, still comes as its record. - if let Some(id) = t.imports.iter().find(|(_, (r, _, _))| *r == record).map(|(i, _)| *i) - && result.iter().any(|r| r["operand"] == json!(format!("b{id}"))) - && let Some((record, trusted, splines)) = t.imports.get(&id) - { + let given = |id: u64, bodies: &mut BTreeMap| { + if let Some((record, trusted, splines)) = t.imports.get(&id) { bodies.insert(format!("b{id}"), json!({"trusted": trusted, "keeping_splines": splines, "record": record})); } + }; + let mut calls: Vec = Vec::new(); + for (op, inputs, params, outputs) in order.iter().filter_map(|&i| t.calls.get(i)) { + for &id in inputs.iter() { + given(id, &mut bodies); + } + let mut c = json!({"op": op, "inputs": inputs.iter().map(|i| format!("b{i}")).collect::>(), "params": params}); + match outputs { + Ok(made) => c["outputs"] = json!(made.iter().map(|i| format!("b{i}")).collect::>()), + Err(e) => c["error"] = json!(e), + } + calls.push(c); + } + for r in &result { + if let Some(id) = r["operand"].as_str().and_then(|b| b.strip_prefix('b')).and_then(|n| n.parse().ok()) { + given(id, &mut bodies); + } } let rig = json!({ "status": out["status"], "error": out["error"], "refused_with": out["refused_with"], "measured": out["measured"], "known_failing": known_failing(dir, &id), }); + let operation_expressible = why_not.is_empty(); let entry = json!({ "case": id, "area": area, "sources": want["sources"], "what": want["what"], - "expressible": why_not.is_empty(), + "expressible": operation_expressible && ungraded.is_empty(), + "operation_expressible": operation_expressible, "why_not": why_not, + "ungraded": ungraded, "expected": want, "rig": rig, + "outcome": outcome, "calls": calls, "bodies": bodies, "result": result, "traced_ops": t.calls.iter().map(|c| c.0).collect::>(), - "status": status, }); let folder: PathBuf = root.join(&area); std::fs::create_dir_all(&folder).map_err(|e| format!("{}: {e}", folder.display()))?; diff --git a/xtask/src/kernel_corpus.rs b/xtask/src/kernel_corpus.rs index 82b1af501..fb366917a 100644 --- a/xtask/src/kernel_corpus.rs +++ b/xtask/src/kernel_corpus.rs @@ -80,6 +80,8 @@ pub(crate) fn manifest(dir: &Path, results: &[Value], golden: Option<&Value>) -> entry = json!({ "case": id, "area": area, "sources": want["sources"], "what": want["what"], "expressible": false, + "operation_expressible": false, + "ungraded": [], "why_not": [format!("the rig's run {}: {}", r["status"].as_str().unwrap_or("?"), r["error"].as_str().unwrap_or(""))], "expected": want, "rig": {"status": r["status"], "error": r["error"]}, @@ -115,20 +117,30 @@ pub(crate) fn manifest(dir: &Path, results: &[Value], golden: Option<&Value>) -> for f in files { let e = read(&f); let area = e["area"].as_str().unwrap_or("?").to_string(); - let a = areas.entry(area.clone()).or_insert_with(|| json!({"cases": 0, "expressible": 0, "not_expressible": 0, "chain": {}, "reasons": {}})); - a["cases"] = json!(a["cases"].as_u64().unwrap_or(0) + 1); + let a = areas.entry(area.clone()).or_insert_with(|| { + json!({"cases": 0, "expressible": 0, "operation_expressible": 0, "not_expressible": 0, "chain": {}, "reasons": {}, "ungraded": {}}) + }); + let bump = |a: &mut Value, k: &str| a[k] = json!(a[k].as_u64().unwrap_or(0) + 1); + bump(a, "cases"); if e["expressible"].as_bool() == Some(true) { - a["expressible"] = json!(a["expressible"].as_u64().unwrap_or(0) + 1); + bump(a, "expressible"); + } + for u in e["ungraded"].as_array().into_iter().flatten().filter_map(Value::as_str) { + let k = u.split(':').next().unwrap_or(u); + bump(&mut a["ungraded"], k); + } + if e["operation_expressible"].as_bool() == Some(true) { + bump(a, "operation_expressible"); let n = e["calls"].as_array().map_or(0, Vec::len).to_string(); - a["chain"][&n] = json!(a["chain"][&n].as_u64().unwrap_or(0) + 1); + bump(&mut a["chain"], &n); } else { - a["not_expressible"] = json!(a["not_expressible"].as_u64().unwrap_or(0) + 1); + bump(a, "not_expressible"); let why: Vec<&str> = e["why_not"].as_array().into_iter().flatten().filter_map(Value::as_str).collect(); let mut kinds: Vec<&str> = why.iter().map(|w| reason_kind(w)).collect(); kinds.sort(); kinds.dedup(); for k in kinds { - a["reasons"][k] = json!(a["reasons"][k].as_u64().unwrap_or(0) + 1); + bump(&mut a["reasons"], k); } not.push(json!({"case": format!("{area}/{}", e["case"].as_str().unwrap_or("?")), "why": why})); } @@ -141,14 +153,10 @@ pub(crate) fn manifest(dir: &Path, results: &[Value], golden: Option<&Value>) -> "not_expressible": not, }); write_json(&dir.join("manifest.json"), &m, true)?; - let mut text = String::from("area cases expressible not\n"); + let mut text = String::from("area cases operation wholly not\n"); for (area, a) in &areas { - text += &format!( - "{area:<20} {:>5} {:>11} {:>3}\n", - a["cases"].as_u64().unwrap_or(0), - a["expressible"].as_u64().unwrap_or(0), - a["not_expressible"].as_u64().unwrap_or(0) - ); + let n = |k: &str| a[k].as_u64().unwrap_or(0); + text += &format!("{area:<27} {:>5} {:>9} {:>6} {:>3}\n", n("cases"), n("operation_expressible"), n("expressible"), n("not_expressible")); } Ok(text) }