diff --git a/crates/doc/src/eval_cost_tests.rs b/crates/doc/src/eval_cost_tests.rs index 146b57416..8139c60a2 100644 --- a/crates/doc/src/eval_cost_tests.rs +++ b/crates/doc/src/eval_cost_tests.rs @@ -647,3 +647,65 @@ fn shared_evaluation_refreshes_live_sketches_and_keeps_unknown_features_conserva assert!(model.results.iter().all(|r| r.error.is_none())); assert!(model.evaluate_shared(&doc), "an unclassified feature must keep the ordinary evaluation path"); } + +/// A saved model writes each set of sketch keys once, as the keys it sets and drops relative to +/// the set written before it (every state repeating its whole set made a 5,484-feature design's +/// model too large for its file, which then kept only some states and evaluated the rest on +/// opening). Every state reads back the keys it was evaluated with, also in a model thinned to +/// fewer states; a set that names no set before it is refused. +#[test] +fn saved_states_read_back_their_sketch_keys() { + let mut doc = Document::new("Sketch keys"); + for i in 0..4 { + doc.add_feature(line_sketch(PlaneRef::Origin { name: "XY".into() }, Vec2::new(f64::from(i) + 1.0, 1.0)), None).unwrap(); + for j in 0..3 { + let corner = point_expr(Vec3::new(f64::from(i) * 20.0, f64::from(j) * 20.0, 0.0)); + let kind = FeatureKind::Box { corner, length: "5".into(), width: "5".into(), height: "5".into(), operation: Operation::NewBody }; + doc.add_feature(kind, None).unwrap(); + } + } + let mut model = Model::new(); + model.evaluate(&doc); + let keys = |m: &Model| m.results.iter().map(|r| r.state.sketch_keys.clone()).collect::>(); + assert_eq!(model.state().sketch_keys.len(), 4); + let saved = model.saved(&doc).unwrap(); + assert_eq!(saved.records["sketch_key_sets"].as_array().unwrap().len(), 4, "one set per sketch"); + assert_eq!(saved.records["states"].as_array().unwrap().len(), doc.features.len()); + assert_eq!(keys(&Model::adopt(&doc, &saved).unwrap()), keys(&model)); + // The final state alone (every body costs more than the budget): the sets are all kept. + let pieces: Vec> = (0..saved.bodies.len()).map(|i| vec![(i, 1 << 20)]).collect(); + let (thinned, _, _) = saved.thinned(&pieces, 0).unwrap(); + assert_eq!(thinned.records["states"].as_array().unwrap().len(), 1); + assert_eq!(thinned.final_state().unwrap().sketch_keys, model.state().sketch_keys); + assert_eq!(keys(&Model::adopt(&doc, &thinned).unwrap()), keys(&model)); + // Sets and drops read as written; a base that is not an earlier set is refused. + let with_sets = |sets: serde_json::Value| SavedModel { + records: { + let mut r = saved.records.clone(); + r["sketch_key_sets"] = sets; + r + }, + bodies: saved.bodies.clone(), + }; + let first = *model.state().sketch_keys.keys().next().unwrap(); + let dropped = with_sets(serde_json::json!([ + {"set": [[first, "0000000000000001"], [7, "0000000000000002"]], "drop": []}, + {"base": 0, "set": [], "drop": []}, + {"base": 1, "set": [[8, "0000000000000003"]], "drop": [first]}, + {"base": 0, "set": [[7, "00000000000000ff"]], "drop": []}, + ])); + let want: BTreeMap = [(7, 0xff), (first, 1)].into(); + assert_eq!(dropped.final_state().unwrap().sketch_keys, want); + let dropped = with_sets(serde_json::json!([ + {"set": [[first, "0000000000000001"], [7, "0000000000000002"]], "drop": []}, + {"base": 0, "set": [], "drop": []}, + {"base": 0, "set": [], "drop": []}, + {"base": 1, "set": [[8, "0000000000000003"]], "drop": [first]}, + ])); + assert_eq!(dropped.final_state().unwrap().sketch_keys, [(7, 2), (8, 3)].into()); + for base in [3, 4] { + let sets = + serde_json::json!([{"set": [], "drop": []}, {"set": [], "drop": []}, {"set": [], "drop": []}, {"base": base, "set": [], "drop": []}]); + assert!(with_sets(sets).final_state().is_err(), "base {base}"); + } +} diff --git a/crates/doc/src/model_cache.rs b/crates/doc/src/model_cache.rs index 71b1f4b63..569563ea5 100644 --- a/crates/doc/src/model_cache.rs +++ b/crates/doc/src/model_cache.rs @@ -22,7 +22,8 @@ pub struct SavedModel { /// Records refer to these at most. const MAX_ITEMS: usize = 1_000_000; -/// Bodies, sketches and sheet bodies all the states hold together, at most (a state is a list of clones). +/// Bodies, sketches, sheet bodies and sketch keys all the states hold together, at most (a state is +/// a list of clones). const MAX_HELD: usize = 50_000_000; /// Longest text a record holds (bytes). A feature's warning can quote every generated name of /// the edges it picked: 122,701 characters in one design, whose saved model was refused, so it @@ -48,6 +49,11 @@ fn unhex(v: &Json) -> Result { v.as_str().and_then(|s| u64::from_str_radix(s, 16).ok()).ok_or_else(|| bad("saved model: key")) } +/// A sketch-key set's list `key` (`set` or `drop`). +fn keys_list<'a>(set: &'a Json, key: &str) -> Result<&'a Vec> { + set.get(key).and_then(Json::as_array).filter(|l| l.len() <= MAX_ITEMS).ok_or_else(|| bad("saved model: sketch keys")) +} + fn bad(what: &str) -> DocError { DocError::Invalid(what.into()) } @@ -128,9 +134,36 @@ struct Writer { sheet_index: HashMap, states: Vec, state_index: HashMap, + /// The states' sketch keys, each distinct map once, written as the changes from the set + /// written before it: every state used to repeat the whole map (one entry per solved sketch), + /// 85 % of a 5,484-feature design's records, which then did not fit the cache's manifest. + sketch_key_sets: Vec, + sketch_key_index: HashMap, usize>, + last_sketch_keys: Option<(BTreeMap, usize)>, } impl Writer { + /// The place of `keys` among the sketch-key sets, written the first time a state holds them + /// (as the changes from the set written last: a later state mostly adds a sketch or two). + fn sketch_key_set(&mut self, keys: &BTreeMap) -> usize { + if let Some(i) = self.sketch_key_index.get(keys) { + return *i; + } + let record = match &self.last_sketch_keys { + Some((base, at)) => { + let set: Vec = keys.iter().filter(|(f, k)| base.get(f) != Some(k)).map(|(f, k)| json!([f, hex(*k)])).collect(); + let drop: Vec = base.keys().filter(|f| !keys.contains_key(f)).copied().collect(); + json!({"base": at, "set": set, "drop": drop}) + } + None => json!({"set": keys.iter().map(|(f, k)| json!([f, hex(*k)])).collect::>(), "drop": []}), + }; + self.sketch_key_sets.push(record); + let i = self.sketch_key_sets.len() - 1; + self.sketch_key_index.insert(keys.clone(), i); + self.last_sketch_keys = Some((keys.clone(), i)); + i + } + fn names(&mut self, b: &ModelBody) -> usize { let id = b.names.id(); if let Some(i) = self.names_index.get(&id) { @@ -216,7 +249,7 @@ impl Writer { "reserved_body_names": st.reserved_body_names.iter().collect::>(), "sheets": sheets, "construct": serde_json::to_value(&st.construct).ok()?, "threads": serde_json::to_value(&st.threads).ok()?, - "sketch_keys": st.sketch_keys.iter().map(|(f, k)| json!([f, hex(*k)])).collect::>(), + "sketch_key_set": self.sketch_key_set(&st.sketch_keys), "blend_edges": serde_json::to_value(&st.blend_edges).ok()?, "fitted": serde_json::to_value(&st.fitted).ok()?, "tapered": serde_json::to_value(&st.tapered).ok()?, }); @@ -250,7 +283,7 @@ impl Model { } let records = json!({ "features": features, "states": w.states, "model_bodies": w.model_bodies, "names": w.names, "sketches": w.sketches, - "sheets": w.sheets, + "sheets": w.sheets, "sketch_key_sets": w.sketch_key_sets, }); Ok(SavedModel { records, bodies: w.bodies }) }) @@ -331,9 +364,49 @@ impl SavedModel { for s in list("sheets")? { sheets.push(serde_json::from_value::(s.clone()).map_err(|_| bad("saved model: sheet"))?); } - let mut states = Vec::new(); + // The sketch-key sets, each the changes from an earlier one (`base`): a short record can + // stand for a large set, so what they hold counts towards MAX_HELD as they are built. + let mut key_sets: Vec> = Vec::new(); let mut held = 0usize; - for s in list("states")? { + for k in list("sketch_key_sets")? { + // A set is the changes from one before it. + let base = match k.get("base") { + Some(b) => Some(key_sets.get(index(Some(b), key_sets.len())?).ok_or_else(|| bad("saved model: sketch keys"))?), + None => None, + }; + let (drop, set) = (keys_list(k, "drop")?, keys_list(k, "set")?); + // Counted before the base is copied: at most what the set can hold. + held = held.saturating_add(base.map_or(0, BTreeMap::len)).saturating_add(set.len()); + if held > MAX_HELD { + return Err(bad("saved model: sketch keys")); + } + let mut keys = base.cloned().unwrap_or_default(); + for f in drop { + keys.remove(&f.as_u64().ok_or_else(|| bad("saved model: sketch keys"))?); + } + for e in set { + keys.insert(e.get(0).and_then(Json::as_u64).ok_or_else(|| bad("saved model: sketch keys"))?, unhex(e.get(1).unwrap_or(&Json::Null))?); + } + key_sets.push(keys); + } + let records = list("states")?; + // What every state holds, counted from its runs and its sketch-key set before any is + // built (a hostile file cannot make the states clone more than MAX_HELD items). + for s in records { + for key in ["bodies", "sketches"] { + for run in s.get(key).and_then(Json::as_array).ok_or_else(|| bad("saved model: state"))? { + held = held.saturating_add(run.get(1).and_then(Json::as_u64).and_then(|n| usize::try_from(n).ok()).unwrap_or(usize::MAX)); + } + } + held = held.saturating_add(s.get("sheets").and_then(Json::as_array).map_or(0, Vec::len)); + held = held.saturating_add(key_sets.get(index(s.get("sketch_key_set"), key_sets.len())?).map_or(0, BTreeMap::len)); + if held > MAX_HELD { + return Err(bad("saved model: states")); + } + } + // Each state is independent of the others once the records they share are read: built + // on threads (a 5,484-feature design spent 3.5 s building them one after another). + let built = kernel::map_on_threads(records, 16, |s: &Json| -> Result { // Runs of consecutive indices, each within the list it indexes. let ids = |key: &str, len: usize| -> Result> { let mut out = Vec::new(); @@ -349,12 +422,8 @@ impl SavedModel { }; let field = |key: &str| parse(s.get(key)); let invalid = |_| bad("saved model: state"); - let mut sketch_keys = BTreeMap::new(); - for k in s.get("sketch_keys").and_then(Json::as_array).ok_or_else(|| bad("saved model: state"))? { - sketch_keys - .insert(k.get(0).and_then(Json::as_u64).ok_or_else(|| bad("saved model: state"))?, unhex(k.get(1).unwrap_or(&Json::Null))?); - } - let st = ModelState { + let sketch_keys = key_sets.get(index(s.get("sketch_key_set"), key_sets.len())?).cloned().ok_or_else(|| bad("saved model: state"))?; + Ok(ModelState { frame_bodies: Default::default(), bodies: ids("bodies", model_bodies.len())?.into_iter().filter_map(|i| model_bodies.get(i).cloned()).collect(), sketches: ids("sketches", sketches.len())?.into_iter().filter_map(|i| sketches.get(i).cloned()).collect(), @@ -377,13 +446,9 @@ impl SavedModel { blend_edges: serde_json::from_value(field("blend_edges")?).map_err(invalid)?, fitted: serde_json::from_value(field("fitted")?).map_err(invalid)?, tapered: serde_json::from_value(field("tapered")?).map_err(invalid)?, - }; - held = held.saturating_add(st.bodies.len()).saturating_add(st.sketches.len()).saturating_add(st.sheets.len()); - if held > MAX_HELD { - return Err(bad("saved model: states")); - } - states.push(Arc::new(st)); - } + }) + }); + let states = built.into_iter().map(|r| r.unwrap_or_else(|| Err(bad("saved model: state"))).map(Arc::new)).collect::>>()?; Ok(states) } @@ -457,7 +522,8 @@ impl SavedModel { let mut stored = std::collections::HashSet::new(); // The sheet bodies, each written once and named by place in the states: all kept. let sheets = list("sheets").cloned().unwrap_or_default(); - let mut used = bytes(sheets.as_slice()).iter().fold(0usize, |t, b| t.saturating_add(*b)); + let key_sets = list("sketch_key_sets").cloned().unwrap_or_default(); + let mut used = bytes(sheets.as_slice()).iter().chain(bytes(key_sets.as_slice()).iter()).fold(0usize, |t, b| t.saturating_add(*b)); for s in std::iter::once(last).chain(order) { let (mut adds, mut new_names, mut new_sketches) = (HashMap::new(), std::collections::HashSet::new(), std::collections::HashSet::new()); let mut records = *state_bytes.get(s)?; @@ -549,7 +615,7 @@ impl SavedModel { } let records = json!({ "features": out_features, "states": out_states, "model_bodies": out_bodies, "names": kept_of(names.as_slice(), &names_used), - "sketches": kept_of(sketches.as_slice(), &sketches_used), "sheets": sheets, + "sketches": kept_of(sketches.as_slice(), &sketches_used), "sheets": sheets, "sketch_key_sets": key_sets, }); let kept: Vec = (0..self.bodies.len()).filter(|i| kernel_used.get(*i) == Some(&true)).collect(); let bodies = kept.iter().filter_map(|i| self.bodies.get(*i).cloned()).collect(); diff --git a/crates/engine/src/cmd/file.rs b/crates/engine/src/cmd/file.rs index cb675a195..e6fd8067d 100644 --- a/crates/engine/src/cmd/file.rs +++ b/crates/engine/src/cmd/file.rs @@ -1520,6 +1520,54 @@ mod tests { std::fs::remove_file(path).unwrap(); } + /// A design's saved model keeps each set of sketch keys once, not again with every state: + /// every state wrote the key of each sketch before it, so a design of many sketches and many + /// features after them (5,484 features) passed the cache's size limit and kept only some of + /// its states. A set is written as the changes from the one before it; the model opens from + /// the file with every state (the keys read back: `eval_cost_tests`). + #[test] + fn a_saved_model_keeps_each_set_of_sketch_keys_once() { + let mut s = Session::new(Document::new("Sketches")); + let run = |s: &mut Session, id: &str, p: Value| s.execute(id, &p).unwrap(); + for i in 0..6 { + run(&mut s, "sketch.create", json!({"plane":"XY","name":format!("Pad{i}")})); + run(&mut s, "sketch.rectangle.two_point", json!({"p0":[20 * i,0],"p1":[20 * i + 10,10]})); + run(&mut s, "solid.extrude", json!({"distance":5,"operation":"new"})); + } + for i in 0..10 { + run(&mut s, "solid.box", json!({"length":5,"width":5,"height":5,"corner":[20 * i,40,0],"operation":"new"})); + } + let saved = saved_geometry(&s, &s.doc).unwrap(); + let features = saved.records["features"].as_array().unwrap(); + assert_eq!(features.len(), s.doc.features.len()); + assert!(features.iter().all(|f| f.get("state").is_some()), "every state kept"); + let states = saved.records["states"].as_array().unwrap(); + let sets = saved.records["sketch_key_sets"].as_array().unwrap(); + // One set per sketch: each later one adds that sketch's key to the one before it. + assert_eq!(sets.len(), 6, "{sets:?}"); + assert_eq!(sets[0]["set"].as_array().unwrap().len(), 1); + for (i, set) in sets.iter().enumerate().skip(1) { + assert_eq!( + (set["base"].as_u64(), set["set"].as_array().unwrap().len(), set["drop"].as_array().unwrap().len()), + (Some(i as u64 - 1), 1, 0) + ); + } + let last = sets.len() - 1; + assert_eq!( + states.iter().filter(|st| st["sketch_key_set"] == json!(last)).count(), + 2 + 10, + "the last sketch, its extrude and every box share one set" + ); + let path = std::env::temp_dir().join(format!("solidcraft-sketch-key-cache-{}.solidcraft", std::process::id())); + let bytes = solidcraft_io::write_design_container_with_cache(&s.doc, solidcraft_io::DesignCompression::Fast, None, Some(&saved)).unwrap(); + std::fs::write(&path, bytes).unwrap(); + let mut reopened = Session::default(); + reopened.execute("doc.open", &json!({"path": path})).unwrap(); + assert_eq!(reopened.model.last_recomputed, 0, "taken from the file, not evaluated"); + assert!(reopened.model.results.iter().zip(&s.model.results).all(|(a, b)| a.ms == b.ms)); + std::fs::remove_file(path).unwrap(); + } + #[test] #[ignore = "manual native cache save benchmark; generate issue303-step.json first"] fn benchmark_native_cache_save() { diff --git a/crates/io/src/design.rs b/crates/io/src/design.rs index 1576e36f5..2b3a0b648 100644 --- a/crates/io/src/design.rs +++ b/crates/io/src/design.rs @@ -1087,12 +1087,12 @@ mod tests { .collect(); let state = json!({ "bodies": [[0, bodies.len()]], "sketches": [], "body_counter": 7, "sheets": [], "construct": [], "threads": [], - "sketch_keys": [], "blend_edges": {}, "fitted": {}, "tapered": {}, + "sketch_key_set": 0, "blend_edges": {}, "fitted": {}, "tapered": {}, }); solidcraft_doc::SavedModel { records: json!({ "features": [{"id": 1, "fingerprint": "0000000000000000", "ms": 0.0, "skipped": false, "state": 0}], - "states": [state], "model_bodies": records, "names": [{"faces": [], "info": []}], "sketches": [], "sheets": [], + "states": [state], "model_bodies": records, "names": [{"faces": [], "info": []}], "sketches": [], "sheets": [], "sketch_key_sets": [{"set": [], "drop": []}], }), bodies, } @@ -1220,7 +1220,7 @@ mod tests { let bodies = if i == 0 { json!([[0, 16]]) } else { json!([[0, 8], [8 * (i + 1), 8]]) }; json!({ "bodies": bodies, "sketches": [], "body_counter": 7, "sheets": [], "construct": [], "threads": [], - "sketch_keys": [], "blend_edges": {}, "fitted": {}, "tapered": {}, + "sketch_key_set": 0, "blend_edges": {}, "fitted": {}, "tapered": {}, }) }) .collect(); @@ -1230,7 +1230,7 @@ mod tests { .map(|(i, ms)| json!({"id": i + 1, "fingerprint": "0000000000000000", "ms": ms, "skipped": false, "state": i})) .collect(); solidcraft_doc::SavedModel { - records: json!({"features": features, "states": states, "model_bodies": records, "names": [{"faces": [], "info": []}], "sketches": [], "sheets": []}), + records: json!({"features": features, "states": states, "model_bodies": records, "names": [{"faces": [], "info": []}], "sketches": [], "sheets": [], "sketch_key_sets": [{"set": [], "drop": []}]}), bodies, } } @@ -1280,7 +1280,7 @@ mod tests { one.records["states"], json!([{ "bodies": [[0, 16]], "sketches": [], "body_counter": 7, "sheets": [], "construct": [], "threads": [], - "sketch_keys": [], "blend_edges": {}, "fitted": {}, "tapered": {}, + "sketch_key_set": 0, "blend_edges": {}, "fitted": {}, "tapered": {}, }]) ); } @@ -1356,6 +1356,24 @@ mod tests { assert!(read_design_preview(&bytes).is_none()); assert_eq!(crate::read_design(&bytes).unwrap().to_json(), doc.to_json()); } + // A cache of the format before sketch-key sets (/4) is a miss, and so are records of + // that shape (each state's keys inline) under the current format: never misread. + let manifest: Value = serde_json::from_slice(&original["cache/model.json"]).unwrap(); + let mut older = manifest.clone(); + older["format"] = json!("solidcraft-model-cache/4"); + let mut inline = manifest.clone(); + let model = inline["model"].as_object_mut().unwrap(); + model.remove("sketch_key_sets"); + let state = model["states"][0].as_object_mut().unwrap(); + state.remove("sketch_key_set"); + state.insert("sketch_keys".into(), json!([])); + for manifest in [older, inline] { + let mut entries = original.clone(); + entries.insert("cache/model.json".into(), serde_json::to_vec(&manifest).unwrap()); + let bytes = package(&entries); + assert!(read_design_geometry(&bytes).is_none(), "{}", manifest["format"]); + assert_eq!(crate::read_design(&bytes).unwrap().to_json(), doc.to_json()); + } let mut entries = original; entries.get_mut("cache/bodies.bin").unwrap()[0] ^= 1; let bytes = package(&entries); diff --git a/crates/io/src/design/native_cache.rs b/crates/io/src/design/native_cache.rs index 37667d5ff..7d85266a3 100644 --- a/crates/io/src/design/native_cache.rs +++ b/crates/io/src/design/native_cache.rs @@ -5,7 +5,7 @@ use super::*; use solidcraft_doc::{ModelState, SavedModel}; -const FORMAT: &str = "solidcraft-model-cache/4"; +const FORMAT: &str = "solidcraft-model-cache/5"; const PRODUCER: &str = "solidcraft-truck-adapter/3"; const MANIFEST: &str = "cache/model.json"; const PAYLOAD: &str = "cache/bodies.bin"; diff --git a/docs/design-container.md b/docs/design-container.md index 91bfa6633..7b1c9d521 100644 --- a/docs/design-container.md +++ b/docs/design-container.md @@ -147,7 +147,7 @@ SHA-256 of their encoding), and a placed copy of a repeated part (an instance fr a STEP assembly) records the part and its matrix, so it shows that part's display again instead of being meshed on its own. -`cache/model.json` records `solidcraft-model-cache/4`, the producer revision +`cache/model.json` records `solidcraft-model-cache/5`, the producer revision `solidcraft-truck-adapter/3`, the application version, the build (source commit, CPU architecture and OS), the evaluation source (below, with the CPU architecture and OS), the SHA-256 of the exact `document.json` (whose blob references pin every @@ -183,7 +183,10 @@ repeat for longest), each while what it adds still fits: the bodies no state kep so far holds (a body is stored once however many states hold it) and the records of the state, those bodies, their names and sketches. Sheet bodies, like sketches, are written once and named by place in the states (most features leave the sheets as they were); a -thinned cache keeps all of them. When the rest of the +thinned cache keeps all of them. So are the states' sketch keys (the evaluation key of each +sketch before the state): each distinct set is written once, as the keys it sets and drops +relative to the set written before it, and a state names its set by place; a thinned cache +keeps all of them too. When the rest of the manifest (the bodies' table, the slices) then leaves too little room, the choice is made again within that much less, and the final state alone is always tried. A feature whose state is left out records none. The cache is left out (and the