From 8456aa7f2dc1f6df1878c791d56c47c9d9c199d7 Mon Sep 17 00:00:00 2001 From: lxp <2770281812@qq.com> Date: Mon, 21 Sep 2026 18:16:44 +0800 Subject: [PATCH] =?UTF-8?q?O30=E6=A0=87=E5=AE=9A=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/artifacts/evidence_journal.py | 116 +++++--- .../core/domain/capture_plan.py | 22 +- .../core/fitting/session.py | 8 +- .../core/fitting/training_quality.py | 28 +- .../core/geometry/frozen_evidence.py | 7 + .../core/geometry/tag_pose/rigid_tag_pair.py | 14 +- .../profiles/observer_policy.py | 15 +- .../profiles/observer_preparation.py | 11 +- .../profiles/preparation.py | 39 ++- .../profiles/validator.py | 12 +- .../runtime/artifacts/capture_validation.py | 5 + .../runtime/branch_initialization.py | 32 ++- .../linkerhand_calibration/runtime/capture.py | 5 + .../runtime/capture_index.py | 23 +- .../runtime/checkpoint_observer_upgrade.py | 99 +++++-- .../runtime/checkpoint_training_upgrade.py | 173 ++++++++++++ .../runtime/coordinator.py | 249 +++++++++++++++--- .../linkerhand_calibration/runtime/engine.py | 22 +- .../runtime/execution.py | 48 +++- .../runtime/image_reference.py | 7 + .../runtime/joint_resume.py | 45 +++- .../runtime/journal_durability.py | 22 ++ .../runtime/motion_execution.py | 19 +- .../runtime/motion_provenance.py | 3 +- .../runtime/observer_preparation.py | 75 +++++- .../runtime/observer_preparation_evidence.py | 26 +- .../runtime/observer_reference.py | 24 +- .../runtime/parent_reference.py | 62 +++++ .../runtime/passed_resume.py | 47 +++- .../linkerhand_calibration/runtime/resume.py | 5 + .../runtime/resume_storage.py | 43 ++- .../linkerhand_calibration/runtime/runner.py | 30 ++- .../linkerhand_calibration/runtime/session.py | 121 +++++++-- .../runtime/shared_pose_handoff.py | 3 +- .../runtime/staged_resume.py | 62 ++++- .../linkerhand_calibration/runtime/status.py | 2 + .../runtime/task_deferral.py | 237 +++++++++++++++++ .../runtime/training.py | 6 +- .../runtime/zero_recovery.py | 16 ++ .../test/test_checkpoint_observer_upgrade.py | 17 +- .../test/test_o30_profile.py | 7 +- .../test/test_parent_reference_reuse.py | 38 +++ .../test/test_passed_resume.py | 80 +++++- .../test/test_resume_storage.py | 94 ++++++- .../test/test_rigid_pair_observer.py | 12 + .../test/test_staged_training.py | 50 +++- .../test/test_trajectory_observation_v3.py | 207 +++++++++++++++ 47 files changed, 2043 insertions(+), 245 deletions(-) create mode 100644 src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_training_upgrade.py create mode 100644 src/linkerhand_calibration/linkerhand_calibration/runtime/journal_durability.py create mode 100644 src/linkerhand_calibration/linkerhand_calibration/runtime/task_deferral.py create mode 100644 src/linkerhand_calibration/test/test_trajectory_observation_v3.py diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/artifacts/evidence_journal.py b/src/linkerhand_calibration/linkerhand_calibration/core/artifacts/evidence_journal.py index 2f8ffdc..ac3d153 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/artifacts/evidence_journal.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/artifacts/evidence_journal.py @@ -22,19 +22,10 @@ import zlib FORMAT = "linkerhand_indexed_evidence_v1" REFERENCE_KEY = "$evidence_record" FIELDS_KEY = "$record_fields" +SCHEMA_KEY = "$record_schema" CANONICAL_SHA256_KEY = "$record_canonical_sha256" DEFAULT_THRESHOLD_BYTES = 16 * 1024 MAXIMUM_BLOCK_BYTES = 4 * 1024 * 1024 -INDEXED_SEQUENCE_FIELDS = { - "relative_quaternion_xyzw", "relative_translation_xyz_m", - "parent_pose_common", "child_pose_common", "view_normal_common_xyz", - "camera_center_common_xyz_m", "state_u8", "state_rad", - "command_vector_u8", "command_vector_rad", "steady_training_nodes", - "command_direction_by_index", -} -MOTION_INDEX_FIELDS = { - "zero_joints", "command_vector", "feedback_vector", -} INDEXED_SCALAR_FIELDS = { "kind", "profile_id", "serial_number", "task_name", "joint", "observation_joint", "view", "image_stamp_ns", "cycle", "direction", @@ -60,7 +51,7 @@ CANONICAL_INDEX_KINDS = { "capture_scope_extension", "capture_observation_revision", "passed_tasks_reused", "passed_pose_models_restored", "staged_pose_models_restored", "resume_verification", "joint_zero_recovery", "visual_motion_observation", - "visual_motion_settled", + "visual_motion_settled", "task_observation_state", } @@ -113,7 +104,8 @@ def archive_path(journal_path: str | Path) -> Path: return path.with_name(path.stem + ".evidence.zlib") -def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEFAULT_THRESHOLD_BYTES) -> None: +def initialize_evidence_journal(journal_path: str | Path, *, + threshold_bytes=DEFAULT_THRESHOLD_BYTES, compression_level=1) -> None: """Enable indexed storage for a new journal without changing old captures.""" journal = Path(journal_path) marker = manifest_path(journal) @@ -125,11 +117,16 @@ def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEF if journal.exists() and journal.stat().st_size: raise ValueError("indexed_evidence_requires_new_or_migrated_journal") journal.parent.mkdir(parents=True, exist_ok=True) + if type(compression_level) is not int or not 1 <= compression_level <= 9: + raise ValueError("invalid_evidence_compression_level") from .storage import atomic_write_json atomic_write_json(marker, { "format": FORMAT, "archive": archive_path(journal).name, - "codec": "zlib_level_1_independent_blocks", + "codec": f"zlib_level_{compression_level}_independent_blocks", + "compression_level": compression_level, + "envelope_fields": "archive_bound_v2", + "field_schemas": {}, "large_record_threshold_bytes": int(threshold_bytes), "maximum_uncompressed_block_bytes": MAXIMUM_BLOCK_BYTES, }) @@ -153,24 +150,33 @@ def is_indexed_journal(journal_path: str | Path) -> bool: def _resume_index(record: Mapping) -> dict: - indexed = {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record + return {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record and (record[key] is None or isinstance(record[key], (str, int, float, bool)))} - if record.get("kind") in {"joint_sample", "secondary_joint_sample"}: - for key in INDEXED_SEQUENCE_FIELDS: - if key in record: - indexed[key] = record[key] - if record.get("kind") == "motion_branch_observation": - for key in MOTION_INDEX_FIELDS: - if key in record: - indexed[key] = record[key] - return indexed -def _envelope(record: Mapping, fields, reference) -> dict: +def _schema_identity(fields) -> str: + return hashlib.sha256(json.dumps(list(fields), separators=(",", ":")).encode()).hexdigest()[:16] + + +@lru_cache(maxsize=512) +def _schema_fields(journal_path: str | Path, identity: str): + marker = _journal_configuration(str(Path(journal_path).resolve())) or {} + fields = marker.get("field_schemas", {}).get(identity) + if not isinstance(fields, list) or not all(isinstance(field, str) for field in fields): + raise ValueError("compressed_evidence_schema_missing") + if _schema_identity(fields) != identity: + raise ValueError("compressed_evidence_schema_changed") + return tuple(fields) + + +def _envelope(record: Mapping, reference, schema_identity) -> dict: indexed = _resume_index(record) - if REFERENCE_KEY in indexed or FIELDS_KEY in indexed: + if REFERENCE_KEY in indexed or FIELDS_KEY in indexed or SCHEMA_KEY in indexed: raise ValueError("calibration_record_uses_reserved_storage_field") - result = {REFERENCE_KEY: reference, FIELDS_KEY: list(fields), **indexed} + # record_sha256 binds the complete archived object. Repeating its field + # names in every one-line resume index added tens of MB without adding an + # independent integrity check. + result = {REFERENCE_KEY: reference, SCHEMA_KEY: schema_identity, **indexed} if record.get("kind") in CANONICAL_INDEX_KINDS: result[CANONICAL_SHA256_KEY] = hashlib.sha256(json.dumps(record, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()).hexdigest() @@ -190,42 +196,64 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping], if marker is None: raise ValueError("unsupported_evidence_journal_format") threshold = int(marker["large_record_threshold_bytes"]) + compression_level = int(marker.get("compression_level", 1)) + if not 1 <= compression_level <= 9: + raise ValueError("invalid_evidence_compression_level") prepared = [] imported = {} for record in records: - if REFERENCE_KEY in record or FIELDS_KEY in record: + if REFERENCE_KEY in record or FIELDS_KEY in record or SCHEMA_KEY in record: raise ValueError("calibration_record_uses_reserved_storage_field") reference = getattr(record, "reference", None) + field_names = getattr(record, "field_names", None) + fields = tuple(record) if field_names is None else tuple(field_names) + schema_identity = _schema_identity(fields) if isinstance(reference, EvidenceRecordReference): - imported[len(prepared)] = _linked_envelope(journal, record) + imported[len(prepared)] = (record, schema_identity) line = None else: line = json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode("utf-8") - prepared.append((record, line)) + prepared.append((record, line, schema_identity, fields)) if not prepared: return + schemas = dict(marker.get("field_schemas", {})) + changed = False + for _, _, identity, fields in prepared: + existing = schemas.get(identity) + if existing is not None and tuple(existing) != fields: + raise ValueError("compressed_evidence_schema_collision") + if existing is None: + schemas[identity] = list(fields) + changed = True + if changed: + marker = {**marker, "field_schemas": schemas} + from .storage import atomic_write_json + atomic_write_json(manifest_path(journal), marker) + _journal_configuration.cache_clear() + evidence = archive_path(journal) evidence.parent.mkdir(parents=True, exist_ok=True) output: list[bytes | None] = [None] * len(prepared) - for index, envelope in imported.items(): + for index, (record, schema_identity) in imported.items(): + envelope = _linked_envelope(journal, record, schema_identity) output[index] = json.dumps(envelope, ensure_ascii=False, separators=(",", ":")).encode("utf-8") with evidence.open("ab") as archive: - large: list[tuple[int, Mapping, bytes]] = [] + large: list[tuple[int, Mapping, bytes, str]] = [] size = 0 def flush_block(): nonlocal large, size if not large: return - block = b"".join(line for _, _, line in large) - compressed = zlib.compress(block, 1) + block = b"".join(line for _, _, line, _ in large) + compressed = zlib.compress(block, compression_level) offset = archive.tell() _write_all(archive, compressed) block_digest = hashlib.sha256(compressed).hexdigest() local = 0 - for index, record, line in large: + for index, record, line, schema_identity in large: reference = { "version": 1, "offset": offset, "compressed_length": len(compressed), @@ -233,12 +261,12 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping], "record_offset": local, "record_length": len(line), "record_sha256": hashlib.sha256(line).hexdigest(), } - output[index] = json.dumps(_envelope(record, record.keys(), reference), + output[index] = json.dumps(_envelope(record, reference, schema_identity), ensure_ascii=False, separators=(",", ":")).encode("utf-8") local += len(line) large, size = [], 0 - for index, (record, line) in enumerate(prepared): + for index, (record, line, schema_identity, _) in enumerate(prepared): if line is None: continue if len(line) < threshold and record.get("kind") not in DETAILED_RECORD_KINDS: @@ -246,7 +274,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping], continue if large and size + len(line) > MAXIMUM_BLOCK_BYTES: flush_block() - large.append((index, record, line)) + large.append((index, record, line, schema_identity)) size += len(line) flush_block() archive.flush() @@ -260,7 +288,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping], os.fsync(stream.fileno()) -def _linked_envelope(destination: Path, record) -> dict: +def _linked_envelope(destination: Path, record, schema_identity) -> dict: """Hard-link an immutable source archive and retain its indexed row.""" reference = record.reference source_archive = (record.path.parent / reference.archive @@ -287,8 +315,7 @@ def _linked_envelope(destination: Path, record) -> dict: linked = EvidenceRecordReference(reference.offset, reference.compressed_length, reference.block_sha256, reference.record_offset, reference.record_length, reference.record_sha256, target.name) - result = {REFERENCE_KEY: linked.as_mapping(), FIELDS_KEY: list(record.field_names), - **record.metadata} + result = {REFERENCE_KEY: linked.as_mapping(), SCHEMA_KEY: schema_identity, **record.metadata} if record.canonical_sha256: result[CANONICAL_SHA256_KEY] = record.canonical_sha256 return result @@ -336,7 +363,10 @@ def decode_envelope(journal_path: str | Path, envelope: Mapping, decoder): if REFERENCE_KEY not in envelope: return envelope value = decoder.loads(evidence_record_bytes(journal_path, envelope[REFERENCE_KEY])) - if not isinstance(value, dict) or tuple(value) != tuple(envelope.get(FIELDS_KEY, ())): + expected_fields = (envelope.get(FIELDS_KEY) if FIELDS_KEY in envelope + else _schema_fields(journal_path, envelope[SCHEMA_KEY]) if SCHEMA_KEY in envelope else None) + if (not isinstance(value, dict) or expected_fields is not None + and tuple(value) != tuple(expected_fields)): raise ValueError("compressed_evidence_record_shape_changed") return value @@ -359,7 +389,9 @@ def compact_existing_journal(source_path: str | Path, *, replace=False) -> dict: for path in (compact, compact_archive, manifest_path(compact)): if path.exists(): path.unlink() - initialize_evidence_journal(compact) + # Closed checkpoints favor disk footprint. Live capture remains level 1 so + # evidence persistence cannot stall the control loop. + initialize_evidence_journal(compact, compression_level=6) source_digest = hashlib.sha256() record_count = 0 diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/domain/capture_plan.py b/src/linkerhand_calibration/linkerhand_calibration/core/domain/capture_plan.py index 50a60d0..ba05063 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/domain/capture_plan.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/domain/capture_plan.py @@ -13,6 +13,8 @@ PLAN_VERSION = "capture_plan_v1" FIXED = "fixed" ADAPTIVE = "adaptive_2_to_3" STAGED = "staged_2_to_3" +TASKWISE_STAGED = "taskwise_staged_2_to_3" +STAGED_POLICIES = frozenset((STAGED, TASKWISE_STAGED)) LEGACY_TRAINING = (0, 1, 2) LEGACY_HOLDOUT = 3 @@ -67,19 +69,25 @@ class CapturePlan: return next(task for task in self.tasks if task.task_key == key) def as_dict(self): - result = {"version": "capture_plan_v2" if self.policy == STAGED else PLAN_VERSION, "policy": self.policy, + version = ("capture_plan_v3" if self.policy == TASKWISE_STAGED else + "capture_plan_v2" if self.policy == STAGED else PLAN_VERSION) + result = {"version": version, "policy": self.policy, "command_capture_mode": self.command_capture_mode, "motion_contract": json.loads(self.motion_contract_json), "tasks": {t.task_key: {"training_cycles": list(t.training), "holdout_cycle": t.holdout, "command_training_cycles": list(t.command_training), "command_holdout_cycle": t.command_holdout} for t in self.tasks}} - if self.policy == STAGED: + if self.policy in STAGED_POLICIES: result["stages"] = [ {"name": name, "cycle": cycle, "tasks": [t.task_key for t in self.tasks if cycle in t.motion_cycles]} for name, cycle in (("first_training", 0), ("second_training", 1), ("supplement", 2), ("validation", 3))] + if self.policy == TASKWISE_STAGED: + result["schedule"] = ( + "first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation" + ) return result @property @@ -101,7 +109,8 @@ def task_partition(profile, task=None, *, joint=None): if joint is not None: task = task_for_joint(profile, joint) key = getattr(task, "key", task) - default = LEGACY_TRAINING[:2] if profile.quality.training_policy == STAGED else profile.quality.training_cycles + default = (LEGACY_TRAINING[:2] if profile.quality.training_policy in STAGED_POLICIES + else profile.quality.training_cycles) training = tuple(profile.quality.task_training_cycles.get(key, default)) holdout = profile.quality.holdout_cycle separate = profile.acquisition.command_capture_mode == "separate" @@ -119,20 +128,21 @@ def select_task_training(profile, task_key, cycles): raise ValueError("capture_plan_unknown_task") if cycles not in (LEGACY_TRAINING, LEGACY_TRAINING[:2]): raise ValueError("capture_plan_invalid_training_partition") - if cycles != tuple(profile.quality.training_cycles) and profile.quality.training_policy not in {ADAPTIVE, STAGED}: + if (cycles != tuple(profile.quality.training_cycles) + and profile.quality.training_policy not in {ADAPTIVE, *STAGED_POLICIES}): raise ValueError("capture_plan_fixed_training_cannot_change") selections = {**profile.quality.task_training_cycles, task_key: cycles} return replace(profile, quality=replace(profile.quality, task_training_cycles=selections)) def configure_training(profile, policy): - if policy not in {FIXED, ADAPTIVE, STAGED}: + if policy not in {FIXED, ADAPTIVE, *STAGED_POLICIES}: raise ValueError("unknown_training_policy") if profile.artifacts.output_schema_version in {2, 3} and ( tuple(profile.quality.training_cycles) != LEGACY_TRAINING or profile.quality.holdout_cycle != LEGACY_HOLDOUT): raise ValueError("production_partition_requires_versioned_training_policy") - if policy in {ADAPTIVE, STAGED} and (profile.artifacts.output_schema_version < 3 + if policy in {ADAPTIVE, *STAGED_POLICIES} and (profile.artifacts.output_schema_version < 3 or not profile.command_based_release or profile.acquisition.command_capture_mode != "interleaved" or tuple(profile.quality.training_cycles) != LEGACY_TRAINING diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/fitting/session.py b/src/linkerhand_calibration/linkerhand_calibration/core/fitting/session.py index 5d13ad3..394fe8b 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/fitting/session.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/fitting/session.py @@ -9,7 +9,8 @@ from typing import Mapping from ..domain.profile import CalibrationProfile, validate_profile from ..domain.result import CalibrationResult, JointMapping -from ..domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, training_cycles +from ..domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED, TASKWISE_STAGED, + training_cycles) from ..urdf.kinematics import UrdfKinematicModel from .motion_fit import channel_for_joint, fit_profile_motion, joint_input_direction from .spatial import ZeroCalibrationProfile, fit_joint_axis_measurement, solve_urdf_zero_offsets, with_depth_free_axis_projection @@ -98,7 +99,8 @@ def build_axis_observations(profile, model, geometry_records, motion, zero_profi rows = [dict(row, command_u8=int(round(float(row[profile.curve_input_domain])))) for row in rows] parent = zero_profile.phase_parent_joint.get(joint) if parent is not None and parent not in by_joint: - if cycle != plan.task(task.key).holdout and profile.quality.training_policy in {ADAPTIVE, STAGED}: + if (cycle != plan.task(task.key).holdout + and profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}): continue # No same-cycle parent observation: do not fabricate a geometric sample. raise ValueError("spatial axis order must place phase parents first") task = next(task for task in profile.motion.tasks if joint in task.joints) @@ -182,7 +184,7 @@ def fit_profile_calibration(profile: CalibrationProfile, source_urdf: Path, reco zero = solve_urdf_zero_offsets(source_urdf=source_urdf, measurements=observations, curves=spatial_curves, motor_by_joint={joint: channel_for_joint(profile, joint) for joint in curves}, zero_profile=zero_profile, training_cycles=geometry_training, validation_cycle=geometry_holdout, - allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED}, + allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}, frozen_training=None if frozen_training is None else frozen_training["zero"], maximum_offset_rad=math.radians(20), finger_maximum_offset_rad=math.radians(20), maximum_validation_mae_rad=math.radians(1), maximum_validation_p95_rad=math.radians(2), diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/fitting/training_quality.py b/src/linkerhand_calibration/linkerhand_calibration/core/fitting/training_quality.py index fb697e9..ec058ea 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/fitting/training_quality.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/fitting/training_quality.py @@ -9,7 +9,8 @@ from pathlib import Path import numpy as np from scipy.stats import t as student_t -from ..domain.capture_plan import CapturePlan, evidence_digest, select_task_training, training_cycles +from ..domain.capture_plan import (CapturePlan, TASKWISE_STAGED, evidence_digest, + select_task_training, training_cycles) from ..domain.motion_path import record_scan_identity from ..domain.sampling import command_nodes from ..urdf.kinematics import UrdfKinematicModel @@ -94,16 +95,22 @@ def _curve_repeatability(profile, task, motion, steady, cycles): def assess_training(profile, task_key, cycles, rows, references, source_urdf): - """Assess an immutable training snapshot. Missing future geometry adds a round. + """Assess one immutable task snapshot without reading validation data. -The local angle confidence is NOT a zero-offset certificate. When the full -spatial dependency graph is available, the production training-only solver -also checks real zero confidence. Otherwise the conservative third round is -retained and the final spatial gate remains mandatory. -""" + Adaptive collection keeps the older conservative behavior: missing + full-hand geometry requests a third round. Taskwise staged collection uses + local two-round repeatability to decide immediately; its full-hand spatial + fit remains a separate mandatory gate after every training task is frozen. + """ cycles = tuple(cycles) chosen = select_task_training(profile, task_key, cycles) task = next(t for t in profile.motion.tasks if t.key == task_key) + taskwise = profile.quality.training_policy == TASKWISE_STAGED + if taskwise: + # A task decision is deliberately local and therefore remains stable + # when later fingers are acquired. Full-hand zero geometry is checked + # by the final training-only fit before any artifact can be published. + rows = tuple(row for row in rows if row.get("task_name") == task_key) if any(row["cycle"] not in training_cycles(chosen, row["task_name"]) for row in rows): raise ValueError("training_assessment_received_nontraining_data") sweep, steady = _by_joint(rows, "sweep"), _by_joint(rows, "steady") @@ -125,8 +132,8 @@ retained and the final spatial gate remains mandatory. fitted_model = {"curves": {name: asdict(curve) for name, curve in local.coordinate_curves.items()}, "mappings": {name: asdict(mapping) for name, mapping in local.mappings.items()}} failures, metrics = _curve_repeatability(chosen, task, local, steady, cycles) - required = set(profile.zero.axis_joints) - ready = required <= sweep.keys() & steady.keys() & references.keys() + required = set() if taskwise else set(profile.zero.axis_joints) + ready = bool(required) and required <= sweep.keys() & steady.keys() & references.keys() if ready: from .session import compile_spatial_profile, build_axis_observations from .spatial_solver.solve import fit_urdf_zero_training @@ -154,7 +161,8 @@ retained and the final spatial gate remains mandatory. failures.extend(zero.failures) except (ValueError, KeyError, np.linalg.LinAlgError) as error: failures.append(str(error)) - decision = ("add_training" if len(cycles) == 2 and (failures or geometry["status"] == "deferred") + decision = ("add_training" if len(cycles) == 2 and (failures + or (geometry["status"] == "deferred" and not taskwise)) else "fail" if failures else "freeze") record = {"kind": "training_decision", "version": TRAINING_DECISION_VERSION, "task_name": task_key, "assessed_cycles": list(cycles), "decision": decision, diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/geometry/frozen_evidence.py b/src/linkerhand_calibration/linkerhand_calibration/core/geometry/frozen_evidence.py index 6526b32..6ae9c11 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/geometry/frozen_evidence.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/geometry/frozen_evidence.py @@ -95,4 +95,11 @@ class EvidenceDecoder: return row def loads(self, line): + # Most evidence rows contain observations only. Avoid recursively + # invoking the model interning hook for every nested pose/candidate + # dictionary when neither interned object can be present. + frozen_key = b'"frozen_image_model"' if isinstance(line, bytes) else '"frozen_image_model"' + reference_key = b'"reference_frame"' if isinstance(line, bytes) else '"reference_frame"' + if frozen_key not in line and reference_key not in line: + return json.loads(line) return json.loads(line, object_hook=self.object_hook) diff --git a/src/linkerhand_calibration/linkerhand_calibration/core/geometry/tag_pose/rigid_tag_pair.py b/src/linkerhand_calibration/linkerhand_calibration/core/geometry/tag_pose/rigid_tag_pair.py index 28efcb9..8a5c3a7 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/core/geometry/tag_pose/rigid_tag_pair.py +++ b/src/linkerhand_calibration/linkerhand_calibration/core/geometry/tag_pose/rigid_tag_pair.py @@ -19,6 +19,7 @@ from .observer_pose import ObserverPose, OBSERVER_POSE_LIMITS from .image_model_selection import compare_image_losses, holm_adjusted_probabilities RIGID_PAIR_CONDITIONING = 'rigid_tag_pair_pose_v1' +RIGID_PAIR_TRAJECTORY_POLICY = 'rigid_pair_full_trajectory_v3' def _pose_vector(pose): @@ -144,12 +145,23 @@ def _pose_bounds(fit, count): def fit_rigid_tag_pair(parents, candidates, corners, matrices, size, stamps, *, parent_corners, parent_size, - parent_candidates=None): + parent_candidates=None, trajectory_states=None): count = len(stamps) if count < 24 or tuple(stamps) != tuple(sorted(set(stamps))) or any(not c for c in candidates): raise ValueError('rigid_pair_independent_images_missing') if parent_candidates is not None and (len(parent_candidates) != count or any(not p for p in parent_candidates)): raise ValueError('rigid_pair_parent_candidates_missing') + if trajectory_states is not None: + states = tuple(trajectory_states) + if len(states) != count or any(len(value) != 2 for value in states): + raise ValueError('rigid_pair_trajectory_state_missing') + phases = np.asarray([value[0] for value in states], dtype=int) + commands = np.asarray([value[1] for value in states], dtype=float) + differences = np.diff(commands) + if (not np.all(np.isfinite(commands)) or np.ptp(commands) <= 0 + or not {1, 2} <= set(phases) + or not np.any(differences > 0) or not np.any(differences < 0)): + raise ValueError('rigid_pair_trajectory_direction_insufficient') images = _PairImages(np.asarray(parent_corners), np.asarray(corners), np.asarray(matrices), square_object_points(parent_size), square_object_points(size), np.asarray([_pose_vector(p) for p in parents])) if (images.parent_pixels.shape != (count, 4, 2) or images.child_pixels.shape != (count, 4, 2) diff --git a/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_policy.py b/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_policy.py index 2e4d21b..9b8831a 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_policy.py +++ b/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_policy.py @@ -9,11 +9,18 @@ from dataclasses import replace LEGACY_OBSERVER = 'frozen_parent_v1' RIGID_OBSERVER = 'rigid_pair_v1' +TRAJECTORY_OBSERVER = 'rigid_pair_trajectory_v3' + + +def is_rigid_observer(policy): + """Return whether a policy requires independent rigid-pair evidence.""" + return policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER} def rigid_observer_dependencies(profile): """Return recipient -> independently measured parent hinge dependencies.""" - if profile.vision_motion or profile.quality.training_policy != 'staged_2_to_3': + from ..core.domain.capture_plan import STAGED_POLICIES + if profile.vision_motion or profile.quality.training_policy not in STAGED_POLICIES: return {} measurements = profile.measurement.measurements tasks = {joint: task for task in profile.motion.tasks for joint in task.joints} @@ -34,13 +41,15 @@ def rigid_observer_dependencies(profile): return dependencies -def configure_observer_policy(profile, *, preserved_joints=()): +def configure_observer_policy(profile, *, preserved_joints=(), policy=RIGID_OBSERVER): """Upgrade every supported unmeasured recipient in a single graph pass.""" + if policy not in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}: + raise ValueError('observer_policy_unknown_strategy') preserved = frozenset(preserved_joints) measurements = dict(profile.measurement.measurements) if not preserved <= measurements.keys(): raise ValueError('observer_policy_unknown_preserved_joint') for joint in rigid_observer_dependencies(profile): if joint not in preserved: - measurements[joint] = replace(measurements[joint], observer_pose_policy=RIGID_OBSERVER) + measurements[joint] = replace(measurements[joint], observer_pose_policy=policy) return replace(profile, measurement=replace(profile.measurement, measurements=measurements)) diff --git a/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_preparation.py b/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_preparation.py index 7ba3a92..55a2564 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_preparation.py +++ b/src/linkerhand_calibration/linkerhand_calibration/profiles/observer_preparation.py @@ -35,7 +35,12 @@ def _bounded_actions(profile, first): def configure_observer_preparations(profile, model, *, preserved_joints=()): - profile = configure_observer_policy(profile, preserved_joints=preserved_joints) + from .observer_policy import TRAJECTORY_OBSERVER + strategy = (TRAJECTORY_OBSERVER if any( + spec.observer_pose_policy == TRAJECTORY_OBSERVER + for spec in profile.measurement.measurements.values()) else None) + profile = configure_observer_policy(profile, preserved_joints=preserved_joints, + **({"policy": strategy} if strategy else {})) channels = compile_tag_feedback_channels(profile, model) by_channel = {channel_for_joint(profile, name): name for name in profile.measurement.measurements} plans = dict(profile.motion.observer_preparations) @@ -88,7 +93,9 @@ def validate_observer_preparations(profile, model): def _validate_action(profile, joint, plan, channels, dependencies, index): - if joint not in dependencies or profile.measurement.measurements[joint].observer_pose_policy != 'rigid_pair_v1': + from .observer_policy import is_rigid_observer + if joint not in dependencies or not is_rigid_observer( + profile.measurement.measurements[joint].observer_pose_policy): raise ValueError('observer_preparation_dependency_invalid') spec = profile.measurement.measurements[joint] channel = channel_for_joint(profile, plan.joint) diff --git a/src/linkerhand_calibration/linkerhand_calibration/profiles/preparation.py b/src/linkerhand_calibration/linkerhand_calibration/profiles/preparation.py index 70e94be..2b3d5fe 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/profiles/preparation.py +++ b/src/linkerhand_calibration/linkerhand_calibration/profiles/preparation.py @@ -15,8 +15,9 @@ def preparation_image_budget(profile, *, joint_count=1): from ..core.geometry.tag_pose.production_image_motion import ( DENSE_PREPARATION_SAMPLING, image_parameters_for_sampling, ) + from ..core.domain.capture_plan import STAGED_POLICIES dense = (profile.key.model == 'O30' - and profile.quality.training_policy == 'staged_2_to_3' + and profile.quality.training_policy in STAGED_POLICIES and bool(profile.motion.preparation_witnesses) and joint_count == 1) return ((256, image_parameters_for_sampling(DENSE_PREPARATION_SAMPLING)) if dense else (128, image_parameters_for_sampling())) @@ -50,9 +51,20 @@ def preparation_contract(profile): value['observer_pose_policies'] = observers if profile.motion.observer_preparations: from ..core.domain.profile import ObserverPreparationPlan + from .observer_policy import TRAJECTORY_OBSERVER sequence = any(isinstance(p, ObserverPreparationPlan) for p in profile.motion.observer_preparations.values()) - value['observer_preparations'] = dict(policy='bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1', + trajectory = any(spec.observer_pose_policy == TRAJECTORY_OBSERVER + for spec in profile.measurement.measurements.values()) + value['observer_preparations'] = dict(policy='occlusion_aware_trajectory_v3' if trajectory + else 'bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1', paths={name: asdict(spec) for name, spec in profile.motion.observer_preparations.items()}) + if trajectory: + value['observer_preparations']['trajectory_solver'] = { + 'policy': 'rigid_pair_full_trajectory_v3', + 'training_validation_split': 'chronological_even_odd', + 'maximum_actions_per_task': 2, + 'holdout_selects_model': False, + } # The live header and decoded JSON must compare identically (including # tuple-valued poses), without relying on incidental Python container types. return json.loads(json.dumps(value, allow_nan=False)) @@ -72,11 +84,18 @@ def validate_preparation_contract(profile, records): if len(declared) != 1 or declared[0] != preparation_contract(profile): raise ValueError('preparation_contract_profile_changed') tasks = {task.key: set(task.joints) for task in profile.motion.tasks} + refreshed_parents = {(row.get('task_name'), row.get('pose_joint'), + row.get('image_model_sha256')) for row in records + if row.get('kind') == 'parent_reference_model_refreshed' + and row.get('passed_zero_preserved') is True} for row in records: if row.get('kind') != 'motion_branch_initialization' or row.get('status') != 'resolved': continue joints = row.get('zero_joints', ()) - if not joints or not set(joints) <= tasks.get(row.get('task_name'), set()): + ordinary = bool(joints) and set(joints) <= tasks.get(row.get('task_name'), set()) + refreshed = (len(joints) == 1 and (row.get('task_name'), joints[0], + row.get('image_model_sha256')) in refreshed_parents) + if not ordinary and not refreshed: raise ValueError('preparation_contract_model_dependency_changed') _, expected = preparation_image_budget(profile, joint_count=len(joints)) if image_parameters_for_sampling(row.get('image_sampling_policy')) != expected: @@ -84,9 +103,12 @@ def validate_preparation_contract(profile, records): def configure_preparation(profile, policy='occlusion_aware_witness_v2'): - if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2'}: + if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2', + 'occlusion_aware_trajectory_v3'}: raise ValueError('unknown_preparation_policy') - if profile.key.profile_id != 'O30/right/o30_right_18/v1' or profile.quality.training_policy != 'staged_2_to_3': + from ..core.domain.capture_plan import STAGED_POLICIES + if (profile.key.profile_id != 'O30/right/o30_right_18/v1' + or profile.quality.training_policy not in STAGED_POLICIES): raise ValueError('preparation_witness_requires_o30_staged_session') from ..core.fitting.motion_fit import channel_for_joint witnesses = {} @@ -102,9 +124,10 @@ def configure_preparation(profile, policy='occlusion_aware_witness_v2'): excursion[channel] = command[channel]-step if command[channel]-step >= lower else command[channel]+step witnesses[target] = ModelPreparation(donor, command, (command, tuple(excursion))) prepared = replace(profile, motion=replace(profile.motion, preparation_witnesses=witnesses)) - if policy == 'occlusion_aware_witness_v2': - from .observer_policy import configure_observer_policy - prepared = configure_observer_policy(prepared) + if policy in {'occlusion_aware_witness_v2', 'occlusion_aware_trajectory_v3'}: + from .observer_policy import configure_observer_policy, RIGID_OBSERVER, TRAJECTORY_OBSERVER + strategy = TRAJECTORY_OBSERVER if policy == 'occlusion_aware_trajectory_v3' else RIGID_OBSERVER + prepared = configure_observer_policy(prepared, policy=strategy) return prepared diff --git a/src/linkerhand_calibration/linkerhand_calibration/profiles/validator.py b/src/linkerhand_calibration/linkerhand_calibration/profiles/validator.py index a7785e9..bd67697 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/profiles/validator.py +++ b/src/linkerhand_calibration/linkerhand_calibration/profiles/validator.py @@ -36,18 +36,20 @@ def validate_executable_profile(profile, source_urdf): if joint == observer: raise ValueError(f"absolute_zero_unobservable:{joint}:a free Tag on its own hinge is not a CAD datum") measured = {joint for task in profile.motion.tasks for joint in task.joints} - from .observer_policy import rigid_observer_dependencies, RIGID_OBSERVER + from .observer_policy import rigid_observer_dependencies, is_rigid_observer observer_dependencies = rigid_observer_dependencies(profile) if not measured <= model.joints.keys() or not (profile.zero.active_joints | profile.zero.passive_joints) <= model.joints.keys(): raise ValueError("Profile joint keys must name URDF joints; SDK names belong in command.names") for name, spec in profile.measurement.measurements.items(): - if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1"}: + if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1", + "rigid_pair_trajectory_v3"}: raise ValueError(f"unknown_observer_pose_policy:{name}") - if spec.observer_pose_policy == "rigid_pair_v1" and ( + from ..core.domain.capture_plan import STAGED_POLICIES + if is_rigid_observer(spec.observer_pose_policy) and ( spec.kind != "relative_rotation" or profile.vision_motion - or profile.quality.training_policy != "staged_2_to_3"): + or profile.quality.training_policy not in STAGED_POLICIES): raise ValueError(f"rigid_pair_requires_staged_relative_measurement:{name}") - if spec.observer_pose_policy == RIGID_OBSERVER and name not in observer_dependencies: + if is_rigid_observer(spec.observer_pose_policy) and name not in observer_dependencies: raise ValueError(f"rigid_pair_requires_independent_parent_hinge:{name}") if spec.kind not in {"relative_rotation", "urdf_axis_chain", "curve", "axis_cross_view_validation"}: raise ValueError(f"unsupported measurement primitive:{name}:{spec.kind}") diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/artifacts/capture_validation.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/artifacts/capture_validation.py index 30762bd..65371c4 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/artifacts/capture_validation.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/artifacts/capture_validation.py @@ -53,6 +53,11 @@ def validate_capture_provenance(profile, serial_number, hashes, records): def validate_capture_units(profile, records): """Offline and resumed data must satisfy the actual post-sweep policy.""" + from ..task_deferral import unresolved_observation_records + blocked = unresolved_observation_records(profile, records) + if blocked: + raise ValueError("capture_unobservable_tasks:" + ",".join(sorted( + f"{row.get('task_name')}:{row.get('cycle')}:{row.get('state')}" for row in blocked))) from ...core.fitting.command_sampling import validate_sampling_plans from ..training import resolve_capture_plan profile = resolve_capture_plan(profile, records, require_complete=True) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/branch_initialization.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/branch_initialization.py index 3c98a67..aaec29e 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/branch_initialization.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/branch_initialization.py @@ -8,6 +8,7 @@ training/holdout observations to revise an existing reference. from dataclasses import asdict, dataclass, replace from collections import deque +from collections.abc import Mapping import hashlib import json import math @@ -89,7 +90,11 @@ def _image_references(record, roles): def _finite_report(value): if isinstance(value, float) and not math.isfinite(value): return None - if isinstance(value, dict): + # Resume keeps large evidence rows as lazy Mapping objects. Solver reports + # are durable journal records, so materialize any nested mapping at this + # single ownership boundary instead of leaking storage-layer objects into + # the JSON encoder. + if isinstance(value, Mapping): return {key: _finite_report(item) for key, item in value.items()} if isinstance(value, (list, tuple)): return [_finite_report(item) for item in value] @@ -125,8 +130,9 @@ def solve_initialization(request: BranchInitializationRequest): return replace(result, preparation_transitions=(record,)) bridges, records, rejections = [], [], [] if request.observer_profile is not None: + from ..profiles.observer_policy import is_rigid_observer required = {joint for joint, spec in request.observer_profile.measurement.measurements.items() - if joint in request.zero_joints and spec.observer_pose_policy == 'rigid_pair_v1'} + if joint in request.zero_joints and is_rigid_observer(spec.observer_pose_policy)} if not required <= {source.joint for source in request.observer_sources}: return ImageMotionResolution(False, 'rigid_pair_source_missing') if request.observer_sources: @@ -137,7 +143,7 @@ def solve_initialization(request: BranchInitializationRequest): bridges.append(bridge) records.append(record) except (ValueError, KeyError, TypeError, IndexError) as error: - if request.observer_profile.measurement.measurements[source.joint].observer_pose_policy == 'rigid_pair_v1': + if is_rigid_observer(request.observer_profile.measurement.measurements[source.joint].observer_pose_policy): return ImageMotionResolution(False, str(error)) rejections.append(str(error)) witness_bridges, witness_records = [], [] @@ -228,7 +234,8 @@ def initialization_artifacts(request, resolution): if record.get("observer_pose_bridges"): from ..core.geometry.tag_pose.pose_bridge import observer_pose_selection_policy, RIGID_OBSERVER_SELECTION_POLICY record["image_model_selection_policy"] = (RIGID_OBSERVER_SELECTION_POLICY - if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges']) + if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3', + 'observer_pose_bridge_v4'} for r in record['observer_pose_bridges']) else observer_pose_selection_policy(request.pose_bridges)) if record.get("preparation_transitions"): from ..core.geometry.tag_pose.transition_image_solver import TRANSITION_SELECTION_POLICY @@ -241,6 +248,7 @@ def initialization_artifacts(request, resolution): if any(r.get('conditioning') for r in record['preparation_witnesses']) else WITNESS_POLICY) record["image_geometry_constraints"] = [asdict(item) for item in request.geometry_constraints] record["source_urdf_sha256"] = request.source_urdf_sha256 + record = _finite_report(record) if not resolution.resolved: return None, record existing = dict(request.existing_evidence_ids) @@ -272,7 +280,8 @@ def initialization_artifacts(request, resolution): payload['image_model_selection_policy'] = record['image_model_selection_policy'] if record.get("observer_pose_bridges"): payload["observer_pose_bridges"] = record["observer_pose_bridges"] - if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges']): + if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3', + 'observer_pose_bridge_v4'} for r in record['observer_pose_bridges']): payload['image_model_selection_policy'] = record['image_model_selection_policy'] if record.get("preparation_transitions"): payload["preparation_transitions"] = record["preparation_transitions"] @@ -340,6 +349,7 @@ class BranchInitialization: self._invalid_reason = "" self._requested = set() self.reference_spec = None + self.parent_model_refresh_for = "" self.reports = {} self.path_index = None @@ -367,6 +377,7 @@ class BranchInitialization: self._invalid_reason = "motion_approach_path_discontinuous" self.path_index = index self.reference_spec = (motion.reference_command, motion.reference_approach) if motion.reference_only else None + self.parent_model_refresh_for = motion.parent_model_refresh_for self.task_name = motion.task_key self.zero_joints = motion.reference_joints self.motion_version = motion_version @@ -378,6 +389,7 @@ class BranchInitialization: self.task_name, self.zero_joints = motion.task_key, motion.zero_joints self.session_epoch = session_epoch self._invalid_reason = "motion_approach_evidence_missing" + self.parent_model_refresh_for = motion.parent_model_refresh_for elif self.path_index is not None: spec = self.profile.motion.joint_zero_references[motion.zero_joints[0]] if spec.evidence_scope != "path" or self.path_index != len(spec.approach_commands): @@ -539,7 +551,7 @@ class BranchInitialization: reason = reason or 'shared_pose_bridge_invalid:' + str(error) source_hinges, parent_reference_sha256 = (), "" task = next(task for task in self.profile.motion.tasks if task.key == self.task_name) - if task.parent_reference is not None: + if task.parent_reference is not None and not self.parent_model_refresh_for: try: source_hinges, parent_reference_sha256 = self.parent_references.geometry(task, session_epoch) except ValueError as error: @@ -550,15 +562,17 @@ class BranchInitialization: tuple(item.geometry.relation for item in source_hinges) + relations) from .observer_reference import observer_inputs observer_sources = observer_inputs(self, view, self._endpoint_records.get(view, ()), - {f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else () + {f.stamp_ns for f in frames}, session_epoch) if ( + self.image_geometry and not self.parent_model_refresh_for) else () transition_source = None - if self.image_geometry: + if self.image_geometry and not self.parent_model_refresh_for: try: transition_source = self.transition_capture.snapshot(view) except ValueError as error: reason = reason or str(error) witness_sources = self.witness_capture.inputs(view, self.zero_joints, - self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else () + self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if ( + self.image_geometry and not self.parent_model_refresh_for) else () return BranchInitializationRequest(self.task_name, view, self.zero_joints, session_epoch, self.motion_version if self.motion_version is not None else motion_version, self.generation, frames, diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/capture.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/capture.py index 1597047..1f3935f 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/capture.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/capture.py @@ -157,6 +157,11 @@ class ObservationCapture: self._preparation_identity = None self.install_motion_model(model) + def current_image_reference_stable(self, view): + """Check the independent current-session anchor, not an active old model.""" + tracker = self._image_references.get(view) + return tracker is not None and tracker.tracking_current_image() + def install_motion_model(self, model: MotionBranchModel): """Add evidence before zero capture without replacing existing roles.""" key = (model.task_name, model.view) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/capture_index.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/capture_index.py index 930e9df..fe75ac8 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/capture_index.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/capture_index.py @@ -64,14 +64,27 @@ class CaptureRecordIndex(Sequence): source = getattr(row, "metadata", row) indexed = {key: value for key, value in source.items() if value is None or isinstance(value, (str, int, float, bool))} - for key in ('relative_quaternion_xyzw', 'steady_training_nodes', 'command_direction_by_index'): - if key in row: - indexed[key] = tuple(row[key]) if self.retain_training_geometry: from ..core.fitting.training_quality import TRAINING_FIELDS from ..core.domain.capture_plan import evidence_digest - indexed.update({key: row[key] for key in TRAINING_FIELDS if key in row}) - indexed['source_record_sha256'] = evidence_digest(row) + from .resume_storage import materialize_plain_record + # A deferred row decompresses and parses its evidence when a + # non-indexed field is accessed. Materialize it exactly once; + # probing every training field through Mapping.__contains__ + # used to decode the same sample dozens of times. + materialized = materialize_plain_record(row) + indexed.update({key: materialized[key] for key in TRAINING_FIELDS + if key in materialized}) + # DeferredCaptureRecord implements Mapping for indexed reads, + # but json.dumps intentionally accepts only concrete JSON + # containers. Hash the immutable materialized record exactly + # as an eager capture would. + indexed['source_record_sha256'] = evidence_digest(materialized) + else: + for key in ('relative_quaternion_xyzw', 'steady_training_nodes', + 'command_direction_by_index'): + if key in row: + indexed[key] = tuple(row[key]) self._store(indexed) elif kind in {'command_sampling_plan', 'training_decision', 'staged_training_decision'}: self._store(row) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_observer_upgrade.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_observer_upgrade.py index a4fd6b7..ec12d48 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_observer_upgrade.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_observer_upgrade.py @@ -8,10 +8,12 @@ from dataclasses import dataclass, replace import hashlib import json from pathlib import Path +import shutil from ..core.domain.profile import CalibrationProfile from ..profiles import load_hand_profile -from ..profiles.observer_policy import configure_observer_policy +from ..profiles.observer_policy import (configure_observer_policy, + RIGID_OBSERVER, TRAJECTORY_OBSERVER) from ..profiles.preparation import preparation_contract from ..profiles.validator import validate_executable_profile from .engine import CalibrationEngine @@ -19,6 +21,23 @@ from .joint_resume import PreparedResume, JointResume from .staged_resume import stage_visits +def _copy_indexed_evidence(source_path, output_dir): + """Copy content-addressed evidence before exposing the migrated index.""" + from ..core.artifacts.evidence_journal import ( + is_indexed_journal, manifest_path, archive_path, + ) + if not is_indexed_journal(source_path): + return + for source_evidence in (manifest_path(source_path), archive_path(source_path)): + target_evidence = output_dir/source_evidence.name + shutil.copy2(source_evidence, target_evidence) + with source_evidence.open('rb') as source_stream, target_evidence.open('rb') as target_stream: + source_hash = hashlib.file_digest(source_stream, 'sha256').hexdigest() + target_hash = hashlib.file_digest(target_stream, 'sha256').hexdigest() + if source_hash != target_hash: + raise ValueError('observer_upgrade_evidence_archive_changed') + + def preserved_observer_joints(checkpoint): if any(r.get('kind') == 'staged_training_decision' for r in checkpoint.rows): raise ValueError('observer_upgrade_training_already_decided') @@ -45,14 +64,23 @@ class ObserverPolicyUpgrade: return cls(source, target, Path(source_path).read_bytes(), Path(target_path).read_bytes()) def metadata(self, checkpoint, protected_inputs, source_path, source_digest): + from .resume_storage import materialize_record + source_header = materialize_record(checkpoint.header) + source_reference = materialize_record(checkpoint.fixed_reference) preserved = preserved_observer_joints(checkpoint) - expected = configure_observer_policy(self.source, preserved_joints=preserved) + target_strategies = {spec.observer_pose_policy + for spec in self.target.measurement.measurements.values() + if spec.observer_pose_policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}} + strategy = (TRAJECTORY_OBSERVER if TRAJECTORY_OBSERVER in target_strategies + else RIGID_OBSERVER) + expected = configure_observer_policy(self.source, preserved_joints=preserved, + policy=strategy) if self.target.motion.observer_preparations != self.source.motion.observer_preparations: if self.source_urdf is None: raise ValueError('observer_upgrade_source_geometry_missing') from ..core.urdf.kinematics import UrdfKinematicModel from ..profiles.observer_preparation import configure_observer_preparations - expected = configure_observer_preparations(self.source, UrdfKinematicModel(self.source_urdf), + expected = configure_observer_preparations(expected, UrdfKinematicModel(self.source_urdf), preserved_joints=preserved) if self.target != expected: raise ValueError('observer_upgrade_changes_existing_contract') @@ -63,19 +91,19 @@ class ObserverPolicyUpgrade: if protected_inputs.get('profile_config_sha256') != new_hash: raise ValueError('observer_upgrade_target_hash_invalid') old_inputs = {**protected_inputs, 'profile_config_sha256': old_hash} - for row in (checkpoint.header, checkpoint.fixed_reference): + for row in (source_header, source_reference): actual = row.get('protected_hashes', row) if any(actual.get(key) != value for key, value in old_inputs.items()): raise ValueError('observer_upgrade_protected_inputs_changed') - if checkpoint.header.get('preparation_contract') != preparation_contract(self.source): + if source_header.get('preparation_contract') != preparation_contract(self.source): raise ValueError('observer_upgrade_source_preparation_changed') - header = {**checkpoint.header, **protected_inputs, + header = {**source_header, **protected_inputs, 'preparation_contract': preparation_contract(self.target), 'resume_checkpoint_requested': True} if 'protected_hashes' in header: header['protected_hashes'] = {**header['protected_hashes'], **protected_inputs} - reference = {**checkpoint.fixed_reference, 'protected_hashes': { - **checkpoint.fixed_reference['protected_hashes'], **protected_inputs}} + reference = {**source_reference, 'protected_hashes': { + **source_reference['protected_hashes'], **protected_inputs}} changes = {joint: dict(previous=spec.observer_pose_policy, current=self.target.measurement.measurements[joint].observer_pose_policy) for joint, spec in self.source.measurement.measurements.items() @@ -83,7 +111,7 @@ class ObserverPolicyUpgrade: audit = dict(kind='capture_observation_revision', policy='unmeasured_observer_upgrade_v1', source_raw_samples=str(Path(source_path).resolve()), source_raw_sha256=source_digest, source_profile_sha256=old_hash, target_profile_sha256=new_hash, - original_header=checkpoint.header, original_fixed_reference=checkpoint.fixed_reference, + original_header=source_header, original_fixed_reference=source_reference, preserved_joints=sorted(preserved), observer_policy_changes=changes, completed_units=[list(unit) for unit in sorted(checkpoint.joints.units)], physical_zero_reverification_performed=False) @@ -92,6 +120,11 @@ class ObserverPolicyUpgrade: audit.update(policy='unmeasured_observer_upgrade_v2', observer_preparations={ name: asdict(plan) for name, plan in self.target.motion.observer_preparations.items() if self.source.motion.observer_preparations.get(name) != plan}) + if strategy == TRAJECTORY_OBSERVER: + audit.update(policy='unmeasured_observer_upgrade_v3', + target_observation_policy='occlusion_aware_trajectory_v3', + accepted_evidence_semantics='preserved_without_reinterpretation', + pending_evidence_semantics='full_trajectory_with_bounded_observation') return header, reference, audit @@ -117,26 +150,34 @@ def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number, or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans): raise ValueError('observer_upgrade_changes_reusable_evidence') output_dir.mkdir(parents=True, exist_ok=False) - temporary = output_dir/'raw_samples.jsonl.tmp' - digest = hashlib.sha256() - with source_path.open('rb') as source, temporary.open('xb') as target: - for line in source: - digest.update(line) - row = json.loads(line) if line.strip() else {} - if row.get('kind') in replacement: - target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode()) - else: - target.write(line) - target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode()) - if digest.hexdigest() != source_digest: - temporary.unlink() - raise ValueError('observer_upgrade_source_changed_during_replay') - (output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml) - (output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml) - report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'} - (output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') - temporary.rename(output_dir/'raw_samples.jsonl') - return report + try: + _copy_indexed_evidence(source_path, output_dir) + temporary = output_dir/'raw_samples.jsonl.tmp' + digest = hashlib.sha256() + with source_path.open('rb') as source, temporary.open('xb') as target: + for line in source: + digest.update(line) + row = json.loads(line) if line.strip() else {} + if row.get('kind') in replacement: + target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode()) + else: + target.write(line) + target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode()) + if digest.hexdigest() != source_digest: + raise ValueError('observer_upgrade_source_changed_during_replay') + (output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml) + (output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml) + report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'} + installed = output_dir/'raw_samples.jsonl' + temporary.rename(installed) + if (output_dir/'raw_samples.storage.json').is_file(): + from ..core.artifacts.evidence_journal import compact_existing_journal + report['storage_compaction'] = compact_existing_journal(installed, replace=True) + (output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') + return report + except Exception: + shutil.rmtree(output_dir, ignore_errors=True) + raise def main(): diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_training_upgrade.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_training_upgrade.py new file mode 100644 index 0000000..519944a --- /dev/null +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/checkpoint_training_upgrade.py @@ -0,0 +1,173 @@ +"""Audited migration from whole-hand staged training to taskwise staged training. + +The measurement contract, motion paths, accepted scan identities and raw image +evidence stay unchanged. Only the ordering of future training visits and the +time at which a task-local repeatability decision is made are revised. The +source journal is immutable and the target journal links its content-addressed +evidence blocks. +""" + +from dataclasses import dataclass +import hashlib +import json +from pathlib import Path +import shutil + +from ..core.domain.capture_plan import (CapturePlan, STAGED, TASKWISE_STAGED, + configure_training) +from ..profiles import load_hand_profile +from ..profiles.export import dump_hand_profile +from ..profiles.preparation import preparation_contract +from ..profiles.validator import validate_executable_profile +from .engine import CalibrationEngine, capture_schedule_version +from .joint_resume import PreparedResume, JointResume +from .resume_storage import materialize_record +from .staged_resume import stage_passed_visits + + +@dataclass(frozen=True) +class TaskwiseTrainingUpgrade: + source: object + target: object + source_yaml: bytes + target_yaml: bytes + + @classmethod + def load(cls, source_profile): + source = load_hand_profile(source_profile) + if source.quality.training_policy != STAGED: + raise ValueError("taskwise_upgrade_requires_staged_source") + target = configure_training(source, TASKWISE_STAGED) + return cls(source, target, Path(source_profile).read_bytes(), + dump_hand_profile(target).encode()) + + def replacement_records(self, checkpoint, *, source_path, source_digest): + if any(row.get("kind") in {"staged_training_decision", "training_decision"} + for row in checkpoint.rows): + raise ValueError("taskwise_upgrade_training_decision_already_recorded") + source_header = materialize_record(checkpoint.header) + source_reference = materialize_record(checkpoint.fixed_reference) + old_hash = hashlib.sha256(self.source_yaml).hexdigest() + new_hash = hashlib.sha256(self.target_yaml).hexdigest() + actual = source_header.get("protected_hashes", source_header) + if actual.get("profile_config_sha256") != old_hash: + raise ValueError("taskwise_upgrade_source_profile_hash_changed") + if source_header.get("preparation_contract") != preparation_contract(self.source): + raise ValueError("taskwise_upgrade_source_preparation_changed") + if preparation_contract(self.target) != preparation_contract(self.source): + raise ValueError("taskwise_upgrade_changes_preparation_contract") + + header = {**source_header, + "capture_schedule_version": capture_schedule_version(self.target), + "capture_plan": CapturePlan.from_profile(self.target).as_dict(), + "profile_config_sha256": new_hash, + "resume_checkpoint_requested": True} + if "protected_hashes" in header: + header["protected_hashes"] = {**header["protected_hashes"], + "profile_config_sha256": new_hash} + reference = {**source_reference, "protected_hashes": { + **source_reference["protected_hashes"], "profile_config_sha256": new_hash}} + audit = { + "kind": "capture_schedule_revision", + "version": "taskwise_staged_training_upgrade_v1", + "source_raw_samples": str(Path(source_path).resolve()), + "source_raw_sha256": source_digest, + "source_profile_sha256": old_hash, + "target_profile_sha256": new_hash, + "source_policy": STAGED, + "target_policy": TASKWISE_STAGED, + "accepted_evidence_semantics": "preserved_without_reinterpretation", + "future_schedule": ( + "first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation" + ), + "completed_units": [list(unit) for unit in sorted(checkpoint.joints.units)], + } + return header, reference, audit + + +def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number, + source_urdf, image_replay_workers=1): + source_path, output_dir = Path(source_path).resolve(), Path(output_dir).resolve() + if output_dir.exists(): + raise FileExistsError(output_dir) + for profile in (upgrade.source, upgrade.target): + validate_executable_profile(profile, source_urdf) + with source_path.open("rb") as stream: + source_digest = hashlib.file_digest(stream, "sha256").hexdigest() + checkpoint = PreparedResume.load(upgrade.source, CalibrationEngine(upgrade.source), + source_path, serial_number, image_replay_workers=image_replay_workers, + resume_mode="passed", source_urdf=source_urdf) + header, reference, audit = upgrade.replacement_records(checkpoint, + source_path=source_path, source_digest=source_digest) + replacement = {"session_start": header, "fixed_base_reference_locked": reference} + rows = tuple(replacement.get(row.get("kind"), row) for row in checkpoint.rows) + (audit,) + + # The source is itself an unchanged-installation ``passed`` checkpoint. + # Its compact journal intentionally omits historical witness windows that + # strict replay would require. Preserve and compare the same durable pass + # decisions here; finalization still performs the complete evidence replay + # before it may publish JSON/URDF artifacts. + verified = JointResume(upgrade.target, image_replay_workers=image_replay_workers) + stage_passed_visits(verified, CalibrationEngine(upgrade.target), rows) + if (verified.units != checkpoint.joints.units + or verified.references != checkpoint.joints.references + or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans): + raise ValueError("taskwise_upgrade_changes_reusable_evidence") + + output_dir.mkdir(parents=True, exist_ok=False) + try: + raw = output_dir / "raw_samples.jsonl" + from ..core.artifacts.evidence_journal import initialize_evidence_journal + from ..core.artifacts.storage import append_jsonl_many, sync_jsonl + initialize_evidence_journal(raw) + for offset in range(0, len(rows), 4096): + append_jsonl_many(raw, rows[offset:offset + 4096], durable=False) + sync_jsonl(raw) + (output_dir / "source_profile.yaml").write_bytes(upgrade.source_yaml) + target_profile = output_dir / "target_profile.yaml" + target_profile.write_bytes(upgrade.target_yaml) + prepared = PreparedResume.load(upgrade.target, CalibrationEngine(upgrade.target), + raw, serial_number, image_replay_workers=image_replay_workers, + resume_mode="passed", source_urdf=source_urdf) + if (prepared.completed_units != frozenset(verified.units) + or prepared.joints.references != verified.references + or prepared.joints.first_cycle_spans != verified.first_cycle_spans): + raise ValueError("taskwise_upgrade_written_checkpoint_changed") + report = {**audit, "reusable_units": len(verified.units), + "zero_references": len(verified.references), + "target_profile": str(target_profile), + "resume_validation": "passed_mode_loaded", + "historical_replay_performed": False, + "full_replay_required_before_publication": True, + "result": "PASSED_EVIDENCE_PRESERVATION_VERIFIED"} + (output_dir / "training_upgrade.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n") + return report + except Exception: + shutil.rmtree(output_dir, ignore_errors=True) + raise + + +def main(): + import argparse + from ..product import load_product_config + from .image_replay import checkpoint_replay_workers + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-profile", required=True) + parser.add_argument("--target-product", required=True) + parser.add_argument("--source-checkpoint", required=True) + parser.add_argument("--output-dir", required=True) + args = parser.parse_args() + product = load_product_config(args.target_product, check_can=False) + upgrade = TaskwiseTrainingUpgrade.load(args.source_profile) + report = upgrade_checkpoint(upgrade, args.source_checkpoint, args.output_dir, + serial_number=product.serial_number, source_urdf=product.source_urdf, + image_replay_workers=checkpoint_replay_workers()) + print(json.dumps({key: report[key] for key in ( + "result", "reusable_units", "zero_references", "target_profile" + )}, ensure_ascii=False), flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/coordinator.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/coordinator.py index af0df24..335f291 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/coordinator.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/coordinator.py @@ -42,7 +42,8 @@ from .resume import ResumeDecision, ResumeVerifier, fingerprint_from_mapping from .safety import SafetyPolicy, SafetySample from .session import CalibrationPhase as Phase from .snapshot import DeviceReadiness, CameraReadiness, RuntimeSnapshot, build_snapshot, legacy_state -from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED +from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED, + TASKWISE_STAGED, STAGED_POLICIES) _TERMINAL_STATES = frozenset({"PASSED", "DIAGNOSTIC_COMPLETE", "PAUSED", "ABORTED", "FAILED"}) @@ -73,7 +74,7 @@ class CalibrationCoordinator: self.training_worker = None self._training_request = None self._frozen_training_sha256 = None - if profile.quality.training_policy in {ADAPTIVE, STAGED}: + if profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}: if self.diagnostic_capture is not None: raise ValueError("adaptive_training_cannot_run_as_diagnostic") from .training import TrainingWorker @@ -91,10 +92,12 @@ class CalibrationCoordinator: self.command_direction_history = deque(maxlen=2000) self._command_directions = [""]*self.command_count self.finalization = finalization or FinalizationController(profile, parameters.session_dir) + from .journal_durability import JournalDurabilityWorker + self.journal_durability = JournalDurabilityWorker() from .capture_index import CaptureRecordIndex self.capture_index = CaptureRecordIndex(retain_complete=( self.diagnostic_capture is not None or not getattr(self.finalization, "uses_journal", False)), - retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED}) + retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}) self._image_record_stamps = {} self.last_command = None self.commanded_speed = None @@ -585,6 +588,7 @@ class CalibrationCoordinator: self._observation_epoch += 1 self.branch_initialization.clear() try: + self.journal_durability.close() sync_jsonl(self.raw_path) finally: self.finalization.close() @@ -640,28 +644,7 @@ class CalibrationCoordinator: if self._passed_reference_import is not None: self._publish_command(list(current)) pending = self._passed_reference_import - self._write_resume_batch(pending) - if pending.complete: - from .passed_resume import restore_parent_models, restored_model_record - binding = restored_model_record(pending.model_records, - self._observation_epoch, self.ports.clock_ns()) - if self.profile.quality.training_policy == STAGED and self.parameters.resume_mode != "passed": - binding.update(kind="staged_pose_models_restored", resume_mode="verify", - physical_zero_reverification_required=True) - binding.pop("unchanged_installation_declared", None) - append_jsonl(self.raw_path, binding) - self.capture_index.append(binding) - restore_parent_models(self.branch_initialization.parent_references, - pending.model_records, self._observation_epoch) - from .observer_reference import restore_observer_records - restore_observer_records(self.branch_initialization, pending.rows) - self.branch_initialization.observer_preparation.restore(pending.rows) - for row in pending.rows: - if row.get("kind") == "motion_branch_observation": - self.branch_initialization.witness_capture.observe(row) - self.execution.zero_references.update(pending.references) - if self.visual_motion is not None: - self.visual_motion.freeze_zeros(pending.references) + if self._write_resume_batch(pending) and self._restore_passed_reference_batch(pending): self._passed_reference_import = None return if self._reference_import is not None and not self._reference_import.complete: @@ -674,7 +657,7 @@ class CalibrationCoordinator: return if phase in {Phase.PREPARE, Phase.MAPPING_PROBE} and self.segment is None and session.current_unit is not None: unit = session.current_unit - if self.profile.quality.training_policy == STAGED: + if self.profile.quality.training_policy in STAGED_POLICIES: used, imported = self.joint_resume.take_task(unit.task_key, cycle=unit.cycle) else: used, imported = self.joint_resume.take_task(unit.task_key) @@ -751,6 +734,17 @@ class CalibrationCoordinator: if self.execution.last_diagnostic_quality is not None else {})} self.capture_index.append(record) append_jsonl(self.raw_path, record) + next_unit = session.current_unit + visit_complete = (quality.passed and (next_unit is None + or (next_unit.task_key, next_unit.cycle) != ( + action.scan_unit.task_key, action.scan_unit.cycle))) + if visit_complete: + resolved_deferral = session.task_deferrals.resolve( + action.scan_unit.task_key, action.scan_unit.cycle) + if resolved_deferral is not None: + resolved_record = resolved_deferral.as_record(stamp_ns=self.ports.clock_ns()) + self.capture_index.append(resolved_record) + append_jsonl(self.raw_path, resolved_record) if self.execution.last_training_decision is not None: decision = self.execution.last_training_decision append_jsonl(self.raw_path, decision) @@ -903,6 +897,8 @@ class CalibrationCoordinator: elif geometry_error: if self._recover_joint_zero(motion, now, geometry_error): return + if self._defer_unobservable_task(motion, geometry_error): + return if self.diagnostic_capture is not None: geometry_error += ":diagnostic_reference=" + self._zero_reference_error self._pause("joint_zero_motion_unresolved:" + geometry_error) @@ -910,7 +906,10 @@ class CalibrationCoordinator: else: if self._recover_joint_zero(motion, now, ""): return - self._pause("joint_zero_unavailable:"+self._zero_reference_error+self._joint_zero_branch_details(motion)) + unavailable = self._zero_reference_error+self._joint_zero_branch_details(motion) + if self._defer_unobservable_task(motion, unavailable): + return + self._pause("joint_zero_unavailable:"+unavailable) return if (self.segment.arrived(now) and motion.phase != "joint_zero") or steady_complete: if motion.phase in {'observer_preflight', 'observer_probe'}: @@ -1028,6 +1027,14 @@ class CalibrationCoordinator: raise ValueError("parent_reference_zero_missing") record = self.branch_initialization.parent_references.confirm(task, self._observation_epoch, self._segment_number, reference, rows) + elif motion.parent_model_refresh_for: + task = next(task for task in self.profile.motion.tasks + if task.key == motion.parent_model_refresh_for) + reference = self.execution.zero_references.get(task.parent_reference.pose_joint) + if reference is None: + raise ValueError("parent_reference_zero_missing") + record = self.branch_initialization.parent_references.authorize_refreshed_pose( + task, self._observation_epoch, self._segment_number, reference) except ValueError as error: self._zero_reference_error = str(error) return False @@ -1172,6 +1179,38 @@ class CalibrationCoordinator: def _recover_joint_zero(self, motion, now, geometry_error): if self.diagnostic_capture is not None: return False + branch_details = self._joint_zero_branch_details(motion) + task = next((task for task in self.profile.motion.tasks + if task.key == motion.task_key), None) + stale_parent_anchor = (motion.reference_reuse + and task is not None and self.capture.current_image_reference_stable(task.view) + and self._zero_reference_error.startswith("parent_reference_samples_missing:") + and "image_reference_current_image_moved" in branch_details) + action = self.zero_recovery.take_parent_model_refresh( + motion, stale_image_anchor=stale_parent_anchor) + if action is not None: + from .zero_recovery import FailureCategory + record = {"kind": "joint_zero_recovery", "task_name": motion.task_key, + "failure_category": FailureCategory.REFERENCE_CHANGED.value, + "capture_plan_sha256": CapturePlan.from_profile(self.execution.profile).sha256, + "joints": list(motion.zero_joints), "action": action, + "reason": self._zero_reference_error+branch_details, + "previous_session_epoch": self._observation_epoch, + "session_epoch": self._observation_epoch+1, + "attempt": 1, "stamp_ns": self.ports.clock_ns()} + append_jsonl(self.raw_path, record) + self.capture_index.append(record) + self._observation_epoch += 1 + self.branch_initialization.parent_references.advance_recovery_epoch( + self._observation_epoch-1, self._observation_epoch) + self.execution.refresh_parent_model(self.last_command or motion.target) + self.branch_initialization.clear() + self._steady_rows.clear() + self._zero_capture_started_at = self._steady_capture_after_ns = None + self._motion = self.segment = None + self._zero_geometry_started_at = self._zero_geometry_completed_at = None + self.reason = "joint_zero_recovery:"+action+":"+str(motion.task_key) + return True if geometry_error and self._recover_observer_preparation(motion): return True action = self.zero_recovery.take(motion, geometry_error=geometry_error, @@ -1279,6 +1318,41 @@ class CalibrationCoordinator: self.reason = 'observer_preparation:' + motion.zero_joints[0] return True + def _defer_unobservable_task(self, motion, geometry_error): + """Continue independent collection after bounded v3 observation fails.""" + from .task_deferral import recoverable_observation_reason + from ..profiles.observer_policy import TRAJECTORY_OBSERVER + refresh = bool(motion.parent_model_refresh_for) + if ((motion.reference_only and not (refresh or motion.reference_reuse)) + or motion.reference_check or len(motion.zero_joints) != 1 + or not recoverable_observation_reason(geometry_error) + or not any(spec.observer_pose_policy == TRAJECTORY_OBSERVER + for spec in self.profile.measurement.measurements.values())): + return False + import hashlib + import json + reports = tuple(self.branch_initialization.reports.values()) + hashes = {report.get('view', ''): report.get('source_frame_hashes', {}) + for report in reports} + corner_digest = hashlib.sha256(json.dumps(hashes, sort_keys=True, + separators=(',', ':'), allow_nan=False).encode()).hexdigest() + decision = self.execution.defer_current_task(geometry_error) + record = decision.as_record(stamp_ns=self.ports.clock_ns(), + corner_evidence_sha256=corner_digest) + append_jsonl(self.raw_path, record) + self.capture_index.append(record) + self._observation_epoch += 1 + self.branch_initialization.parent_references.advance_recovery_epoch( + self._observation_epoch-1, self._observation_epoch) + self.branch_initialization.clear() + self._steady_rows.clear() + self._zero_capture_started_at = None + self._steady_capture_after_ns = None + self._zero_geometry_started_at = self._zero_geometry_completed_at = None + self.segment, self._motion = None, None + self.reason = f"task_observation_{decision.state.value}:{decision.task_name}" + return True + def _joint_zero_branch_details(self, motion): task = next(task for task in self.profile.motion.tasks if task.key == motion.task_key) if motion.reference_only: @@ -1598,12 +1672,113 @@ class CalibrationCoordinator: append_jsonl(self.raw_path, {"kind": "fixed_base_reference_locked", **self._current_fingerprint}) def _write_resume_batch(self, pending): - """Keep both zero-evidence and scan imports outside an unbounded tick.""" - rows = pending.take_batch() - append_jsonl_many(self.raw_path, rows, durable=False) - self.capture_index.extend(rows) - if pending.complete: - sync_jsonl(self.raw_path) + """Transfer bounded rows and poll a non-blocking durability barrier.""" + if not pending.transfer_complete: + rows = pending.take_batch() + append_jsonl_many(self.raw_path, rows, durable=False) + self.capture_index.extend(rows) + if not pending.transfer_complete: + return False + if pending.durability_future is None: + pending.durability_future = self.journal_durability.submit( + lambda: sync_jsonl(self.raw_path)) + return False + if not pending.durability_future.done(): + return False + pending.durability_future.result() + pending.durable = True + return True + + def _restore_passed_reference_batch(self, pending): + """Restore imported runtime indexes in bounded control-loop slices.""" + if pending.restore_phase == "binding": + from .passed_resume import restored_model_record + binding = restored_model_record(pending.model_records, + self._observation_epoch, self.ports.clock_ns()) + if self.profile.quality.training_policy in STAGED_POLICIES and self.parameters.resume_mode != "passed": + binding.update(kind="staged_pose_models_restored", resume_mode="verify", + physical_zero_reverification_required=True) + binding.pop("unchanged_installation_declared", None) + append_jsonl_many(self.raw_path, (binding,), durable=False) + self.capture_index.append(binding) + pending.durability_future = self.journal_durability.submit( + lambda: sync_jsonl(self.raw_path)) + pending.restore_phase = "binding_sync" + return False + + if pending.restore_phase == "binding_sync": + if not pending.durability_future.done(): + return False + pending.durability_future.result() + pending.restore_phase = "models" + return False + + if pending.restore_phase == "models": + from .passed_resume import restore_parent_models + restore_parent_models(self.branch_initialization.parent_references, + pending.model_records, self._observation_epoch, pending.rows) + pending.restore_rows = tuple(row for row in pending.rows + if row.get("kind") == "motion_branch_observation") + pending.restore_offset = 0 + pending.restore_phase = "observer_records" + return False + + if pending.restore_phase == "observer_records": + from .observer_reference import restore_observer_records + batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 512] + restore_observer_records(self.branch_initialization, batch) + pending.restore_offset += len(batch) + if pending.restore_offset == len(pending.restore_rows): + if self.parameters.resume_mode == "passed": + # Passed tasks contribute immutable zero/model evidence, + # never unfinished live actions. Replaying an old active + # observer decision can move an already accepted joint + # while the next task is preparing. + pending.restore_rows = tuple(row for row in pending.rows + if row.get("kind") == "motion_branch_observation" + and row.get("preparation_witness_joint")) + pending.restore_offset = 0 + pending.restore_phase = "witness" + return False + preparation_kinds = {"observer_preparation_decision", + "image_observation_frame", "observer_preparation_segment_complete", + "observer_preparation_excursion_started", + "observer_preparation_cancelled_before_excursion"} + pending.restore_rows = tuple(row for row in pending.rows + if row.get("kind") in preparation_kinds) + pending.restore_offset = 0 + pending.restore_phase = "observer_preparation" + return False + + if pending.restore_phase == "observer_preparation": + batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 256] + self.branch_initialization.observer_preparation.restore(batch) + pending.restore_offset += len(batch) + if pending.restore_offset == len(pending.restore_rows): + pending.restore_rows = tuple(row for row in pending.rows + if row.get("kind") == "motion_branch_observation" + and row.get("preparation_witness_joint")) + pending.restore_offset = 0 + pending.restore_phase = "witness" + return False + + if pending.restore_phase == "witness": + batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 64] + for row in batch: + self.branch_initialization.witness_capture.observe(row) + pending.restore_offset += len(batch) + if pending.restore_offset < len(pending.restore_rows): + return False + pending.restore_rows = () + pending.restore_phase = "complete" + self.execution.zero_references.update(pending.references) + if self.visual_motion is not None: + self.visual_motion.freeze_zeros(pending.references) + return True + + if pending.restore_phase == "complete": + return True + raise RuntimeError("unknown_passed_reference_restore_phase") def _import_resume_batch(self): pending = self._resume_import @@ -1666,7 +1841,7 @@ class CalibrationCoordinator: if self.profile.artifacts.output_schema_version >= 3: self.joint_resume = checkpoint.joints decision = replace(decision, reason="整场参考通过，等待逐关节验证原零位后复用") - if self.profile.quality.training_policy == STAGED: + if self.profile.quality.training_policy in STAGED_POLICIES: from .staged_resume import restore_recovery self._passed_reference_import = checkpoint.passed_import restore_recovery(self.execution.session, self.zero_recovery, rows) @@ -1676,7 +1851,7 @@ class CalibrationCoordinator: decision = replace(decision, completed_units=tuple(sorted(complete))) except (OSError, ValueError, KeyError, TypeError) as error: decision = ResumeDecision(False, "旧断点校验未通过，已重新采集", (str(error),)) - if self.profile.quality.training_policy == STAGED and not decision.reuse: + if self.profile.quality.training_policy in STAGED_POLICIES and not decision.reuse: self._pause("staged_resume_reference_or_evidence_changed:" + str(decision.incompatible_fields)) return decision = self.execution.restore(decision, rows) @@ -1701,7 +1876,7 @@ class CalibrationCoordinator: return self.joint_resume = checkpoint.joints self._passed_reference_import = checkpoint.passed_import - if self.profile.quality.training_policy == STAGED: + if self.profile.quality.training_policy in STAGED_POLICIES: from .staged_resume import restore_recovery restore_recovery(self.execution.session, self.zero_recovery, checkpoint.rows) if checkpoint.frozen_training is not None: @@ -1709,7 +1884,7 @@ class CalibrationCoordinator: # The existing bounded importer skips each task only after its original # rows and zero references are durable. No old task is moved or scanned. self._resume_message = "按已通过记录续采，跳过历史回放和已完成关节零位复核" - if self.profile.quality.training_policy == STAGED: + if self.profile.quality.training_policy in STAGED_POLICIES: self._resume_message = "断点证据校验通过，直接复用已完成访问；不重复运动或零位复核" self.execution.restore(ResumeDecision(True, self._resume_message), ()) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/engine.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/engine.py index 96bab10..022c6f5 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/engine.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/engine.py @@ -15,7 +15,8 @@ from .diagnostic_capture import DiagnosticCapturePlan from ..core.domain.profile import ACQUISITION_POLICY_VERSION from ..core.domain.motion_path import task_segments, scan_identity -from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, LEGACY_TRAINING, LEGACY_HOLDOUT +from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED, + TASKWISE_STAGED, STAGED_POLICIES, LEGACY_TRAINING, LEGACY_HOLDOUT) TRAINING_CYCLES = LEGACY_TRAINING HOLDOUT_CYCLE = LEGACY_HOLDOUT CAPTURE_SCHEDULE_VERSION = "unified_schedule_v7_single_pass_endpoint_images" @@ -27,8 +28,10 @@ PARENT_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v10_verified_parent_geomet IMAGE_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v11_stationary_image_reference" ADAPTIVE_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v12_adaptive_training" STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v13_staged_training" +TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v14_taskwise_staged_training" SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION, - STAGED_CAPTURE_SCHEDULE_VERSION, ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}" + STAGED_CAPTURE_SCHEDULE_VERSION, TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION, + ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}" for mode in ("interleaved", "separate")), *(f"{PATH_CAPTURE_SCHEDULE_VERSION}_{mode}" for mode in ("interleaved", "separate")), *(f"{VISUAL_CAPTURE_SCHEDULE_VERSION}_{mode}" for mode in ("interleaved", "separate")), *(f"{PARENT_REFERENCE_SCHEDULE_VERSION}_{feedback}_{mode}" @@ -40,6 +43,8 @@ SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION, def capture_schedule_version(profile): if profile.quality.training_policy == STAGED: return STAGED_CAPTURE_SCHEDULE_VERSION + if profile.quality.training_policy == TASKWISE_STAGED: + return TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION if profile.quality.training_policy == ADAPTIVE: return ADAPTIVE_CAPTURE_SCHEDULE_VERSION if profile.acquisition.fixed_reference_mode == "stationary_image": @@ -202,6 +207,17 @@ class CalibrationEngine: segment.end, speed, "steady", segment.key) for segment in task_segments(task)) if self.profile.quality.training_policy == STAGED: units.sort(key=lambda unit: unit.cycle) # Stable: declared task/segment order within a pass. + elif self.profile.quality.training_policy == TASKWISE_STAGED: + # Keep the already collected first whole-hand pass reusable. During + # the second pass each task's optional third training visit is + # placed immediately after that task. Validation remains a final, + # isolated whole-hand pass and can never precede future training. + task_order = {task.key: index for index, task in enumerate(self.profile.motion.tasks)} + units.sort(key=lambda unit: ( + 0 if unit.cycle == 0 else 1 if unit.cycle in {1, 2} else 2, + task_order[unit.task_key], + unit.cycle, + )) return tuple(units) def mapping_probe_delta(self, task: TaskSpec) -> float | None: @@ -287,7 +303,7 @@ class CalibrationEngine: and session_start.get("acquisition_policy_version") == ACQUISITION_POLICY_VERSION and session_start.get("capture_schedule_version") in ( - (capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, STAGED} or self.profile.acquisition.fixed_reference_mode == "stationary_image" + (capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, *STAGED_POLICIES} or self.profile.acquisition.fixed_reference_mode == "stationary_image" or self.profile.vision_motion or any(task.segments for task in self.profile.motion.tasks) or any(task.parent_reference is not None for task in self.profile.motion.tasks) or any(spec.evidence_scope == "path" for spec in self.profile.motion.joint_zero_references.values()) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/execution.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/execution.py index 53cec39..ad8fe2f 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/execution.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/execution.py @@ -55,8 +55,8 @@ class SessionExecution: def _make_effects(self, current): action = self.action - from ..core.domain.capture_plan import STAGED - staged = self.profile.quality.training_policy == STAGED + from ..core.domain.capture_plan import STAGED_POLICIES + staged = self.profile.quality.training_policy in STAGED_POLICIES visit = None if action.scan_unit is None else (action.scan_unit.task_key, action.scan_unit.cycle) new_visit = staged and visit != self._entered_visit previous_visit = self._entered_visit @@ -298,6 +298,38 @@ class SessionExecution: current = target self._effects[:1] = [*effects, motion] + def refresh_parent_model(self, current): + """Acquire a current image model without replacing a passed zero. + + Parent-reference reuse deliberately starts with the immutable passed + model. If only its stationary image anchor is stale, run the same + reviewed approach used by that parent's original zero acquisition and + fit a session-local model before retrying the held-pose check. + """ + motion = self._effects[0] if self._effects else None + if (self.session.phase != Phase.PREPARE or motion is None + or motion.phase != "joint_zero" or not motion.reference_reuse + or len(motion.zero_joints) != 1): + raise ValueError("parent_model_refresh_outside_reference_reuse") + name = motion.zero_joints[0] + spec = self.profile.motion.joint_zero_references[name] + source_task = next(task for task in self.profile.motion.tasks + if task.key == spec.task_key) + metadata = dict(task_key=motion.task_key, command_index=motion.command_index, + cycle=motion.cycle, attempt=motion.attempt, reference_only=True, + reference_command=spec.command, reference_approach=spec.approach_commands, + parent_model_refresh_for=motion.task_key) + effects = [] + for target in (*spec.approach_commands, spec.command): + effects.extend(MotionCommand("zero_approach", point, motion.speed, + reference_joints=(name,), **metadata) + for point in build_joint_reference_waypoints( + source_task, target, profile=self.profile, current_command=current)) + current = target + effects.append(MotionCommand("joint_zero", spec.command, motion.speed, + zero_joints=(name,), **metadata)) + self._effects[:0] = effects + def prepend_observer_effects(self, effects): if (self.session.phase != Phase.PREPARE or not self._effects or not effects or any(effect.phase != 'observer_probe' or len(effect.reference_joints) != 1 @@ -313,6 +345,12 @@ class SessionExecution: zero_joints=(task.parent_reference.pose_joint,))) self.prepend_observer_effects(effects) + def defer_current_task(self, reason): + decision = self.session.defer_current_task(reason) + self._effects.clear() + self._action_key = None + return decision + def record_diagnostic_zero_attempt(self, attempt: DiagnosticZeroAttempt): """A diagnostic observation closes an attempt, never creates a zero.""" motion = self._effects[0] if self._effects else None @@ -325,13 +363,13 @@ class SessionExecution: self.diagnostic_zero_attempts.update({name: attempt for name in attempt.joint_names}) def training_check_due(self): - from ..core.domain.capture_plan import ADAPTIVE, STAGED + from ..core.domain.capture_plan import ADAPTIVE, STAGED, TASKWISE_STAGED unit = self.action.scan_unit - if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED} or self.diagnostic_capture is not None + if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED, TASKWISE_STAGED} or self.diagnostic_capture is not None or unit is None or unit.cycle not in (1, 2) or (unit.task_key, unit.cycle) in self._training_checks): return False - round_units = [u for u in self.session.engine.scan_units() + round_units = [u for u in self.session.scheduled_units if (self.profile.quality.training_policy == STAGED or u.task_key == unit.task_key) and u.cycle == unit.cycle] return unit.identity == round_units[-1].identity diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/image_reference.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/image_reference.py index bd13c1a..a772d96 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/image_reference.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/image_reference.py @@ -118,6 +118,13 @@ class ImageReferenceTracker: return (snapshot is self._motion_snapshot and snapshot.epoch == self._epoch and snapshot.reference is self.reference and snapshot.stamp_ns == self._last_stamp) + def tracking_current_image(self): + """Whether the latest frame still matches this tracker's own anchor.""" + with self._lock: + return (self.reference is not None and self._snapshot is not None + and self._snapshot[3].get("branch_status") == "tracking" + and self._snapshot[1] == self._last_stamp) + def reference_confirmed(self, role, revision): with self._lock: snapshot = self._motion_snapshot diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/joint_resume.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/joint_resume.py index e83e0d3..079245c 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/joint_resume.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/joint_resume.py @@ -19,22 +19,46 @@ class ResumeJournalImport: rows: tuple offset: int = field(default=0, init=False) + durability_future: object | None = field(default=None, init=False, repr=False) + durable: bool = field(default=False, init=False) def take_batch(self): from .resume_storage import DeferredCaptureRecord, materialize_record linked = (self.offset < len(self.rows) and isinstance(self.rows[self.offset], DeferredCaptureRecord) and self.rows[self.offset].reference is not None) - limit = 512 if linked else 64 + # Linked evidence can normally be copied without decompression. Scan + # samples additionally feed the live training index, which reads their + # geometry and digest. Find that boundary across the whole candidate + # batch so a cheap image row cannot hide dozens of expensive samples. + limit = 64 + if linked: + geometry_kinds = {"joint_sample", "secondary_joint_sample"} + candidate = self.rows[self.offset:self.offset + 512] + first_geometry = next((index for index, row in enumerate(candidate) + if row.get("kind") in geometry_kinds), None) + if first_geometry is None: + limit = 512 + elif first_geometry > 0: + limit = first_geometry + else: + # Training geometry is materialized once per row by the live + # index. A 512-row slice remains bounded while avoiding one + # control-loop turn for every 64 historical records. + limit = 512 batch = tuple(row if isinstance(row, DeferredCaptureRecord) and row.reference is not None else materialize_record(row) for row in self.rows[self.offset:self.offset + limit]) self.offset += len(batch) return batch @property - def complete(self): + def transfer_complete(self): return self.offset == len(self.rows) + @property + def complete(self): + return self.durable + @dataclass class TaskResumeImport(ResumeJournalImport): @@ -86,17 +110,18 @@ class PreparedResume: raise ValueError("checkpoint identity/policy changed") from .training import resolve_capture_plan from .engine import CalibrationEngine - from ..core.domain.capture_plan import STAGED + from ..core.domain.capture_plan import STAGED, STAGED_POLICIES profile = resolve_capture_plan(profile, rows, source_urdf=source_urdf, - verify_decisions=profile.quality.training_policy == STAGED and source_urdf is not None) + verify_decisions=profile.quality.training_policy in STAGED_POLICIES and source_urdf is not None) engine = CalibrationEngine(profile) complete = frozenset(record_scan_identity(r) for r in rows if r.get("kind") == "scan_unit_complete" and r.get("passed") is True) joints = JointResume(profile, image_replay_workers=image_replay_workers) - if profile.quality.training_policy == STAGED: + if profile.quality.training_policy in STAGED_POLICIES: from .staged_resume import stage_visits, stage_passed_visits, load_frozen_checkpoint, reference_import (stage_passed_visits if resume_mode == "passed" else stage_visits)(joints, engine, rows) - frozen = load_frozen_checkpoint(profile, rows, path, source_urdf) + frozen = (load_frozen_checkpoint(profile, rows, path, source_urdf) + if profile.quality.training_policy == STAGED else None) # Index/hash historical provenance before live callbacks start. # The control loop only authorizes and drains this bounded import. pending = reference_import(joints, reuse_passed_visits=resume_mode == "passed") @@ -134,10 +159,12 @@ def _index_motion_provenance(rows): kinds = {"motion_branch_observation", "motion_branch_initialization", "observer_preparation_decision", "observer_preparation_segment_complete", "observer_preparation_excursion_started", "observer_preparation_cancelled_before_excursion", + "task_observation_state", "capture_scope_extension", "capture_observation_revision", "passed_tasks_reused", "passed_pose_models_restored", "staged_pose_models_restored", "resume_verification", - "parent_reference_verified", "joint_zero_recovery", + "parent_reference_verified", "parent_reference_model_refreshed", + "joint_zero_recovery", "joint_zero_motion_reference_verified", "joint_zero_sample", "visual_motion_observation", "visual_motion_settled"} selected = (row for row in rows @@ -313,8 +340,8 @@ class JointResume: return tuple(rows) def take_task(self, task_key, cycle=None): - from ..core.domain.capture_plan import STAGED - if self.profile.quality.training_policy == STAGED: + from ..core.domain.capture_plan import STAGED_POLICIES + if self.profile.quality.training_policy in STAGED_POLICIES: from .staged_resume import visit_rows return visit_rows(self, task_key, cycle) task = next(t for t in self.profile.motion.tasks if t.key == task_key) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/journal_durability.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/journal_durability.py new file mode 100644 index 0000000..dd6e3c9 --- /dev/null +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/journal_durability.py @@ -0,0 +1,22 @@ +"""Run journal persistence barriers without blocking the control callback.""" + +from concurrent.futures import ThreadPoolExecutor + + +class JournalDurabilityWorker: + """Single ordered worker for resume-import fsync barriers. + + Resume rows are appended by the coordinator in bounded batches. The + final fsync can still take seconds on a large journal, so it runs here + while the coordinator continues feedback, safety and hold commands. + """ + + def __init__(self): + self._executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="calibration-journal-sync") + + def submit(self, operation): + return self._executor.submit(operation) + + def close(self): + self._executor.shutdown(wait=True, cancel_futures=False) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_execution.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_execution.py index d6349a9..8f41465 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_execution.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_execution.py @@ -59,6 +59,10 @@ class MotionCommand: visual_return_joints: tuple[str, ...] = () defer_settling_to_hold: bool = False observer_preparation_id: str = '' + # A session-local model acquisition for a parent whose historical image + # anchor no longer matches the current, stable fixed-Tag image. The + # accepted joint zero and completed scan stay immutable. + parent_model_refresh_for: str = '' @property def recording(self): @@ -135,8 +139,21 @@ class MotionExecution: if self.profile.command.unit == "u8": duration_key = ("scan_trajectory_full_range_seconds" if self.command.phase in {"sweep", "steady"} else "command_trajectory_full_range_seconds") + full_range_duration = float(params.get(duration_key, + params.get("command_trajectory_full_range_seconds", 6.0))) + if len(self.command.measured_channels) > 1: + # O30's reviewed range-calibration group motion pairs SDK + # speed 200 with a 20 u8/s command path. Preserve that rate + # for coupled calibration segments: a faster continuous ramp + # can let the middle finger's feedback lag behind the intended + # 80-unit clearance while the ring finger is already loaded. + coupled_rate = float(self.command.speed)/10.0 + if coupled_rate <= 0.0: + raise ValueError("coupled u8 motion speed must be positive") + full_range_duration = max(full_range_duration, + math.pi*255.0/(2.0*coupled_rate)) travelled, phase, duration = cosine_position_trajectory_u8(0, distance, - elapsed, float(params.get(duration_key, params.get("command_trajectory_full_range_seconds", 6.0)))) + elapsed, full_range_duration) else: speed = self.command.speed caps = self.profile.command.maximum_velocity diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_provenance.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_provenance.py index 8bd79fc..22a77d2 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_provenance.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/motion_provenance.py @@ -385,8 +385,9 @@ def validate_motion_provenance(profile, records, *, required=False, require_comp from .shared_tag_evidence import validate_shared_tag_references if any(payload.get('shared_tag_pose_bridges') for payload in evidence.values()): validate_shared_tag_references(profile, records, evidence) + from ..profiles.observer_policy import is_rigid_observer if (any(payload.get('observer_pose_bridges') for payload in evidence.values()) - or any(s.observer_pose_policy == 'rigid_pair_v1' for s in profile.measurement.measurements.values())): + or any(is_rigid_observer(s.observer_pose_policy) for s in profile.measurement.measurements.values())): from .observer_reference import validate_observer_references validate_observer_references(profile, records, evidence) image_frames, image_samples = _image_frames(records), {} diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation.py index 6bc371e..c95309b 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation.py @@ -4,6 +4,8 @@ Decisions and completed segments are journaled. A restart cannot replenish the motion budget or relabel an incomplete excursion as usable evidence. """ from dataclasses import dataclass, replace +import hashlib +import json import math import numpy as np from scipy.spatial.transform import Rotation @@ -16,6 +18,7 @@ from ..profiles.observer_preparation import observer_action, observer_actions POLICY = 'bounded_rigid_observation_v1' SEQUENCE_POLICY = 'bounded_rigid_observation_v2' +TRAJECTORY_POLICY = 'occlusion_aware_trajectory_v3' PHASES = frozenset({'observer_preflight', 'observer_probe'}) RECOVERABLE = frozenset({'rigid_pair_candidates_unresolved', 'rigid_pair_pose_uncertain', 'rigid_pair_viewpoint_span_insufficient', 'rigid_pair_source_missing', 'observer_images_missing'}) @@ -44,6 +47,66 @@ validation or a statement that a pose is observable. needs_excitation=len(selected) < 24 or span < 15.) +def trajectory_observability(rows, parent_role, child_role): + """Measure whether a monocular trajectory can distinguish planar branches. + + This is an acquisition decision only. It never marks geometry as passed; + the independent rigid-pair fit and final holdout still own acceptance. + """ + rows = tuple(sorted(rows, key=lambda row: row['image_stamp_ns'])) + coverage = normal_coverage(rows, parent_role) + paired = [row for row in rows if {parent_role, child_role} <= set(row.get('tags', {}))] + separations, diagonals = [], [] + for row in paired[::max(1, len(paired)//36)]: + tag = row['tags'][parent_role] + poses = solve_square_tag_ippe(np.asarray(tag['corners_xy']), tag_size_m=tag['tag_size_m'], + camera_matrix=np.asarray(row['camera_matrix'])) + if len(poses) > 1: + separations.append(math.degrees((Rotation.from_quat(poses[0].quaternion_xyzw).inv() + * Rotation.from_quat(poses[1].quaternion_xyzw)).magnitude())) + corners = np.asarray(tag['corners_xy'], dtype=float) + diagonals.append(float(np.mean((np.linalg.norm(corners[0]-corners[2]), + np.linalg.norm(corners[1]-corners[3]))))) + completeness = len(paired)/max(1, len(rows)) + branch_separation = float(np.median(separations)) if separations else 0. + tag_diagonal = float(np.median(diagonals)) if diagonals else 0. + needs = (coverage['needs_excitation'] or completeness < .9 + or branch_separation < 1.) + return {**coverage, 'paired_frames': len(paired), + 'pair_detection_fraction': completeness, + 'median_planar_branch_separation_deg': branch_separation, + 'median_tag_diagonal_px': tag_diagonal, + 'needs_excitation': needs} + + +def observation_action_utility(profile, joint, plan, rows): + """Rank a declared safe action by expected out-of-plane information.""" + spec = profile.measurement.measurements[joint] + metrics = trajectory_observability(rows, spec.parent_role, spec.child_role) + channel = channel_for_joint(profile, plan.joint) + lower, upper = profile.command.minimum_values[channel], profile.command.maximum_values[channel] + excursion = max(abs(point[channel]-plan.command[channel]) for point in plan.approach_commands) + fraction = excursion/(upper-lower) + # The donor was selected from the URDF dependency graph for a transverse + # axis. This conservative estimate is only used to choose acquisition; + # it is never reported as a measured joint angle. + expected_normal_gain = 60.*fraction + expected_pixel_separation = metrics['median_tag_diagonal_px'] * math.sin(math.radians(expected_normal_gain)) + reliability = metrics['pair_detection_fraction'] + return {**metrics, 'donor': plan.joint, 'excursion_fraction': fraction, + 'expected_normal_gain_deg': expected_normal_gain, + 'expected_candidate_separation_px': expected_pixel_separation, + 'utility': expected_pixel_separation*reliability} + + +def corner_evidence_sha256(rows, roles): + payload = [{"stamp": row['image_stamp_ns'], "camera_matrix": row.get('camera_matrix'), + "tags": {role: row.get('tags', {}).get(role, {}).get('corners_xy') for role in roles}} + for row in rows] + return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':'), + allow_nan=False).encode()).hexdigest() + + def latest_source_rows(owner, joint, epoch): spec = owner.profile.measurement.measurements[joint] reports = [report for (source_epoch, view, _), (model, report) in owner.parent_references.models.items() @@ -90,7 +153,8 @@ class ObserverPreparation: previous = self.decisions.get(joint) if previous is not None and previous != row: index = previous.get('action_index', 0)+1 - if (row.get('policy') != SEQUENCE_POLICY or row.get('action_index') != index + if (row.get('policy') not in {SEQUENCE_POLICY, TRAJECTORY_POLICY} + or row.get('action_index') != index or index >= len(observer_actions(self.owner.profile, joint)) or row.get('previous_preparation_id') != source_frame_sha256(previous) or self.source(joint) is None): @@ -175,13 +239,20 @@ class ObserverPreparation: plan = actions[index] spec = self.owner.profile.measurement.measurements[joint] rows = self.source(joint).rows if previous is not None else latest_source_rows(self.owner, joint, epoch) - coverage = normal_coverage(rows, spec.parent_role) + coverage = (trajectory_observability(rows, spec.parent_role, spec.child_role) + if spec.observer_pose_policy == 'rigid_pair_trajectory_v3' + else normal_coverage(rows, spec.parent_role)) + utility = observation_action_utility(self.owner.profile, joint, plan, rows) + policy = (TRAJECTORY_POLICY if spec.observer_pose_policy == 'rigid_pair_trajectory_v3' + else POLICY if len(actions)==1 else SEQUENCE_POLICY) record = dict(kind='observer_preparation_decision', policy=POLICY if len(actions)==1 else SEQUENCE_POLICY, joint=joint, task_name=motion.task_key, session_epoch=epoch, stamp_ns=stamp_ns, action='excite' if reason or coverage['needs_excitation'] else 'reuse_source', reason=reason or 'normal_coverage_preflight', coverage=coverage, source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in rows}, donor=plan.joint, path=[list(p) for p in (*plan.approach_commands, plan.command)]) + record.update(policy=policy, observability=coverage, action_utility=utility, + corner_evidence_sha256=corner_evidence_sha256(rows, (spec.parent_role, spec.child_role))) if len(actions)>1: record.update(action_index=index, action_limit=len(actions), previous_preparation_id=None if previous is None else source_frame_sha256(previous)) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation_evidence.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation_evidence.py index 494e303..a4c8520 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation_evidence.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_preparation_evidence.py @@ -11,6 +11,7 @@ from .motion_execution import feedback_tolerances from .motion_provenance import source_frame_sha256 from .shared_tag_reference import bridge_frames from ..profiles.observer_preparation import observer_action, observer_actions +from ..profiles.observer_policy import TRAJECTORY_OBSERVER def stationary_probe_tail(profile, joint, rows, target): @@ -30,16 +31,16 @@ def stationary_probe_tail(profile, joint, rows, target): def validate_probe_evidence(profile, joint, evidence, *, admission=None): - from .observer_preparation import POLICY, SEQUENCE_POLICY + from .observer_preparation import POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY decision, rows, segments = evidence.decision, evidence.rows, evidence.segments plan = observer_action(profile, joint, decision) identity = source_frame_sha256(decision) - if (decision.get('policy') == SEQUENCE_POLICY + if (decision.get('policy') in {SEQUENCE_POLICY, TRAJECTORY_POLICY} and decision.get('action_limit') != len(observer_actions(profile, joint))): raise ValueError('observer_preparation_action_budget_changed') path = (*plan.approach_commands, plan.command) spec = profile.measurement.measurements[joint] - if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY} or decision.get('action') != 'excite' + if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY} or decision.get('action') != 'excite' or decision.get('joint') != joint or decision.get('donor') != plan.joint or decision.get('task_name') != profile.motion.joint_zero_references[joint].task_key or decision.get('path') != [list(p) for p in path] @@ -139,9 +140,14 @@ def build_probe_bridge(profile, source): parents.append(pair[0]); children.append(pair[1]); matrices.append(row['camera_matrix']) parent_pixels.append(row['tags'][spec.parent_role]['corners_xy']) child_pixels.append(row['tags'][spec.child_role]['corners_xy']) + plan = observer_action(profile, source.joint, source.preparation.decision) + donor_channel = channel_for_joint(profile, plan.joint) + trajectory_states = (tuple((row['reference_path_index'], row['command_vector'][donor_channel]) + for row in rows) if spec.observer_pose_policy == TRAJECTORY_OBSERVER else None) pose = fit_rigid_tag_pair([p[0] for p in parents], children, child_pixels, matrices, tags[spec.child_role].size_m, tuple(r['image_stamp_ns'] for r in rows), - parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m, parent_candidates=parents) + parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m, + parent_candidates=parents, trajectory_states=trajectory_states) relation = MotionRelation(source.joint, spec.parent_role, spec.child_role) bridge = SharedTagPoseBridge(relation, pose.quaternion_xyzw, pose.translation_xyz_m, bridge_frames(endpoints, relation), rotation_uncertainty_rad=pose.rotation_uncertainty_rad, @@ -158,6 +164,18 @@ def build_probe_bridge(profile, source): preparation_started=source.preparation.started, source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in source.preparation.rows}, endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints}) + if spec.observer_pose_policy == TRAJECTORY_OBSERVER: + from .observer_preparation import trajectory_observability, corner_evidence_sha256 + record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={ + 'policy': 'rigid_pair_full_trajectory_v3', + 'selection_source': 'training_trajectory', + 'holdout_role': 'veto_only', + 'observability': trajectory_observability(rows, spec.parent_role, spec.child_role), + 'adjusted_candidate_probabilities': [list(value) + for value in pose.comparison_adjusted_p_values], + 'corner_evidence_sha256': corner_evidence_sha256( + source.preparation.rows, (spec.parent_role, spec.child_role)), + }) return bridge, record diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_reference.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_reference.py index c0be995..06f7d21 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_reference.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/observer_reference.py @@ -12,6 +12,7 @@ from .branch_initialization import _pose from .shared_tag_reference import bridge_frames from .motion_execution import feedback_tolerances from .motion_provenance import source_frame_sha256 +from ..profiles.observer_policy import is_rigid_observer def observer_roles(profile, motion): @@ -43,7 +44,7 @@ def build_observer_bridge(profile, source): raise ValueError('observer_parent_model_invalid') uncertainty = {name: (angle, point) for name, angle, point in source.parent_report['geometry_uncertainty']} inherited = uncertainty[parent_model.geometry[0].relation.joint] - rigid = spec.observer_pose_policy == 'rigid_pair_v1' + rigid = is_rigid_observer(spec.observer_pose_policy) if not rigid: require_observer_precision_budget(inherited) target = profile.motion.joint_zero_references[source.joint].command @@ -109,6 +110,18 @@ def build_observer_bridge(profile, source): endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints}) if rigid: record['conditioning'] = RIGID_PAIR_CONDITIONING + from ..profiles.observer_policy import TRAJECTORY_OBSERVER + if spec.observer_pose_policy == TRAJECTORY_OBSERVER: + from .observer_preparation import trajectory_observability, corner_evidence_sha256 + record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={ + 'policy': 'rigid_pair_full_trajectory_v3', + 'selection_source': 'training_trajectory', 'holdout_role': 'veto_only', + 'observability': trajectory_observability(rows, spec.parent_role, spec.child_role), + 'adjusted_candidate_probabilities': [list(value) + for value in pose.comparison_adjusted_p_values], + 'corner_evidence_sha256': corner_evidence_sha256( + rows, (spec.parent_role, spec.child_role)), + }) return bridge, record @@ -139,7 +152,7 @@ def observer_inputs(owner, view, endpoint_rows, arc_stamps, epoch): continue held = tuple(sorted(owner.tag_feedback_channels[spec.child_role]-owner.tag_feedback_channels[spec.parent_role])) result.append(ObserverBridgeInput(joint, model.image_model, report, rows, endpoints, held)) - if spec.observer_pose_policy == 'rigid_pair_v1': + if is_rigid_observer(spec.observer_pose_policy): # Select the latest eligible source before fitting. A failed # fit cannot search older sources for a more favorable result. break @@ -182,9 +195,10 @@ def validate_observer_references(profile, records, evidence): checked = set() for payload in evidence.values(): required = {joint for joint, spec in profile.measurement.measurements.items() - if joint in payload['zero_joints'] and spec.observer_pose_policy == 'rigid_pair_v1'} + if joint in payload['zero_joints'] and is_rigid_observer(spec.observer_pose_policy)} declared = [r['joint'] for r in payload.get('observer_pose_bridges', ()) - if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'}] + if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3', + 'observer_pose_bridge_v4'}] if set(declared) != required or len(declared) != len(required): raise ValueError('rigid_pair_required_evidence_missing') for payload in evidence.values(): @@ -204,7 +218,7 @@ def validate_observer_references(profile, records, evidence): if (evidence_id not in evidence or evidence[evidence_id]['tag_role'] != role or evidence[evidence_id]['image_model_sha256'] != record['parent_model_sha256']): raise ValueError('observer_parent_identity_changed') - if record.get('version') == 'observer_pose_bridge_v3': + if record.get('version') in {'observer_pose_bridge_v3', 'observer_pose_bridge_v4'}: from .observer_preparation_evidence import validate_probe_reference validate_probe_reference(profile, records, payload, record, parent, observations) continue diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/parent_reference.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/parent_reference.py index 2f91eb5..a019d70 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/parent_reference.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/parent_reference.py @@ -5,6 +5,7 @@ upstream pose change invalidates the former, but not a rigid Tag's mounting. No curve, joint zero or candidate selection is copied between joints. """ +from dataclasses import dataclass import math import numpy as np @@ -15,6 +16,14 @@ from ..core.geometry.tag_pose.source_hinge import SourceHinge from .resume import ResumeVerifier +@dataclass(frozen=True) +class RefreshedPoseSource: + """Bind a refreshed model to its immutable source epoch across restarts.""" + + image_model_sha256: str + source_epoch: int + + def check_parent_pose(profile, task, reference, rows, channels, *, verification='staged_revisit_v1', feedback_limits=None): """Same pure check for live execution and persisted evidence read-back.""" @@ -78,6 +87,7 @@ class ParentReferenceRegistry: self.models = {} self.verified = {} self.zero_references = {} + self.refreshed_pose_sources = {} def remember_zero(self, reference, epoch): for view in {sample.view for sample in reference.samples}: @@ -104,6 +114,8 @@ class ParentReferenceRegistry: for (epoch, view, joint), value in tuple(self.models.items()) if epoch == previous}) self.verified.update({(current, task): value for (epoch, task), value in tuple(self.verified.items()) if epoch == previous}) + self.refreshed_pose_sources.update({(current, task): value + for (epoch, task), value in tuple(self.refreshed_pose_sources.items()) if epoch == previous}) self.zero_references.update({(current, view, joint): value for (epoch, view, joint), value in tuple(self.zero_references.items()) if epoch == previous}) @@ -129,6 +141,12 @@ class ParentReferenceRegistry: """ from ..core.geometry.tag_pose.pose_bridge import is_handoff_policy pose, geometry = self.sources(task, epoch) + refreshed = self.refreshed_pose_sources.get((epoch, task.key)) + if refreshed is not None: + if (refreshed.image_model_sha256 != pose[1].get("image_model_sha256") + or refreshed.source_epoch != pose[1].get("session_epoch")): + raise ValueError("parent_reference_refreshed_pose_source_changed") + return pose if not is_handoff_policy(geometry[1].get('image_model_selection_policy')): return pose bridges = geometry[1].get('shared_tag_pose_bridges', ()) @@ -138,6 +156,50 @@ class ParentReferenceRegistry: raise ValueError('parent_reference_handoff_source_changed') return geometry + def authorize_refreshed_pose(self, task, epoch, version, reference): + """Prefer a freshly measured pose model while retaining passed geometry.""" + joint = task.parent_reference.pose_joint + model, report = self.models.get((epoch, task.view, joint), (None, None)) + if (model is None or report is None or report.get("session_epoch") != epoch + or model.image_model is None + or {item.relation.joint for item in model.image_model.geometry} != {joint}): + raise ValueError("parent_reference_refresh_model_missing_or_changed") + identity = report.get("image_model_sha256") + if not isinstance(identity, str) or not identity: + raise ValueError("parent_reference_refresh_model_missing_or_changed") + self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource(identity, epoch) + from .motion_provenance import source_frame_sha256 + return {"kind": "parent_reference_model_refreshed", "schema_version": 1, + "task_name": task.key, "view": task.view, "pose_joint": joint, + "session_epoch": epoch, "motion_version": version, + "image_model_sha256": identity, + "source_zero_sha256": source_frame_sha256(reference.as_record()), + "passed_zero_preserved": True, + "source_frame_hashes": dict(report.get("source_frame_hashes", {}))} + + def restore_refreshed_pose(self, record, epoch): + """Restore an explicit refresh grant without relabelling its source.""" + task = next((item for item in self.profile.motion.tasks + if item.key == record.get("task_name")), None) + if (task is None or task.parent_reference is None + or record.get("schema_version") != 1 + or record.get("view") != task.view + or record.get("pose_joint") != task.parent_reference.pose_joint + or record.get("passed_zero_preserved") is not True + or type(record.get("session_epoch")) is not int): + raise ValueError("parent_reference_refresh_record_changed") + model, report = self.models.get((epoch, task.view, + task.parent_reference.pose_joint), (None, None)) + identity = record.get("image_model_sha256") + if (model is None or report is None or model.image_model is None + or identity != report.get("image_model_sha256") + or record["session_epoch"] != report.get("session_epoch") + or dict(record.get("source_frame_hashes", {})) + != dict(report.get("source_frame_hashes", {}))): + raise ValueError("parent_reference_refresh_model_missing_or_changed") + self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource( + identity, record["session_epoch"]) + def activate_joints(self, joints, epoch, capture): installed = set() for joint in joints: diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/passed_resume.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/passed_resume.py index 04832c5..55c113b 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/passed_resume.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/passed_resume.py @@ -5,7 +5,7 @@ reverification runs here. New acquisitions and final artifact validation keep their normal gates. An incomplete task is restarted as a whole. """ -from dataclasses import dataclass +from dataclasses import dataclass, field from ..core.domain.motion_path import record_scan_identity from ..core.domain.reference import read_joint_zero_references @@ -16,6 +16,29 @@ from .joint_resume import ResumeJournalImport, _index_motion_provenance class PassedReferenceImport(ResumeJournalImport): references: dict model_records: tuple + restore_phase: str = field(default="binding", init=False) + restore_rows: tuple = field(default=(), init=False, repr=False) + restore_offset: int = field(default=0, init=False) + + +def coherent_observer_preparation_rows(rows): + """Discard transition fragments whose originating decision is absent. + + Compact checkpoints may contain a cancellation after an earlier migration + already removed the decision it cancelled. Such a fragment represents no + live state and cannot be replayed on its own. Decisions and all transitions + still present together remain byte-for-byte unchanged. + """ + from .motion_provenance import source_frame_sha256 + + decisions = {source_frame_sha256(row) for row in rows + if row.get("kind") == "observer_preparation_decision" + and row.get("action") == "excite"} + bound_kinds = {"observer_preparation_segment_complete", + "observer_preparation_excursion_started", + "observer_preparation_cancelled_before_excursion"} + return tuple(row for row in rows if row.get("kind") not in bound_kinds + or row.get("observer_preparation_id") in decisions) def stage_passed_tasks(resume, engine, rows): @@ -48,15 +71,25 @@ def reference_import(resume): """Import prior evidence durably before skipping any completed task.""" provenance = _index_motion_provenance(resume.rows) references = resume.references + tasks = {reference.task_key for reference in references.values()} parent_images = {(row["view"], int(stamp)) for row in provenance.values() - if row.get("kind") == "parent_reference_verified" for stamp in row["source_frame_hashes"]} + if row.get("kind") == "parent_reference_verified" + and row.get("task_name") in tasks for stamp in row["source_frame_hashes"]} parent_images.update((sample["view"], sample["image_stamp_ns"]) for row in provenance.values() if row.get("kind") == "parent_reference_verified" + and row.get("task_name") in tasks for sample in row["source_reference"]["samples"]) zero_images = parent_images | {(row.get("view"), row.get("image_stamp_ns")) for row in provenance.values() if row.get("kind") == "joint_zero_sample" and row.get("joint") in references} selected = [] for row in provenance.values(): + # Passed mode restarts every incomplete task as a whole. Its old + # branch models and preparation images belong to an abandoned visit; + # restoring them would bind the new session's current image anchor to + # a previous camera epoch. Task-independent audit rows remain useful. + task_name = row.get("task_name") + if task_name is not None and task_name not in tasks: + continue image = row.get("view"), row.get("image_stamp_ns") if (row.get("kind") in {"joint_zero_sample", "joint_zero_motion_reference_verified"} and row.get("joint") not in references and image not in parent_images): @@ -64,7 +97,7 @@ def reference_import(resume): if row.get("kind") == "pnp_candidate_frame" and image not in zero_images: continue selected.append(row) - tasks = {reference.task_key for reference in references.values()} + selected = coherent_observer_preparation_rows(selected) models = tuple(row for row in selected if row.get("kind") == "motion_branch_initialization" and row.get("status") == "resolved" and row.get("task_name") in tasks) audit = {"kind": "passed_tasks_reused", "resume_mode": "passed", @@ -76,7 +109,7 @@ def reference_import(resume): references=references, model_records=models) -def restore_parent_models(registry, records, epoch): +def restore_parent_models(registry, records, epoch, provenance_records=()): """Keep source identities while making passed installations available to new tasks.""" from ..core.geometry.image_motion_replay import frozen_image_model from .branch_initialization import MotionBranchModel @@ -87,6 +120,12 @@ def restore_parent_models(registry, records, epoch): model = MotionBranchModel(row["task_name"], row["view"], (), tuple(sorted(row["evidence_ids"].items())), frozen_image_model(row)) registry.remember(model, row, epoch=epoch) + latest_refreshes = {} + for row in provenance_records: + if row.get("kind") == "parent_reference_model_refreshed": + latest_refreshes[row.get("task_name")] = row + for record in latest_refreshes.values(): + registry.restore_refreshed_pose(record, epoch) def restored_model_record(records, epoch, stamp_ns): diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/resume.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/resume.py index 3eddba8..81593fe 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/resume.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/resume.py @@ -93,6 +93,11 @@ def discover_resume_candidate( reject_diagnostic_capture((row,)) kind = str(row.get("kind", "")) if kind == "session_start": + from ..core.artifacts.evidence_journal import REFERENCE_KEY, decode_envelope + if REFERENCE_KEY in row: + from ..core.geometry.frozen_evidence import EvidenceDecoder + row = decode_envelope(candidate / "raw_samples.jsonl", row, + EvidenceDecoder(verify_hashes=False)) if start is not None: raise ValueError("multiple checkpoint session headers") start = row diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/resume_storage.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/resume_storage.py index 2ff4148..c80549d 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/resume_storage.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/resume_storage.py @@ -35,7 +35,9 @@ class DeferredCaptureRecord(Mapping): @classmethod def from_envelope(cls, path, envelope, decoder): - from ..core.artifacts.evidence_journal import CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY + from ..core.artifacts.evidence_journal import ( + CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY, SCHEMA_KEY, _schema_fields, + ) value = object.__new__(cls) value.path, value.offset, value.length = path, 0, 0 from ..core.artifacts.evidence_journal import EvidenceRecordReference @@ -43,41 +45,54 @@ class DeferredCaptureRecord(Mapping): value.digest = bytes.fromhex(value.reference.record_sha256) value.canonical_sha256 = envelope.get(CANONICAL_SHA256_KEY) value.metadata = {key: item for key, item in envelope.items() - if key not in {REFERENCE_KEY, FIELDS_KEY, CANONICAL_SHA256_KEY}} - value.field_names = _shared_field_names(tuple(envelope[FIELDS_KEY])) + if key not in {REFERENCE_KEY, FIELDS_KEY, SCHEMA_KEY, CANONICAL_SHA256_KEY}} + fields = envelope.get(FIELDS_KEY) + if fields is None and SCHEMA_KEY in envelope: + fields = _schema_fields(path, envelope[SCHEMA_KEY]) + value.field_names = None if fields is None else _shared_field_names(tuple(fields)) value.decoder = decoder return value def materialize(self): + return self.decoder.loads(self._record_bytes()) + + def materialize_plain(self): + """Decode one integrity-checked row without building shared models.""" + return json.loads(self._record_bytes()) + + def _record_bytes(self): if self.reference is not None: from ..core.artifacts.evidence_journal import evidence_record_bytes - line = evidence_record_bytes(self.path, self.reference) - return self.decoder.loads(line) + return evidence_record_bytes(self.path, self.reference) with self.path.open("rb") as stream: stream.seek(self.offset) line = stream.read(self.length) if hashlib.sha256(line).digest() != self.digest: raise ValueError("resume_source_record_changed") - return self.decoder.loads(line) + return line def __getitem__(self, key): if key in self.metadata: return self.metadata[key] - if key not in self.field_names: + if self.field_names is not None and key not in self.field_names: raise KeyError(key) return self.materialize()[key] def __iter__(self): - return iter(self.field_names) + return iter(self.materialize() if self.field_names is None else self.field_names) def __len__(self): - return len(self.field_names) + return len(self.materialize()) if self.field_names is None else len(self.field_names) def materialize_record(row): return row.materialize() if isinstance(row, DeferredCaptureRecord) else row +def materialize_plain_record(row): + return row.materialize_plain() if isinstance(row, DeferredCaptureRecord) else row + + def load_passed_capture(path): """Read once without retaining every scan's expanded image dictionaries. @@ -101,7 +116,15 @@ def load_passed_capture(path): if not line.strip(): continue try: - row = decoder.loads(line) + # Indexed envelopes contain only scalar lookup fields and an + # evidence reference. Running the recursive evidence decoder + # over tens of thousands of these small rows dominated resume + # startup even though there is no model to intern. Preserve + # the decoder for the few inline records that carry shared + # frozen objects. + needs_evidence_decoder = (b'"frozen_image_model"' in line + or b'"reference_frame"' in line) + row = decoder.loads(line) if needs_evidence_decoder else json.loads(line) except json.JSONDecodeError as error: raise ValueError(f"invalid calibration JSONL at line {number}") from error if not isinstance(row, dict): diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/runner.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/runner.py index 87192db..c6166e1 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/runner.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/runner.py @@ -7,6 +7,7 @@ hardware. Pending legacy status/artifact contracts are isolated in compat. from __future__ import annotations import argparse +import hashlib import json import math import os @@ -136,6 +137,21 @@ def run_online( if diagnostic is not None and not commands_enabled: raise ValueError("diagnostic_capture requires commands_enabled") session = create_session_directory(config.session_root) + # A CLI-selected preparation policy is a real protected profile, not an + # in-process hint. Materialize its canonical bytes so the launched node, + # resume selector and journal all bind the identical effective contract. + from ..profiles.export import dump_hand_profile + effective_profile = dump_hand_profile(profile).encode() + effective_digest = hashlib.sha256(effective_profile).hexdigest() + from ..profiles import load_hand_profile + runtime_profile_changed = (config.profile_config is None + or load_hand_profile(config.profile_config) != profile) + if runtime_profile_changed: + effective_path = session / "effective_profile.yaml" + effective_path.write_bytes(effective_profile) + from dataclasses import replace + config = replace(config, profile_config=effective_path, + profile_config_sha256=effective_digest) from ..profiles.validator import validate_executable_profile, require_release_geometry contract_report = validate_executable_profile(profile, config.source_urdf) atomic_write_json(session/"measurement_contract.json", contract_report) @@ -307,7 +323,8 @@ def main(args: list[str] | None = None) -> None: parser.add_argument("--record-bag", action="store_true") parser.add_argument("--commands-disabled", action="store_true") parser.add_argument("--no-resume", action="store_true") - parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3"), default=None, + parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3", + "taskwise_staged_2_to_3"), default=None, help="试验采集策略：两轮训练，必要时补第三轮；保留独立验证，默认沿用配置") parser.add_argument("--startup-timeout-seconds", type=float, default=None, help="等待节点初始化及设备就绪的秒数（默认 120）；大断点逐帧校验可显式延长，不改变运动保护") @@ -322,9 +339,10 @@ def main(args: list[str] | None = None) -> None: parser.add_argument("--offline-raw", default="") parser.add_argument("--offline-output", default="") parser.add_argument("--publish-offline", action="store_true") - parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_witness_v2', - choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2"), - help="新 O30 staged 准备策略；省略取值使用 v2，按观测依赖统一启用刚体双 Tag 求解；v1 仅保留历史兼容") + parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_trajectory_v3', + choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2", + "occlusion_aware_trajectory_v3"), + help="O30 staged 准备策略；省略取值使用 v3 整段轨迹消歧和有界自动观察；v1/v2 仅保留历史兼容") selected = parser.parse_args(args) if selected.startup_timeout_seconds is not None and ( not math.isfinite(selected.startup_timeout_seconds) or selected.startup_timeout_seconds <= 0 @@ -357,6 +375,10 @@ def main(args: list[str] | None = None) -> None: from ..profiles.preparation import configure_preparation try: profile = configure_preparation(config.calibration_contract.typed_profile, selected.preparation_policy) + if selected.preparation_policy == 'occlusion_aware_trajectory_v3': + from ..core.urdf.kinematics import UrdfKinematicModel + from ..profiles.observer_preparation import configure_observer_preparations + profile = configure_observer_preparations(profile, UrdfKinematicModel(config.source_urdf)) config = replace(config, calibration_contract=ProductCalibrationContract(declarative=profile)) except ValueError as error: parser.error(str(error)) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/session.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/session.py index b158e04..38b754b 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/session.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/session.py @@ -62,7 +62,7 @@ class CalibrationSession: self.required_observation_views = required_observation_views(profile, diagnostic_capture) self.engine = CalibrationEngine(profile, diagnostic_capture=diagnostic_capture) self.phase = CalibrationPhase.WAIT_DEVICE - self._units = self.engine.scan_units() + self._units = list(self.engine.scan_units()) self._unit_index = 0 self._retry_by_unit: dict[tuple[str, int, str], int] = {} self._mapping_probed_channels: set[int] = set() @@ -72,6 +72,8 @@ class CalibrationSession: self._pause = PauseStatus() self._reference_locked = False self._rescan_pending = False + from .task_deferral import TaskDeferralQueue + self.task_deferrals = TaskDeferralQueue(profile) @property def current_unit(self) -> ScanUnit | None: @@ -79,13 +81,17 @@ class CalibrationSession: return None return self._units[self._unit_index] + @property + def scheduled_units(self): + return tuple(self._units) + def select_training(self, task_key, cycles): """Replace only future optional units; evidence IDs never change.""" from ..core.domain.capture_plan import select_task_training current = self.current_unit self.profile = select_task_training(self.profile, task_key, cycles) self.engine.profile = self.profile - self._units = self.engine.scan_units() + self._units = list(self.engine.scan_units()) if current is not None: self._unit_index = next(i for i, unit in enumerate(self._units) if unit.identity == current.identity) @@ -161,7 +167,7 @@ class CalibrationSession: self._require(CalibrationPhase.PREPARE) unit = self.current_unit if unit is None: - self.phase = CalibrationPhase.RETURN_BASELINE + self._finish_capture_schedule() return task = self._task(unit.task_key) if ( @@ -227,6 +233,79 @@ class CalibrationSession: # The executor elides already-satisfied waypoints without motion. self.phase = CalibrationPhase.PREPARE return + self._finish_capture_schedule() + + def defer_current_task(self, reason: str): + """Move one recoverable visual visit to the end of its current stage. + + A second failure records the visit as unobservable and removes only + its still-uncompleted units. Finalization is then blocked explicitly. + """ + self._require(CalibrationPhase.PREPARE) + unit = self.current_unit + if unit is None: + raise ValueError("task_deferral_without_current_unit") + from .task_deferral import dependency_closure, TaskObservationState + affected = dependency_closure(self.profile, unit.task_key) + moved = self._move_visit_to_stage_end(unit.cycle, affected) + attempt = self.task_deferrals.attempts.get((unit.task_key, unit.cycle), 0)+1 + decision = self.task_deferrals.decide(unit.task_key, unit.cycle, reason, len(moved)) + if decision.state == TaskObservationState.PENDING: + self._insert_at_stage_end(unit.cycle, moved) + else: + # Descendants cannot be accepted without the missing parent + # geometry. Remove their future visits from every later cycle. + self._units = self._units[:self._unit_index] + [candidate + for candidate in self._units[self._unit_index:] + if candidate.task_key not in set(decision.affected_tasks)] + self._rescan_pending = False + if self.current_unit is None: + self._finish_capture_schedule() + else: + self.phase = CalibrationPhase.PREPARE + return decision + + def restore_task_deferrals(self, records): + """Restore retry budgets and the exact stage-local pending order.""" + self.task_deferrals.restore(records) + from .task_deferral import TaskObservationState, unresolved_observation_records + for row in unresolved_observation_records(self.profile, records): + if row.get("state") != TaskObservationState.PENDING.value: + continue + moved = self._move_visit_to_stage_end( + int(row["cycle"]), frozenset(row["affected_tasks"])) + self._insert_at_stage_end(int(row["cycle"]), moved) + + def _stage_index(self, cycle): + from ..core.domain.capture_plan import TASKWISE_STAGED + if self.profile.quality.training_policy == TASKWISE_STAGED: + return 0 if cycle == 0 else 1 if cycle in {1, 2} else 2 + return cycle + + def _move_visit_to_stage_end(self, cycle, affected_tasks): + indices = [index for index in range(self._unit_index, len(self._units)) + if self._units[index].cycle == cycle + and self._units[index].task_key in affected_tasks] + if not indices: + raise ValueError("task_deferral_has_no_pending_units") + moved = [self._units[index] for index in indices] + for index in reversed(indices): + del self._units[index] + return moved + + def _insert_at_stage_end(self, cycle, moved): + stage = self._stage_index(cycle) + insertion = next((index for index in range(self._unit_index, len(self._units)) + if self._stage_index(self._units[index].cycle) > stage), len(self._units)) + self._units[insertion:insertion] = moved + + def _finish_capture_schedule(self): + if self.task_deferrals.unobservable: + details = {f"{task}:{cycle}": reason + for (task, cycle), reason in self.task_deferrals.unobservable.items()} + self.fail("unobservable_tasks", "安全观察和一次阶段回访后仍不可观测", + details, suggestion="保留数据；调整 Tag 朝向、增加非共面 Tag 或改变机位后重试") + return self.phase = CalibrationPhase.RETURN_BASELINE def return_complete(self) -> None: @@ -331,6 +410,21 @@ class CalibrationSession: ids = tuple(tag.tag_id for tag in view.tags if tag.role in roles) if ids: required[view.name] = ids + from ..core.domain.capture_plan import TASKWISE_STAGED + taskwise = self.profile.quality.training_policy == TASKWISE_STAGED + displayed_cycle = (None if unit is None or unit.sample_phase == "steady" else + ({0: 1, 1: 2, 2: 2, 3: 3}[unit.cycle] if taskwise else + list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key + and u.sample_phase == "sweep")).index(unit.cycle) + 1)) + displayed_cycle_count = (3 if taskwise else + len(self.diagnostic_capture.cycles) if self.diagnostic_capture is not None else + len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key + and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1) + capture_stage = ("training_supplement" if taskwise and unit is not None and unit.cycle == 2 + else "command_validation" if unit is not None and unit.sample_phase == "steady" + and unit.cycle == self.profile.quality.holdout_cycle + 2 + else "command_training" if unit is not None and unit.sample_phase == "steady" + else "motion") return CalibrationStatus( state=self.phase.value, phase=self.phase.value, @@ -344,22 +438,17 @@ class CalibrationSession: task=TaskStatus( key="" if task is None else task.key, label="" if task is None else " / ".join(task.joints), - cycle=(None if unit is None or unit.sample_phase == "steady" else - list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key - and u.sample_phase == "sweep")).index(unit.cycle) + 1), - cycle_count=( - len(self.diagnostic_capture.cycles) - if self.diagnostic_capture is not None - else len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key - and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1 - ), + cycle=displayed_cycle, + cycle_count=displayed_cycle_count, direction="" if unit is None else unit.direction, - capture_stage=("command_validation" if unit is not None and unit.sample_phase == "steady" - and unit.cycle == self.profile.quality.holdout_cycle + 2 else "command_training" if unit is not None - and unit.sample_phase == "steady" else "motion"), + capture_stage=capture_stage, required_tag_ids_by_view=required, ), - resume={"message": self._resume_message}, + resume={"message": self._resume_message, + "deferred_tasks": {f"{task}:{cycle}": attempt + for (task, cycle), attempt in self.task_deferrals.attempts.items()}, + "unobservable_tasks": [f"{task}:{cycle}" + for task, cycle in self.task_deferrals.unobservable]}, pause=self._pause, ) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/shared_pose_handoff.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/shared_pose_handoff.py index 3063ae5..0a49a11 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/shared_pose_handoff.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/shared_pose_handoff.py @@ -18,7 +18,8 @@ SOURCE_PHASES = frozenset({'sweep', 'steady', 'joint_zero'}) def enabled(profile): - return (profile.quality.training_policy == 'staged_2_to_3' + from ..core.domain.capture_plan import STAGED_POLICIES + return (profile.quality.training_policy in STAGED_POLICIES and bool(profile.motion.preparation_witnesses)) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/staged_resume.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/staged_resume.py index eb617ec..c111b9b 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/staged_resume.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/staged_resume.py @@ -12,6 +12,8 @@ from .scan_quality import evaluate_capture_unit def stage_visits(resume, engine, rows): + from .task_deferral import deferred_visit_keys + deferred = deferred_visit_keys(resume.profile, rows) references = read_joint_zero_references(resume.profile, rows) validate_motion_provenance(resume.profile, rows, references=references, image_replay_workers=resume.image_replay_workers) @@ -22,11 +24,15 @@ def stage_visits(resume, engine, rows): if row.get('kind') == 'scan_unit_complete': latest[record_scan_identity(row)] = row spans, valid = {}, set() - for (key, _), units in visits.items(): + for (key, cycle), units in visits.items(): task = next(t for t in resume.profile.motion.tasks if t.key == key) if not set(task.joints) <= references.keys(): + if (key, cycle) in deferred: + continue break if any(latest.get(u.identity, {}).get('passed') is not True for u in units): + if (key, cycle) in deferred: + continue break if not all(evaluate_capture_unit(resume.profile, u, latest[u.identity].get('attempt', 1), rows, first_cycle_spans=spans).passed for u in units): @@ -49,6 +55,8 @@ def stage_passed_visits(resume, engine, rows): Every imported evidence record remains content-bound and finalization still performs the complete image/provenance replay before publication. """ + from .task_deferral import deferred_visit_keys + deferred = deferred_visit_keys(resume.profile, rows) references = read_joint_zero_references(resume.profile, rows) visits, latest = OrderedDict(), {} for unit in engine.scan_units(): @@ -63,9 +71,13 @@ def stage_passed_visits(resume, engine, rows): for (task_key, cycle), units in visits.items(): task = next(task for task in resume.profile.motion.tasks if task.key == task_key) if not set(task.joints) <= references.keys(): + if (task_key, cycle) in deferred: + continue break completions = [latest.get(unit.identity) for unit in units] if any(row is None or row.get("passed") is not True for row in completions): + if (task_key, cycle) in deferred: + continue break valid.update(unit.identity for unit in units) if cycle == 0: @@ -98,10 +110,23 @@ def load_frozen_checkpoint(profile, rows, path, source): def reference_import(resume, *, reuse_passed_visits=False): from .joint_resume import _index_motion_provenance - from .passed_resume import PassedReferenceImport + from .passed_resume import PassedReferenceImport, coherent_observer_preparation_rows provenance = tuple(_index_motion_provenance(resume.rows).values()) if reuse_passed_visits: + reused_tasks = {identity[0] for identity in resume.units} + reused_joints = {joint for task in resume.profile.motion.tasks + if task.key in reused_tasks for joint in task.joints} + references = {joint: reference for joint, reference in resume.references.items() + if joint in reused_joints} + # A visit that did not pass is restarted with a fresh preparation + # model in the current image epoch. Reusing its old candidate path is + # neither a passed-unit optimization nor valid cross-session evidence. + provenance = tuple(row for row in provenance + if row.get("task_name") is None or row.get("task_name") in reused_tasks) provenance = _essential_indexed_provenance(provenance) + provenance = coherent_observer_preparation_rows(provenance) + else: + references = resume.references models = tuple(r for r in provenance if r.get('kind') == 'motion_branch_initialization' and r.get('status') == 'resolved') audit = () if reuse_passed_visits: @@ -111,17 +136,19 @@ def reference_import(resume, *, reuse_passed_visits=False): resume.verified_visits.update((name, cycle) for task in resume.profile.motion.tasks for key, cycle in visits if key == task.key for name in task.joints) audit = ({'kind': 'passed_tasks_reused', 'resume_mode': 'passed', - 'unchanged_installation_declared': True, 'training_policy': 'staged_2_to_3', + 'unchanged_installation_declared': True, + 'training_policy': resume.profile.quality.training_policy, 'completed_units': [list(key) for key in sorted(resume.units)], 'historical_replay_performed': False, 'physical_zero_reverification_performed': False, 'full_replay_required_before_publication': True},) - return PassedReferenceImport(rows=(*(r.as_record() for r in resume.references.values()), *provenance, *audit), - references=resume.references, model_records=models) + return PassedReferenceImport(rows=(*(r.as_record() for r in references.values()), *provenance, *audit), + references=references, model_records=models) def _essential_indexed_provenance(rows): """Keep source frames actually named by accepted models in compact journals.""" import re + from collections.abc import Mapping from .resume_storage import DeferredCaptureRecord, materialize_record if not any(isinstance(row, DeferredCaptureRecord) and row.canonical_sha256 for row in rows): @@ -129,17 +156,21 @@ def _essential_indexed_provenance(rows): hashes = set() def collect(value, key=""): - if isinstance(value, dict): + if isinstance(value, Mapping): for name, item in value.items(): - collect(item, name) + # Digest maps are keyed by image stamp. Preserve the parent + # field name while descending into them; replacing it with the + # numeric stamp made every digest invisible to this filter. + collect(item, key if "hash" in key.lower() else str(name)) elif isinstance(value, (list, tuple)): for item in value: collect(item, key) - elif (isinstance(value, str) and "hash" in key + elif (isinstance(value, str) and "hash" in key.lower() and re.fullmatch(r"[0-9a-f]{64}", value)): hashes.add(value) evidence_kinds = {"motion_branch_initialization", "parent_reference_verified", + "parent_reference_model_refreshed", "observer_preparation_decision", "observer_preparation_segment_complete", "joint_zero_motion_reference_verified"} for row in rows: @@ -152,6 +183,7 @@ def _essential_indexed_provenance(rows): def restore_recovery(session, recovery, rows): + session.restore_task_deferrals(rows) for row in rows: if row.get('kind') == 'joint_zero_recovery': recovery.restore(row['task_name'], row['joints']) @@ -171,9 +203,21 @@ def visit_rows(resume, task_key, cycle): rows = [r for r in resume.rows if record_scan_identity(r) in units and r.get('kind') not in {'joint_zero_reference', 'joint_zero_sample'}] if units: + from ..core.domain.capture_plan import TASKWISE_STAGED from .engine import CalibrationEngine + if resume.profile.quality.training_policy == TASKWISE_STAGED and cycle in {1, 2}: + # Persist the causal decision beside the visit that produced it. + # This lets a later restart reconstruct an optional cycle 2 from + # the new journal without depending on in-memory profile state. + assessed = list(range(cycle + 1)) + decisions = [r for r in resume.rows if r.get('kind') == 'training_decision' + and r.get('task_name') == task_key and r.get('assessed_cycles') == assessed] + if len(decisions) != 1: + raise ValueError('taskwise_resume_training_decision_missing') + rows.extend(decisions) stage = [u for u in CalibrationEngine(resume.profile).scan_units() if u.cycle == cycle] - if cycle in {1, 2} and stage[-1].task_key == task_key: + if (resume.profile.quality.training_policy != TASKWISE_STAGED + and cycle in {1, 2} and stage[-1].task_key == task_key): decisions = [r for r in resume.rows if r.get('kind') == 'staged_training_decision'] candidates = decisions[:1] if cycle == 1 else decisions[1:] rows.extend(candidates) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/status.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/status.py index 5f79b9b..5a70a6a 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/status.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/status.py @@ -144,6 +144,8 @@ def render_status_zh(value: Mapping[str, Any], estimator=None) -> str: stage_line = f"阶段：{phase_label}（第 {task.get('cycle') or '-'}/{task.get('cycle_count', 4)} 轮）" if capture_stage in {"command_training", "command_validation"}: stage_line = "阶段：" + ("稳态指令映射训练" if capture_stage == "command_training" else "稳态指令映射独立验证") + elif capture_stage == "training_supplement": + stage_line = f"阶段：{phase_label}（当前任务训练补采 1/1）" lines = [ f"[{value.get('model', '未知型号')} / {value.get('side', '?')} / {value.get('serial_number', '?')}] {progress:6.1%} 预计剩余 {eta_text}", stage_line, diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/task_deferral.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/task_deferral.py new file mode 100644 index 0000000..3ac301c --- /dev/null +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/task_deferral.py @@ -0,0 +1,237 @@ +"""Bounded task deferral for recoverable visual observability failures. + +The queue only reorders uncompleted scan units inside one staged cycle. It +never changes accepted units, reference models, quality limits or holdout +semantics. Safety failures are deliberately outside this module. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum + + +POLICY = "dependency_ordered_observation_deferral_v1" +RECOVERABLE_REASONS = frozenset({ + "rigid_pair_candidates_unresolved", + "rigid_pair_pose_uncertain", + "rigid_pair_viewpoint_span_insufficient", + "rigid_pair_source_missing", + "observer_images_missing", + "observer_preparation_candidates_missing", + "motion_candidates_missing", + "motion_candidate_assignment_ambiguous", + "motion_candidate_family_discontinuous", + "motion_candidate_family_interrupted", + "motion_geometry_ambiguous", + "image_motion_alternative_angles_unresolved", + "image_motion_families_not_distinguishable", + "image_motion_frame_angles_ambiguous", + "image_motion_frame_angles_uncertain", + "image_motion_frame_angles_unobservable", + "image_motion_insufficient_independent_frames", + "image_motion_no_observable_image_model", + "image_motion_shared_pose_unresolved", + "parent_reference_samples_missing", + "image_reference_current_image_moved", + "pose_image_motion_root_missing", +}) + + +class TaskObservationState(str, Enum): + PASSED = "passed" + PENDING = "pending" + DEPENDENCY_PENDING = "dependency_pending" + UNOBSERVABLE = "unobservable" + + +def task_dependencies(profile): + """Compile parent task -> descendant tasks from observed Tag roles.""" + task_by_joint = {joint: task.key for task in profile.motion.tasks for joint in task.joints} + owners = {} + for joint, spec in profile.measurement.measurements.items(): + task = task_by_joint.get(joint) + if task is not None and spec.child_role: + owners.setdefault((spec.view, spec.child_role), set()).add(task) + graph = {task.key: set() for task in profile.motion.tasks} + for joint, spec in profile.measurement.measurements.items(): + child_task = task_by_joint.get(joint) + if child_task is None or not spec.parent_role: + continue + for parent_task in owners.get((spec.view, spec.parent_role), ()): + if parent_task != child_task: + graph[parent_task].add(child_task) + return {key: frozenset(value) for key, value in graph.items()} + + +def dependency_closure(profile, task_key): + graph = task_dependencies(profile) + pending, result = [task_key], set() + while pending: + current = pending.pop() + if current in result: + continue + result.add(current) + pending.extend(graph.get(current, ())) + return frozenset(result) + + +def recoverable_observation_reason(reason): + text = str(reason) + return any(code in text for code in RECOVERABLE_REASONS) + + +@dataclass(frozen=True) +class DeferralDecision: + task_name: str + cycle: int + attempt: int + state: TaskObservationState + affected_tasks: tuple[str, ...] + reason: str + moved_units: int + + def as_record(self, *, stamp_ns, corner_evidence_sha256=""): + dependency_state = (TaskObservationState.DEPENDENCY_PENDING.value + if self.state == TaskObservationState.PENDING + else TaskObservationState.UNOBSERVABLE.value) + return { + "kind": "task_observation_state", + "schema_version": 1, + "policy": POLICY, + "task_name": self.task_name, + "cycle": self.cycle, + "attempt": self.attempt, + "state": self.state.value, + "affected_tasks": list(self.affected_tasks), + "dependency_states": ({task: dependency_state + for task in self.affected_tasks if task != self.task_name} + if self.state != TaskObservationState.PASSED else {}), + "reason": self.reason, + "moved_units": self.moved_units, + "corner_evidence_sha256": corner_evidence_sha256, + "stamp_ns": int(stamp_ns), + } + + +class TaskDeferralQueue: + """Own the finite retry state independently of motion execution.""" + + def __init__(self, profile, *, revisit_limit=1): + if revisit_limit != 1: + raise ValueError("task_deferral_revisit_limit_must_be_one") + self.profile = profile + self.revisit_limit = revisit_limit + self.attempts = {} + self.unobservable = {} + self.resolved = set() + self.inferred_resolutions = set() + + def decide(self, task_name, cycle, reason, moved_units): + key = (str(task_name), int(cycle)) + if key in self.resolved: + raise ValueError("task_deferral_already_resolved") + attempt = self.attempts.get(key, 0) + 1 + if attempt > self.revisit_limit + 1: + raise ValueError("task_deferral_budget_exhausted") + self.attempts[key] = attempt + affected = tuple(task.key for task in self.profile.motion.tasks + if task.key in dependency_closure(self.profile, task_name)) + state = (TaskObservationState.PENDING if attempt <= self.revisit_limit + else TaskObservationState.UNOBSERVABLE) + if state == TaskObservationState.UNOBSERVABLE: + self.unobservable[key] = str(reason) + return DeferralDecision(str(task_name), int(cycle), attempt, state, + affected, str(reason), int(moved_units)) + + def resolve(self, task_name, cycle): + key = (str(task_name), int(cycle)) + attempt = self.attempts.get(key) + if attempt is None or key in self.resolved: + return None + self.resolved.add(key) + self.unobservable.pop(key, None) + return DeferralDecision(key[0], key[1], attempt, TaskObservationState.PASSED, + (key[0],), "trajectory_observation_resolved", 0) + + def restore(self, records): + records = tuple(records) + state_positions = {} + for position, row in enumerate(records): + if row.get("kind") != "task_observation_state": + continue + if row.get("policy") != POLICY or row.get("state") not in { + state.value for state in TaskObservationState}: + raise ValueError("task_deferral_checkpoint_policy_changed") + key = (str(row["task_name"]), int(row["cycle"])) + state_positions[key] = position + attempt = int(row["attempt"]) + expected_affected = tuple(task.key for task in self.profile.motion.tasks + if task.key in dependency_closure(self.profile, key[0])) + if tuple(row.get("affected_tasks", ())) != expected_affected: + raise ValueError("task_deferral_checkpoint_dependencies_changed") + if row["state"] == TaskObservationState.PASSED.value: + if self.attempts.get(key) != attempt or key in self.resolved: + raise ValueError("task_deferral_checkpoint_resolution_changed") + self.resolved.add(key) + self.unobservable.pop(key, None) + continue + if attempt != self.attempts.get(key, 0)+1 or attempt not in (1, 2): + raise ValueError("task_deferral_checkpoint_budget_changed") + self.attempts[key] = attempt + if row["state"] == TaskObservationState.UNOBSERVABLE.value: + self.unobservable[key] = str(row.get("reason", "")) + # If a crash happened after the final scan-unit commit but before the + # adjacent PASSED state record, the complete visit itself closes the + # pending state. A partial direction can never do so. + from .engine import CalibrationEngine + from ..core.domain.motion_path import record_scan_identity + completion_positions = {} + for position, row in enumerate(records): + if row.get("kind") == "scan_unit_complete" and row.get("passed") is True: + completion_positions[record_scan_identity(row)] = position + units = CalibrationEngine(self.profile).scan_units() + for key in set(self.attempts) - self.resolved - set(self.unobservable): + required = {unit.identity for unit in units + if (unit.task_key, unit.cycle) == key} + boundary = state_positions[key] + if required and all(completion_positions.get(identity, -1) > boundary + for identity in required): + self.resolved.add(key) + self.inferred_resolutions.add(key) + + +def deferred_visit_keys(profile, records): + """Return incomplete visits that may be followed by durable later visits. + + A non-contiguous resume prefix is legal only when a versioned deferral + record explains the gap. Replaying the queue also validates retry order, + dependency closure and terminal state before the gap is trusted. + """ + rows = [row for row in records if row.get("kind") == "task_observation_state"] + if not rows: + return frozenset() + queue = TaskDeferralQueue(profile) + queue.restore(records) + latest = {} + for row in rows: + latest[(str(row["task_name"]), int(row["cycle"]))] = row + blocked = set() + for key, row in latest.items(): + if row["state"] == TaskObservationState.PASSED.value or key in queue.resolved: + continue + blocked.update((task, key[1]) for task in row["affected_tasks"]) + return frozenset(blocked) + + +def unresolved_observation_records(profile, records): + """Return latest states that are not closed by a full durable visit.""" + rows = [row for row in records if row.get("kind") == "task_observation_state"] + if not rows: + return () + queue = TaskDeferralQueue(profile) + queue.restore(records) + latest = {} + for row in rows: + latest[(str(row["task_name"]), int(row["cycle"]))] = row + return tuple(row for key, row in latest.items() if key not in queue.resolved) diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/training.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/training.py index 4e9ae8d..4d23a92 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/training.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/training.py @@ -6,8 +6,8 @@ import threading import time import traceback -from ..core.domain.capture_plan import (ADAPTIVE, STAGED, CapturePlan, configure_training, - evidence_digest, select_task_training) +from ..core.domain.capture_plan import (ADAPTIVE, STAGED, TASKWISE_STAGED, + CapturePlan, configure_training, evidence_digest, select_task_training) from ..core.domain.reference import JointZeroReference from ..core.fitting.training_quality import (TRAINING_DECISION_VERSION, assess_training, training_snapshot) @@ -22,7 +22,7 @@ def resolve_capture_plan(profile, records, *, source_urdf=None, verify_decisions base = configure_training(profile, profile.quality.training_policy) headers = [row for row in records if row.get("kind") == "session_start"] plan = CapturePlan.from_profile(base) - if base.quality.training_policy != ADAPTIVE: + if base.quality.training_policy not in {ADAPTIVE, TASKWISE_STAGED}: if any(row.get("kind") in {"training_decision", "staged_training_decision"} for row in records): raise ValueError("adaptive_decisions_in_fixed_capture") if headers and "capture_plan" in headers[0] and headers[0]["capture_plan"] != plan.as_dict(): diff --git a/src/linkerhand_calibration/linkerhand_calibration/runtime/zero_recovery.py b/src/linkerhand_calibration/linkerhand_calibration/runtime/zero_recovery.py index b817fef..7446d42 100644 --- a/src/linkerhand_calibration/linkerhand_calibration/runtime/zero_recovery.py +++ b/src/linkerhand_calibration/linkerhand_calibration/runtime/zero_recovery.py @@ -63,6 +63,22 @@ class ZeroRecovery: def restore(self, task_key, joints): self._used.add((task_key, tuple(joints))) + def take_parent_model_refresh(self, motion, *, stale_image_anchor): + """Spend one fresh-model allowance for a resumed parent reference. + + This does not reacquire its joint zero or scan. It only permits the + already declared safe approach to rebuild the image-domain model in + the current session's stationary camera coordinate frame. + """ + if (not motion.reference_reuse or not stale_image_anchor + or len(motion.zero_joints) != 1): + return None + key = (motion.task_key, tuple(motion.zero_joints)) + if key in self._used: + return None + self._used.add(key) + return "refresh_parent_model" + def take(self, motion, *, geometry_error, reports, referenced_joints): if motion.reference_reuse: return None diff --git a/src/linkerhand_calibration/test/test_checkpoint_observer_upgrade.py b/src/linkerhand_calibration/test/test_checkpoint_observer_upgrade.py index 29f0a5b..d289b8a 100644 --- a/src/linkerhand_calibration/test/test_checkpoint_observer_upgrade.py +++ b/src/linkerhand_calibration/test/test_checkpoint_observer_upgrade.py @@ -11,7 +11,7 @@ from linkerhand_calibration.profiles import load_bundled_hand_profile from linkerhand_calibration.profiles.export import dump_hand_profile from linkerhand_calibration.profiles.observer_policy import configure_observer_policy from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract -from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade +from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade, _copy_indexed_evidence from linkerhand_calibration.runtime.joint_resume import _index_motion_provenance @@ -81,3 +81,18 @@ def test_active_observation_migration_only_adds_unmeasured_bounded_paths(migrati checkpoint.joints.references['index_dip']=object() with pytest.raises(ValueError,match='changes_existing_contract'): upgrade.metadata(checkpoint,inputs,'source.jsonl','c'*64) + + +def test_indexed_checkpoint_migration_copies_content_bound_archive(tmp_path): + from linkerhand_calibration.core.artifacts.evidence_journal import ( + initialize_evidence_journal, append_indexed_records, archive_path, manifest_path, + ) + source = tmp_path/'source/raw_samples.jsonl' + source.parent.mkdir() + initialize_evidence_journal(source, threshold_bytes=128) + append_indexed_records(source, ({'kind': 'pnp_candidate_frame', 'view': 'side', + 'image_stamp_ns': 1, 'pixels': 'x'*2048},), durable=True) + target = tmp_path/'target'; target.mkdir() + _copy_indexed_evidence(source, target) + assert (target/archive_path(source).name).read_bytes() == archive_path(source).read_bytes() + assert (target/manifest_path(source).name).read_bytes() == manifest_path(source).read_bytes() diff --git a/src/linkerhand_calibration/test/test_o30_profile.py b/src/linkerhand_calibration/test/test_o30_profile.py index 26a5bc7..e402da4 100644 --- a/src/linkerhand_calibration/test/test_o30_profile.py +++ b/src/linkerhand_calibration/test/test_o30_profile.py @@ -117,7 +117,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile): segment = task.segments[0] start = build_calibration_motion_command(task, segment.start, profile=profile, segment_key=segment.key) end = build_calibration_motion_command(task, segment.end, profile=profile, segment_key=segment.key) - execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200), + execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200, + measured_channels=segment.moving_channels), initial_command=start, initial_feedback=start, now=0., identity='linked', visual_observer=VisualMotionObserver(profile)) previous = execution.sample(0) @@ -127,8 +128,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile): assert actual[3] == pytest.approx(80+175*progress, abs=1.) assert actual[2] == actual[4] == actual[5] # Integer bytes carry at most one unit of quantization error. - assert max(abs(a-b) for a,b in zip(actual,previous)) <= 30*.01+1. - assert math.pi*255/(2*execution.duration) <= 30.000001 + assert max(abs(a-b) for a,b in zip(actual,previous)) <= 20*.01+1. + assert math.pi*255/(2*execution.duration) <= 20.000001 previous = actual diff --git a/src/linkerhand_calibration/test/test_parent_reference_reuse.py b/src/linkerhand_calibration/test/test_parent_reference_reuse.py index abb8982..35c261d 100644 --- a/src/linkerhand_calibration/test/test_parent_reference_reuse.py +++ b/src/linkerhand_calibration/test/test_parent_reference_reuse.py @@ -22,6 +22,8 @@ from linkerhand_calibration.runtime.parent_reference import ParentReferenceRegis from linkerhand_calibration.runtime.capture import ObservationCapture from linkerhand_calibration.runtime.reference_lock import ReferenceLock from linkerhand_calibration.runtime.motion_execution import MotionCommand +from linkerhand_calibration.runtime.execution import SessionExecution +from linkerhand_calibration.runtime.session import CalibrationPhase as Phase from linkerhand_calibration.runtime.zero_recovery import ZeroRecovery from o30_capture_fixture import SOURCE @@ -101,6 +103,11 @@ def test_registry_needs_both_sources_and_fresh_check_and_preserves_zeros(setup): with pytest.raises(ValueError, match="source_missing"): owner.sources(task, 2) with pytest.raises(ValueError, match="unverified"): owner.geometry(task, 1) + refresh = owner.authorize_refreshed_pose(task, 1, 9, reference) + assert refresh["passed_zero_preserved"] is True + assert refresh["image_model_sha256"] == roll_report["image_model_sha256"] + assert owner.pose_source(task, 1) == (roll, roll_report) + for row in rows: row["pnp_observation_evidence"] = {"child": dict(tag_role="thumb_mcp", image_model_sha256=roll_report["image_model_sha256"], @@ -168,11 +175,42 @@ def test_stationary_reference_cannot_trigger_a_new_solver_or_extra_motion_retry( assert owner.request("front", motion, session_epoch=1, motion_version=10) is None recovery = ZeroRecovery() assert recovery.take(motion, geometry_error="", reports={}, referenced_joints=()) is None + assert recovery.take_parent_model_refresh(motion, stale_image_anchor=False) is None ordinary = replace(motion, reference_only=False, reference_reuse=False, zero_joints=("thumb_ip",)) assert recovery.take(ordinary, geometry_error="ambiguous", reports={"front": { "status": "unresolved", "reason": "image_motion_families_not_distinguishable"}}, referenced_joints=()) is None +def test_stale_cross_session_anchor_gets_one_model_refresh_without_replacing_zero(setup): + profile, task, _ = setup + parent = task.parent_reference.pose_joint + motion = MotionCommand("joint_zero", + profile.motion.joint_zero_references[task.joints[0]].command, 200, + task_key=task.key, cycle=0, zero_joints=(parent,), + reference_only=True, reference_reuse=True) + recovery = ZeroRecovery() + assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) == "refresh_parent_model" + assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) is None + + execution = SessionExecution(profile) + execution.session.phase = Phase.PREPARE + frozen = object() + execution.zero_references[parent] = frozen + execution._effects = [motion] + execution.refresh_parent_model(motion.target) + assert execution.zero_references[parent] is frozen + assert execution._effects[-1] is motion + refresh = execution._effects[:-1] + assert refresh and refresh[-1].phase == "joint_zero" + assert refresh[-1].zero_joints == (parent,) + assert all(item.parent_model_refresh_for == task.key for item in refresh) + assert all(item.reference_only and not item.reference_reuse for item in refresh) + moving = [item for item in refresh if item.phase == "zero_approach"] + from linkerhand_calibration.core.fitting.motion_fit import channel_for_joint + channel = channel_for_joint(profile, parent) + assert max(item.target[channel] for item in moving) != min(item.target[channel] for item in moving) + + def test_coordinator_waits_for_current_parent_images_and_keeps_both_joint_zeros(setup, tmp_path, monkeypatch): from linkerhand_calibration.runtime.coordinator import CalibrationCoordinator from linkerhand_calibration.runtime import coordinator as module diff --git a/src/linkerhand_calibration/test/test_passed_resume.py b/src/linkerhand_calibration/test/test_passed_resume.py index 26031ac..ca9c9e1 100644 --- a/src/linkerhand_calibration/test/test_passed_resume.py +++ b/src/linkerhand_calibration/test/test_passed_resume.py @@ -44,6 +44,73 @@ def test_passed_parent_model_keeps_original_source_in_new_observation_epoch(): assert dict(pose_source[0].evidence_ids)['moving'] == 'a' * 64 +def test_passed_resume_does_not_restore_incomplete_task_preparation(monkeypatch): + profile = load_bundled_hand_profile('o30_right_18') + engine, resume = CalibrationEngine(profile), JointResume(profile) + passed, incomplete = profile.motion.tasks[:2] + rows = [*passed_records(engine, passed), + dict(kind='motion_branch_observation', task_name=passed.key, + view=passed.view, image_stamp_ns=1, session_epoch=1), + dict(kind='motion_branch_observation', task_name=incomplete.key, + view=incomplete.view, image_stamp_ns=2, session_epoch=1), + dict(kind='motion_branch_initialization', task_name=incomplete.key, + view=incomplete.view, status='resolved', session_epoch=1)] + monkeypatch.setattr(passed_resume, 'read_joint_zero_references', + lambda *args: stub_references([passed, incomplete])) + + passed_resume.stage_passed_tasks(resume, engine, rows) + pending = passed_resume.reference_import(resume) + + assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key} + assert not pending.model_records + + +def test_taskwise_passed_resume_does_not_restore_unpassed_visit_model(): + from linkerhand_calibration.runtime import staged_resume + + profile = load_bundled_hand_profile('o30_right_18') + passed, incomplete = profile.motion.tasks[:2] + references = stub_references([passed, incomplete]) + rows = [ + dict(kind='motion_branch_observation', task_name=passed.key, + view=passed.view, image_stamp_ns=1, session_epoch=1), + dict(kind='motion_branch_observation', task_name=incomplete.key, + view=incomplete.view, image_stamp_ns=2, session_epoch=1), + dict(kind='motion_branch_initialization', task_name=incomplete.key, + view=incomplete.view, status='resolved', session_epoch=1), + ] + accepted = next(unit for unit in CalibrationEngine(profile).scan_units() + if unit.task_key == passed.key) + resume = JointResume(profile) + resume.rows, resume.references, resume.units = tuple(rows), references, {accepted.identity} + + pending = staged_resume.reference_import(resume, reuse_passed_visits=True) + + assert set(pending.references) == set(passed.joints) + assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key} + assert not pending.model_records + + +def test_passed_resume_omits_orphaned_observer_cancellation(): + from linkerhand_calibration.runtime import staged_resume + + profile = load_bundled_hand_profile('o30_right_18') + task = profile.motion.tasks[0] + accepted = next(unit for unit in CalibrationEngine(profile).scan_units() + if unit.task_key == task.key) + orphan = dict(kind='observer_preparation_cancelled_before_excursion', + joint=task.joints[0], observer_preparation_id='a'*64, + session_epoch=1, stamp_ns=1) + resume = JointResume(profile) + resume.rows = (orphan,) + resume.references = stub_references([task]) + resume.units = {accepted.identity} + + pending = staged_resume.reference_import(resume, reuse_passed_visits=True) + + assert orphan not in pending.rows + + @pytest.mark.parametrize('failure', [None, 'missing', 'failed', 'support', 'new_attempt']) def test_last_group_failure_keeps_128_prior_passes_without_recalculating(monkeypatch, failure): profile = load_bundled_hand_profile('o30_right_18') @@ -81,7 +148,10 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon expected_units = sum(unit.task_key == task.key for unit in host.execution.session.engine.scan_units()) rows = [dict(kind='motion_branch_observation', view=task.view, image_stamp_ns=i, session_epoch=1, task_name=task.key) - for i in range(257)] + passed_records(host.execution.session.engine, task) + for i in range(257)] + [dict(kind='observer_preparation_decision', + task_name=task.key, joint=task.joints[0], action='excite', + action_index=0, session_epoch=1, stamp_ns=1)] + passed_records( + host.execution.session.engine, task) references = stub_references([task]) monkeypatch.setattr(passed_resume, 'read_joint_zero_references', lambda *args: references) resume = JointResume(host.profile) @@ -123,13 +193,19 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon assert host.state in {'ABORTED', 'PAUSED'} assert not host.execution.zero_references and host._resumed_count == 0 else: - for _ in range(20): + # Reference restoration is intentionally sliced across callbacks + # and journal barriers run on a worker. Give that worker real + # scheduling opportunities instead of spinning only the fake clock. + import time + for _ in range(400): host.receive_feedback(host.profile.command.names, host.profile.command.baseline_values, host.ports.clock_ns()) host.tick() if host.execution.session.current_unit.task_key != task.key: break + time.sleep(.001) assert host._resumed_count == expected_units assert host.execution.zero_references == references + assert not host.branch_initialization.observer_preparation.decisions assert host.segment is None import json imported = [json.loads(line) for line in host.raw_path.read_text().splitlines()] diff --git a/src/linkerhand_calibration/test/test_resume_storage.py b/src/linkerhand_calibration/test/test_resume_storage.py index 432d762..636c7e7 100644 --- a/src/linkerhand_calibration/test/test_resume_storage.py +++ b/src/linkerhand_calibration/test/test_resume_storage.py @@ -10,6 +10,8 @@ from linkerhand_calibration.runtime.resume_storage import ( DeferredCaptureRecord, load_passed_capture, ) from linkerhand_calibration.runtime.joint_resume import ResumeJournalImport, _index_motion_provenance +from linkerhand_calibration.runtime.staged_resume import _essential_indexed_provenance +from linkerhand_calibration.runtime.motion_provenance import source_frame_sha256 from linkerhand_calibration.core.domain.motion_path import record_scan_identity from linkerhand_calibration.runtime.diagnostic_capture import reject_diagnostic_capture from linkerhand_calibration.core.artifacts.evidence_journal import ( @@ -18,6 +20,8 @@ from linkerhand_calibration.core.artifacts.evidence_journal import ( ) from linkerhand_calibration.storage import append_jsonl, append_jsonl_many from linkerhand_calibration.runtime.acquisition import load_capture +from linkerhand_calibration.runtime.capture_index import CaptureRecordIndex +from linkerhand_calibration.runtime.resume import discover_resume_candidate def capture(tmp_path, rows): @@ -49,7 +53,7 @@ def test_scalar_index_never_reads_image_payload_and_import_is_bounded(tmp_path, assert len(batch) == size and len(calls)-before == size assert all(type(r) is dict for r in batch) imported.extend(batch) - assert pending.complete and imported == original + assert pending.transfer_complete and not pending.complete and imported == original assert all("pnp_observation_evidence" not in r.metadata for r in rows) @@ -101,6 +105,7 @@ def test_indexed_journal_keeps_resume_fields_inline_and_restores_full_evidence(t deferred = load_passed_capture(path) assert all(isinstance(row, DeferredCaptureRecord) for row in deferred[1:]) assert deferred[1]["relative_quaternion_xyzw"] == [0., 0., 0., 1.] + assert "relative_quaternion_xyzw" not in deferred[1].metadata assert deferred[1].materialize() == samples[0] @@ -162,3 +167,90 @@ def test_resume_import_hard_links_compressed_evidence_without_copying_records(tm archive_path(source).unlink() source.unlink() assert load_capture(target) == original + + +def test_linked_joint_samples_use_bounded_batches_and_concrete_training_digest(tmp_path): + path = tmp_path / "raw_samples.jsonl" + initialize_evidence_journal(path, threshold_bytes=1) + leading = [dict(kind="image_observation_frame", task_name="index_dip_side", + cycle=0, direction="increasing", image_stamp_ns=-stamp-1, + pixels=list(range(100))) for stamp in range(10)] + samples = [dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip", + cycle=0, direction="increasing", image_stamp_ns=stamp, + command_u8=stamp, feedback_u8=stamp, relative_quaternion_xyzw=[0., 0., 0., 1.]) + for stamp in range(100)] + append_jsonl_many(path, (*leading, *samples)) + pending = ResumeJournalImport(tuple(load_passed_capture(path))) + + assert len(pending.take_batch()) == len(leading) + first = pending.take_batch() + assert len(first) == len(samples) and all(row.get("kind") == "joint_sample" for row in first) + index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True) + index.extend(first) + assert len(index) == len(samples) + assert all(len(row["source_record_sha256"]) == 64 for row in index) + + +def test_training_index_materializes_each_deferred_sample_once(tmp_path, monkeypatch): + path = tmp_path / "raw_samples.jsonl" + initialize_evidence_journal(path, threshold_bytes=1) + sample = dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip", + cycle=0, direction="increasing", image_stamp_ns=1, command_u8=20, + feedback_u8=19, relative_quaternion_xyzw=[0., 0., 0., 1.], + relative_translation_xyz_m=[0., 0., .01]) + append_jsonl(path, sample) + row = load_passed_capture(path)[0] + original = DeferredCaptureRecord.materialize_plain + calls = [] + + def observed(record): + calls.append(record.reference.record_sha256) + return original(record) + + monkeypatch.setattr(DeferredCaptureRecord, "materialize_plain", observed) + index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True) + index.append(row) + + assert len(calls) == 1 + assert index[0]["relative_translation_xyz_m"] == [0., 0., .01] + + +def test_compact_passed_resume_keeps_only_digest_referenced_motion_frames(tmp_path): + path = tmp_path / "raw_samples.jsonl" + initialize_evidence_journal(path, threshold_bytes=1) + used = {"kind": "motion_branch_observation", "task_name": "index_dip_side", + "view": "side", "image_stamp_ns": 10, "pixels": list(range(100))} + unused = {**used, "image_stamp_ns": 20, "pixels": list(range(101))} + accepted = {"kind": "motion_branch_initialization", "status": "resolved", + "task_name": "index_dip_side", "view": "side", + "source_frame_hashes": {"10": source_frame_sha256(used)}} + append_jsonl_many(path, (used, unused, accepted)) + + indexed = tuple(_index_motion_provenance(load_passed_capture(path)).values()) + essential = _essential_indexed_provenance(indexed) + + observations = [row for row in essential if row.get("kind") == "motion_branch_observation"] + assert len(observations) == 1 + assert observations[0].canonical_sha256 == source_frame_sha256(used) + + +def test_discovery_reads_large_compressed_session_header(tmp_path): + from linkerhand_calibration.runtime.engine import ACQUISITION_POLICY_VERSION + from linkerhand_calibration.core.geometry.pnp import POSE_TRACKING_POLICY_VERSION + session = tmp_path / "SERIAL" / "20260101_000000" + session.mkdir(parents=True) + path = session / "raw_samples.jsonl" + initialize_evidence_journal(path, threshold_bytes=128) + hashes = {"profile_config_sha256": "a" * 64} + contract = {"large": list(range(100))} + append_jsonl_many(path, [{"kind": "session_start", "profile_id": "O30/right/layout/v1", + "serial_number": "SERIAL", "acquisition_policy_version": ACQUISITION_POLICY_VERSION, + "capture_schedule_version": "unified_schedule_v13_staged_training", + "pose_tracking_policy_version": POSE_TRACKING_POLICY_VERSION, "protected_hashes": hashes, + "capture_plan": {"policy": "staged_2_to_3"}, "preparation_contract": contract}, + {"kind": "fixed_base_reference_locked"}, + {"kind": "scan_unit_complete", "task_name": "index_dip_side", "cycle": 0, + "direction": "increasing", "passed": True}]) + assert discover_resume_candidate(tmp_path / "SERIAL", profile_id="O30/right/layout/v1", + serial_number="SERIAL", protected_hashes=hashes, training_policy="staged_2_to_3", + preparation_contract=contract) == session diff --git a/src/linkerhand_calibration/test/test_rigid_pair_observer.py b/src/linkerhand_calibration/test/test_rigid_pair_observer.py index a524958..b292fbc 100644 --- a/src/linkerhand_calibration/test/test_rigid_pair_observer.py +++ b/src/linkerhand_calibration/test/test_rigid_pair_observer.py @@ -10,6 +10,7 @@ from linkerhand_calibration.profiles import load_hand_profile from linkerhand_calibration.profiles.export import write_hand_profile from linkerhand_calibration.profiles.validator import validate_executable_profile from linkerhand_calibration.profiles.preparation import preparation_contract +from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER from linkerhand_calibration.runtime.observer_reference import ObserverBridgeInput,build_observer_bridge,validate_observer_references from linkerhand_calibration.runtime.shared_tag_reference import bridge_frames from linkerhand_calibration.runtime.branch_initialization import BranchInitializationRequest,solve_initialization,initialization_artifacts @@ -72,6 +73,17 @@ def test_independent_parent_branches_converge_on_real_bounded_probe(tmp_path): assert bridge.translation_uncertainty_m<.001 assert all(p==0 for _,p in record['pose']['comparison_adjusted_p_values']) assert set(record['pose']['training_stamps']).isdisjoint(record['pose']['validation_stamps']) + measurements=dict(profile.measurement.measurements) + measurements['index_dip']=replace(measurements['index_dip'], + observer_pose_policy=TRAJECTORY_OBSERVER) + trajectory_profile=replace(profile,measurement=replace(profile.measurement, + measurements=measurements)) + trajectory_bridge,trajectory_record=build_observer_bridge(trajectory_profile, + ObserverBridgeInput('index_dip',model,parent,(),endpoints,(16,),evidence)) + assert trajectory_record['version']=='observer_pose_bridge_v4' + assert trajectory_record['trajectory_branch_decision']['holdout_role']=='veto_only' + assert trajectory_record['trajectory_branch_decision']['corner_evidence_sha256'] + assert trajectory_bridge.rotation_uncertainty_rad<.017453293 @pytest.fixture(scope='module') diff --git a/src/linkerhand_calibration/test/test_staged_training.py b/src/linkerhand_calibration/test/test_staged_training.py index 70478d2..e6414b6 100644 --- a/src/linkerhand_calibration/test/test_staged_training.py +++ b/src/linkerhand_calibration/test/test_staged_training.py @@ -7,7 +7,9 @@ from pathlib import Path import pytest -from linkerhand_calibration.core.domain.capture_plan import STAGED, CapturePlan, configure_training, select_task_training +from linkerhand_calibration.core.domain.capture_plan import ( + STAGED, TASKWISE_STAGED, CapturePlan, configure_training, select_task_training, +) from linkerhand_calibration.core.domain.reference import read_joint_zero_references from linkerhand_calibration.profiles import load_bundled_hand_profile from linkerhand_calibration.runtime.engine import CalibrationEngine @@ -41,6 +43,52 @@ def test_all_first_visits_precede_second_and_validation_keeps_original_motion(): assert fixed.quality.training_policy == 'fixed' +def test_taskwise_schedule_keeps_first_pass_and_validation_global_but_supplements_locally(): + profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED) + plan = CapturePlan.from_profile(profile).as_dict() + assert plan['version'] == 'capture_plan_v3' + assert plan['schedule'] == ( + 'first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation' + ) + first_task = profile.motion.tasks[0] + profile = select_task_training(profile, first_task.key, (0, 1, 2)) + units = CalibrationEngine(profile).scan_units() + visits = [] + for unit in units: + visit = unit.task_key, unit.cycle + if not visits or visits[-1] != visit: + visits.append(visit) + task_keys = [task.key for task in profile.motion.tasks] + assert visits[:len(task_keys)] == [(key, 0) for key in task_keys] + training_visits = visits[len(task_keys):-len(task_keys)] + assert training_visits[:2] == [(first_task.key, 1), (first_task.key, 2)] + assert training_visits[2:] == [(key, 1) for key in task_keys[1:]] + assert visits[-len(task_keys):] == [(key, 3) for key in task_keys] + + +def test_taskwise_resume_carries_the_decision_that_created_optional_round(): + from linkerhand_calibration.runtime.joint_resume import JointResume + from linkerhand_calibration.runtime.staged_resume import visit_rows + + profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED) + task = profile.motion.tasks[0] + profile = select_task_training(profile, task.key, (0, 1, 2)) + units = [unit for unit in CalibrationEngine(profile).scan_units() + if unit.task_key == task.key and unit.cycle == 2] + decision = {'kind': 'training_decision', 'task_name': task.key, + 'assessed_cycles': [0, 1, 2], 'decision': 'freeze'} + rows = [dict(kind='scan_unit_complete', task_name=unit.task_key, + cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key, + passed=True) for unit in units] + resume = JointResume(profile) + resume.rows = (*rows, decision) + resume.units = {unit.identity for unit in units} + resume.verified_visits.update((joint, 2) for joint in task.joints) + used, imported = visit_rows(resume, task.key, 2) + assert used == tuple(sorted(unit.identity for unit in units)) + assert [row for row in imported if row.get('kind') == 'training_decision'] == [decision] + + def test_revisit_reuses_model_but_checks_zero_via_existing_path(): profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED) execution = SessionExecution(profile) diff --git a/src/linkerhand_calibration/test/test_trajectory_observation_v3.py b/src/linkerhand_calibration/test/test_trajectory_observation_v3.py new file mode 100644 index 0000000..22dda0b --- /dev/null +++ b/src/linkerhand_calibration/test/test_trajectory_observation_v3.py @@ -0,0 +1,207 @@ +"""v3 trajectory observation, bounded deferral and publication refusal.""" + +import hashlib +from types import SimpleNamespace + +import pytest + +from linkerhand_calibration.core.domain.capture_plan import ( + STAGED, TASKWISE_STAGED, configure_training, select_task_training, +) +from linkerhand_calibration.core.urdf.kinematics import UrdfKinematicModel +from linkerhand_calibration.profiles import load_bundled_hand_profile +from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER +from linkerhand_calibration.profiles.observer_policy import configure_observer_policy +from linkerhand_calibration.profiles.observer_preparation import configure_observer_preparations +from linkerhand_calibration.profiles.export import dump_hand_profile +from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract +from linkerhand_calibration.runtime.artifacts.capture_validation import validate_capture_units +from linkerhand_calibration.runtime.session import CalibrationPhase, CalibrationSession +from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade +from linkerhand_calibration.runtime.task_deferral import ( + POLICY, TaskDeferralQueue, TaskObservationState, deferred_visit_keys, + dependency_closure, recoverable_observation_reason, +) +from o30_capture_fixture import SOURCE + + +@pytest.fixture +def trajectory_profile(): + profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED) + profile = configure_preparation(profile, 'occlusion_aware_trajectory_v3') + return configure_observer_preparations(profile, UrdfKinematicModel(SOURCE)) + + +def test_v3_is_a_versioned_profile_contract_with_two_action_bound(trajectory_profile): + policy = preparation_contract(trajectory_profile)['observer_preparations'] + assert policy['policy'] == 'occlusion_aware_trajectory_v3' + assert policy['trajectory_solver']['holdout_selects_model'] is False + assert policy['trajectory_solver']['maximum_actions_per_task'] == 2 + assert all(1 <= len(plan.actions) <= 2 + for plan in trajectory_profile.motion.observer_preparations.values()) + assert trajectory_profile.measurement.measurements['index_dip'].observer_pose_policy \ + == TRAJECTORY_OBSERVER + + +def test_visual_ambiguity_is_deferred_but_hardware_failure_is_not(): + assert recoverable_observation_reason('image_motion_frame_angles_ambiguous') + assert recoverable_observation_reason('motion_candidates_missing') + assert not recoverable_observation_reason('sdk_disconnected') + assert not recoverable_observation_reason('feedback_stalled') + assert not recoverable_observation_reason('over_temperature') + + +def test_parent_deferral_moves_dependency_closure_to_stage_end(trajectory_profile): + assert 'index_dip_side' in dependency_closure(trajectory_profile, 'index_pip_side') + session = CalibrationSession(trajectory_profile) + session.phase = CalibrationPhase.PREPARE + session._unit_index = next(i for i, unit in enumerate(session._units) + if unit.task_key == 'index_pip_side' and unit.cycle == 0) + original_prefix = tuple(unit.identity for unit in session._units[:session._unit_index]) + decision = session.defer_current_task('rigid_pair_candidates_unresolved') + assert decision.state == TaskObservationState.PENDING + assert {'index_pip_side', 'index_dip_side'} <= set(decision.affected_tasks) + record = decision.as_record(stamp_ns=1) + assert record['dependency_states']['index_dip_side'] == 'dependency_pending' + assert tuple(unit.identity for unit in session._units[:session._unit_index]) == original_prefix + same_cycle = [unit.task_key for unit in session._units if unit.cycle == 0] + assert same_cycle[-1] == 'index_dip_side' + + +def test_staged_assessment_waits_for_deferred_visit(trajectory_profile): + from linkerhand_calibration.runtime.execution import SessionExecution + execution = SessionExecution(trajectory_profile) + session = execution.session + session.phase = CalibrationPhase.PREPARE + session._unit_index = next(i for i, unit in enumerate(session._units) + if unit.task_key == 'index_pip_side' and unit.cycle == 1) + session.defer_current_task('rigid_pair_candidates_unresolved') + cycle = [unit for unit in session.scheduled_units if unit.cycle == 1] + assert cycle[-1].task_key == 'index_dip_side' + session._unit_index = session._units.index(cycle[-2]) + assert not execution.training_check_due() + session._unit_index = session._units.index(cycle[-1]) + assert execution.training_check_due() + + +def test_second_failure_is_terminal_and_budget_survives_restart(trajectory_profile): + queue = TaskDeferralQueue(trajectory_profile) + first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2) + records = [first.as_record(stamp_ns=1, corner_evidence_sha256='a'*64)] + restored = TaskDeferralQueue(trajectory_profile) + restored.restore(records) + second = restored.decide('index_dip_side', 0, + 'rigid_pair_candidates_unresolved', 2) + assert second.state == TaskObservationState.UNOBSERVABLE + records.append(second.as_record(stamp_ns=2, corner_evidence_sha256='b'*64)) + again = TaskDeferralQueue(trajectory_profile) + again.restore(records) + assert again.attempts[('index_dip_side', 0)] == 2 + assert ('index_dip_side', 0) in again.unobservable + with pytest.raises(ValueError, match='budget_exhausted'): + again.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 1) + with pytest.raises(ValueError, match='budget_changed'): + again.restore([first.as_record(stamp_ns=3)]) + + +def test_restart_restores_pending_visit_at_end_of_taskwise_stage(trajectory_profile): + profile = configure_training(trajectory_profile, TASKWISE_STAGED) + profile = select_task_training(profile, profile.motion.tasks[0].key, (0, 1, 2)) + queue = TaskDeferralQueue(profile) + pending = queue.decide('index_dip_side', 0, + 'image_motion_no_observable_image_model', 2).as_record(stamp_ns=1) + session = CalibrationSession(profile) + session.restore_task_deferrals([pending]) + first_stage = [unit for unit in session.scheduled_units if unit.cycle == 0] + assert first_stage[-1].task_key == 'index_dip_side' + assert session.task_deferrals.attempts[('index_dip_side', 0)] == 1 + + # The same rule treats cycle 1 and optional cycle 2 as one training stage, + # so a restored revisit cannot jump ahead of later training or into holdout. + queue = TaskDeferralQueue(profile) + pending = queue.decide('index_dip_side', 1, + 'image_motion_no_observable_image_model', 2).as_record(stamp_ns=2) + session = CalibrationSession(profile) + session.restore_task_deferrals([pending]) + training_stage = [unit for unit in session.scheduled_units if unit.cycle in {1, 2}] + assert training_stage[-1].task_key == 'index_dip_side' + assert session.scheduled_units[session.scheduled_units.index(training_stage[-1]) + 1].cycle == 3 + + +def test_deferral_audit_allows_only_declared_noncontiguous_resume_gap(trajectory_profile): + queue = TaskDeferralQueue(trajectory_profile) + pending = queue.decide('index_pip_side', 1, + 'rigid_pair_candidates_unresolved', 4).as_record(stamp_ns=1) + keys = deferred_visit_keys(trajectory_profile, [pending]) + assert ('index_pip_side', 1) in keys + assert ('index_dip_side', 1) in keys + changed = {**pending, 'affected_tasks': ['index_pip_side']} + with pytest.raises(ValueError, match='dependencies_changed'): + deferred_visit_keys(trajectory_profile, [changed]) + + +def test_full_visit_commit_closes_crash_window_but_partial_visit_does_not(trajectory_profile): + from linkerhand_calibration.runtime.engine import CalibrationEngine + queue = TaskDeferralQueue(trajectory_profile) + pending = queue.decide('index_dip_side', 1, + 'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1) + units = [unit for unit in CalibrationEngine(trajectory_profile).scan_units() + if (unit.task_key, unit.cycle) == ('index_dip_side', 1)] + completed = [dict(kind='scan_unit_complete', task_name=unit.task_key, + cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key, + passed=True) for unit in units] + partial = TaskDeferralQueue(trajectory_profile) + partial.restore([pending, *completed[:-1]]) + assert ('index_dip_side', 1) not in partial.resolved + restored = TaskDeferralQueue(trajectory_profile) + restored.restore([pending, *completed]) + assert ('index_dip_side', 1) in restored.inferred_resolutions + + +def test_unobservable_record_blocks_release_before_fitting(trajectory_profile): + queue = TaskDeferralQueue(trajectory_profile) + first = queue.decide('index_dip_side', 0, + 'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1) + row = queue.decide('index_dip_side', 0, + 'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=2) + with pytest.raises(ValueError, match='capture_unobservable_tasks:index_dip_side:0:unobservable'): + validate_capture_units(trajectory_profile, [first, row]) + + +def test_resolved_revisit_clears_release_blocker_state(trajectory_profile): + queue = TaskDeferralQueue(trajectory_profile) + first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2) + passed = queue.resolve('index_dip_side', 0) + assert passed is not None and passed.state == TaskObservationState.PASSED + restored = TaskDeferralQueue(trajectory_profile) + restored.restore([first.as_record(stamp_ns=1), passed.as_record(stamp_ns=2)]) + assert ('index_dip_side', 0) in restored.resolved + assert not restored.unobservable + + +def test_v3_checkpoint_upgrade_preserves_accepted_joint_semantics(): + base = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED) + source = configure_preparation(base, 'occlusion_aware_witness_v2') + source = configure_observer_preparations(source, UrdfKinematicModel(SOURCE)) + preserved = set(source.measurement.measurements)-{'index_dip'} + target = configure_observer_policy(source, preserved_joints=preserved, + policy=TRAJECTORY_OBSERVER) + source_yaml, target_yaml = (dump_hand_profile(profile).encode() + for profile in (source, target)) + upgrade = ObserverPolicyUpgrade(source, target, source_yaml, target_yaml, + source_urdf=SOURCE) + inputs = {'profile_config_sha256': hashlib.sha256(target_yaml).hexdigest(), + 'source_urdf_sha256': 'a'*64} + old_inputs = {**inputs, 'profile_config_sha256': hashlib.sha256(source_yaml).hexdigest()} + header = {'kind': 'session_start', **old_inputs, + 'preparation_contract': preparation_contract(source)} + reference = {'kind': 'fixed_base_reference_locked', + 'protected_hashes': {**old_inputs, 'intrinsics_sha256': 'b'*64}} + checkpoint = SimpleNamespace(header=header, fixed_reference=reference, + rows=(header, reference), joints=SimpleNamespace( + references={joint: object() for joint in preserved}, units=set())) + _, _, audit = upgrade.metadata(checkpoint, inputs, 'source.jsonl', 'c'*64) + assert audit['policy'] == 'unmeasured_observer_upgrade_v3' + assert audit['preserved_joints'] == sorted(preserved) + assert audit['observer_policy_changes'] == { + 'index_dip': {'previous': 'rigid_pair_v1', 'current': TRAJECTORY_OBSERVER}}