O30标定修改

This commit is contained in:
lxp
2026-09-21 18:16:44 +08:00
parent 54b8f66b2f
commit 8456aa7f2d
47 changed files with 2043 additions and 245 deletions
@@ -22,19 +22,10 @@ import zlib
FORMAT = "linkerhand_indexed_evidence_v1" FORMAT = "linkerhand_indexed_evidence_v1"
REFERENCE_KEY = "$evidence_record" REFERENCE_KEY = "$evidence_record"
FIELDS_KEY = "$record_fields" FIELDS_KEY = "$record_fields"
SCHEMA_KEY = "$record_schema"
CANONICAL_SHA256_KEY = "$record_canonical_sha256" CANONICAL_SHA256_KEY = "$record_canonical_sha256"
DEFAULT_THRESHOLD_BYTES = 16 * 1024 DEFAULT_THRESHOLD_BYTES = 16 * 1024
MAXIMUM_BLOCK_BYTES = 4 * 1024 * 1024 MAXIMUM_BLOCK_BYTES = 4 * 1024 * 1024
INDEXED_SEQUENCE_FIELDS = {
"relative_quaternion_xyzw", "relative_translation_xyz_m",
"parent_pose_common", "child_pose_common", "view_normal_common_xyz",
"camera_center_common_xyz_m", "state_u8", "state_rad",
"command_vector_u8", "command_vector_rad", "steady_training_nodes",
"command_direction_by_index",
}
MOTION_INDEX_FIELDS = {
"zero_joints", "command_vector", "feedback_vector",
}
INDEXED_SCALAR_FIELDS = { INDEXED_SCALAR_FIELDS = {
"kind", "profile_id", "serial_number", "task_name", "joint", "kind", "profile_id", "serial_number", "task_name", "joint",
"observation_joint", "view", "image_stamp_ns", "cycle", "direction", "observation_joint", "view", "image_stamp_ns", "cycle", "direction",
@@ -60,7 +51,7 @@ CANONICAL_INDEX_KINDS = {
"capture_scope_extension", "capture_observation_revision", "capture_scope_extension", "capture_observation_revision",
"passed_tasks_reused", "passed_pose_models_restored", "staged_pose_models_restored", "passed_tasks_reused", "passed_pose_models_restored", "staged_pose_models_restored",
"resume_verification", "joint_zero_recovery", "visual_motion_observation", "resume_verification", "joint_zero_recovery", "visual_motion_observation",
"visual_motion_settled", "visual_motion_settled", "task_observation_state",
} }
@@ -113,7 +104,8 @@ def archive_path(journal_path: str | Path) -> Path:
return path.with_name(path.stem + ".evidence.zlib") return path.with_name(path.stem + ".evidence.zlib")
def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEFAULT_THRESHOLD_BYTES) -> None: def initialize_evidence_journal(journal_path: str | Path, *,
threshold_bytes=DEFAULT_THRESHOLD_BYTES, compression_level=1) -> None:
"""Enable indexed storage for a new journal without changing old captures.""" """Enable indexed storage for a new journal without changing old captures."""
journal = Path(journal_path) journal = Path(journal_path)
marker = manifest_path(journal) marker = manifest_path(journal)
@@ -125,11 +117,16 @@ def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEF
if journal.exists() and journal.stat().st_size: if journal.exists() and journal.stat().st_size:
raise ValueError("indexed_evidence_requires_new_or_migrated_journal") raise ValueError("indexed_evidence_requires_new_or_migrated_journal")
journal.parent.mkdir(parents=True, exist_ok=True) journal.parent.mkdir(parents=True, exist_ok=True)
if type(compression_level) is not int or not 1 <= compression_level <= 9:
raise ValueError("invalid_evidence_compression_level")
from .storage import atomic_write_json from .storage import atomic_write_json
atomic_write_json(marker, { atomic_write_json(marker, {
"format": FORMAT, "format": FORMAT,
"archive": archive_path(journal).name, "archive": archive_path(journal).name,
"codec": "zlib_level_1_independent_blocks", "codec": f"zlib_level_{compression_level}_independent_blocks",
"compression_level": compression_level,
"envelope_fields": "archive_bound_v2",
"field_schemas": {},
"large_record_threshold_bytes": int(threshold_bytes), "large_record_threshold_bytes": int(threshold_bytes),
"maximum_uncompressed_block_bytes": MAXIMUM_BLOCK_BYTES, "maximum_uncompressed_block_bytes": MAXIMUM_BLOCK_BYTES,
}) })
@@ -153,24 +150,33 @@ def is_indexed_journal(journal_path: str | Path) -> bool:
def _resume_index(record: Mapping) -> dict: def _resume_index(record: Mapping) -> dict:
indexed = {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record return {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record
and (record[key] is None or isinstance(record[key], (str, int, float, bool)))} and (record[key] is None or isinstance(record[key], (str, int, float, bool)))}
if record.get("kind") in {"joint_sample", "secondary_joint_sample"}:
for key in INDEXED_SEQUENCE_FIELDS:
if key in record:
indexed[key] = record[key]
if record.get("kind") == "motion_branch_observation":
for key in MOTION_INDEX_FIELDS:
if key in record:
indexed[key] = record[key]
return indexed
def _envelope(record: Mapping, fields, reference) -> dict: def _schema_identity(fields) -> str:
return hashlib.sha256(json.dumps(list(fields), separators=(",", ":")).encode()).hexdigest()[:16]
@lru_cache(maxsize=512)
def _schema_fields(journal_path: str | Path, identity: str):
marker = _journal_configuration(str(Path(journal_path).resolve())) or {}
fields = marker.get("field_schemas", {}).get(identity)
if not isinstance(fields, list) or not all(isinstance(field, str) for field in fields):
raise ValueError("compressed_evidence_schema_missing")
if _schema_identity(fields) != identity:
raise ValueError("compressed_evidence_schema_changed")
return tuple(fields)
def _envelope(record: Mapping, reference, schema_identity) -> dict:
indexed = _resume_index(record) indexed = _resume_index(record)
if REFERENCE_KEY in indexed or FIELDS_KEY in indexed: if REFERENCE_KEY in indexed or FIELDS_KEY in indexed or SCHEMA_KEY in indexed:
raise ValueError("calibration_record_uses_reserved_storage_field") raise ValueError("calibration_record_uses_reserved_storage_field")
result = {REFERENCE_KEY: reference, FIELDS_KEY: list(fields), **indexed} # record_sha256 binds the complete archived object. Repeating its field
# names in every one-line resume index added tens of MB without adding an
# independent integrity check.
result = {REFERENCE_KEY: reference, SCHEMA_KEY: schema_identity, **indexed}
if record.get("kind") in CANONICAL_INDEX_KINDS: if record.get("kind") in CANONICAL_INDEX_KINDS:
result[CANONICAL_SHA256_KEY] = hashlib.sha256(json.dumps(record, sort_keys=True, result[CANONICAL_SHA256_KEY] = hashlib.sha256(json.dumps(record, sort_keys=True,
separators=(",", ":"), allow_nan=False).encode()).hexdigest() separators=(",", ":"), allow_nan=False).encode()).hexdigest()
@@ -190,42 +196,64 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
if marker is None: if marker is None:
raise ValueError("unsupported_evidence_journal_format") raise ValueError("unsupported_evidence_journal_format")
threshold = int(marker["large_record_threshold_bytes"]) threshold = int(marker["large_record_threshold_bytes"])
compression_level = int(marker.get("compression_level", 1))
if not 1 <= compression_level <= 9:
raise ValueError("invalid_evidence_compression_level")
prepared = [] prepared = []
imported = {} imported = {}
for record in records: for record in records:
if REFERENCE_KEY in record or FIELDS_KEY in record: if REFERENCE_KEY in record or FIELDS_KEY in record or SCHEMA_KEY in record:
raise ValueError("calibration_record_uses_reserved_storage_field") raise ValueError("calibration_record_uses_reserved_storage_field")
reference = getattr(record, "reference", None) reference = getattr(record, "reference", None)
field_names = getattr(record, "field_names", None)
fields = tuple(record) if field_names is None else tuple(field_names)
schema_identity = _schema_identity(fields)
if isinstance(reference, EvidenceRecordReference): if isinstance(reference, EvidenceRecordReference):
imported[len(prepared)] = _linked_envelope(journal, record) imported[len(prepared)] = (record, schema_identity)
line = None line = None
else: else:
line = json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode("utf-8") line = json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
prepared.append((record, line)) prepared.append((record, line, schema_identity, fields))
if not prepared: if not prepared:
return return
schemas = dict(marker.get("field_schemas", {}))
changed = False
for _, _, identity, fields in prepared:
existing = schemas.get(identity)
if existing is not None and tuple(existing) != fields:
raise ValueError("compressed_evidence_schema_collision")
if existing is None:
schemas[identity] = list(fields)
changed = True
if changed:
marker = {**marker, "field_schemas": schemas}
from .storage import atomic_write_json
atomic_write_json(manifest_path(journal), marker)
_journal_configuration.cache_clear()
evidence = archive_path(journal) evidence = archive_path(journal)
evidence.parent.mkdir(parents=True, exist_ok=True) evidence.parent.mkdir(parents=True, exist_ok=True)
output: list[bytes | None] = [None] * len(prepared) output: list[bytes | None] = [None] * len(prepared)
for index, envelope in imported.items(): for index, (record, schema_identity) in imported.items():
envelope = _linked_envelope(journal, record, schema_identity)
output[index] = json.dumps(envelope, ensure_ascii=False, output[index] = json.dumps(envelope, ensure_ascii=False,
separators=(",", ":")).encode("utf-8") separators=(",", ":")).encode("utf-8")
with evidence.open("ab") as archive: with evidence.open("ab") as archive:
large: list[tuple[int, Mapping, bytes]] = [] large: list[tuple[int, Mapping, bytes, str]] = []
size = 0 size = 0
def flush_block(): def flush_block():
nonlocal large, size nonlocal large, size
if not large: if not large:
return return
block = b"".join(line for _, _, line in large) block = b"".join(line for _, _, line, _ in large)
compressed = zlib.compress(block, 1) compressed = zlib.compress(block, compression_level)
offset = archive.tell() offset = archive.tell()
_write_all(archive, compressed) _write_all(archive, compressed)
block_digest = hashlib.sha256(compressed).hexdigest() block_digest = hashlib.sha256(compressed).hexdigest()
local = 0 local = 0
for index, record, line in large: for index, record, line, schema_identity in large:
reference = { reference = {
"version": 1, "offset": offset, "version": 1, "offset": offset,
"compressed_length": len(compressed), "compressed_length": len(compressed),
@@ -233,12 +261,12 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
"record_offset": local, "record_length": len(line), "record_offset": local, "record_length": len(line),
"record_sha256": hashlib.sha256(line).hexdigest(), "record_sha256": hashlib.sha256(line).hexdigest(),
} }
output[index] = json.dumps(_envelope(record, record.keys(), reference), output[index] = json.dumps(_envelope(record, reference, schema_identity),
ensure_ascii=False, separators=(",", ":")).encode("utf-8") ensure_ascii=False, separators=(",", ":")).encode("utf-8")
local += len(line) local += len(line)
large, size = [], 0 large, size = [], 0
for index, (record, line) in enumerate(prepared): for index, (record, line, schema_identity, _) in enumerate(prepared):
if line is None: if line is None:
continue continue
if len(line) < threshold and record.get("kind") not in DETAILED_RECORD_KINDS: if len(line) < threshold and record.get("kind") not in DETAILED_RECORD_KINDS:
@@ -246,7 +274,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
continue continue
if large and size + len(line) > MAXIMUM_BLOCK_BYTES: if large and size + len(line) > MAXIMUM_BLOCK_BYTES:
flush_block() flush_block()
large.append((index, record, line)) large.append((index, record, line, schema_identity))
size += len(line) size += len(line)
flush_block() flush_block()
archive.flush() archive.flush()
@@ -260,7 +288,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
os.fsync(stream.fileno()) os.fsync(stream.fileno())
def _linked_envelope(destination: Path, record) -> dict: def _linked_envelope(destination: Path, record, schema_identity) -> dict:
"""Hard-link an immutable source archive and retain its indexed row.""" """Hard-link an immutable source archive and retain its indexed row."""
reference = record.reference reference = record.reference
source_archive = (record.path.parent / reference.archive source_archive = (record.path.parent / reference.archive
@@ -287,8 +315,7 @@ def _linked_envelope(destination: Path, record) -> dict:
linked = EvidenceRecordReference(reference.offset, reference.compressed_length, linked = EvidenceRecordReference(reference.offset, reference.compressed_length,
reference.block_sha256, reference.record_offset, reference.record_length, reference.block_sha256, reference.record_offset, reference.record_length,
reference.record_sha256, target.name) reference.record_sha256, target.name)
result = {REFERENCE_KEY: linked.as_mapping(), FIELDS_KEY: list(record.field_names), result = {REFERENCE_KEY: linked.as_mapping(), SCHEMA_KEY: schema_identity, **record.metadata}
**record.metadata}
if record.canonical_sha256: if record.canonical_sha256:
result[CANONICAL_SHA256_KEY] = record.canonical_sha256 result[CANONICAL_SHA256_KEY] = record.canonical_sha256
return result return result
@@ -336,7 +363,10 @@ def decode_envelope(journal_path: str | Path, envelope: Mapping, decoder):
if REFERENCE_KEY not in envelope: if REFERENCE_KEY not in envelope:
return envelope return envelope
value = decoder.loads(evidence_record_bytes(journal_path, envelope[REFERENCE_KEY])) value = decoder.loads(evidence_record_bytes(journal_path, envelope[REFERENCE_KEY]))
if not isinstance(value, dict) or tuple(value) != tuple(envelope.get(FIELDS_KEY, ())): expected_fields = (envelope.get(FIELDS_KEY) if FIELDS_KEY in envelope
else _schema_fields(journal_path, envelope[SCHEMA_KEY]) if SCHEMA_KEY in envelope else None)
if (not isinstance(value, dict) or expected_fields is not None
and tuple(value) != tuple(expected_fields)):
raise ValueError("compressed_evidence_record_shape_changed") raise ValueError("compressed_evidence_record_shape_changed")
return value return value
@@ -359,7 +389,9 @@ def compact_existing_journal(source_path: str | Path, *, replace=False) -> dict:
for path in (compact, compact_archive, manifest_path(compact)): for path in (compact, compact_archive, manifest_path(compact)):
if path.exists(): if path.exists():
path.unlink() path.unlink()
initialize_evidence_journal(compact) # Closed checkpoints favor disk footprint. Live capture remains level 1 so
# evidence persistence cannot stall the control loop.
initialize_evidence_journal(compact, compression_level=6)
source_digest = hashlib.sha256() source_digest = hashlib.sha256()
record_count = 0 record_count = 0
@@ -13,6 +13,8 @@ PLAN_VERSION = "capture_plan_v1"
FIXED = "fixed" FIXED = "fixed"
ADAPTIVE = "adaptive_2_to_3" ADAPTIVE = "adaptive_2_to_3"
STAGED = "staged_2_to_3" STAGED = "staged_2_to_3"
TASKWISE_STAGED = "taskwise_staged_2_to_3"
STAGED_POLICIES = frozenset((STAGED, TASKWISE_STAGED))
LEGACY_TRAINING = (0, 1, 2) LEGACY_TRAINING = (0, 1, 2)
LEGACY_HOLDOUT = 3 LEGACY_HOLDOUT = 3
@@ -67,19 +69,25 @@ class CapturePlan:
return next(task for task in self.tasks if task.task_key == key) return next(task for task in self.tasks if task.task_key == key)
def as_dict(self): def as_dict(self):
result = {"version": "capture_plan_v2" if self.policy == STAGED else PLAN_VERSION, "policy": self.policy, version = ("capture_plan_v3" if self.policy == TASKWISE_STAGED else
"capture_plan_v2" if self.policy == STAGED else PLAN_VERSION)
result = {"version": version, "policy": self.policy,
"command_capture_mode": self.command_capture_mode, "command_capture_mode": self.command_capture_mode,
"motion_contract": json.loads(self.motion_contract_json), "motion_contract": json.loads(self.motion_contract_json),
"tasks": {t.task_key: {"training_cycles": list(t.training), "tasks": {t.task_key: {"training_cycles": list(t.training),
"holdout_cycle": t.holdout, "holdout_cycle": t.holdout,
"command_training_cycles": list(t.command_training), "command_training_cycles": list(t.command_training),
"command_holdout_cycle": t.command_holdout} for t in self.tasks}} "command_holdout_cycle": t.command_holdout} for t in self.tasks}}
if self.policy == STAGED: if self.policy in STAGED_POLICIES:
result["stages"] = [ result["stages"] = [
{"name": name, "cycle": cycle, "tasks": [t.task_key for t in self.tasks {"name": name, "cycle": cycle, "tasks": [t.task_key for t in self.tasks
if cycle in t.motion_cycles]} if cycle in t.motion_cycles]}
for name, cycle in (("first_training", 0), ("second_training", 1), for name, cycle in (("first_training", 0), ("second_training", 1),
("supplement", 2), ("validation", 3))] ("supplement", 2), ("validation", 3))]
if self.policy == TASKWISE_STAGED:
result["schedule"] = (
"first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation"
)
return result return result
@property @property
@@ -101,7 +109,8 @@ def task_partition(profile, task=None, *, joint=None):
if joint is not None: if joint is not None:
task = task_for_joint(profile, joint) task = task_for_joint(profile, joint)
key = getattr(task, "key", task) key = getattr(task, "key", task)
default = LEGACY_TRAINING[:2] if profile.quality.training_policy == STAGED else profile.quality.training_cycles default = (LEGACY_TRAINING[:2] if profile.quality.training_policy in STAGED_POLICIES
else profile.quality.training_cycles)
training = tuple(profile.quality.task_training_cycles.get(key, default)) training = tuple(profile.quality.task_training_cycles.get(key, default))
holdout = profile.quality.holdout_cycle holdout = profile.quality.holdout_cycle
separate = profile.acquisition.command_capture_mode == "separate" separate = profile.acquisition.command_capture_mode == "separate"
@@ -119,20 +128,21 @@ def select_task_training(profile, task_key, cycles):
raise ValueError("capture_plan_unknown_task") raise ValueError("capture_plan_unknown_task")
if cycles not in (LEGACY_TRAINING, LEGACY_TRAINING[:2]): if cycles not in (LEGACY_TRAINING, LEGACY_TRAINING[:2]):
raise ValueError("capture_plan_invalid_training_partition") raise ValueError("capture_plan_invalid_training_partition")
if cycles != tuple(profile.quality.training_cycles) and profile.quality.training_policy not in {ADAPTIVE, STAGED}: if (cycles != tuple(profile.quality.training_cycles)
and profile.quality.training_policy not in {ADAPTIVE, *STAGED_POLICIES}):
raise ValueError("capture_plan_fixed_training_cannot_change") raise ValueError("capture_plan_fixed_training_cannot_change")
selections = {**profile.quality.task_training_cycles, task_key: cycles} selections = {**profile.quality.task_training_cycles, task_key: cycles}
return replace(profile, quality=replace(profile.quality, task_training_cycles=selections)) return replace(profile, quality=replace(profile.quality, task_training_cycles=selections))
def configure_training(profile, policy): def configure_training(profile, policy):
if policy not in {FIXED, ADAPTIVE, STAGED}: if policy not in {FIXED, ADAPTIVE, *STAGED_POLICIES}:
raise ValueError("unknown_training_policy") raise ValueError("unknown_training_policy")
if profile.artifacts.output_schema_version in {2, 3} and ( if profile.artifacts.output_schema_version in {2, 3} and (
tuple(profile.quality.training_cycles) != LEGACY_TRAINING tuple(profile.quality.training_cycles) != LEGACY_TRAINING
or profile.quality.holdout_cycle != LEGACY_HOLDOUT): or profile.quality.holdout_cycle != LEGACY_HOLDOUT):
raise ValueError("production_partition_requires_versioned_training_policy") raise ValueError("production_partition_requires_versioned_training_policy")
if policy in {ADAPTIVE, STAGED} and (profile.artifacts.output_schema_version < 3 if policy in {ADAPTIVE, *STAGED_POLICIES} and (profile.artifacts.output_schema_version < 3
or not profile.command_based_release or not profile.command_based_release
or profile.acquisition.command_capture_mode != "interleaved" or profile.acquisition.command_capture_mode != "interleaved"
or tuple(profile.quality.training_cycles) != LEGACY_TRAINING or tuple(profile.quality.training_cycles) != LEGACY_TRAINING
@@ -9,7 +9,8 @@ from typing import Mapping
from ..domain.profile import CalibrationProfile, validate_profile from ..domain.profile import CalibrationProfile, validate_profile
from ..domain.result import CalibrationResult, JointMapping from ..domain.result import CalibrationResult, JointMapping
from ..domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, training_cycles from ..domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED, TASKWISE_STAGED,
training_cycles)
from ..urdf.kinematics import UrdfKinematicModel from ..urdf.kinematics import UrdfKinematicModel
from .motion_fit import channel_for_joint, fit_profile_motion, joint_input_direction from .motion_fit import channel_for_joint, fit_profile_motion, joint_input_direction
from .spatial import ZeroCalibrationProfile, fit_joint_axis_measurement, solve_urdf_zero_offsets, with_depth_free_axis_projection from .spatial import ZeroCalibrationProfile, fit_joint_axis_measurement, solve_urdf_zero_offsets, with_depth_free_axis_projection
@@ -98,7 +99,8 @@ def build_axis_observations(profile, model, geometry_records, motion, zero_profi
rows = [dict(row, command_u8=int(round(float(row[profile.curve_input_domain])))) for row in rows] rows = [dict(row, command_u8=int(round(float(row[profile.curve_input_domain])))) for row in rows]
parent = zero_profile.phase_parent_joint.get(joint) parent = zero_profile.phase_parent_joint.get(joint)
if parent is not None and parent not in by_joint: if parent is not None and parent not in by_joint:
if cycle != plan.task(task.key).holdout and profile.quality.training_policy in {ADAPTIVE, STAGED}: if (cycle != plan.task(task.key).holdout
and profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}):
continue # No same-cycle parent observation: do not fabricate a geometric sample. continue # No same-cycle parent observation: do not fabricate a geometric sample.
raise ValueError("spatial axis order must place phase parents first") raise ValueError("spatial axis order must place phase parents first")
task = next(task for task in profile.motion.tasks if joint in task.joints) task = next(task for task in profile.motion.tasks if joint in task.joints)
@@ -182,7 +184,7 @@ def fit_profile_calibration(profile: CalibrationProfile, source_urdf: Path, reco
zero = solve_urdf_zero_offsets(source_urdf=source_urdf, measurements=observations, zero = solve_urdf_zero_offsets(source_urdf=source_urdf, measurements=observations,
curves=spatial_curves, motor_by_joint={joint: channel_for_joint(profile, joint) for joint in curves}, curves=spatial_curves, motor_by_joint={joint: channel_for_joint(profile, joint) for joint in curves},
zero_profile=zero_profile, training_cycles=geometry_training, validation_cycle=geometry_holdout, zero_profile=zero_profile, training_cycles=geometry_training, validation_cycle=geometry_holdout,
allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED}, allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED},
frozen_training=None if frozen_training is None else frozen_training["zero"], frozen_training=None if frozen_training is None else frozen_training["zero"],
maximum_offset_rad=math.radians(20), finger_maximum_offset_rad=math.radians(20), maximum_offset_rad=math.radians(20), finger_maximum_offset_rad=math.radians(20),
maximum_validation_mae_rad=math.radians(1), maximum_validation_p95_rad=math.radians(2), maximum_validation_mae_rad=math.radians(1), maximum_validation_p95_rad=math.radians(2),
@@ -9,7 +9,8 @@ from pathlib import Path
import numpy as np import numpy as np
from scipy.stats import t as student_t from scipy.stats import t as student_t
from ..domain.capture_plan import CapturePlan, evidence_digest, select_task_training, training_cycles from ..domain.capture_plan import (CapturePlan, TASKWISE_STAGED, evidence_digest,
select_task_training, training_cycles)
from ..domain.motion_path import record_scan_identity from ..domain.motion_path import record_scan_identity
from ..domain.sampling import command_nodes from ..domain.sampling import command_nodes
from ..urdf.kinematics import UrdfKinematicModel from ..urdf.kinematics import UrdfKinematicModel
@@ -94,16 +95,22 @@ def _curve_repeatability(profile, task, motion, steady, cycles):
def assess_training(profile, task_key, cycles, rows, references, source_urdf): def assess_training(profile, task_key, cycles, rows, references, source_urdf):
"""Assess an immutable training snapshot. Missing future geometry adds a round. """Assess one immutable task snapshot without reading validation data.
The local angle confidence is NOT a zero-offset certificate. When the full Adaptive collection keeps the older conservative behavior: missing
spatial dependency graph is available, the production training-only solver full-hand geometry requests a third round. Taskwise staged collection uses
also checks real zero confidence. Otherwise the conservative third round is local two-round repeatability to decide immediately; its full-hand spatial
retained and the final spatial gate remains mandatory. fit remains a separate mandatory gate after every training task is frozen.
""" """
cycles = tuple(cycles) cycles = tuple(cycles)
chosen = select_task_training(profile, task_key, cycles) chosen = select_task_training(profile, task_key, cycles)
task = next(t for t in profile.motion.tasks if t.key == task_key) task = next(t for t in profile.motion.tasks if t.key == task_key)
taskwise = profile.quality.training_policy == TASKWISE_STAGED
if taskwise:
# A task decision is deliberately local and therefore remains stable
# when later fingers are acquired. Full-hand zero geometry is checked
# by the final training-only fit before any artifact can be published.
rows = tuple(row for row in rows if row.get("task_name") == task_key)
if any(row["cycle"] not in training_cycles(chosen, row["task_name"]) for row in rows): if any(row["cycle"] not in training_cycles(chosen, row["task_name"]) for row in rows):
raise ValueError("training_assessment_received_nontraining_data") raise ValueError("training_assessment_received_nontraining_data")
sweep, steady = _by_joint(rows, "sweep"), _by_joint(rows, "steady") sweep, steady = _by_joint(rows, "sweep"), _by_joint(rows, "steady")
@@ -125,8 +132,8 @@ retained and the final spatial gate remains mandatory.
fitted_model = {"curves": {name: asdict(curve) for name, curve in local.coordinate_curves.items()}, fitted_model = {"curves": {name: asdict(curve) for name, curve in local.coordinate_curves.items()},
"mappings": {name: asdict(mapping) for name, mapping in local.mappings.items()}} "mappings": {name: asdict(mapping) for name, mapping in local.mappings.items()}}
failures, metrics = _curve_repeatability(chosen, task, local, steady, cycles) failures, metrics = _curve_repeatability(chosen, task, local, steady, cycles)
required = set(profile.zero.axis_joints) required = set() if taskwise else set(profile.zero.axis_joints)
ready = required <= sweep.keys() & steady.keys() & references.keys() ready = bool(required) and required <= sweep.keys() & steady.keys() & references.keys()
if ready: if ready:
from .session import compile_spatial_profile, build_axis_observations from .session import compile_spatial_profile, build_axis_observations
from .spatial_solver.solve import fit_urdf_zero_training from .spatial_solver.solve import fit_urdf_zero_training
@@ -154,7 +161,8 @@ retained and the final spatial gate remains mandatory.
failures.extend(zero.failures) failures.extend(zero.failures)
except (ValueError, KeyError, np.linalg.LinAlgError) as error: except (ValueError, KeyError, np.linalg.LinAlgError) as error:
failures.append(str(error)) failures.append(str(error))
decision = ("add_training" if len(cycles) == 2 and (failures or geometry["status"] == "deferred") decision = ("add_training" if len(cycles) == 2 and (failures
or (geometry["status"] == "deferred" and not taskwise))
else "fail" if failures else "freeze") else "fail" if failures else "freeze")
record = {"kind": "training_decision", "version": TRAINING_DECISION_VERSION, record = {"kind": "training_decision", "version": TRAINING_DECISION_VERSION,
"task_name": task_key, "assessed_cycles": list(cycles), "decision": decision, "task_name": task_key, "assessed_cycles": list(cycles), "decision": decision,
@@ -95,4 +95,11 @@ class EvidenceDecoder:
return row return row
def loads(self, line): def loads(self, line):
# Most evidence rows contain observations only. Avoid recursively
# invoking the model interning hook for every nested pose/candidate
# dictionary when neither interned object can be present.
frozen_key = b'"frozen_image_model"' if isinstance(line, bytes) else '"frozen_image_model"'
reference_key = b'"reference_frame"' if isinstance(line, bytes) else '"reference_frame"'
if frozen_key not in line and reference_key not in line:
return json.loads(line)
return json.loads(line, object_hook=self.object_hook) return json.loads(line, object_hook=self.object_hook)
@@ -19,6 +19,7 @@ from .observer_pose import ObserverPose, OBSERVER_POSE_LIMITS
from .image_model_selection import compare_image_losses, holm_adjusted_probabilities from .image_model_selection import compare_image_losses, holm_adjusted_probabilities
RIGID_PAIR_CONDITIONING = 'rigid_tag_pair_pose_v1' RIGID_PAIR_CONDITIONING = 'rigid_tag_pair_pose_v1'
RIGID_PAIR_TRAJECTORY_POLICY = 'rigid_pair_full_trajectory_v3'
def _pose_vector(pose): def _pose_vector(pose):
@@ -144,12 +145,23 @@ def _pose_bounds(fit, count):
def fit_rigid_tag_pair(parents, candidates, corners, matrices, size, stamps, *, parent_corners, parent_size, def fit_rigid_tag_pair(parents, candidates, corners, matrices, size, stamps, *, parent_corners, parent_size,
parent_candidates=None): parent_candidates=None, trajectory_states=None):
count = len(stamps) count = len(stamps)
if count < 24 or tuple(stamps) != tuple(sorted(set(stamps))) or any(not c for c in candidates): if count < 24 or tuple(stamps) != tuple(sorted(set(stamps))) or any(not c for c in candidates):
raise ValueError('rigid_pair_independent_images_missing') raise ValueError('rigid_pair_independent_images_missing')
if parent_candidates is not None and (len(parent_candidates) != count or any(not p for p in parent_candidates)): if parent_candidates is not None and (len(parent_candidates) != count or any(not p for p in parent_candidates)):
raise ValueError('rigid_pair_parent_candidates_missing') raise ValueError('rigid_pair_parent_candidates_missing')
if trajectory_states is not None:
states = tuple(trajectory_states)
if len(states) != count or any(len(value) != 2 for value in states):
raise ValueError('rigid_pair_trajectory_state_missing')
phases = np.asarray([value[0] for value in states], dtype=int)
commands = np.asarray([value[1] for value in states], dtype=float)
differences = np.diff(commands)
if (not np.all(np.isfinite(commands)) or np.ptp(commands) <= 0
or not {1, 2} <= set(phases)
or not np.any(differences > 0) or not np.any(differences < 0)):
raise ValueError('rigid_pair_trajectory_direction_insufficient')
images = _PairImages(np.asarray(parent_corners), np.asarray(corners), np.asarray(matrices), images = _PairImages(np.asarray(parent_corners), np.asarray(corners), np.asarray(matrices),
square_object_points(parent_size), square_object_points(size), np.asarray([_pose_vector(p) for p in parents])) square_object_points(parent_size), square_object_points(size), np.asarray([_pose_vector(p) for p in parents]))
if (images.parent_pixels.shape != (count, 4, 2) or images.child_pixels.shape != (count, 4, 2) if (images.parent_pixels.shape != (count, 4, 2) or images.child_pixels.shape != (count, 4, 2)
@@ -9,11 +9,18 @@ from dataclasses import replace
LEGACY_OBSERVER = 'frozen_parent_v1' LEGACY_OBSERVER = 'frozen_parent_v1'
RIGID_OBSERVER = 'rigid_pair_v1' RIGID_OBSERVER = 'rigid_pair_v1'
TRAJECTORY_OBSERVER = 'rigid_pair_trajectory_v3'
def is_rigid_observer(policy):
"""Return whether a policy requires independent rigid-pair evidence."""
return policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}
def rigid_observer_dependencies(profile): def rigid_observer_dependencies(profile):
"""Return recipient -> independently measured parent hinge dependencies.""" """Return recipient -> independently measured parent hinge dependencies."""
if profile.vision_motion or profile.quality.training_policy != 'staged_2_to_3': from ..core.domain.capture_plan import STAGED_POLICIES
if profile.vision_motion or profile.quality.training_policy not in STAGED_POLICIES:
return {} return {}
measurements = profile.measurement.measurements measurements = profile.measurement.measurements
tasks = {joint: task for task in profile.motion.tasks for joint in task.joints} tasks = {joint: task for task in profile.motion.tasks for joint in task.joints}
@@ -34,13 +41,15 @@ def rigid_observer_dependencies(profile):
return dependencies return dependencies
def configure_observer_policy(profile, *, preserved_joints=()): def configure_observer_policy(profile, *, preserved_joints=(), policy=RIGID_OBSERVER):
"""Upgrade every supported unmeasured recipient in a single graph pass.""" """Upgrade every supported unmeasured recipient in a single graph pass."""
if policy not in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}:
raise ValueError('observer_policy_unknown_strategy')
preserved = frozenset(preserved_joints) preserved = frozenset(preserved_joints)
measurements = dict(profile.measurement.measurements) measurements = dict(profile.measurement.measurements)
if not preserved <= measurements.keys(): if not preserved <= measurements.keys():
raise ValueError('observer_policy_unknown_preserved_joint') raise ValueError('observer_policy_unknown_preserved_joint')
for joint in rigid_observer_dependencies(profile): for joint in rigid_observer_dependencies(profile):
if joint not in preserved: if joint not in preserved:
measurements[joint] = replace(measurements[joint], observer_pose_policy=RIGID_OBSERVER) measurements[joint] = replace(measurements[joint], observer_pose_policy=policy)
return replace(profile, measurement=replace(profile.measurement, measurements=measurements)) return replace(profile, measurement=replace(profile.measurement, measurements=measurements))
@@ -35,7 +35,12 @@ def _bounded_actions(profile, first):
def configure_observer_preparations(profile, model, *, preserved_joints=()): def configure_observer_preparations(profile, model, *, preserved_joints=()):
profile = configure_observer_policy(profile, preserved_joints=preserved_joints) from .observer_policy import TRAJECTORY_OBSERVER
strategy = (TRAJECTORY_OBSERVER if any(
spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in profile.measurement.measurements.values()) else None)
profile = configure_observer_policy(profile, preserved_joints=preserved_joints,
**({"policy": strategy} if strategy else {}))
channels = compile_tag_feedback_channels(profile, model) channels = compile_tag_feedback_channels(profile, model)
by_channel = {channel_for_joint(profile, name): name for name in profile.measurement.measurements} by_channel = {channel_for_joint(profile, name): name for name in profile.measurement.measurements}
plans = dict(profile.motion.observer_preparations) plans = dict(profile.motion.observer_preparations)
@@ -88,7 +93,9 @@ def validate_observer_preparations(profile, model):
def _validate_action(profile, joint, plan, channels, dependencies, index): def _validate_action(profile, joint, plan, channels, dependencies, index):
if joint not in dependencies or profile.measurement.measurements[joint].observer_pose_policy != 'rigid_pair_v1': from .observer_policy import is_rigid_observer
if joint not in dependencies or not is_rigid_observer(
profile.measurement.measurements[joint].observer_pose_policy):
raise ValueError('observer_preparation_dependency_invalid') raise ValueError('observer_preparation_dependency_invalid')
spec = profile.measurement.measurements[joint] spec = profile.measurement.measurements[joint]
channel = channel_for_joint(profile, plan.joint) channel = channel_for_joint(profile, plan.joint)
@@ -15,8 +15,9 @@ def preparation_image_budget(profile, *, joint_count=1):
from ..core.geometry.tag_pose.production_image_motion import ( from ..core.geometry.tag_pose.production_image_motion import (
DENSE_PREPARATION_SAMPLING, image_parameters_for_sampling, DENSE_PREPARATION_SAMPLING, image_parameters_for_sampling,
) )
from ..core.domain.capture_plan import STAGED_POLICIES
dense = (profile.key.model == 'O30' dense = (profile.key.model == 'O30'
and profile.quality.training_policy == 'staged_2_to_3' and profile.quality.training_policy in STAGED_POLICIES
and bool(profile.motion.preparation_witnesses) and joint_count == 1) and bool(profile.motion.preparation_witnesses) and joint_count == 1)
return ((256, image_parameters_for_sampling(DENSE_PREPARATION_SAMPLING)) if dense return ((256, image_parameters_for_sampling(DENSE_PREPARATION_SAMPLING)) if dense
else (128, image_parameters_for_sampling())) else (128, image_parameters_for_sampling()))
@@ -50,9 +51,20 @@ def preparation_contract(profile):
value['observer_pose_policies'] = observers value['observer_pose_policies'] = observers
if profile.motion.observer_preparations: if profile.motion.observer_preparations:
from ..core.domain.profile import ObserverPreparationPlan from ..core.domain.profile import ObserverPreparationPlan
from .observer_policy import TRAJECTORY_OBSERVER
sequence = any(isinstance(p, ObserverPreparationPlan) for p in profile.motion.observer_preparations.values()) sequence = any(isinstance(p, ObserverPreparationPlan) for p in profile.motion.observer_preparations.values())
value['observer_preparations'] = dict(policy='bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1', trajectory = any(spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in profile.measurement.measurements.values())
value['observer_preparations'] = dict(policy='occlusion_aware_trajectory_v3' if trajectory
else 'bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1',
paths={name: asdict(spec) for name, spec in profile.motion.observer_preparations.items()}) paths={name: asdict(spec) for name, spec in profile.motion.observer_preparations.items()})
if trajectory:
value['observer_preparations']['trajectory_solver'] = {
'policy': 'rigid_pair_full_trajectory_v3',
'training_validation_split': 'chronological_even_odd',
'maximum_actions_per_task': 2,
'holdout_selects_model': False,
}
# The live header and decoded JSON must compare identically (including # The live header and decoded JSON must compare identically (including
# tuple-valued poses), without relying on incidental Python container types. # tuple-valued poses), without relying on incidental Python container types.
return json.loads(json.dumps(value, allow_nan=False)) return json.loads(json.dumps(value, allow_nan=False))
@@ -72,11 +84,18 @@ def validate_preparation_contract(profile, records):
if len(declared) != 1 or declared[0] != preparation_contract(profile): if len(declared) != 1 or declared[0] != preparation_contract(profile):
raise ValueError('preparation_contract_profile_changed') raise ValueError('preparation_contract_profile_changed')
tasks = {task.key: set(task.joints) for task in profile.motion.tasks} tasks = {task.key: set(task.joints) for task in profile.motion.tasks}
refreshed_parents = {(row.get('task_name'), row.get('pose_joint'),
row.get('image_model_sha256')) for row in records
if row.get('kind') == 'parent_reference_model_refreshed'
and row.get('passed_zero_preserved') is True}
for row in records: for row in records:
if row.get('kind') != 'motion_branch_initialization' or row.get('status') != 'resolved': if row.get('kind') != 'motion_branch_initialization' or row.get('status') != 'resolved':
continue continue
joints = row.get('zero_joints', ()) joints = row.get('zero_joints', ())
if not joints or not set(joints) <= tasks.get(row.get('task_name'), set()): ordinary = bool(joints) and set(joints) <= tasks.get(row.get('task_name'), set())
refreshed = (len(joints) == 1 and (row.get('task_name'), joints[0],
row.get('image_model_sha256')) in refreshed_parents)
if not ordinary and not refreshed:
raise ValueError('preparation_contract_model_dependency_changed') raise ValueError('preparation_contract_model_dependency_changed')
_, expected = preparation_image_budget(profile, joint_count=len(joints)) _, expected = preparation_image_budget(profile, joint_count=len(joints))
if image_parameters_for_sampling(row.get('image_sampling_policy')) != expected: if image_parameters_for_sampling(row.get('image_sampling_policy')) != expected:
@@ -84,9 +103,12 @@ def validate_preparation_contract(profile, records):
def configure_preparation(profile, policy='occlusion_aware_witness_v2'): def configure_preparation(profile, policy='occlusion_aware_witness_v2'):
if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2'}: if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2',
'occlusion_aware_trajectory_v3'}:
raise ValueError('unknown_preparation_policy') raise ValueError('unknown_preparation_policy')
if profile.key.profile_id != 'O30/right/o30_right_18/v1' or profile.quality.training_policy != 'staged_2_to_3': from ..core.domain.capture_plan import STAGED_POLICIES
if (profile.key.profile_id != 'O30/right/o30_right_18/v1'
or profile.quality.training_policy not in STAGED_POLICIES):
raise ValueError('preparation_witness_requires_o30_staged_session') raise ValueError('preparation_witness_requires_o30_staged_session')
from ..core.fitting.motion_fit import channel_for_joint from ..core.fitting.motion_fit import channel_for_joint
witnesses = {} witnesses = {}
@@ -102,9 +124,10 @@ def configure_preparation(profile, policy='occlusion_aware_witness_v2'):
excursion[channel] = command[channel]-step if command[channel]-step >= lower else command[channel]+step excursion[channel] = command[channel]-step if command[channel]-step >= lower else command[channel]+step
witnesses[target] = ModelPreparation(donor, command, (command, tuple(excursion))) witnesses[target] = ModelPreparation(donor, command, (command, tuple(excursion)))
prepared = replace(profile, motion=replace(profile.motion, preparation_witnesses=witnesses)) prepared = replace(profile, motion=replace(profile.motion, preparation_witnesses=witnesses))
if policy == 'occlusion_aware_witness_v2': if policy in {'occlusion_aware_witness_v2', 'occlusion_aware_trajectory_v3'}:
from .observer_policy import configure_observer_policy from .observer_policy import configure_observer_policy, RIGID_OBSERVER, TRAJECTORY_OBSERVER
prepared = configure_observer_policy(prepared) strategy = TRAJECTORY_OBSERVER if policy == 'occlusion_aware_trajectory_v3' else RIGID_OBSERVER
prepared = configure_observer_policy(prepared, policy=strategy)
return prepared return prepared
@@ -36,18 +36,20 @@ def validate_executable_profile(profile, source_urdf):
if joint == observer: if joint == observer:
raise ValueError(f"absolute_zero_unobservable:{joint}:a free Tag on its own hinge is not a CAD datum") raise ValueError(f"absolute_zero_unobservable:{joint}:a free Tag on its own hinge is not a CAD datum")
measured = {joint for task in profile.motion.tasks for joint in task.joints} measured = {joint for task in profile.motion.tasks for joint in task.joints}
from .observer_policy import rigid_observer_dependencies, RIGID_OBSERVER from .observer_policy import rigid_observer_dependencies, is_rigid_observer
observer_dependencies = rigid_observer_dependencies(profile) observer_dependencies = rigid_observer_dependencies(profile)
if not measured <= model.joints.keys() or not (profile.zero.active_joints | profile.zero.passive_joints) <= model.joints.keys(): if not measured <= model.joints.keys() or not (profile.zero.active_joints | profile.zero.passive_joints) <= model.joints.keys():
raise ValueError("Profile joint keys must name URDF joints; SDK names belong in command.names") raise ValueError("Profile joint keys must name URDF joints; SDK names belong in command.names")
for name, spec in profile.measurement.measurements.items(): for name, spec in profile.measurement.measurements.items():
if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1"}: if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1",
"rigid_pair_trajectory_v3"}:
raise ValueError(f"unknown_observer_pose_policy:{name}") raise ValueError(f"unknown_observer_pose_policy:{name}")
if spec.observer_pose_policy == "rigid_pair_v1" and ( from ..core.domain.capture_plan import STAGED_POLICIES
if is_rigid_observer(spec.observer_pose_policy) and (
spec.kind != "relative_rotation" or profile.vision_motion spec.kind != "relative_rotation" or profile.vision_motion
or profile.quality.training_policy != "staged_2_to_3"): or profile.quality.training_policy not in STAGED_POLICIES):
raise ValueError(f"rigid_pair_requires_staged_relative_measurement:{name}") raise ValueError(f"rigid_pair_requires_staged_relative_measurement:{name}")
if spec.observer_pose_policy == RIGID_OBSERVER and name not in observer_dependencies: if is_rigid_observer(spec.observer_pose_policy) and name not in observer_dependencies:
raise ValueError(f"rigid_pair_requires_independent_parent_hinge:{name}") raise ValueError(f"rigid_pair_requires_independent_parent_hinge:{name}")
if spec.kind not in {"relative_rotation", "urdf_axis_chain", "curve", "axis_cross_view_validation"}: if spec.kind not in {"relative_rotation", "urdf_axis_chain", "curve", "axis_cross_view_validation"}:
raise ValueError(f"unsupported measurement primitive:{name}:{spec.kind}") raise ValueError(f"unsupported measurement primitive:{name}:{spec.kind}")
@@ -53,6 +53,11 @@ def validate_capture_provenance(profile, serial_number, hashes, records):
def validate_capture_units(profile, records): def validate_capture_units(profile, records):
"""Offline and resumed data must satisfy the actual post-sweep policy.""" """Offline and resumed data must satisfy the actual post-sweep policy."""
from ..task_deferral import unresolved_observation_records
blocked = unresolved_observation_records(profile, records)
if blocked:
raise ValueError("capture_unobservable_tasks:" + ",".join(sorted(
f"{row.get('task_name')}:{row.get('cycle')}:{row.get('state')}" for row in blocked)))
from ...core.fitting.command_sampling import validate_sampling_plans from ...core.fitting.command_sampling import validate_sampling_plans
from ..training import resolve_capture_plan from ..training import resolve_capture_plan
profile = resolve_capture_plan(profile, records, require_complete=True) profile = resolve_capture_plan(profile, records, require_complete=True)
@@ -8,6 +8,7 @@ training/holdout observations to revise an existing reference.
from dataclasses import asdict, dataclass, replace from dataclasses import asdict, dataclass, replace
from collections import deque from collections import deque
from collections.abc import Mapping
import hashlib import hashlib
import json import json
import math import math
@@ -89,7 +90,11 @@ def _image_references(record, roles):
def _finite_report(value): def _finite_report(value):
if isinstance(value, float) and not math.isfinite(value): if isinstance(value, float) and not math.isfinite(value):
return None return None
if isinstance(value, dict): # Resume keeps large evidence rows as lazy Mapping objects. Solver reports
# are durable journal records, so materialize any nested mapping at this
# single ownership boundary instead of leaking storage-layer objects into
# the JSON encoder.
if isinstance(value, Mapping):
return {key: _finite_report(item) for key, item in value.items()} return {key: _finite_report(item) for key, item in value.items()}
if isinstance(value, (list, tuple)): if isinstance(value, (list, tuple)):
return [_finite_report(item) for item in value] return [_finite_report(item) for item in value]
@@ -125,8 +130,9 @@ def solve_initialization(request: BranchInitializationRequest):
return replace(result, preparation_transitions=(record,)) return replace(result, preparation_transitions=(record,))
bridges, records, rejections = [], [], [] bridges, records, rejections = [], [], []
if request.observer_profile is not None: if request.observer_profile is not None:
from ..profiles.observer_policy import is_rigid_observer
required = {joint for joint, spec in request.observer_profile.measurement.measurements.items() required = {joint for joint, spec in request.observer_profile.measurement.measurements.items()
if joint in request.zero_joints and spec.observer_pose_policy == 'rigid_pair_v1'} if joint in request.zero_joints and is_rigid_observer(spec.observer_pose_policy)}
if not required <= {source.joint for source in request.observer_sources}: if not required <= {source.joint for source in request.observer_sources}:
return ImageMotionResolution(False, 'rigid_pair_source_missing') return ImageMotionResolution(False, 'rigid_pair_source_missing')
if request.observer_sources: if request.observer_sources:
@@ -137,7 +143,7 @@ def solve_initialization(request: BranchInitializationRequest):
bridges.append(bridge) bridges.append(bridge)
records.append(record) records.append(record)
except (ValueError, KeyError, TypeError, IndexError) as error: except (ValueError, KeyError, TypeError, IndexError) as error:
if request.observer_profile.measurement.measurements[source.joint].observer_pose_policy == 'rigid_pair_v1': if is_rigid_observer(request.observer_profile.measurement.measurements[source.joint].observer_pose_policy):
return ImageMotionResolution(False, str(error)) return ImageMotionResolution(False, str(error))
rejections.append(str(error)) rejections.append(str(error))
witness_bridges, witness_records = [], [] witness_bridges, witness_records = [], []
@@ -228,7 +234,8 @@ def initialization_artifacts(request, resolution):
if record.get("observer_pose_bridges"): if record.get("observer_pose_bridges"):
from ..core.geometry.tag_pose.pose_bridge import observer_pose_selection_policy, RIGID_OBSERVER_SELECTION_POLICY from ..core.geometry.tag_pose.pose_bridge import observer_pose_selection_policy, RIGID_OBSERVER_SELECTION_POLICY
record["image_model_selection_policy"] = (RIGID_OBSERVER_SELECTION_POLICY record["image_model_selection_policy"] = (RIGID_OBSERVER_SELECTION_POLICY
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges']) if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'} for r in record['observer_pose_bridges'])
else observer_pose_selection_policy(request.pose_bridges)) else observer_pose_selection_policy(request.pose_bridges))
if record.get("preparation_transitions"): if record.get("preparation_transitions"):
from ..core.geometry.tag_pose.transition_image_solver import TRANSITION_SELECTION_POLICY from ..core.geometry.tag_pose.transition_image_solver import TRANSITION_SELECTION_POLICY
@@ -241,6 +248,7 @@ def initialization_artifacts(request, resolution):
if any(r.get('conditioning') for r in record['preparation_witnesses']) else WITNESS_POLICY) if any(r.get('conditioning') for r in record['preparation_witnesses']) else WITNESS_POLICY)
record["image_geometry_constraints"] = [asdict(item) for item in request.geometry_constraints] record["image_geometry_constraints"] = [asdict(item) for item in request.geometry_constraints]
record["source_urdf_sha256"] = request.source_urdf_sha256 record["source_urdf_sha256"] = request.source_urdf_sha256
record = _finite_report(record)
if not resolution.resolved: if not resolution.resolved:
return None, record return None, record
existing = dict(request.existing_evidence_ids) existing = dict(request.existing_evidence_ids)
@@ -272,7 +280,8 @@ def initialization_artifacts(request, resolution):
payload['image_model_selection_policy'] = record['image_model_selection_policy'] payload['image_model_selection_policy'] = record['image_model_selection_policy']
if record.get("observer_pose_bridges"): if record.get("observer_pose_bridges"):
payload["observer_pose_bridges"] = record["observer_pose_bridges"] payload["observer_pose_bridges"] = record["observer_pose_bridges"]
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges']): if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'} for r in record['observer_pose_bridges']):
payload['image_model_selection_policy'] = record['image_model_selection_policy'] payload['image_model_selection_policy'] = record['image_model_selection_policy']
if record.get("preparation_transitions"): if record.get("preparation_transitions"):
payload["preparation_transitions"] = record["preparation_transitions"] payload["preparation_transitions"] = record["preparation_transitions"]
@@ -340,6 +349,7 @@ class BranchInitialization:
self._invalid_reason = "" self._invalid_reason = ""
self._requested = set() self._requested = set()
self.reference_spec = None self.reference_spec = None
self.parent_model_refresh_for = ""
self.reports = {} self.reports = {}
self.path_index = None self.path_index = None
@@ -367,6 +377,7 @@ class BranchInitialization:
self._invalid_reason = "motion_approach_path_discontinuous" self._invalid_reason = "motion_approach_path_discontinuous"
self.path_index = index self.path_index = index
self.reference_spec = (motion.reference_command, motion.reference_approach) if motion.reference_only else None self.reference_spec = (motion.reference_command, motion.reference_approach) if motion.reference_only else None
self.parent_model_refresh_for = motion.parent_model_refresh_for
self.task_name = motion.task_key self.task_name = motion.task_key
self.zero_joints = motion.reference_joints self.zero_joints = motion.reference_joints
self.motion_version = motion_version self.motion_version = motion_version
@@ -378,6 +389,7 @@ class BranchInitialization:
self.task_name, self.zero_joints = motion.task_key, motion.zero_joints self.task_name, self.zero_joints = motion.task_key, motion.zero_joints
self.session_epoch = session_epoch self.session_epoch = session_epoch
self._invalid_reason = "motion_approach_evidence_missing" self._invalid_reason = "motion_approach_evidence_missing"
self.parent_model_refresh_for = motion.parent_model_refresh_for
elif self.path_index is not None: elif self.path_index is not None:
spec = self.profile.motion.joint_zero_references[motion.zero_joints[0]] spec = self.profile.motion.joint_zero_references[motion.zero_joints[0]]
if spec.evidence_scope != "path" or self.path_index != len(spec.approach_commands): if spec.evidence_scope != "path" or self.path_index != len(spec.approach_commands):
@@ -539,7 +551,7 @@ class BranchInitialization:
reason = reason or 'shared_pose_bridge_invalid:' + str(error) reason = reason or 'shared_pose_bridge_invalid:' + str(error)
source_hinges, parent_reference_sha256 = (), "" source_hinges, parent_reference_sha256 = (), ""
task = next(task for task in self.profile.motion.tasks if task.key == self.task_name) task = next(task for task in self.profile.motion.tasks if task.key == self.task_name)
if task.parent_reference is not None: if task.parent_reference is not None and not self.parent_model_refresh_for:
try: try:
source_hinges, parent_reference_sha256 = self.parent_references.geometry(task, session_epoch) source_hinges, parent_reference_sha256 = self.parent_references.geometry(task, session_epoch)
except ValueError as error: except ValueError as error:
@@ -550,15 +562,17 @@ class BranchInitialization:
tuple(item.geometry.relation for item in source_hinges) + relations) tuple(item.geometry.relation for item in source_hinges) + relations)
from .observer_reference import observer_inputs from .observer_reference import observer_inputs
observer_sources = observer_inputs(self, view, self._endpoint_records.get(view, ()), observer_sources = observer_inputs(self, view, self._endpoint_records.get(view, ()),
{f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else () {f.stamp_ns for f in frames}, session_epoch) if (
self.image_geometry and not self.parent_model_refresh_for) else ()
transition_source = None transition_source = None
if self.image_geometry: if self.image_geometry and not self.parent_model_refresh_for:
try: try:
transition_source = self.transition_capture.snapshot(view) transition_source = self.transition_capture.snapshot(view)
except ValueError as error: except ValueError as error:
reason = reason or str(error) reason = reason or str(error)
witness_sources = self.witness_capture.inputs(view, self.zero_joints, witness_sources = self.witness_capture.inputs(view, self.zero_joints,
self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else () self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if (
self.image_geometry and not self.parent_model_refresh_for) else ()
return BranchInitializationRequest(self.task_name, view, self.zero_joints, return BranchInitializationRequest(self.task_name, view, self.zero_joints,
session_epoch, self.motion_version if self.motion_version is not None else motion_version, self.generation, session_epoch, self.motion_version if self.motion_version is not None else motion_version, self.generation,
frames, frames,
@@ -157,6 +157,11 @@ class ObservationCapture:
self._preparation_identity = None self._preparation_identity = None
self.install_motion_model(model) self.install_motion_model(model)
def current_image_reference_stable(self, view):
"""Check the independent current-session anchor, not an active old model."""
tracker = self._image_references.get(view)
return tracker is not None and tracker.tracking_current_image()
def install_motion_model(self, model: MotionBranchModel): def install_motion_model(self, model: MotionBranchModel):
"""Add evidence before zero capture without replacing existing roles.""" """Add evidence before zero capture without replacing existing roles."""
key = (model.task_name, model.view) key = (model.task_name, model.view)
@@ -64,14 +64,27 @@ class CaptureRecordIndex(Sequence):
source = getattr(row, "metadata", row) source = getattr(row, "metadata", row)
indexed = {key: value for key, value in source.items() indexed = {key: value for key, value in source.items()
if value is None or isinstance(value, (str, int, float, bool))} if value is None or isinstance(value, (str, int, float, bool))}
for key in ('relative_quaternion_xyzw', 'steady_training_nodes', 'command_direction_by_index'):
if key in row:
indexed[key] = tuple(row[key])
if self.retain_training_geometry: if self.retain_training_geometry:
from ..core.fitting.training_quality import TRAINING_FIELDS from ..core.fitting.training_quality import TRAINING_FIELDS
from ..core.domain.capture_plan import evidence_digest from ..core.domain.capture_plan import evidence_digest
indexed.update({key: row[key] for key in TRAINING_FIELDS if key in row}) from .resume_storage import materialize_plain_record
indexed['source_record_sha256'] = evidence_digest(row) # A deferred row decompresses and parses its evidence when a
# non-indexed field is accessed. Materialize it exactly once;
# probing every training field through Mapping.__contains__
# used to decode the same sample dozens of times.
materialized = materialize_plain_record(row)
indexed.update({key: materialized[key] for key in TRAINING_FIELDS
if key in materialized})
# DeferredCaptureRecord implements Mapping for indexed reads,
# but json.dumps intentionally accepts only concrete JSON
# containers. Hash the immutable materialized record exactly
# as an eager capture would.
indexed['source_record_sha256'] = evidence_digest(materialized)
else:
for key in ('relative_quaternion_xyzw', 'steady_training_nodes',
'command_direction_by_index'):
if key in row:
indexed[key] = tuple(row[key])
self._store(indexed) self._store(indexed)
elif kind in {'command_sampling_plan', 'training_decision', 'staged_training_decision'}: elif kind in {'command_sampling_plan', 'training_decision', 'staged_training_decision'}:
self._store(row) self._store(row)
@@ -8,10 +8,12 @@ from dataclasses import dataclass, replace
import hashlib import hashlib
import json import json
from pathlib import Path from pathlib import Path
import shutil
from ..core.domain.profile import CalibrationProfile from ..core.domain.profile import CalibrationProfile
from ..profiles import load_hand_profile from ..profiles import load_hand_profile
from ..profiles.observer_policy import configure_observer_policy from ..profiles.observer_policy import (configure_observer_policy,
RIGID_OBSERVER, TRAJECTORY_OBSERVER)
from ..profiles.preparation import preparation_contract from ..profiles.preparation import preparation_contract
from ..profiles.validator import validate_executable_profile from ..profiles.validator import validate_executable_profile
from .engine import CalibrationEngine from .engine import CalibrationEngine
@@ -19,6 +21,23 @@ from .joint_resume import PreparedResume, JointResume
from .staged_resume import stage_visits from .staged_resume import stage_visits
def _copy_indexed_evidence(source_path, output_dir):
"""Copy content-addressed evidence before exposing the migrated index."""
from ..core.artifacts.evidence_journal import (
is_indexed_journal, manifest_path, archive_path,
)
if not is_indexed_journal(source_path):
return
for source_evidence in (manifest_path(source_path), archive_path(source_path)):
target_evidence = output_dir/source_evidence.name
shutil.copy2(source_evidence, target_evidence)
with source_evidence.open('rb') as source_stream, target_evidence.open('rb') as target_stream:
source_hash = hashlib.file_digest(source_stream, 'sha256').hexdigest()
target_hash = hashlib.file_digest(target_stream, 'sha256').hexdigest()
if source_hash != target_hash:
raise ValueError('observer_upgrade_evidence_archive_changed')
def preserved_observer_joints(checkpoint): def preserved_observer_joints(checkpoint):
if any(r.get('kind') == 'staged_training_decision' for r in checkpoint.rows): if any(r.get('kind') == 'staged_training_decision' for r in checkpoint.rows):
raise ValueError('observer_upgrade_training_already_decided') raise ValueError('observer_upgrade_training_already_decided')
@@ -45,14 +64,23 @@ class ObserverPolicyUpgrade:
return cls(source, target, Path(source_path).read_bytes(), Path(target_path).read_bytes()) return cls(source, target, Path(source_path).read_bytes(), Path(target_path).read_bytes())
def metadata(self, checkpoint, protected_inputs, source_path, source_digest): def metadata(self, checkpoint, protected_inputs, source_path, source_digest):
from .resume_storage import materialize_record
source_header = materialize_record(checkpoint.header)
source_reference = materialize_record(checkpoint.fixed_reference)
preserved = preserved_observer_joints(checkpoint) preserved = preserved_observer_joints(checkpoint)
expected = configure_observer_policy(self.source, preserved_joints=preserved) target_strategies = {spec.observer_pose_policy
for spec in self.target.measurement.measurements.values()
if spec.observer_pose_policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}}
strategy = (TRAJECTORY_OBSERVER if TRAJECTORY_OBSERVER in target_strategies
else RIGID_OBSERVER)
expected = configure_observer_policy(self.source, preserved_joints=preserved,
policy=strategy)
if self.target.motion.observer_preparations != self.source.motion.observer_preparations: if self.target.motion.observer_preparations != self.source.motion.observer_preparations:
if self.source_urdf is None: if self.source_urdf is None:
raise ValueError('observer_upgrade_source_geometry_missing') raise ValueError('observer_upgrade_source_geometry_missing')
from ..core.urdf.kinematics import UrdfKinematicModel from ..core.urdf.kinematics import UrdfKinematicModel
from ..profiles.observer_preparation import configure_observer_preparations from ..profiles.observer_preparation import configure_observer_preparations
expected = configure_observer_preparations(self.source, UrdfKinematicModel(self.source_urdf), expected = configure_observer_preparations(expected, UrdfKinematicModel(self.source_urdf),
preserved_joints=preserved) preserved_joints=preserved)
if self.target != expected: if self.target != expected:
raise ValueError('observer_upgrade_changes_existing_contract') raise ValueError('observer_upgrade_changes_existing_contract')
@@ -63,19 +91,19 @@ class ObserverPolicyUpgrade:
if protected_inputs.get('profile_config_sha256') != new_hash: if protected_inputs.get('profile_config_sha256') != new_hash:
raise ValueError('observer_upgrade_target_hash_invalid') raise ValueError('observer_upgrade_target_hash_invalid')
old_inputs = {**protected_inputs, 'profile_config_sha256': old_hash} old_inputs = {**protected_inputs, 'profile_config_sha256': old_hash}
for row in (checkpoint.header, checkpoint.fixed_reference): for row in (source_header, source_reference):
actual = row.get('protected_hashes', row) actual = row.get('protected_hashes', row)
if any(actual.get(key) != value for key, value in old_inputs.items()): if any(actual.get(key) != value for key, value in old_inputs.items()):
raise ValueError('observer_upgrade_protected_inputs_changed') raise ValueError('observer_upgrade_protected_inputs_changed')
if checkpoint.header.get('preparation_contract') != preparation_contract(self.source): if source_header.get('preparation_contract') != preparation_contract(self.source):
raise ValueError('observer_upgrade_source_preparation_changed') raise ValueError('observer_upgrade_source_preparation_changed')
header = {**checkpoint.header, **protected_inputs, header = {**source_header, **protected_inputs,
'preparation_contract': preparation_contract(self.target), 'preparation_contract': preparation_contract(self.target),
'resume_checkpoint_requested': True} 'resume_checkpoint_requested': True}
if 'protected_hashes' in header: if 'protected_hashes' in header:
header['protected_hashes'] = {**header['protected_hashes'], **protected_inputs} header['protected_hashes'] = {**header['protected_hashes'], **protected_inputs}
reference = {**checkpoint.fixed_reference, 'protected_hashes': { reference = {**source_reference, 'protected_hashes': {
**checkpoint.fixed_reference['protected_hashes'], **protected_inputs}} **source_reference['protected_hashes'], **protected_inputs}}
changes = {joint: dict(previous=spec.observer_pose_policy, changes = {joint: dict(previous=spec.observer_pose_policy,
current=self.target.measurement.measurements[joint].observer_pose_policy) current=self.target.measurement.measurements[joint].observer_pose_policy)
for joint, spec in self.source.measurement.measurements.items() for joint, spec in self.source.measurement.measurements.items()
@@ -83,7 +111,7 @@ class ObserverPolicyUpgrade:
audit = dict(kind='capture_observation_revision', policy='unmeasured_observer_upgrade_v1', audit = dict(kind='capture_observation_revision', policy='unmeasured_observer_upgrade_v1',
source_raw_samples=str(Path(source_path).resolve()), source_raw_sha256=source_digest, source_raw_samples=str(Path(source_path).resolve()), source_raw_sha256=source_digest,
source_profile_sha256=old_hash, target_profile_sha256=new_hash, source_profile_sha256=old_hash, target_profile_sha256=new_hash,
original_header=checkpoint.header, original_fixed_reference=checkpoint.fixed_reference, original_header=source_header, original_fixed_reference=source_reference,
preserved_joints=sorted(preserved), observer_policy_changes=changes, preserved_joints=sorted(preserved), observer_policy_changes=changes,
completed_units=[list(unit) for unit in sorted(checkpoint.joints.units)], completed_units=[list(unit) for unit in sorted(checkpoint.joints.units)],
physical_zero_reverification_performed=False) physical_zero_reverification_performed=False)
@@ -92,6 +120,11 @@ class ObserverPolicyUpgrade:
audit.update(policy='unmeasured_observer_upgrade_v2', observer_preparations={ audit.update(policy='unmeasured_observer_upgrade_v2', observer_preparations={
name: asdict(plan) for name, plan in self.target.motion.observer_preparations.items() name: asdict(plan) for name, plan in self.target.motion.observer_preparations.items()
if self.source.motion.observer_preparations.get(name) != plan}) if self.source.motion.observer_preparations.get(name) != plan})
if strategy == TRAJECTORY_OBSERVER:
audit.update(policy='unmeasured_observer_upgrade_v3',
target_observation_policy='occlusion_aware_trajectory_v3',
accepted_evidence_semantics='preserved_without_reinterpretation',
pending_evidence_semantics='full_trajectory_with_bounded_observation')
return header, reference, audit return header, reference, audit
@@ -117,26 +150,34 @@ def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number,
or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans): or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans):
raise ValueError('observer_upgrade_changes_reusable_evidence') raise ValueError('observer_upgrade_changes_reusable_evidence')
output_dir.mkdir(parents=True, exist_ok=False) output_dir.mkdir(parents=True, exist_ok=False)
temporary = output_dir/'raw_samples.jsonl.tmp' try:
digest = hashlib.sha256() _copy_indexed_evidence(source_path, output_dir)
with source_path.open('rb') as source, temporary.open('xb') as target: temporary = output_dir/'raw_samples.jsonl.tmp'
for line in source: digest = hashlib.sha256()
digest.update(line) with source_path.open('rb') as source, temporary.open('xb') as target:
row = json.loads(line) if line.strip() else {} for line in source:
if row.get('kind') in replacement: digest.update(line)
target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode()) row = json.loads(line) if line.strip() else {}
else: if row.get('kind') in replacement:
target.write(line) target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode())
target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode()) else:
if digest.hexdigest() != source_digest: target.write(line)
temporary.unlink() target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode())
raise ValueError('observer_upgrade_source_changed_during_replay') if digest.hexdigest() != source_digest:
(output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml) raise ValueError('observer_upgrade_source_changed_during_replay')
(output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml) (output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml)
report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'} (output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml)
(output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n') report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'}
temporary.rename(output_dir/'raw_samples.jsonl') installed = output_dir/'raw_samples.jsonl'
return report temporary.rename(installed)
if (output_dir/'raw_samples.storage.json').is_file():
from ..core.artifacts.evidence_journal import compact_existing_journal
report['storage_compaction'] = compact_existing_journal(installed, replace=True)
(output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except Exception:
shutil.rmtree(output_dir, ignore_errors=True)
raise
def main(): def main():
@@ -0,0 +1,173 @@
"""Audited migration from whole-hand staged training to taskwise staged training.
The measurement contract, motion paths, accepted scan identities and raw image
evidence stay unchanged. Only the ordering of future training visits and the
time at which a task-local repeatability decision is made are revised. The
source journal is immutable and the target journal links its content-addressed
evidence blocks.
"""
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import shutil
from ..core.domain.capture_plan import (CapturePlan, STAGED, TASKWISE_STAGED,
configure_training)
from ..profiles import load_hand_profile
from ..profiles.export import dump_hand_profile
from ..profiles.preparation import preparation_contract
from ..profiles.validator import validate_executable_profile
from .engine import CalibrationEngine, capture_schedule_version
from .joint_resume import PreparedResume, JointResume
from .resume_storage import materialize_record
from .staged_resume import stage_passed_visits
@dataclass(frozen=True)
class TaskwiseTrainingUpgrade:
source: object
target: object
source_yaml: bytes
target_yaml: bytes
@classmethod
def load(cls, source_profile):
source = load_hand_profile(source_profile)
if source.quality.training_policy != STAGED:
raise ValueError("taskwise_upgrade_requires_staged_source")
target = configure_training(source, TASKWISE_STAGED)
return cls(source, target, Path(source_profile).read_bytes(),
dump_hand_profile(target).encode())
def replacement_records(self, checkpoint, *, source_path, source_digest):
if any(row.get("kind") in {"staged_training_decision", "training_decision"}
for row in checkpoint.rows):
raise ValueError("taskwise_upgrade_training_decision_already_recorded")
source_header = materialize_record(checkpoint.header)
source_reference = materialize_record(checkpoint.fixed_reference)
old_hash = hashlib.sha256(self.source_yaml).hexdigest()
new_hash = hashlib.sha256(self.target_yaml).hexdigest()
actual = source_header.get("protected_hashes", source_header)
if actual.get("profile_config_sha256") != old_hash:
raise ValueError("taskwise_upgrade_source_profile_hash_changed")
if source_header.get("preparation_contract") != preparation_contract(self.source):
raise ValueError("taskwise_upgrade_source_preparation_changed")
if preparation_contract(self.target) != preparation_contract(self.source):
raise ValueError("taskwise_upgrade_changes_preparation_contract")
header = {**source_header,
"capture_schedule_version": capture_schedule_version(self.target),
"capture_plan": CapturePlan.from_profile(self.target).as_dict(),
"profile_config_sha256": new_hash,
"resume_checkpoint_requested": True}
if "protected_hashes" in header:
header["protected_hashes"] = {**header["protected_hashes"],
"profile_config_sha256": new_hash}
reference = {**source_reference, "protected_hashes": {
**source_reference["protected_hashes"], "profile_config_sha256": new_hash}}
audit = {
"kind": "capture_schedule_revision",
"version": "taskwise_staged_training_upgrade_v1",
"source_raw_samples": str(Path(source_path).resolve()),
"source_raw_sha256": source_digest,
"source_profile_sha256": old_hash,
"target_profile_sha256": new_hash,
"source_policy": STAGED,
"target_policy": TASKWISE_STAGED,
"accepted_evidence_semantics": "preserved_without_reinterpretation",
"future_schedule": (
"first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation"
),
"completed_units": [list(unit) for unit in sorted(checkpoint.joints.units)],
}
return header, reference, audit
def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number,
source_urdf, image_replay_workers=1):
source_path, output_dir = Path(source_path).resolve(), Path(output_dir).resolve()
if output_dir.exists():
raise FileExistsError(output_dir)
for profile in (upgrade.source, upgrade.target):
validate_executable_profile(profile, source_urdf)
with source_path.open("rb") as stream:
source_digest = hashlib.file_digest(stream, "sha256").hexdigest()
checkpoint = PreparedResume.load(upgrade.source, CalibrationEngine(upgrade.source),
source_path, serial_number, image_replay_workers=image_replay_workers,
resume_mode="passed", source_urdf=source_urdf)
header, reference, audit = upgrade.replacement_records(checkpoint,
source_path=source_path, source_digest=source_digest)
replacement = {"session_start": header, "fixed_base_reference_locked": reference}
rows = tuple(replacement.get(row.get("kind"), row) for row in checkpoint.rows) + (audit,)
# The source is itself an unchanged-installation ``passed`` checkpoint.
# Its compact journal intentionally omits historical witness windows that
# strict replay would require. Preserve and compare the same durable pass
# decisions here; finalization still performs the complete evidence replay
# before it may publish JSON/URDF artifacts.
verified = JointResume(upgrade.target, image_replay_workers=image_replay_workers)
stage_passed_visits(verified, CalibrationEngine(upgrade.target), rows)
if (verified.units != checkpoint.joints.units
or verified.references != checkpoint.joints.references
or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans):
raise ValueError("taskwise_upgrade_changes_reusable_evidence")
output_dir.mkdir(parents=True, exist_ok=False)
try:
raw = output_dir / "raw_samples.jsonl"
from ..core.artifacts.evidence_journal import initialize_evidence_journal
from ..core.artifacts.storage import append_jsonl_many, sync_jsonl
initialize_evidence_journal(raw)
for offset in range(0, len(rows), 4096):
append_jsonl_many(raw, rows[offset:offset + 4096], durable=False)
sync_jsonl(raw)
(output_dir / "source_profile.yaml").write_bytes(upgrade.source_yaml)
target_profile = output_dir / "target_profile.yaml"
target_profile.write_bytes(upgrade.target_yaml)
prepared = PreparedResume.load(upgrade.target, CalibrationEngine(upgrade.target),
raw, serial_number, image_replay_workers=image_replay_workers,
resume_mode="passed", source_urdf=source_urdf)
if (prepared.completed_units != frozenset(verified.units)
or prepared.joints.references != verified.references
or prepared.joints.first_cycle_spans != verified.first_cycle_spans):
raise ValueError("taskwise_upgrade_written_checkpoint_changed")
report = {**audit, "reusable_units": len(verified.units),
"zero_references": len(verified.references),
"target_profile": str(target_profile),
"resume_validation": "passed_mode_loaded",
"historical_replay_performed": False,
"full_replay_required_before_publication": True,
"result": "PASSED_EVIDENCE_PRESERVATION_VERIFIED"}
(output_dir / "training_upgrade.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n")
return report
except Exception:
shutil.rmtree(output_dir, ignore_errors=True)
raise
def main():
import argparse
from ..product import load_product_config
from .image_replay import checkpoint_replay_workers
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-profile", required=True)
parser.add_argument("--target-product", required=True)
parser.add_argument("--source-checkpoint", required=True)
parser.add_argument("--output-dir", required=True)
args = parser.parse_args()
product = load_product_config(args.target_product, check_can=False)
upgrade = TaskwiseTrainingUpgrade.load(args.source_profile)
report = upgrade_checkpoint(upgrade, args.source_checkpoint, args.output_dir,
serial_number=product.serial_number, source_urdf=product.source_urdf,
image_replay_workers=checkpoint_replay_workers())
print(json.dumps({key: report[key] for key in (
"result", "reusable_units", "zero_references", "target_profile"
)}, ensure_ascii=False), flush=True)
if __name__ == "__main__":
main()
@@ -42,7 +42,8 @@ from .resume import ResumeDecision, ResumeVerifier, fingerprint_from_mapping
from .safety import SafetyPolicy, SafetySample from .safety import SafetyPolicy, SafetySample
from .session import CalibrationPhase as Phase from .session import CalibrationPhase as Phase
from .snapshot import DeviceReadiness, CameraReadiness, RuntimeSnapshot, build_snapshot, legacy_state from .snapshot import DeviceReadiness, CameraReadiness, RuntimeSnapshot, build_snapshot, legacy_state
from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED,
TASKWISE_STAGED, STAGED_POLICIES)
_TERMINAL_STATES = frozenset({"PASSED", "DIAGNOSTIC_COMPLETE", "PAUSED", "ABORTED", "FAILED"}) _TERMINAL_STATES = frozenset({"PASSED", "DIAGNOSTIC_COMPLETE", "PAUSED", "ABORTED", "FAILED"})
@@ -73,7 +74,7 @@ class CalibrationCoordinator:
self.training_worker = None self.training_worker = None
self._training_request = None self._training_request = None
self._frozen_training_sha256 = None self._frozen_training_sha256 = None
if profile.quality.training_policy in {ADAPTIVE, STAGED}: if profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}:
if self.diagnostic_capture is not None: if self.diagnostic_capture is not None:
raise ValueError("adaptive_training_cannot_run_as_diagnostic") raise ValueError("adaptive_training_cannot_run_as_diagnostic")
from .training import TrainingWorker from .training import TrainingWorker
@@ -91,10 +92,12 @@ class CalibrationCoordinator:
self.command_direction_history = deque(maxlen=2000) self.command_direction_history = deque(maxlen=2000)
self._command_directions = [""]*self.command_count self._command_directions = [""]*self.command_count
self.finalization = finalization or FinalizationController(profile, parameters.session_dir) self.finalization = finalization or FinalizationController(profile, parameters.session_dir)
from .journal_durability import JournalDurabilityWorker
self.journal_durability = JournalDurabilityWorker()
from .capture_index import CaptureRecordIndex from .capture_index import CaptureRecordIndex
self.capture_index = CaptureRecordIndex(retain_complete=( self.capture_index = CaptureRecordIndex(retain_complete=(
self.diagnostic_capture is not None or not getattr(self.finalization, "uses_journal", False)), self.diagnostic_capture is not None or not getattr(self.finalization, "uses_journal", False)),
retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED}) retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED})
self._image_record_stamps = {} self._image_record_stamps = {}
self.last_command = None self.last_command = None
self.commanded_speed = None self.commanded_speed = None
@@ -585,6 +588,7 @@ class CalibrationCoordinator:
self._observation_epoch += 1 self._observation_epoch += 1
self.branch_initialization.clear() self.branch_initialization.clear()
try: try:
self.journal_durability.close()
sync_jsonl(self.raw_path) sync_jsonl(self.raw_path)
finally: finally:
self.finalization.close() self.finalization.close()
@@ -640,28 +644,7 @@ class CalibrationCoordinator:
if self._passed_reference_import is not None: if self._passed_reference_import is not None:
self._publish_command(list(current)) self._publish_command(list(current))
pending = self._passed_reference_import pending = self._passed_reference_import
self._write_resume_batch(pending) if self._write_resume_batch(pending) and self._restore_passed_reference_batch(pending):
if pending.complete:
from .passed_resume import restore_parent_models, restored_model_record
binding = restored_model_record(pending.model_records,
self._observation_epoch, self.ports.clock_ns())
if self.profile.quality.training_policy == STAGED and self.parameters.resume_mode != "passed":
binding.update(kind="staged_pose_models_restored", resume_mode="verify",
physical_zero_reverification_required=True)
binding.pop("unchanged_installation_declared", None)
append_jsonl(self.raw_path, binding)
self.capture_index.append(binding)
restore_parent_models(self.branch_initialization.parent_references,
pending.model_records, self._observation_epoch)
from .observer_reference import restore_observer_records
restore_observer_records(self.branch_initialization, pending.rows)
self.branch_initialization.observer_preparation.restore(pending.rows)
for row in pending.rows:
if row.get("kind") == "motion_branch_observation":
self.branch_initialization.witness_capture.observe(row)
self.execution.zero_references.update(pending.references)
if self.visual_motion is not None:
self.visual_motion.freeze_zeros(pending.references)
self._passed_reference_import = None self._passed_reference_import = None
return return
if self._reference_import is not None and not self._reference_import.complete: if self._reference_import is not None and not self._reference_import.complete:
@@ -674,7 +657,7 @@ class CalibrationCoordinator:
return return
if phase in {Phase.PREPARE, Phase.MAPPING_PROBE} and self.segment is None and session.current_unit is not None: if phase in {Phase.PREPARE, Phase.MAPPING_PROBE} and self.segment is None and session.current_unit is not None:
unit = session.current_unit unit = session.current_unit
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy in STAGED_POLICIES:
used, imported = self.joint_resume.take_task(unit.task_key, cycle=unit.cycle) used, imported = self.joint_resume.take_task(unit.task_key, cycle=unit.cycle)
else: else:
used, imported = self.joint_resume.take_task(unit.task_key) used, imported = self.joint_resume.take_task(unit.task_key)
@@ -751,6 +734,17 @@ class CalibrationCoordinator:
if self.execution.last_diagnostic_quality is not None else {})} if self.execution.last_diagnostic_quality is not None else {})}
self.capture_index.append(record) self.capture_index.append(record)
append_jsonl(self.raw_path, record) append_jsonl(self.raw_path, record)
next_unit = session.current_unit
visit_complete = (quality.passed and (next_unit is None
or (next_unit.task_key, next_unit.cycle) != (
action.scan_unit.task_key, action.scan_unit.cycle)))
if visit_complete:
resolved_deferral = session.task_deferrals.resolve(
action.scan_unit.task_key, action.scan_unit.cycle)
if resolved_deferral is not None:
resolved_record = resolved_deferral.as_record(stamp_ns=self.ports.clock_ns())
self.capture_index.append(resolved_record)
append_jsonl(self.raw_path, resolved_record)
if self.execution.last_training_decision is not None: if self.execution.last_training_decision is not None:
decision = self.execution.last_training_decision decision = self.execution.last_training_decision
append_jsonl(self.raw_path, decision) append_jsonl(self.raw_path, decision)
@@ -903,6 +897,8 @@ class CalibrationCoordinator:
elif geometry_error: elif geometry_error:
if self._recover_joint_zero(motion, now, geometry_error): if self._recover_joint_zero(motion, now, geometry_error):
return return
if self._defer_unobservable_task(motion, geometry_error):
return
if self.diagnostic_capture is not None: if self.diagnostic_capture is not None:
geometry_error += ":diagnostic_reference=" + self._zero_reference_error geometry_error += ":diagnostic_reference=" + self._zero_reference_error
self._pause("joint_zero_motion_unresolved:" + geometry_error) self._pause("joint_zero_motion_unresolved:" + geometry_error)
@@ -910,7 +906,10 @@ class CalibrationCoordinator:
else: else:
if self._recover_joint_zero(motion, now, ""): if self._recover_joint_zero(motion, now, ""):
return return
self._pause("joint_zero_unavailable:"+self._zero_reference_error+self._joint_zero_branch_details(motion)) unavailable = self._zero_reference_error+self._joint_zero_branch_details(motion)
if self._defer_unobservable_task(motion, unavailable):
return
self._pause("joint_zero_unavailable:"+unavailable)
return return
if (self.segment.arrived(now) and motion.phase != "joint_zero") or steady_complete: if (self.segment.arrived(now) and motion.phase != "joint_zero") or steady_complete:
if motion.phase in {'observer_preflight', 'observer_probe'}: if motion.phase in {'observer_preflight', 'observer_probe'}:
@@ -1028,6 +1027,14 @@ class CalibrationCoordinator:
raise ValueError("parent_reference_zero_missing") raise ValueError("parent_reference_zero_missing")
record = self.branch_initialization.parent_references.confirm(task, record = self.branch_initialization.parent_references.confirm(task,
self._observation_epoch, self._segment_number, reference, rows) self._observation_epoch, self._segment_number, reference, rows)
elif motion.parent_model_refresh_for:
task = next(task for task in self.profile.motion.tasks
if task.key == motion.parent_model_refresh_for)
reference = self.execution.zero_references.get(task.parent_reference.pose_joint)
if reference is None:
raise ValueError("parent_reference_zero_missing")
record = self.branch_initialization.parent_references.authorize_refreshed_pose(
task, self._observation_epoch, self._segment_number, reference)
except ValueError as error: except ValueError as error:
self._zero_reference_error = str(error) self._zero_reference_error = str(error)
return False return False
@@ -1172,6 +1179,38 @@ class CalibrationCoordinator:
def _recover_joint_zero(self, motion, now, geometry_error): def _recover_joint_zero(self, motion, now, geometry_error):
if self.diagnostic_capture is not None: if self.diagnostic_capture is not None:
return False return False
branch_details = self._joint_zero_branch_details(motion)
task = next((task for task in self.profile.motion.tasks
if task.key == motion.task_key), None)
stale_parent_anchor = (motion.reference_reuse
and task is not None and self.capture.current_image_reference_stable(task.view)
and self._zero_reference_error.startswith("parent_reference_samples_missing:")
and "image_reference_current_image_moved" in branch_details)
action = self.zero_recovery.take_parent_model_refresh(
motion, stale_image_anchor=stale_parent_anchor)
if action is not None:
from .zero_recovery import FailureCategory
record = {"kind": "joint_zero_recovery", "task_name": motion.task_key,
"failure_category": FailureCategory.REFERENCE_CHANGED.value,
"capture_plan_sha256": CapturePlan.from_profile(self.execution.profile).sha256,
"joints": list(motion.zero_joints), "action": action,
"reason": self._zero_reference_error+branch_details,
"previous_session_epoch": self._observation_epoch,
"session_epoch": self._observation_epoch+1,
"attempt": 1, "stamp_ns": self.ports.clock_ns()}
append_jsonl(self.raw_path, record)
self.capture_index.append(record)
self._observation_epoch += 1
self.branch_initialization.parent_references.advance_recovery_epoch(
self._observation_epoch-1, self._observation_epoch)
self.execution.refresh_parent_model(self.last_command or motion.target)
self.branch_initialization.clear()
self._steady_rows.clear()
self._zero_capture_started_at = self._steady_capture_after_ns = None
self._motion = self.segment = None
self._zero_geometry_started_at = self._zero_geometry_completed_at = None
self.reason = "joint_zero_recovery:"+action+":"+str(motion.task_key)
return True
if geometry_error and self._recover_observer_preparation(motion): if geometry_error and self._recover_observer_preparation(motion):
return True return True
action = self.zero_recovery.take(motion, geometry_error=geometry_error, action = self.zero_recovery.take(motion, geometry_error=geometry_error,
@@ -1279,6 +1318,41 @@ class CalibrationCoordinator:
self.reason = 'observer_preparation:' + motion.zero_joints[0] self.reason = 'observer_preparation:' + motion.zero_joints[0]
return True return True
def _defer_unobservable_task(self, motion, geometry_error):
"""Continue independent collection after bounded v3 observation fails."""
from .task_deferral import recoverable_observation_reason
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
refresh = bool(motion.parent_model_refresh_for)
if ((motion.reference_only and not (refresh or motion.reference_reuse))
or motion.reference_check or len(motion.zero_joints) != 1
or not recoverable_observation_reason(geometry_error)
or not any(spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in self.profile.measurement.measurements.values())):
return False
import hashlib
import json
reports = tuple(self.branch_initialization.reports.values())
hashes = {report.get('view', ''): report.get('source_frame_hashes', {})
for report in reports}
corner_digest = hashlib.sha256(json.dumps(hashes, sort_keys=True,
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
decision = self.execution.defer_current_task(geometry_error)
record = decision.as_record(stamp_ns=self.ports.clock_ns(),
corner_evidence_sha256=corner_digest)
append_jsonl(self.raw_path, record)
self.capture_index.append(record)
self._observation_epoch += 1
self.branch_initialization.parent_references.advance_recovery_epoch(
self._observation_epoch-1, self._observation_epoch)
self.branch_initialization.clear()
self._steady_rows.clear()
self._zero_capture_started_at = None
self._steady_capture_after_ns = None
self._zero_geometry_started_at = self._zero_geometry_completed_at = None
self.segment, self._motion = None, None
self.reason = f"task_observation_{decision.state.value}:{decision.task_name}"
return True
def _joint_zero_branch_details(self, motion): def _joint_zero_branch_details(self, motion):
task = next(task for task in self.profile.motion.tasks if task.key == motion.task_key) task = next(task for task in self.profile.motion.tasks if task.key == motion.task_key)
if motion.reference_only: if motion.reference_only:
@@ -1598,12 +1672,113 @@ class CalibrationCoordinator:
append_jsonl(self.raw_path, {"kind": "fixed_base_reference_locked", **self._current_fingerprint}) append_jsonl(self.raw_path, {"kind": "fixed_base_reference_locked", **self._current_fingerprint})
def _write_resume_batch(self, pending): def _write_resume_batch(self, pending):
"""Keep both zero-evidence and scan imports outside an unbounded tick.""" """Transfer bounded rows and poll a non-blocking durability barrier."""
rows = pending.take_batch() if not pending.transfer_complete:
append_jsonl_many(self.raw_path, rows, durable=False) rows = pending.take_batch()
self.capture_index.extend(rows) append_jsonl_many(self.raw_path, rows, durable=False)
if pending.complete: self.capture_index.extend(rows)
sync_jsonl(self.raw_path) if not pending.transfer_complete:
return False
if pending.durability_future is None:
pending.durability_future = self.journal_durability.submit(
lambda: sync_jsonl(self.raw_path))
return False
if not pending.durability_future.done():
return False
pending.durability_future.result()
pending.durable = True
return True
def _restore_passed_reference_batch(self, pending):
"""Restore imported runtime indexes in bounded control-loop slices."""
if pending.restore_phase == "binding":
from .passed_resume import restored_model_record
binding = restored_model_record(pending.model_records,
self._observation_epoch, self.ports.clock_ns())
if self.profile.quality.training_policy in STAGED_POLICIES and self.parameters.resume_mode != "passed":
binding.update(kind="staged_pose_models_restored", resume_mode="verify",
physical_zero_reverification_required=True)
binding.pop("unchanged_installation_declared", None)
append_jsonl_many(self.raw_path, (binding,), durable=False)
self.capture_index.append(binding)
pending.durability_future = self.journal_durability.submit(
lambda: sync_jsonl(self.raw_path))
pending.restore_phase = "binding_sync"
return False
if pending.restore_phase == "binding_sync":
if not pending.durability_future.done():
return False
pending.durability_future.result()
pending.restore_phase = "models"
return False
if pending.restore_phase == "models":
from .passed_resume import restore_parent_models
restore_parent_models(self.branch_initialization.parent_references,
pending.model_records, self._observation_epoch, pending.rows)
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation")
pending.restore_offset = 0
pending.restore_phase = "observer_records"
return False
if pending.restore_phase == "observer_records":
from .observer_reference import restore_observer_records
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 512]
restore_observer_records(self.branch_initialization, batch)
pending.restore_offset += len(batch)
if pending.restore_offset == len(pending.restore_rows):
if self.parameters.resume_mode == "passed":
# Passed tasks contribute immutable zero/model evidence,
# never unfinished live actions. Replaying an old active
# observer decision can move an already accepted joint
# while the next task is preparing.
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation"
and row.get("preparation_witness_joint"))
pending.restore_offset = 0
pending.restore_phase = "witness"
return False
preparation_kinds = {"observer_preparation_decision",
"image_observation_frame", "observer_preparation_segment_complete",
"observer_preparation_excursion_started",
"observer_preparation_cancelled_before_excursion"}
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") in preparation_kinds)
pending.restore_offset = 0
pending.restore_phase = "observer_preparation"
return False
if pending.restore_phase == "observer_preparation":
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 256]
self.branch_initialization.observer_preparation.restore(batch)
pending.restore_offset += len(batch)
if pending.restore_offset == len(pending.restore_rows):
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation"
and row.get("preparation_witness_joint"))
pending.restore_offset = 0
pending.restore_phase = "witness"
return False
if pending.restore_phase == "witness":
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 64]
for row in batch:
self.branch_initialization.witness_capture.observe(row)
pending.restore_offset += len(batch)
if pending.restore_offset < len(pending.restore_rows):
return False
pending.restore_rows = ()
pending.restore_phase = "complete"
self.execution.zero_references.update(pending.references)
if self.visual_motion is not None:
self.visual_motion.freeze_zeros(pending.references)
return True
if pending.restore_phase == "complete":
return True
raise RuntimeError("unknown_passed_reference_restore_phase")
def _import_resume_batch(self): def _import_resume_batch(self):
pending = self._resume_import pending = self._resume_import
@@ -1666,7 +1841,7 @@ class CalibrationCoordinator:
if self.profile.artifacts.output_schema_version >= 3: if self.profile.artifacts.output_schema_version >= 3:
self.joint_resume = checkpoint.joints self.joint_resume = checkpoint.joints
decision = replace(decision, reason="整场参考通过,等待逐关节验证原零位后复用") decision = replace(decision, reason="整场参考通过,等待逐关节验证原零位后复用")
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import restore_recovery from .staged_resume import restore_recovery
self._passed_reference_import = checkpoint.passed_import self._passed_reference_import = checkpoint.passed_import
restore_recovery(self.execution.session, self.zero_recovery, rows) restore_recovery(self.execution.session, self.zero_recovery, rows)
@@ -1676,7 +1851,7 @@ class CalibrationCoordinator:
decision = replace(decision, completed_units=tuple(sorted(complete))) decision = replace(decision, completed_units=tuple(sorted(complete)))
except (OSError, ValueError, KeyError, TypeError) as error: except (OSError, ValueError, KeyError, TypeError) as error:
decision = ResumeDecision(False, "旧断点校验未通过,已重新采集", (str(error),)) decision = ResumeDecision(False, "旧断点校验未通过,已重新采集", (str(error),))
if self.profile.quality.training_policy == STAGED and not decision.reuse: if self.profile.quality.training_policy in STAGED_POLICIES and not decision.reuse:
self._pause("staged_resume_reference_or_evidence_changed:" + str(decision.incompatible_fields)) self._pause("staged_resume_reference_or_evidence_changed:" + str(decision.incompatible_fields))
return return
decision = self.execution.restore(decision, rows) decision = self.execution.restore(decision, rows)
@@ -1701,7 +1876,7 @@ class CalibrationCoordinator:
return return
self.joint_resume = checkpoint.joints self.joint_resume = checkpoint.joints
self._passed_reference_import = checkpoint.passed_import self._passed_reference_import = checkpoint.passed_import
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import restore_recovery from .staged_resume import restore_recovery
restore_recovery(self.execution.session, self.zero_recovery, checkpoint.rows) restore_recovery(self.execution.session, self.zero_recovery, checkpoint.rows)
if checkpoint.frozen_training is not None: if checkpoint.frozen_training is not None:
@@ -1709,7 +1884,7 @@ class CalibrationCoordinator:
# The existing bounded importer skips each task only after its original # The existing bounded importer skips each task only after its original
# rows and zero references are durable. No old task is moved or scanned. # rows and zero references are durable. No old task is moved or scanned.
self._resume_message = "按已通过记录续采,跳过历史回放和已完成关节零位复核" self._resume_message = "按已通过记录续采,跳过历史回放和已完成关节零位复核"
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy in STAGED_POLICIES:
self._resume_message = "断点证据校验通过,直接复用已完成访问;不重复运动或零位复核" self._resume_message = "断点证据校验通过,直接复用已完成访问;不重复运动或零位复核"
self.execution.restore(ResumeDecision(True, self._resume_message), ()) self.execution.restore(ResumeDecision(True, self._resume_message), ())
@@ -15,7 +15,8 @@ from .diagnostic_capture import DiagnosticCapturePlan
from ..core.domain.profile import ACQUISITION_POLICY_VERSION from ..core.domain.profile import ACQUISITION_POLICY_VERSION
from ..core.domain.motion_path import task_segments, scan_identity from ..core.domain.motion_path import task_segments, scan_identity
from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, LEGACY_TRAINING, LEGACY_HOLDOUT from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED,
TASKWISE_STAGED, STAGED_POLICIES, LEGACY_TRAINING, LEGACY_HOLDOUT)
TRAINING_CYCLES = LEGACY_TRAINING TRAINING_CYCLES = LEGACY_TRAINING
HOLDOUT_CYCLE = LEGACY_HOLDOUT HOLDOUT_CYCLE = LEGACY_HOLDOUT
CAPTURE_SCHEDULE_VERSION = "unified_schedule_v7_single_pass_endpoint_images" CAPTURE_SCHEDULE_VERSION = "unified_schedule_v7_single_pass_endpoint_images"
@@ -27,8 +28,10 @@ PARENT_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v10_verified_parent_geomet
IMAGE_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v11_stationary_image_reference" IMAGE_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v11_stationary_image_reference"
ADAPTIVE_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v12_adaptive_training" ADAPTIVE_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v12_adaptive_training"
STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v13_staged_training" STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v13_staged_training"
TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v14_taskwise_staged_training"
SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION, SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION,
STAGED_CAPTURE_SCHEDULE_VERSION, ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}" STAGED_CAPTURE_SCHEDULE_VERSION, TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION,
ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{PATH_CAPTURE_SCHEDULE_VERSION}_{mode}" for mode in ("interleaved", "separate")), *(f"{PATH_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{VISUAL_CAPTURE_SCHEDULE_VERSION}_{mode}" for mode in ("interleaved", "separate")), *(f"{VISUAL_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{PARENT_REFERENCE_SCHEDULE_VERSION}_{feedback}_{mode}" for mode in ("interleaved", "separate")), *(f"{PARENT_REFERENCE_SCHEDULE_VERSION}_{feedback}_{mode}"
@@ -40,6 +43,8 @@ SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION,
def capture_schedule_version(profile): def capture_schedule_version(profile):
if profile.quality.training_policy == STAGED: if profile.quality.training_policy == STAGED:
return STAGED_CAPTURE_SCHEDULE_VERSION return STAGED_CAPTURE_SCHEDULE_VERSION
if profile.quality.training_policy == TASKWISE_STAGED:
return TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION
if profile.quality.training_policy == ADAPTIVE: if profile.quality.training_policy == ADAPTIVE:
return ADAPTIVE_CAPTURE_SCHEDULE_VERSION return ADAPTIVE_CAPTURE_SCHEDULE_VERSION
if profile.acquisition.fixed_reference_mode == "stationary_image": if profile.acquisition.fixed_reference_mode == "stationary_image":
@@ -202,6 +207,17 @@ class CalibrationEngine:
segment.end, speed, "steady", segment.key) for segment in task_segments(task)) segment.end, speed, "steady", segment.key) for segment in task_segments(task))
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy == STAGED:
units.sort(key=lambda unit: unit.cycle) # Stable: declared task/segment order within a pass. units.sort(key=lambda unit: unit.cycle) # Stable: declared task/segment order within a pass.
elif self.profile.quality.training_policy == TASKWISE_STAGED:
# Keep the already collected first whole-hand pass reusable. During
# the second pass each task's optional third training visit is
# placed immediately after that task. Validation remains a final,
# isolated whole-hand pass and can never precede future training.
task_order = {task.key: index for index, task in enumerate(self.profile.motion.tasks)}
units.sort(key=lambda unit: (
0 if unit.cycle == 0 else 1 if unit.cycle in {1, 2} else 2,
task_order[unit.task_key],
unit.cycle,
))
return tuple(units) return tuple(units)
def mapping_probe_delta(self, task: TaskSpec) -> float | None: def mapping_probe_delta(self, task: TaskSpec) -> float | None:
@@ -287,7 +303,7 @@ class CalibrationEngine:
and session_start.get("acquisition_policy_version") and session_start.get("acquisition_policy_version")
== ACQUISITION_POLICY_VERSION == ACQUISITION_POLICY_VERSION
and session_start.get("capture_schedule_version") in ( and session_start.get("capture_schedule_version") in (
(capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, STAGED} or self.profile.acquisition.fixed_reference_mode == "stationary_image" (capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, *STAGED_POLICIES} or self.profile.acquisition.fixed_reference_mode == "stationary_image"
or self.profile.vision_motion or any(task.segments for task in self.profile.motion.tasks) or self.profile.vision_motion or any(task.segments for task in self.profile.motion.tasks)
or any(task.parent_reference is not None for task in self.profile.motion.tasks) or any(task.parent_reference is not None for task in self.profile.motion.tasks)
or any(spec.evidence_scope == "path" for spec in self.profile.motion.joint_zero_references.values()) or any(spec.evidence_scope == "path" for spec in self.profile.motion.joint_zero_references.values())
@@ -55,8 +55,8 @@ class SessionExecution:
def _make_effects(self, current): def _make_effects(self, current):
action = self.action action = self.action
from ..core.domain.capture_plan import STAGED from ..core.domain.capture_plan import STAGED_POLICIES
staged = self.profile.quality.training_policy == STAGED staged = self.profile.quality.training_policy in STAGED_POLICIES
visit = None if action.scan_unit is None else (action.scan_unit.task_key, action.scan_unit.cycle) visit = None if action.scan_unit is None else (action.scan_unit.task_key, action.scan_unit.cycle)
new_visit = staged and visit != self._entered_visit new_visit = staged and visit != self._entered_visit
previous_visit = self._entered_visit previous_visit = self._entered_visit
@@ -298,6 +298,38 @@ class SessionExecution:
current = target current = target
self._effects[:1] = [*effects, motion] self._effects[:1] = [*effects, motion]
def refresh_parent_model(self, current):
"""Acquire a current image model without replacing a passed zero.
Parent-reference reuse deliberately starts with the immutable passed
model. If only its stationary image anchor is stale, run the same
reviewed approach used by that parent's original zero acquisition and
fit a session-local model before retrying the held-pose check.
"""
motion = self._effects[0] if self._effects else None
if (self.session.phase != Phase.PREPARE or motion is None
or motion.phase != "joint_zero" or not motion.reference_reuse
or len(motion.zero_joints) != 1):
raise ValueError("parent_model_refresh_outside_reference_reuse")
name = motion.zero_joints[0]
spec = self.profile.motion.joint_zero_references[name]
source_task = next(task for task in self.profile.motion.tasks
if task.key == spec.task_key)
metadata = dict(task_key=motion.task_key, command_index=motion.command_index,
cycle=motion.cycle, attempt=motion.attempt, reference_only=True,
reference_command=spec.command, reference_approach=spec.approach_commands,
parent_model_refresh_for=motion.task_key)
effects = []
for target in (*spec.approach_commands, spec.command):
effects.extend(MotionCommand("zero_approach", point, motion.speed,
reference_joints=(name,), **metadata)
for point in build_joint_reference_waypoints(
source_task, target, profile=self.profile, current_command=current))
current = target
effects.append(MotionCommand("joint_zero", spec.command, motion.speed,
zero_joints=(name,), **metadata))
self._effects[:0] = effects
def prepend_observer_effects(self, effects): def prepend_observer_effects(self, effects):
if (self.session.phase != Phase.PREPARE or not self._effects or not effects if (self.session.phase != Phase.PREPARE or not self._effects or not effects
or any(effect.phase != 'observer_probe' or len(effect.reference_joints) != 1 or any(effect.phase != 'observer_probe' or len(effect.reference_joints) != 1
@@ -313,6 +345,12 @@ class SessionExecution:
zero_joints=(task.parent_reference.pose_joint,))) zero_joints=(task.parent_reference.pose_joint,)))
self.prepend_observer_effects(effects) self.prepend_observer_effects(effects)
def defer_current_task(self, reason):
decision = self.session.defer_current_task(reason)
self._effects.clear()
self._action_key = None
return decision
def record_diagnostic_zero_attempt(self, attempt: DiagnosticZeroAttempt): def record_diagnostic_zero_attempt(self, attempt: DiagnosticZeroAttempt):
"""A diagnostic observation closes an attempt, never creates a zero.""" """A diagnostic observation closes an attempt, never creates a zero."""
motion = self._effects[0] if self._effects else None motion = self._effects[0] if self._effects else None
@@ -325,13 +363,13 @@ class SessionExecution:
self.diagnostic_zero_attempts.update({name: attempt for name in attempt.joint_names}) self.diagnostic_zero_attempts.update({name: attempt for name in attempt.joint_names})
def training_check_due(self): def training_check_due(self):
from ..core.domain.capture_plan import ADAPTIVE, STAGED from ..core.domain.capture_plan import ADAPTIVE, STAGED, TASKWISE_STAGED
unit = self.action.scan_unit unit = self.action.scan_unit
if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED} or self.diagnostic_capture is not None if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED, TASKWISE_STAGED} or self.diagnostic_capture is not None
or unit is None or unit.cycle not in (1, 2) or unit is None or unit.cycle not in (1, 2)
or (unit.task_key, unit.cycle) in self._training_checks): or (unit.task_key, unit.cycle) in self._training_checks):
return False return False
round_units = [u for u in self.session.engine.scan_units() round_units = [u for u in self.session.scheduled_units
if (self.profile.quality.training_policy == STAGED or u.task_key == unit.task_key) if (self.profile.quality.training_policy == STAGED or u.task_key == unit.task_key)
and u.cycle == unit.cycle] and u.cycle == unit.cycle]
return unit.identity == round_units[-1].identity return unit.identity == round_units[-1].identity
@@ -118,6 +118,13 @@ class ImageReferenceTracker:
return (snapshot is self._motion_snapshot and snapshot.epoch == self._epoch return (snapshot is self._motion_snapshot and snapshot.epoch == self._epoch
and snapshot.reference is self.reference and snapshot.stamp_ns == self._last_stamp) and snapshot.reference is self.reference and snapshot.stamp_ns == self._last_stamp)
def tracking_current_image(self):
"""Whether the latest frame still matches this tracker's own anchor."""
with self._lock:
return (self.reference is not None and self._snapshot is not None
and self._snapshot[3].get("branch_status") == "tracking"
and self._snapshot[1] == self._last_stamp)
def reference_confirmed(self, role, revision): def reference_confirmed(self, role, revision):
with self._lock: with self._lock:
snapshot = self._motion_snapshot snapshot = self._motion_snapshot
@@ -19,22 +19,46 @@ class ResumeJournalImport:
rows: tuple rows: tuple
offset: int = field(default=0, init=False) offset: int = field(default=0, init=False)
durability_future: object | None = field(default=None, init=False, repr=False)
durable: bool = field(default=False, init=False)
def take_batch(self): def take_batch(self):
from .resume_storage import DeferredCaptureRecord, materialize_record from .resume_storage import DeferredCaptureRecord, materialize_record
linked = (self.offset < len(self.rows) linked = (self.offset < len(self.rows)
and isinstance(self.rows[self.offset], DeferredCaptureRecord) and isinstance(self.rows[self.offset], DeferredCaptureRecord)
and self.rows[self.offset].reference is not None) and self.rows[self.offset].reference is not None)
limit = 512 if linked else 64 # Linked evidence can normally be copied without decompression. Scan
# samples additionally feed the live training index, which reads their
# geometry and digest. Find that boundary across the whole candidate
# batch so a cheap image row cannot hide dozens of expensive samples.
limit = 64
if linked:
geometry_kinds = {"joint_sample", "secondary_joint_sample"}
candidate = self.rows[self.offset:self.offset + 512]
first_geometry = next((index for index, row in enumerate(candidate)
if row.get("kind") in geometry_kinds), None)
if first_geometry is None:
limit = 512
elif first_geometry > 0:
limit = first_geometry
else:
# Training geometry is materialized once per row by the live
# index. A 512-row slice remains bounded while avoiding one
# control-loop turn for every 64 historical records.
limit = 512
batch = tuple(row if isinstance(row, DeferredCaptureRecord) and row.reference is not None batch = tuple(row if isinstance(row, DeferredCaptureRecord) and row.reference is not None
else materialize_record(row) for row in self.rows[self.offset:self.offset + limit]) else materialize_record(row) for row in self.rows[self.offset:self.offset + limit])
self.offset += len(batch) self.offset += len(batch)
return batch return batch
@property @property
def complete(self): def transfer_complete(self):
return self.offset == len(self.rows) return self.offset == len(self.rows)
@property
def complete(self):
return self.durable
@dataclass @dataclass
class TaskResumeImport(ResumeJournalImport): class TaskResumeImport(ResumeJournalImport):
@@ -86,17 +110,18 @@ class PreparedResume:
raise ValueError("checkpoint identity/policy changed") raise ValueError("checkpoint identity/policy changed")
from .training import resolve_capture_plan from .training import resolve_capture_plan
from .engine import CalibrationEngine from .engine import CalibrationEngine
from ..core.domain.capture_plan import STAGED from ..core.domain.capture_plan import STAGED, STAGED_POLICIES
profile = resolve_capture_plan(profile, rows, source_urdf=source_urdf, profile = resolve_capture_plan(profile, rows, source_urdf=source_urdf,
verify_decisions=profile.quality.training_policy == STAGED and source_urdf is not None) verify_decisions=profile.quality.training_policy in STAGED_POLICIES and source_urdf is not None)
engine = CalibrationEngine(profile) engine = CalibrationEngine(profile)
complete = frozenset(record_scan_identity(r) for r in rows complete = frozenset(record_scan_identity(r) for r in rows
if r.get("kind") == "scan_unit_complete" and r.get("passed") is True) if r.get("kind") == "scan_unit_complete" and r.get("passed") is True)
joints = JointResume(profile, image_replay_workers=image_replay_workers) joints = JointResume(profile, image_replay_workers=image_replay_workers)
if profile.quality.training_policy == STAGED: if profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import stage_visits, stage_passed_visits, load_frozen_checkpoint, reference_import from .staged_resume import stage_visits, stage_passed_visits, load_frozen_checkpoint, reference_import
(stage_passed_visits if resume_mode == "passed" else stage_visits)(joints, engine, rows) (stage_passed_visits if resume_mode == "passed" else stage_visits)(joints, engine, rows)
frozen = load_frozen_checkpoint(profile, rows, path, source_urdf) frozen = (load_frozen_checkpoint(profile, rows, path, source_urdf)
if profile.quality.training_policy == STAGED else None)
# Index/hash historical provenance before live callbacks start. # Index/hash historical provenance before live callbacks start.
# The control loop only authorizes and drains this bounded import. # The control loop only authorizes and drains this bounded import.
pending = reference_import(joints, reuse_passed_visits=resume_mode == "passed") pending = reference_import(joints, reuse_passed_visits=resume_mode == "passed")
@@ -134,10 +159,12 @@ def _index_motion_provenance(rows):
kinds = {"motion_branch_observation", "motion_branch_initialization", kinds = {"motion_branch_observation", "motion_branch_initialization",
"observer_preparation_decision", "observer_preparation_segment_complete", "observer_preparation_decision", "observer_preparation_segment_complete",
"observer_preparation_excursion_started", "observer_preparation_cancelled_before_excursion", "observer_preparation_excursion_started", "observer_preparation_cancelled_before_excursion",
"task_observation_state",
"capture_scope_extension", "capture_observation_revision", "capture_scope_extension", "capture_observation_revision",
"passed_tasks_reused", "passed_pose_models_restored", "passed_tasks_reused", "passed_pose_models_restored",
"staged_pose_models_restored", "resume_verification", "staged_pose_models_restored", "resume_verification",
"parent_reference_verified", "joint_zero_recovery", "parent_reference_verified", "parent_reference_model_refreshed",
"joint_zero_recovery",
"joint_zero_motion_reference_verified", "joint_zero_sample", "joint_zero_motion_reference_verified", "joint_zero_sample",
"visual_motion_observation", "visual_motion_settled"} "visual_motion_observation", "visual_motion_settled"}
selected = (row for row in rows selected = (row for row in rows
@@ -313,8 +340,8 @@ class JointResume:
return tuple(rows) return tuple(rows)
def take_task(self, task_key, cycle=None): def take_task(self, task_key, cycle=None):
from ..core.domain.capture_plan import STAGED from ..core.domain.capture_plan import STAGED_POLICIES
if self.profile.quality.training_policy == STAGED: if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import visit_rows from .staged_resume import visit_rows
return visit_rows(self, task_key, cycle) return visit_rows(self, task_key, cycle)
task = next(t for t in self.profile.motion.tasks if t.key == task_key) task = next(t for t in self.profile.motion.tasks if t.key == task_key)
@@ -0,0 +1,22 @@
"""Run journal persistence barriers without blocking the control callback."""
from concurrent.futures import ThreadPoolExecutor
class JournalDurabilityWorker:
"""Single ordered worker for resume-import fsync barriers.
Resume rows are appended by the coordinator in bounded batches. The
final fsync can still take seconds on a large journal, so it runs here
while the coordinator continues feedback, safety and hold commands.
"""
def __init__(self):
self._executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="calibration-journal-sync")
def submit(self, operation):
return self._executor.submit(operation)
def close(self):
self._executor.shutdown(wait=True, cancel_futures=False)
@@ -59,6 +59,10 @@ class MotionCommand:
visual_return_joints: tuple[str, ...] = () visual_return_joints: tuple[str, ...] = ()
defer_settling_to_hold: bool = False defer_settling_to_hold: bool = False
observer_preparation_id: str = '' observer_preparation_id: str = ''
# A session-local model acquisition for a parent whose historical image
# anchor no longer matches the current, stable fixed-Tag image. The
# accepted joint zero and completed scan stay immutable.
parent_model_refresh_for: str = ''
@property @property
def recording(self): def recording(self):
@@ -135,8 +139,21 @@ class MotionExecution:
if self.profile.command.unit == "u8": if self.profile.command.unit == "u8":
duration_key = ("scan_trajectory_full_range_seconds" duration_key = ("scan_trajectory_full_range_seconds"
if self.command.phase in {"sweep", "steady"} else "command_trajectory_full_range_seconds") if self.command.phase in {"sweep", "steady"} else "command_trajectory_full_range_seconds")
full_range_duration = float(params.get(duration_key,
params.get("command_trajectory_full_range_seconds", 6.0)))
if len(self.command.measured_channels) > 1:
# O30's reviewed range-calibration group motion pairs SDK
# speed 200 with a 20 u8/s command path. Preserve that rate
# for coupled calibration segments: a faster continuous ramp
# can let the middle finger's feedback lag behind the intended
# 80-unit clearance while the ring finger is already loaded.
coupled_rate = float(self.command.speed)/10.0
if coupled_rate <= 0.0:
raise ValueError("coupled u8 motion speed must be positive")
full_range_duration = max(full_range_duration,
math.pi*255.0/(2.0*coupled_rate))
travelled, phase, duration = cosine_position_trajectory_u8(0, distance, travelled, phase, duration = cosine_position_trajectory_u8(0, distance,
elapsed, float(params.get(duration_key, params.get("command_trajectory_full_range_seconds", 6.0)))) elapsed, full_range_duration)
else: else:
speed = self.command.speed speed = self.command.speed
caps = self.profile.command.maximum_velocity caps = self.profile.command.maximum_velocity
@@ -385,8 +385,9 @@ def validate_motion_provenance(profile, records, *, required=False, require_comp
from .shared_tag_evidence import validate_shared_tag_references from .shared_tag_evidence import validate_shared_tag_references
if any(payload.get('shared_tag_pose_bridges') for payload in evidence.values()): if any(payload.get('shared_tag_pose_bridges') for payload in evidence.values()):
validate_shared_tag_references(profile, records, evidence) validate_shared_tag_references(profile, records, evidence)
from ..profiles.observer_policy import is_rigid_observer
if (any(payload.get('observer_pose_bridges') for payload in evidence.values()) if (any(payload.get('observer_pose_bridges') for payload in evidence.values())
or any(s.observer_pose_policy == 'rigid_pair_v1' for s in profile.measurement.measurements.values())): or any(is_rigid_observer(s.observer_pose_policy) for s in profile.measurement.measurements.values())):
from .observer_reference import validate_observer_references from .observer_reference import validate_observer_references
validate_observer_references(profile, records, evidence) validate_observer_references(profile, records, evidence)
image_frames, image_samples = _image_frames(records), {} image_frames, image_samples = _image_frames(records), {}
@@ -4,6 +4,8 @@ Decisions and completed segments are journaled. A restart cannot replenish the
motion budget or relabel an incomplete excursion as usable evidence. motion budget or relabel an incomplete excursion as usable evidence.
""" """
from dataclasses import dataclass, replace from dataclasses import dataclass, replace
import hashlib
import json
import math import math
import numpy as np import numpy as np
from scipy.spatial.transform import Rotation from scipy.spatial.transform import Rotation
@@ -16,6 +18,7 @@ from ..profiles.observer_preparation import observer_action, observer_actions
POLICY = 'bounded_rigid_observation_v1' POLICY = 'bounded_rigid_observation_v1'
SEQUENCE_POLICY = 'bounded_rigid_observation_v2' SEQUENCE_POLICY = 'bounded_rigid_observation_v2'
TRAJECTORY_POLICY = 'occlusion_aware_trajectory_v3'
PHASES = frozenset({'observer_preflight', 'observer_probe'}) PHASES = frozenset({'observer_preflight', 'observer_probe'})
RECOVERABLE = frozenset({'rigid_pair_candidates_unresolved', 'rigid_pair_pose_uncertain', RECOVERABLE = frozenset({'rigid_pair_candidates_unresolved', 'rigid_pair_pose_uncertain',
'rigid_pair_viewpoint_span_insufficient', 'rigid_pair_source_missing', 'observer_images_missing'}) 'rigid_pair_viewpoint_span_insufficient', 'rigid_pair_source_missing', 'observer_images_missing'})
@@ -44,6 +47,66 @@ validation or a statement that a pose is observable.
needs_excitation=len(selected) < 24 or span < 15.) needs_excitation=len(selected) < 24 or span < 15.)
def trajectory_observability(rows, parent_role, child_role):
"""Measure whether a monocular trajectory can distinguish planar branches.
This is an acquisition decision only. It never marks geometry as passed;
the independent rigid-pair fit and final holdout still own acceptance.
"""
rows = tuple(sorted(rows, key=lambda row: row['image_stamp_ns']))
coverage = normal_coverage(rows, parent_role)
paired = [row for row in rows if {parent_role, child_role} <= set(row.get('tags', {}))]
separations, diagonals = [], []
for row in paired[::max(1, len(paired)//36)]:
tag = row['tags'][parent_role]
poses = solve_square_tag_ippe(np.asarray(tag['corners_xy']), tag_size_m=tag['tag_size_m'],
camera_matrix=np.asarray(row['camera_matrix']))
if len(poses) > 1:
separations.append(math.degrees((Rotation.from_quat(poses[0].quaternion_xyzw).inv()
* Rotation.from_quat(poses[1].quaternion_xyzw)).magnitude()))
corners = np.asarray(tag['corners_xy'], dtype=float)
diagonals.append(float(np.mean((np.linalg.norm(corners[0]-corners[2]),
np.linalg.norm(corners[1]-corners[3])))))
completeness = len(paired)/max(1, len(rows))
branch_separation = float(np.median(separations)) if separations else 0.
tag_diagonal = float(np.median(diagonals)) if diagonals else 0.
needs = (coverage['needs_excitation'] or completeness < .9
or branch_separation < 1.)
return {**coverage, 'paired_frames': len(paired),
'pair_detection_fraction': completeness,
'median_planar_branch_separation_deg': branch_separation,
'median_tag_diagonal_px': tag_diagonal,
'needs_excitation': needs}
def observation_action_utility(profile, joint, plan, rows):
"""Rank a declared safe action by expected out-of-plane information."""
spec = profile.measurement.measurements[joint]
metrics = trajectory_observability(rows, spec.parent_role, spec.child_role)
channel = channel_for_joint(profile, plan.joint)
lower, upper = profile.command.minimum_values[channel], profile.command.maximum_values[channel]
excursion = max(abs(point[channel]-plan.command[channel]) for point in plan.approach_commands)
fraction = excursion/(upper-lower)
# The donor was selected from the URDF dependency graph for a transverse
# axis. This conservative estimate is only used to choose acquisition;
# it is never reported as a measured joint angle.
expected_normal_gain = 60.*fraction
expected_pixel_separation = metrics['median_tag_diagonal_px'] * math.sin(math.radians(expected_normal_gain))
reliability = metrics['pair_detection_fraction']
return {**metrics, 'donor': plan.joint, 'excursion_fraction': fraction,
'expected_normal_gain_deg': expected_normal_gain,
'expected_candidate_separation_px': expected_pixel_separation,
'utility': expected_pixel_separation*reliability}
def corner_evidence_sha256(rows, roles):
payload = [{"stamp": row['image_stamp_ns'], "camera_matrix": row.get('camera_matrix'),
"tags": {role: row.get('tags', {}).get(role, {}).get('corners_xy') for role in roles}}
for row in rows]
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':'),
allow_nan=False).encode()).hexdigest()
def latest_source_rows(owner, joint, epoch): def latest_source_rows(owner, joint, epoch):
spec = owner.profile.measurement.measurements[joint] spec = owner.profile.measurement.measurements[joint]
reports = [report for (source_epoch, view, _), (model, report) in owner.parent_references.models.items() reports = [report for (source_epoch, view, _), (model, report) in owner.parent_references.models.items()
@@ -90,7 +153,8 @@ class ObserverPreparation:
previous = self.decisions.get(joint) previous = self.decisions.get(joint)
if previous is not None and previous != row: if previous is not None and previous != row:
index = previous.get('action_index', 0)+1 index = previous.get('action_index', 0)+1
if (row.get('policy') != SEQUENCE_POLICY or row.get('action_index') != index if (row.get('policy') not in {SEQUENCE_POLICY, TRAJECTORY_POLICY}
or row.get('action_index') != index
or index >= len(observer_actions(self.owner.profile, joint)) or index >= len(observer_actions(self.owner.profile, joint))
or row.get('previous_preparation_id') != source_frame_sha256(previous) or row.get('previous_preparation_id') != source_frame_sha256(previous)
or self.source(joint) is None): or self.source(joint) is None):
@@ -175,13 +239,20 @@ class ObserverPreparation:
plan = actions[index] plan = actions[index]
spec = self.owner.profile.measurement.measurements[joint] spec = self.owner.profile.measurement.measurements[joint]
rows = self.source(joint).rows if previous is not None else latest_source_rows(self.owner, joint, epoch) rows = self.source(joint).rows if previous is not None else latest_source_rows(self.owner, joint, epoch)
coverage = normal_coverage(rows, spec.parent_role) coverage = (trajectory_observability(rows, spec.parent_role, spec.child_role)
if spec.observer_pose_policy == 'rigid_pair_trajectory_v3'
else normal_coverage(rows, spec.parent_role))
utility = observation_action_utility(self.owner.profile, joint, plan, rows)
policy = (TRAJECTORY_POLICY if spec.observer_pose_policy == 'rigid_pair_trajectory_v3'
else POLICY if len(actions)==1 else SEQUENCE_POLICY)
record = dict(kind='observer_preparation_decision', policy=POLICY if len(actions)==1 else SEQUENCE_POLICY, joint=joint, record = dict(kind='observer_preparation_decision', policy=POLICY if len(actions)==1 else SEQUENCE_POLICY, joint=joint,
task_name=motion.task_key, session_epoch=epoch, stamp_ns=stamp_ns, task_name=motion.task_key, session_epoch=epoch, stamp_ns=stamp_ns,
action='excite' if reason or coverage['needs_excitation'] else 'reuse_source', action='excite' if reason or coverage['needs_excitation'] else 'reuse_source',
reason=reason or 'normal_coverage_preflight', coverage=coverage, reason=reason or 'normal_coverage_preflight', coverage=coverage,
source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in rows}, source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in rows},
donor=plan.joint, path=[list(p) for p in (*plan.approach_commands, plan.command)]) donor=plan.joint, path=[list(p) for p in (*plan.approach_commands, plan.command)])
record.update(policy=policy, observability=coverage, action_utility=utility,
corner_evidence_sha256=corner_evidence_sha256(rows, (spec.parent_role, spec.child_role)))
if len(actions)>1: if len(actions)>1:
record.update(action_index=index, action_limit=len(actions), record.update(action_index=index, action_limit=len(actions),
previous_preparation_id=None if previous is None else source_frame_sha256(previous)) previous_preparation_id=None if previous is None else source_frame_sha256(previous))
@@ -11,6 +11,7 @@ from .motion_execution import feedback_tolerances
from .motion_provenance import source_frame_sha256 from .motion_provenance import source_frame_sha256
from .shared_tag_reference import bridge_frames from .shared_tag_reference import bridge_frames
from ..profiles.observer_preparation import observer_action, observer_actions from ..profiles.observer_preparation import observer_action, observer_actions
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
def stationary_probe_tail(profile, joint, rows, target): def stationary_probe_tail(profile, joint, rows, target):
@@ -30,16 +31,16 @@ def stationary_probe_tail(profile, joint, rows, target):
def validate_probe_evidence(profile, joint, evidence, *, admission=None): def validate_probe_evidence(profile, joint, evidence, *, admission=None):
from .observer_preparation import POLICY, SEQUENCE_POLICY from .observer_preparation import POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY
decision, rows, segments = evidence.decision, evidence.rows, evidence.segments decision, rows, segments = evidence.decision, evidence.rows, evidence.segments
plan = observer_action(profile, joint, decision) plan = observer_action(profile, joint, decision)
identity = source_frame_sha256(decision) identity = source_frame_sha256(decision)
if (decision.get('policy') == SEQUENCE_POLICY if (decision.get('policy') in {SEQUENCE_POLICY, TRAJECTORY_POLICY}
and decision.get('action_limit') != len(observer_actions(profile, joint))): and decision.get('action_limit') != len(observer_actions(profile, joint))):
raise ValueError('observer_preparation_action_budget_changed') raise ValueError('observer_preparation_action_budget_changed')
path = (*plan.approach_commands, plan.command) path = (*plan.approach_commands, plan.command)
spec = profile.measurement.measurements[joint] spec = profile.measurement.measurements[joint]
if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY} or decision.get('action') != 'excite' if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY} or decision.get('action') != 'excite'
or decision.get('joint') != joint or decision.get('donor') != plan.joint or decision.get('joint') != joint or decision.get('donor') != plan.joint
or decision.get('task_name') != profile.motion.joint_zero_references[joint].task_key or decision.get('task_name') != profile.motion.joint_zero_references[joint].task_key
or decision.get('path') != [list(p) for p in path] or decision.get('path') != [list(p) for p in path]
@@ -139,9 +140,14 @@ def build_probe_bridge(profile, source):
parents.append(pair[0]); children.append(pair[1]); matrices.append(row['camera_matrix']) parents.append(pair[0]); children.append(pair[1]); matrices.append(row['camera_matrix'])
parent_pixels.append(row['tags'][spec.parent_role]['corners_xy']) parent_pixels.append(row['tags'][spec.parent_role]['corners_xy'])
child_pixels.append(row['tags'][spec.child_role]['corners_xy']) child_pixels.append(row['tags'][spec.child_role]['corners_xy'])
plan = observer_action(profile, source.joint, source.preparation.decision)
donor_channel = channel_for_joint(profile, plan.joint)
trajectory_states = (tuple((row['reference_path_index'], row['command_vector'][donor_channel])
for row in rows) if spec.observer_pose_policy == TRAJECTORY_OBSERVER else None)
pose = fit_rigid_tag_pair([p[0] for p in parents], children, child_pixels, matrices, pose = fit_rigid_tag_pair([p[0] for p in parents], children, child_pixels, matrices,
tags[spec.child_role].size_m, tuple(r['image_stamp_ns'] for r in rows), tags[spec.child_role].size_m, tuple(r['image_stamp_ns'] for r in rows),
parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m, parent_candidates=parents) parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m,
parent_candidates=parents, trajectory_states=trajectory_states)
relation = MotionRelation(source.joint, spec.parent_role, spec.child_role) relation = MotionRelation(source.joint, spec.parent_role, spec.child_role)
bridge = SharedTagPoseBridge(relation, pose.quaternion_xyzw, pose.translation_xyz_m, bridge = SharedTagPoseBridge(relation, pose.quaternion_xyzw, pose.translation_xyz_m,
bridge_frames(endpoints, relation), rotation_uncertainty_rad=pose.rotation_uncertainty_rad, bridge_frames(endpoints, relation), rotation_uncertainty_rad=pose.rotation_uncertainty_rad,
@@ -158,6 +164,18 @@ def build_probe_bridge(profile, source):
preparation_started=source.preparation.started, preparation_started=source.preparation.started,
source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in source.preparation.rows}, source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in source.preparation.rows},
endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints}) endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints})
if spec.observer_pose_policy == TRAJECTORY_OBSERVER:
from .observer_preparation import trajectory_observability, corner_evidence_sha256
record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={
'policy': 'rigid_pair_full_trajectory_v3',
'selection_source': 'training_trajectory',
'holdout_role': 'veto_only',
'observability': trajectory_observability(rows, spec.parent_role, spec.child_role),
'adjusted_candidate_probabilities': [list(value)
for value in pose.comparison_adjusted_p_values],
'corner_evidence_sha256': corner_evidence_sha256(
source.preparation.rows, (spec.parent_role, spec.child_role)),
})
return bridge, record return bridge, record
@@ -12,6 +12,7 @@ from .branch_initialization import _pose
from .shared_tag_reference import bridge_frames from .shared_tag_reference import bridge_frames
from .motion_execution import feedback_tolerances from .motion_execution import feedback_tolerances
from .motion_provenance import source_frame_sha256 from .motion_provenance import source_frame_sha256
from ..profiles.observer_policy import is_rigid_observer
def observer_roles(profile, motion): def observer_roles(profile, motion):
@@ -43,7 +44,7 @@ def build_observer_bridge(profile, source):
raise ValueError('observer_parent_model_invalid') raise ValueError('observer_parent_model_invalid')
uncertainty = {name: (angle, point) for name, angle, point in source.parent_report['geometry_uncertainty']} uncertainty = {name: (angle, point) for name, angle, point in source.parent_report['geometry_uncertainty']}
inherited = uncertainty[parent_model.geometry[0].relation.joint] inherited = uncertainty[parent_model.geometry[0].relation.joint]
rigid = spec.observer_pose_policy == 'rigid_pair_v1' rigid = is_rigid_observer(spec.observer_pose_policy)
if not rigid: if not rigid:
require_observer_precision_budget(inherited) require_observer_precision_budget(inherited)
target = profile.motion.joint_zero_references[source.joint].command target = profile.motion.joint_zero_references[source.joint].command
@@ -109,6 +110,18 @@ def build_observer_bridge(profile, source):
endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints}) endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints})
if rigid: if rigid:
record['conditioning'] = RIGID_PAIR_CONDITIONING record['conditioning'] = RIGID_PAIR_CONDITIONING
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
if spec.observer_pose_policy == TRAJECTORY_OBSERVER:
from .observer_preparation import trajectory_observability, corner_evidence_sha256
record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={
'policy': 'rigid_pair_full_trajectory_v3',
'selection_source': 'training_trajectory', 'holdout_role': 'veto_only',
'observability': trajectory_observability(rows, spec.parent_role, spec.child_role),
'adjusted_candidate_probabilities': [list(value)
for value in pose.comparison_adjusted_p_values],
'corner_evidence_sha256': corner_evidence_sha256(
rows, (spec.parent_role, spec.child_role)),
})
return bridge, record return bridge, record
@@ -139,7 +152,7 @@ def observer_inputs(owner, view, endpoint_rows, arc_stamps, epoch):
continue continue
held = tuple(sorted(owner.tag_feedback_channels[spec.child_role]-owner.tag_feedback_channels[spec.parent_role])) held = tuple(sorted(owner.tag_feedback_channels[spec.child_role]-owner.tag_feedback_channels[spec.parent_role]))
result.append(ObserverBridgeInput(joint, model.image_model, report, rows, endpoints, held)) result.append(ObserverBridgeInput(joint, model.image_model, report, rows, endpoints, held))
if spec.observer_pose_policy == 'rigid_pair_v1': if is_rigid_observer(spec.observer_pose_policy):
# Select the latest eligible source before fitting. A failed # Select the latest eligible source before fitting. A failed
# fit cannot search older sources for a more favorable result. # fit cannot search older sources for a more favorable result.
break break
@@ -182,9 +195,10 @@ def validate_observer_references(profile, records, evidence):
checked = set() checked = set()
for payload in evidence.values(): for payload in evidence.values():
required = {joint for joint, spec in profile.measurement.measurements.items() required = {joint for joint, spec in profile.measurement.measurements.items()
if joint in payload['zero_joints'] and spec.observer_pose_policy == 'rigid_pair_v1'} if joint in payload['zero_joints'] and is_rigid_observer(spec.observer_pose_policy)}
declared = [r['joint'] for r in payload.get('observer_pose_bridges', ()) declared = [r['joint'] for r in payload.get('observer_pose_bridges', ())
if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'}] if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'}]
if set(declared) != required or len(declared) != len(required): if set(declared) != required or len(declared) != len(required):
raise ValueError('rigid_pair_required_evidence_missing') raise ValueError('rigid_pair_required_evidence_missing')
for payload in evidence.values(): for payload in evidence.values():
@@ -204,7 +218,7 @@ def validate_observer_references(profile, records, evidence):
if (evidence_id not in evidence or evidence[evidence_id]['tag_role'] != role if (evidence_id not in evidence or evidence[evidence_id]['tag_role'] != role
or evidence[evidence_id]['image_model_sha256'] != record['parent_model_sha256']): or evidence[evidence_id]['image_model_sha256'] != record['parent_model_sha256']):
raise ValueError('observer_parent_identity_changed') raise ValueError('observer_parent_identity_changed')
if record.get('version') == 'observer_pose_bridge_v3': if record.get('version') in {'observer_pose_bridge_v3', 'observer_pose_bridge_v4'}:
from .observer_preparation_evidence import validate_probe_reference from .observer_preparation_evidence import validate_probe_reference
validate_probe_reference(profile, records, payload, record, parent, observations) validate_probe_reference(profile, records, payload, record, parent, observations)
continue continue
@@ -5,6 +5,7 @@ upstream pose change invalidates the former, but not a rigid Tag's mounting.
No curve, joint zero or candidate selection is copied between joints. No curve, joint zero or candidate selection is copied between joints.
""" """
from dataclasses import dataclass
import math import math
import numpy as np import numpy as np
@@ -15,6 +16,14 @@ from ..core.geometry.tag_pose.source_hinge import SourceHinge
from .resume import ResumeVerifier from .resume import ResumeVerifier
@dataclass(frozen=True)
class RefreshedPoseSource:
"""Bind a refreshed model to its immutable source epoch across restarts."""
image_model_sha256: str
source_epoch: int
def check_parent_pose(profile, task, reference, rows, channels, *, def check_parent_pose(profile, task, reference, rows, channels, *,
verification='staged_revisit_v1', feedback_limits=None): verification='staged_revisit_v1', feedback_limits=None):
"""Same pure check for live execution and persisted evidence read-back.""" """Same pure check for live execution and persisted evidence read-back."""
@@ -78,6 +87,7 @@ class ParentReferenceRegistry:
self.models = {} self.models = {}
self.verified = {} self.verified = {}
self.zero_references = {} self.zero_references = {}
self.refreshed_pose_sources = {}
def remember_zero(self, reference, epoch): def remember_zero(self, reference, epoch):
for view in {sample.view for sample in reference.samples}: for view in {sample.view for sample in reference.samples}:
@@ -104,6 +114,8 @@ class ParentReferenceRegistry:
for (epoch, view, joint), value in tuple(self.models.items()) if epoch == previous}) for (epoch, view, joint), value in tuple(self.models.items()) if epoch == previous})
self.verified.update({(current, task): value self.verified.update({(current, task): value
for (epoch, task), value in tuple(self.verified.items()) if epoch == previous}) for (epoch, task), value in tuple(self.verified.items()) if epoch == previous})
self.refreshed_pose_sources.update({(current, task): value
for (epoch, task), value in tuple(self.refreshed_pose_sources.items()) if epoch == previous})
self.zero_references.update({(current, view, joint): value self.zero_references.update({(current, view, joint): value
for (epoch, view, joint), value in tuple(self.zero_references.items()) if epoch == previous}) for (epoch, view, joint), value in tuple(self.zero_references.items()) if epoch == previous})
@@ -129,6 +141,12 @@ class ParentReferenceRegistry:
""" """
from ..core.geometry.tag_pose.pose_bridge import is_handoff_policy from ..core.geometry.tag_pose.pose_bridge import is_handoff_policy
pose, geometry = self.sources(task, epoch) pose, geometry = self.sources(task, epoch)
refreshed = self.refreshed_pose_sources.get((epoch, task.key))
if refreshed is not None:
if (refreshed.image_model_sha256 != pose[1].get("image_model_sha256")
or refreshed.source_epoch != pose[1].get("session_epoch")):
raise ValueError("parent_reference_refreshed_pose_source_changed")
return pose
if not is_handoff_policy(geometry[1].get('image_model_selection_policy')): if not is_handoff_policy(geometry[1].get('image_model_selection_policy')):
return pose return pose
bridges = geometry[1].get('shared_tag_pose_bridges', ()) bridges = geometry[1].get('shared_tag_pose_bridges', ())
@@ -138,6 +156,50 @@ class ParentReferenceRegistry:
raise ValueError('parent_reference_handoff_source_changed') raise ValueError('parent_reference_handoff_source_changed')
return geometry return geometry
def authorize_refreshed_pose(self, task, epoch, version, reference):
"""Prefer a freshly measured pose model while retaining passed geometry."""
joint = task.parent_reference.pose_joint
model, report = self.models.get((epoch, task.view, joint), (None, None))
if (model is None or report is None or report.get("session_epoch") != epoch
or model.image_model is None
or {item.relation.joint for item in model.image_model.geometry} != {joint}):
raise ValueError("parent_reference_refresh_model_missing_or_changed")
identity = report.get("image_model_sha256")
if not isinstance(identity, str) or not identity:
raise ValueError("parent_reference_refresh_model_missing_or_changed")
self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource(identity, epoch)
from .motion_provenance import source_frame_sha256
return {"kind": "parent_reference_model_refreshed", "schema_version": 1,
"task_name": task.key, "view": task.view, "pose_joint": joint,
"session_epoch": epoch, "motion_version": version,
"image_model_sha256": identity,
"source_zero_sha256": source_frame_sha256(reference.as_record()),
"passed_zero_preserved": True,
"source_frame_hashes": dict(report.get("source_frame_hashes", {}))}
def restore_refreshed_pose(self, record, epoch):
"""Restore an explicit refresh grant without relabelling its source."""
task = next((item for item in self.profile.motion.tasks
if item.key == record.get("task_name")), None)
if (task is None or task.parent_reference is None
or record.get("schema_version") != 1
or record.get("view") != task.view
or record.get("pose_joint") != task.parent_reference.pose_joint
or record.get("passed_zero_preserved") is not True
or type(record.get("session_epoch")) is not int):
raise ValueError("parent_reference_refresh_record_changed")
model, report = self.models.get((epoch, task.view,
task.parent_reference.pose_joint), (None, None))
identity = record.get("image_model_sha256")
if (model is None or report is None or model.image_model is None
or identity != report.get("image_model_sha256")
or record["session_epoch"] != report.get("session_epoch")
or dict(record.get("source_frame_hashes", {}))
!= dict(report.get("source_frame_hashes", {}))):
raise ValueError("parent_reference_refresh_model_missing_or_changed")
self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource(
identity, record["session_epoch"])
def activate_joints(self, joints, epoch, capture): def activate_joints(self, joints, epoch, capture):
installed = set() installed = set()
for joint in joints: for joint in joints:
@@ -5,7 +5,7 @@ reverification runs here. New acquisitions and final artifact validation keep
their normal gates. An incomplete task is restarted as a whole. their normal gates. An incomplete task is restarted as a whole.
""" """
from dataclasses import dataclass from dataclasses import dataclass, field
from ..core.domain.motion_path import record_scan_identity from ..core.domain.motion_path import record_scan_identity
from ..core.domain.reference import read_joint_zero_references from ..core.domain.reference import read_joint_zero_references
@@ -16,6 +16,29 @@ from .joint_resume import ResumeJournalImport, _index_motion_provenance
class PassedReferenceImport(ResumeJournalImport): class PassedReferenceImport(ResumeJournalImport):
references: dict references: dict
model_records: tuple model_records: tuple
restore_phase: str = field(default="binding", init=False)
restore_rows: tuple = field(default=(), init=False, repr=False)
restore_offset: int = field(default=0, init=False)
def coherent_observer_preparation_rows(rows):
"""Discard transition fragments whose originating decision is absent.
Compact checkpoints may contain a cancellation after an earlier migration
already removed the decision it cancelled. Such a fragment represents no
live state and cannot be replayed on its own. Decisions and all transitions
still present together remain byte-for-byte unchanged.
"""
from .motion_provenance import source_frame_sha256
decisions = {source_frame_sha256(row) for row in rows
if row.get("kind") == "observer_preparation_decision"
and row.get("action") == "excite"}
bound_kinds = {"observer_preparation_segment_complete",
"observer_preparation_excursion_started",
"observer_preparation_cancelled_before_excursion"}
return tuple(row for row in rows if row.get("kind") not in bound_kinds
or row.get("observer_preparation_id") in decisions)
def stage_passed_tasks(resume, engine, rows): def stage_passed_tasks(resume, engine, rows):
@@ -48,15 +71,25 @@ def reference_import(resume):
"""Import prior evidence durably before skipping any completed task.""" """Import prior evidence durably before skipping any completed task."""
provenance = _index_motion_provenance(resume.rows) provenance = _index_motion_provenance(resume.rows)
references = resume.references references = resume.references
tasks = {reference.task_key for reference in references.values()}
parent_images = {(row["view"], int(stamp)) for row in provenance.values() parent_images = {(row["view"], int(stamp)) for row in provenance.values()
if row.get("kind") == "parent_reference_verified" for stamp in row["source_frame_hashes"]} if row.get("kind") == "parent_reference_verified"
and row.get("task_name") in tasks for stamp in row["source_frame_hashes"]}
parent_images.update((sample["view"], sample["image_stamp_ns"]) parent_images.update((sample["view"], sample["image_stamp_ns"])
for row in provenance.values() if row.get("kind") == "parent_reference_verified" for row in provenance.values() if row.get("kind") == "parent_reference_verified"
and row.get("task_name") in tasks
for sample in row["source_reference"]["samples"]) for sample in row["source_reference"]["samples"])
zero_images = parent_images | {(row.get("view"), row.get("image_stamp_ns")) zero_images = parent_images | {(row.get("view"), row.get("image_stamp_ns"))
for row in provenance.values() if row.get("kind") == "joint_zero_sample" and row.get("joint") in references} for row in provenance.values() if row.get("kind") == "joint_zero_sample" and row.get("joint") in references}
selected = [] selected = []
for row in provenance.values(): for row in provenance.values():
# Passed mode restarts every incomplete task as a whole. Its old
# branch models and preparation images belong to an abandoned visit;
# restoring them would bind the new session's current image anchor to
# a previous camera epoch. Task-independent audit rows remain useful.
task_name = row.get("task_name")
if task_name is not None and task_name not in tasks:
continue
image = row.get("view"), row.get("image_stamp_ns") image = row.get("view"), row.get("image_stamp_ns")
if (row.get("kind") in {"joint_zero_sample", "joint_zero_motion_reference_verified"} if (row.get("kind") in {"joint_zero_sample", "joint_zero_motion_reference_verified"}
and row.get("joint") not in references and image not in parent_images): and row.get("joint") not in references and image not in parent_images):
@@ -64,7 +97,7 @@ def reference_import(resume):
if row.get("kind") == "pnp_candidate_frame" and image not in zero_images: if row.get("kind") == "pnp_candidate_frame" and image not in zero_images:
continue continue
selected.append(row) selected.append(row)
tasks = {reference.task_key for reference in references.values()} selected = coherent_observer_preparation_rows(selected)
models = tuple(row for row in selected if row.get("kind") == "motion_branch_initialization" models = tuple(row for row in selected if row.get("kind") == "motion_branch_initialization"
and row.get("status") == "resolved" and row.get("task_name") in tasks) and row.get("status") == "resolved" and row.get("task_name") in tasks)
audit = {"kind": "passed_tasks_reused", "resume_mode": "passed", audit = {"kind": "passed_tasks_reused", "resume_mode": "passed",
@@ -76,7 +109,7 @@ def reference_import(resume):
references=references, model_records=models) references=references, model_records=models)
def restore_parent_models(registry, records, epoch): def restore_parent_models(registry, records, epoch, provenance_records=()):
"""Keep source identities while making passed installations available to new tasks.""" """Keep source identities while making passed installations available to new tasks."""
from ..core.geometry.image_motion_replay import frozen_image_model from ..core.geometry.image_motion_replay import frozen_image_model
from .branch_initialization import MotionBranchModel from .branch_initialization import MotionBranchModel
@@ -87,6 +120,12 @@ def restore_parent_models(registry, records, epoch):
model = MotionBranchModel(row["task_name"], row["view"], (), model = MotionBranchModel(row["task_name"], row["view"], (),
tuple(sorted(row["evidence_ids"].items())), frozen_image_model(row)) tuple(sorted(row["evidence_ids"].items())), frozen_image_model(row))
registry.remember(model, row, epoch=epoch) registry.remember(model, row, epoch=epoch)
latest_refreshes = {}
for row in provenance_records:
if row.get("kind") == "parent_reference_model_refreshed":
latest_refreshes[row.get("task_name")] = row
for record in latest_refreshes.values():
registry.restore_refreshed_pose(record, epoch)
def restored_model_record(records, epoch, stamp_ns): def restored_model_record(records, epoch, stamp_ns):
@@ -93,6 +93,11 @@ def discover_resume_candidate(
reject_diagnostic_capture((row,)) reject_diagnostic_capture((row,))
kind = str(row.get("kind", "")) kind = str(row.get("kind", ""))
if kind == "session_start": if kind == "session_start":
from ..core.artifacts.evidence_journal import REFERENCE_KEY, decode_envelope
if REFERENCE_KEY in row:
from ..core.geometry.frozen_evidence import EvidenceDecoder
row = decode_envelope(candidate / "raw_samples.jsonl", row,
EvidenceDecoder(verify_hashes=False))
if start is not None: if start is not None:
raise ValueError("multiple checkpoint session headers") raise ValueError("multiple checkpoint session headers")
start = row start = row
@@ -35,7 +35,9 @@ class DeferredCaptureRecord(Mapping):
@classmethod @classmethod
def from_envelope(cls, path, envelope, decoder): def from_envelope(cls, path, envelope, decoder):
from ..core.artifacts.evidence_journal import CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY from ..core.artifacts.evidence_journal import (
CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY, SCHEMA_KEY, _schema_fields,
)
value = object.__new__(cls) value = object.__new__(cls)
value.path, value.offset, value.length = path, 0, 0 value.path, value.offset, value.length = path, 0, 0
from ..core.artifacts.evidence_journal import EvidenceRecordReference from ..core.artifacts.evidence_journal import EvidenceRecordReference
@@ -43,41 +45,54 @@ class DeferredCaptureRecord(Mapping):
value.digest = bytes.fromhex(value.reference.record_sha256) value.digest = bytes.fromhex(value.reference.record_sha256)
value.canonical_sha256 = envelope.get(CANONICAL_SHA256_KEY) value.canonical_sha256 = envelope.get(CANONICAL_SHA256_KEY)
value.metadata = {key: item for key, item in envelope.items() value.metadata = {key: item for key, item in envelope.items()
if key not in {REFERENCE_KEY, FIELDS_KEY, CANONICAL_SHA256_KEY}} if key not in {REFERENCE_KEY, FIELDS_KEY, SCHEMA_KEY, CANONICAL_SHA256_KEY}}
value.field_names = _shared_field_names(tuple(envelope[FIELDS_KEY])) fields = envelope.get(FIELDS_KEY)
if fields is None and SCHEMA_KEY in envelope:
fields = _schema_fields(path, envelope[SCHEMA_KEY])
value.field_names = None if fields is None else _shared_field_names(tuple(fields))
value.decoder = decoder value.decoder = decoder
return value return value
def materialize(self): def materialize(self):
return self.decoder.loads(self._record_bytes())
def materialize_plain(self):
"""Decode one integrity-checked row without building shared models."""
return json.loads(self._record_bytes())
def _record_bytes(self):
if self.reference is not None: if self.reference is not None:
from ..core.artifacts.evidence_journal import evidence_record_bytes from ..core.artifacts.evidence_journal import evidence_record_bytes
line = evidence_record_bytes(self.path, self.reference) return evidence_record_bytes(self.path, self.reference)
return self.decoder.loads(line)
with self.path.open("rb") as stream: with self.path.open("rb") as stream:
stream.seek(self.offset) stream.seek(self.offset)
line = stream.read(self.length) line = stream.read(self.length)
if hashlib.sha256(line).digest() != self.digest: if hashlib.sha256(line).digest() != self.digest:
raise ValueError("resume_source_record_changed") raise ValueError("resume_source_record_changed")
return self.decoder.loads(line) return line
def __getitem__(self, key): def __getitem__(self, key):
if key in self.metadata: if key in self.metadata:
return self.metadata[key] return self.metadata[key]
if key not in self.field_names: if self.field_names is not None and key not in self.field_names:
raise KeyError(key) raise KeyError(key)
return self.materialize()[key] return self.materialize()[key]
def __iter__(self): def __iter__(self):
return iter(self.field_names) return iter(self.materialize() if self.field_names is None else self.field_names)
def __len__(self): def __len__(self):
return len(self.field_names) return len(self.materialize()) if self.field_names is None else len(self.field_names)
def materialize_record(row): def materialize_record(row):
return row.materialize() if isinstance(row, DeferredCaptureRecord) else row return row.materialize() if isinstance(row, DeferredCaptureRecord) else row
def materialize_plain_record(row):
return row.materialize_plain() if isinstance(row, DeferredCaptureRecord) else row
def load_passed_capture(path): def load_passed_capture(path):
"""Read once without retaining every scan's expanded image dictionaries. """Read once without retaining every scan's expanded image dictionaries.
@@ -101,7 +116,15 @@ def load_passed_capture(path):
if not line.strip(): if not line.strip():
continue continue
try: try:
row = decoder.loads(line) # Indexed envelopes contain only scalar lookup fields and an
# evidence reference. Running the recursive evidence decoder
# over tens of thousands of these small rows dominated resume
# startup even though there is no model to intern. Preserve
# the decoder for the few inline records that carry shared
# frozen objects.
needs_evidence_decoder = (b'"frozen_image_model"' in line
or b'"reference_frame"' in line)
row = decoder.loads(line) if needs_evidence_decoder else json.loads(line)
except json.JSONDecodeError as error: except json.JSONDecodeError as error:
raise ValueError(f"invalid calibration JSONL at line {number}") from error raise ValueError(f"invalid calibration JSONL at line {number}") from error
if not isinstance(row, dict): if not isinstance(row, dict):
@@ -7,6 +7,7 @@ hardware. Pending legacy status/artifact contracts are isolated in compat.
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import hashlib
import json import json
import math import math
import os import os
@@ -136,6 +137,21 @@ def run_online(
if diagnostic is not None and not commands_enabled: if diagnostic is not None and not commands_enabled:
raise ValueError("diagnostic_capture requires commands_enabled") raise ValueError("diagnostic_capture requires commands_enabled")
session = create_session_directory(config.session_root) session = create_session_directory(config.session_root)
# A CLI-selected preparation policy is a real protected profile, not an
# in-process hint. Materialize its canonical bytes so the launched node,
# resume selector and journal all bind the identical effective contract.
from ..profiles.export import dump_hand_profile
effective_profile = dump_hand_profile(profile).encode()
effective_digest = hashlib.sha256(effective_profile).hexdigest()
from ..profiles import load_hand_profile
runtime_profile_changed = (config.profile_config is None
or load_hand_profile(config.profile_config) != profile)
if runtime_profile_changed:
effective_path = session / "effective_profile.yaml"
effective_path.write_bytes(effective_profile)
from dataclasses import replace
config = replace(config, profile_config=effective_path,
profile_config_sha256=effective_digest)
from ..profiles.validator import validate_executable_profile, require_release_geometry from ..profiles.validator import validate_executable_profile, require_release_geometry
contract_report = validate_executable_profile(profile, config.source_urdf) contract_report = validate_executable_profile(profile, config.source_urdf)
atomic_write_json(session/"measurement_contract.json", contract_report) atomic_write_json(session/"measurement_contract.json", contract_report)
@@ -307,7 +323,8 @@ def main(args: list[str] | None = None) -> None:
parser.add_argument("--record-bag", action="store_true") parser.add_argument("--record-bag", action="store_true")
parser.add_argument("--commands-disabled", action="store_true") parser.add_argument("--commands-disabled", action="store_true")
parser.add_argument("--no-resume", action="store_true") parser.add_argument("--no-resume", action="store_true")
parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3"), default=None, parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3",
"taskwise_staged_2_to_3"), default=None,
help="试验采集策略:两轮训练,必要时补第三轮;保留独立验证,默认沿用配置") help="试验采集策略:两轮训练,必要时补第三轮;保留独立验证,默认沿用配置")
parser.add_argument("--startup-timeout-seconds", type=float, default=None, parser.add_argument("--startup-timeout-seconds", type=float, default=None,
help="等待节点初始化及设备就绪的秒数(默认 120);大断点逐帧校验可显式延长,不改变运动保护") help="等待节点初始化及设备就绪的秒数(默认 120);大断点逐帧校验可显式延长,不改变运动保护")
@@ -322,9 +339,10 @@ def main(args: list[str] | None = None) -> None:
parser.add_argument("--offline-raw", default="") parser.add_argument("--offline-raw", default="")
parser.add_argument("--offline-output", default="") parser.add_argument("--offline-output", default="")
parser.add_argument("--publish-offline", action="store_true") parser.add_argument("--publish-offline", action="store_true")
parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_witness_v2', parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_trajectory_v3',
choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2"), choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2",
help="新 O30 staged 准备策略;省略取值使用 v2,按观测依赖统一启用刚体双 Tag 求解;v1 仅保留历史兼容") "occlusion_aware_trajectory_v3"),
help="O30 staged 准备策略;省略取值使用 v3 整段轨迹消歧和有界自动观察;v1/v2 仅保留历史兼容")
selected = parser.parse_args(args) selected = parser.parse_args(args)
if selected.startup_timeout_seconds is not None and ( if selected.startup_timeout_seconds is not None and (
not math.isfinite(selected.startup_timeout_seconds) or selected.startup_timeout_seconds <= 0 not math.isfinite(selected.startup_timeout_seconds) or selected.startup_timeout_seconds <= 0
@@ -357,6 +375,10 @@ def main(args: list[str] | None = None) -> None:
from ..profiles.preparation import configure_preparation from ..profiles.preparation import configure_preparation
try: try:
profile = configure_preparation(config.calibration_contract.typed_profile, selected.preparation_policy) profile = configure_preparation(config.calibration_contract.typed_profile, selected.preparation_policy)
if selected.preparation_policy == 'occlusion_aware_trajectory_v3':
from ..core.urdf.kinematics import UrdfKinematicModel
from ..profiles.observer_preparation import configure_observer_preparations
profile = configure_observer_preparations(profile, UrdfKinematicModel(config.source_urdf))
config = replace(config, calibration_contract=ProductCalibrationContract(declarative=profile)) config = replace(config, calibration_contract=ProductCalibrationContract(declarative=profile))
except ValueError as error: except ValueError as error:
parser.error(str(error)) parser.error(str(error))
@@ -62,7 +62,7 @@ class CalibrationSession:
self.required_observation_views = required_observation_views(profile, diagnostic_capture) self.required_observation_views = required_observation_views(profile, diagnostic_capture)
self.engine = CalibrationEngine(profile, diagnostic_capture=diagnostic_capture) self.engine = CalibrationEngine(profile, diagnostic_capture=diagnostic_capture)
self.phase = CalibrationPhase.WAIT_DEVICE self.phase = CalibrationPhase.WAIT_DEVICE
self._units = self.engine.scan_units() self._units = list(self.engine.scan_units())
self._unit_index = 0 self._unit_index = 0
self._retry_by_unit: dict[tuple[str, int, str], int] = {} self._retry_by_unit: dict[tuple[str, int, str], int] = {}
self._mapping_probed_channels: set[int] = set() self._mapping_probed_channels: set[int] = set()
@@ -72,6 +72,8 @@ class CalibrationSession:
self._pause = PauseStatus() self._pause = PauseStatus()
self._reference_locked = False self._reference_locked = False
self._rescan_pending = False self._rescan_pending = False
from .task_deferral import TaskDeferralQueue
self.task_deferrals = TaskDeferralQueue(profile)
@property @property
def current_unit(self) -> ScanUnit | None: def current_unit(self) -> ScanUnit | None:
@@ -79,13 +81,17 @@ class CalibrationSession:
return None return None
return self._units[self._unit_index] return self._units[self._unit_index]
@property
def scheduled_units(self):
return tuple(self._units)
def select_training(self, task_key, cycles): def select_training(self, task_key, cycles):
"""Replace only future optional units; evidence IDs never change.""" """Replace only future optional units; evidence IDs never change."""
from ..core.domain.capture_plan import select_task_training from ..core.domain.capture_plan import select_task_training
current = self.current_unit current = self.current_unit
self.profile = select_task_training(self.profile, task_key, cycles) self.profile = select_task_training(self.profile, task_key, cycles)
self.engine.profile = self.profile self.engine.profile = self.profile
self._units = self.engine.scan_units() self._units = list(self.engine.scan_units())
if current is not None: if current is not None:
self._unit_index = next(i for i, unit in enumerate(self._units) if unit.identity == current.identity) self._unit_index = next(i for i, unit in enumerate(self._units) if unit.identity == current.identity)
@@ -161,7 +167,7 @@ class CalibrationSession:
self._require(CalibrationPhase.PREPARE) self._require(CalibrationPhase.PREPARE)
unit = self.current_unit unit = self.current_unit
if unit is None: if unit is None:
self.phase = CalibrationPhase.RETURN_BASELINE self._finish_capture_schedule()
return return
task = self._task(unit.task_key) task = self._task(unit.task_key)
if ( if (
@@ -227,6 +233,79 @@ class CalibrationSession:
# The executor elides already-satisfied waypoints without motion. # The executor elides already-satisfied waypoints without motion.
self.phase = CalibrationPhase.PREPARE self.phase = CalibrationPhase.PREPARE
return return
self._finish_capture_schedule()
def defer_current_task(self, reason: str):
"""Move one recoverable visual visit to the end of its current stage.
A second failure records the visit as unobservable and removes only
its still-uncompleted units. Finalization is then blocked explicitly.
"""
self._require(CalibrationPhase.PREPARE)
unit = self.current_unit
if unit is None:
raise ValueError("task_deferral_without_current_unit")
from .task_deferral import dependency_closure, TaskObservationState
affected = dependency_closure(self.profile, unit.task_key)
moved = self._move_visit_to_stage_end(unit.cycle, affected)
attempt = self.task_deferrals.attempts.get((unit.task_key, unit.cycle), 0)+1
decision = self.task_deferrals.decide(unit.task_key, unit.cycle, reason, len(moved))
if decision.state == TaskObservationState.PENDING:
self._insert_at_stage_end(unit.cycle, moved)
else:
# Descendants cannot be accepted without the missing parent
# geometry. Remove their future visits from every later cycle.
self._units = self._units[:self._unit_index] + [candidate
for candidate in self._units[self._unit_index:]
if candidate.task_key not in set(decision.affected_tasks)]
self._rescan_pending = False
if self.current_unit is None:
self._finish_capture_schedule()
else:
self.phase = CalibrationPhase.PREPARE
return decision
def restore_task_deferrals(self, records):
"""Restore retry budgets and the exact stage-local pending order."""
self.task_deferrals.restore(records)
from .task_deferral import TaskObservationState, unresolved_observation_records
for row in unresolved_observation_records(self.profile, records):
if row.get("state") != TaskObservationState.PENDING.value:
continue
moved = self._move_visit_to_stage_end(
int(row["cycle"]), frozenset(row["affected_tasks"]))
self._insert_at_stage_end(int(row["cycle"]), moved)
def _stage_index(self, cycle):
from ..core.domain.capture_plan import TASKWISE_STAGED
if self.profile.quality.training_policy == TASKWISE_STAGED:
return 0 if cycle == 0 else 1 if cycle in {1, 2} else 2
return cycle
def _move_visit_to_stage_end(self, cycle, affected_tasks):
indices = [index for index in range(self._unit_index, len(self._units))
if self._units[index].cycle == cycle
and self._units[index].task_key in affected_tasks]
if not indices:
raise ValueError("task_deferral_has_no_pending_units")
moved = [self._units[index] for index in indices]
for index in reversed(indices):
del self._units[index]
return moved
def _insert_at_stage_end(self, cycle, moved):
stage = self._stage_index(cycle)
insertion = next((index for index in range(self._unit_index, len(self._units))
if self._stage_index(self._units[index].cycle) > stage), len(self._units))
self._units[insertion:insertion] = moved
def _finish_capture_schedule(self):
if self.task_deferrals.unobservable:
details = {f"{task}:{cycle}": reason
for (task, cycle), reason in self.task_deferrals.unobservable.items()}
self.fail("unobservable_tasks", "安全观察和一次阶段回访后仍不可观测",
details, suggestion="保留数据;调整 Tag 朝向、增加非共面 Tag 或改变机位后重试")
return
self.phase = CalibrationPhase.RETURN_BASELINE self.phase = CalibrationPhase.RETURN_BASELINE
def return_complete(self) -> None: def return_complete(self) -> None:
@@ -331,6 +410,21 @@ class CalibrationSession:
ids = tuple(tag.tag_id for tag in view.tags if tag.role in roles) ids = tuple(tag.tag_id for tag in view.tags if tag.role in roles)
if ids: if ids:
required[view.name] = ids required[view.name] = ids
from ..core.domain.capture_plan import TASKWISE_STAGED
taskwise = self.profile.quality.training_policy == TASKWISE_STAGED
displayed_cycle = (None if unit is None or unit.sample_phase == "steady" else
({0: 1, 1: 2, 2: 2, 3: 3}[unit.cycle] if taskwise else
list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key
and u.sample_phase == "sweep")).index(unit.cycle) + 1))
displayed_cycle_count = (3 if taskwise else
len(self.diagnostic_capture.cycles) if self.diagnostic_capture is not None else
len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key
and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1)
capture_stage = ("training_supplement" if taskwise and unit is not None and unit.cycle == 2
else "command_validation" if unit is not None and unit.sample_phase == "steady"
and unit.cycle == self.profile.quality.holdout_cycle + 2
else "command_training" if unit is not None and unit.sample_phase == "steady"
else "motion")
return CalibrationStatus( return CalibrationStatus(
state=self.phase.value, state=self.phase.value,
phase=self.phase.value, phase=self.phase.value,
@@ -344,22 +438,17 @@ class CalibrationSession:
task=TaskStatus( task=TaskStatus(
key="" if task is None else task.key, key="" if task is None else task.key,
label="" if task is None else " / ".join(task.joints), label="" if task is None else " / ".join(task.joints),
cycle=(None if unit is None or unit.sample_phase == "steady" else cycle=displayed_cycle,
list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key cycle_count=displayed_cycle_count,
and u.sample_phase == "sweep")).index(unit.cycle) + 1),
cycle_count=(
len(self.diagnostic_capture.cycles)
if self.diagnostic_capture is not None
else len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key
and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1
),
direction="" if unit is None else unit.direction, direction="" if unit is None else unit.direction,
capture_stage=("command_validation" if unit is not None and unit.sample_phase == "steady" capture_stage=capture_stage,
and unit.cycle == self.profile.quality.holdout_cycle + 2 else "command_training" if unit is not None
and unit.sample_phase == "steady" else "motion"),
required_tag_ids_by_view=required, required_tag_ids_by_view=required,
), ),
resume={"message": self._resume_message}, resume={"message": self._resume_message,
"deferred_tasks": {f"{task}:{cycle}": attempt
for (task, cycle), attempt in self.task_deferrals.attempts.items()},
"unobservable_tasks": [f"{task}:{cycle}"
for task, cycle in self.task_deferrals.unobservable]},
pause=self._pause, pause=self._pause,
) )
@@ -18,7 +18,8 @@ SOURCE_PHASES = frozenset({'sweep', 'steady', 'joint_zero'})
def enabled(profile): def enabled(profile):
return (profile.quality.training_policy == 'staged_2_to_3' from ..core.domain.capture_plan import STAGED_POLICIES
return (profile.quality.training_policy in STAGED_POLICIES
and bool(profile.motion.preparation_witnesses)) and bool(profile.motion.preparation_witnesses))
@@ -12,6 +12,8 @@ from .scan_quality import evaluate_capture_unit
def stage_visits(resume, engine, rows): def stage_visits(resume, engine, rows):
from .task_deferral import deferred_visit_keys
deferred = deferred_visit_keys(resume.profile, rows)
references = read_joint_zero_references(resume.profile, rows) references = read_joint_zero_references(resume.profile, rows)
validate_motion_provenance(resume.profile, rows, references=references, validate_motion_provenance(resume.profile, rows, references=references,
image_replay_workers=resume.image_replay_workers) image_replay_workers=resume.image_replay_workers)
@@ -22,11 +24,15 @@ def stage_visits(resume, engine, rows):
if row.get('kind') == 'scan_unit_complete': if row.get('kind') == 'scan_unit_complete':
latest[record_scan_identity(row)] = row latest[record_scan_identity(row)] = row
spans, valid = {}, set() spans, valid = {}, set()
for (key, _), units in visits.items(): for (key, cycle), units in visits.items():
task = next(t for t in resume.profile.motion.tasks if t.key == key) task = next(t for t in resume.profile.motion.tasks if t.key == key)
if not set(task.joints) <= references.keys(): if not set(task.joints) <= references.keys():
if (key, cycle) in deferred:
continue
break break
if any(latest.get(u.identity, {}).get('passed') is not True for u in units): if any(latest.get(u.identity, {}).get('passed') is not True for u in units):
if (key, cycle) in deferred:
continue
break break
if not all(evaluate_capture_unit(resume.profile, u, latest[u.identity].get('attempt', 1), rows, if not all(evaluate_capture_unit(resume.profile, u, latest[u.identity].get('attempt', 1), rows,
first_cycle_spans=spans).passed for u in units): first_cycle_spans=spans).passed for u in units):
@@ -49,6 +55,8 @@ def stage_passed_visits(resume, engine, rows):
Every imported evidence record remains content-bound and finalization still Every imported evidence record remains content-bound and finalization still
performs the complete image/provenance replay before publication. performs the complete image/provenance replay before publication.
""" """
from .task_deferral import deferred_visit_keys
deferred = deferred_visit_keys(resume.profile, rows)
references = read_joint_zero_references(resume.profile, rows) references = read_joint_zero_references(resume.profile, rows)
visits, latest = OrderedDict(), {} visits, latest = OrderedDict(), {}
for unit in engine.scan_units(): for unit in engine.scan_units():
@@ -63,9 +71,13 @@ def stage_passed_visits(resume, engine, rows):
for (task_key, cycle), units in visits.items(): for (task_key, cycle), units in visits.items():
task = next(task for task in resume.profile.motion.tasks if task.key == task_key) task = next(task for task in resume.profile.motion.tasks if task.key == task_key)
if not set(task.joints) <= references.keys(): if not set(task.joints) <= references.keys():
if (task_key, cycle) in deferred:
continue
break break
completions = [latest.get(unit.identity) for unit in units] completions = [latest.get(unit.identity) for unit in units]
if any(row is None or row.get("passed") is not True for row in completions): if any(row is None or row.get("passed") is not True for row in completions):
if (task_key, cycle) in deferred:
continue
break break
valid.update(unit.identity for unit in units) valid.update(unit.identity for unit in units)
if cycle == 0: if cycle == 0:
@@ -98,10 +110,23 @@ def load_frozen_checkpoint(profile, rows, path, source):
def reference_import(resume, *, reuse_passed_visits=False): def reference_import(resume, *, reuse_passed_visits=False):
from .joint_resume import _index_motion_provenance from .joint_resume import _index_motion_provenance
from .passed_resume import PassedReferenceImport from .passed_resume import PassedReferenceImport, coherent_observer_preparation_rows
provenance = tuple(_index_motion_provenance(resume.rows).values()) provenance = tuple(_index_motion_provenance(resume.rows).values())
if reuse_passed_visits: if reuse_passed_visits:
reused_tasks = {identity[0] for identity in resume.units}
reused_joints = {joint for task in resume.profile.motion.tasks
if task.key in reused_tasks for joint in task.joints}
references = {joint: reference for joint, reference in resume.references.items()
if joint in reused_joints}
# A visit that did not pass is restarted with a fresh preparation
# model in the current image epoch. Reusing its old candidate path is
# neither a passed-unit optimization nor valid cross-session evidence.
provenance = tuple(row for row in provenance
if row.get("task_name") is None or row.get("task_name") in reused_tasks)
provenance = _essential_indexed_provenance(provenance) provenance = _essential_indexed_provenance(provenance)
provenance = coherent_observer_preparation_rows(provenance)
else:
references = resume.references
models = tuple(r for r in provenance if r.get('kind') == 'motion_branch_initialization' and r.get('status') == 'resolved') models = tuple(r for r in provenance if r.get('kind') == 'motion_branch_initialization' and r.get('status') == 'resolved')
audit = () audit = ()
if reuse_passed_visits: if reuse_passed_visits:
@@ -111,17 +136,19 @@ def reference_import(resume, *, reuse_passed_visits=False):
resume.verified_visits.update((name, cycle) for task in resume.profile.motion.tasks resume.verified_visits.update((name, cycle) for task in resume.profile.motion.tasks
for key, cycle in visits if key == task.key for name in task.joints) for key, cycle in visits if key == task.key for name in task.joints)
audit = ({'kind': 'passed_tasks_reused', 'resume_mode': 'passed', audit = ({'kind': 'passed_tasks_reused', 'resume_mode': 'passed',
'unchanged_installation_declared': True, 'training_policy': 'staged_2_to_3', 'unchanged_installation_declared': True,
'training_policy': resume.profile.quality.training_policy,
'completed_units': [list(key) for key in sorted(resume.units)], 'completed_units': [list(key) for key in sorted(resume.units)],
'historical_replay_performed': False, 'physical_zero_reverification_performed': False, 'historical_replay_performed': False, 'physical_zero_reverification_performed': False,
'full_replay_required_before_publication': True},) 'full_replay_required_before_publication': True},)
return PassedReferenceImport(rows=(*(r.as_record() for r in resume.references.values()), *provenance, *audit), return PassedReferenceImport(rows=(*(r.as_record() for r in references.values()), *provenance, *audit),
references=resume.references, model_records=models) references=references, model_records=models)
def _essential_indexed_provenance(rows): def _essential_indexed_provenance(rows):
"""Keep source frames actually named by accepted models in compact journals.""" """Keep source frames actually named by accepted models in compact journals."""
import re import re
from collections.abc import Mapping
from .resume_storage import DeferredCaptureRecord, materialize_record from .resume_storage import DeferredCaptureRecord, materialize_record
if not any(isinstance(row, DeferredCaptureRecord) and row.canonical_sha256 for row in rows): if not any(isinstance(row, DeferredCaptureRecord) and row.canonical_sha256 for row in rows):
@@ -129,17 +156,21 @@ def _essential_indexed_provenance(rows):
hashes = set() hashes = set()
def collect(value, key=""): def collect(value, key=""):
if isinstance(value, dict): if isinstance(value, Mapping):
for name, item in value.items(): for name, item in value.items():
collect(item, name) # Digest maps are keyed by image stamp. Preserve the parent
# field name while descending into them; replacing it with the
# numeric stamp made every digest invisible to this filter.
collect(item, key if "hash" in key.lower() else str(name))
elif isinstance(value, (list, tuple)): elif isinstance(value, (list, tuple)):
for item in value: for item in value:
collect(item, key) collect(item, key)
elif (isinstance(value, str) and "hash" in key elif (isinstance(value, str) and "hash" in key.lower()
and re.fullmatch(r"[0-9a-f]{64}", value)): and re.fullmatch(r"[0-9a-f]{64}", value)):
hashes.add(value) hashes.add(value)
evidence_kinds = {"motion_branch_initialization", "parent_reference_verified", evidence_kinds = {"motion_branch_initialization", "parent_reference_verified",
"parent_reference_model_refreshed",
"observer_preparation_decision", "observer_preparation_segment_complete", "observer_preparation_decision", "observer_preparation_segment_complete",
"joint_zero_motion_reference_verified"} "joint_zero_motion_reference_verified"}
for row in rows: for row in rows:
@@ -152,6 +183,7 @@ def _essential_indexed_provenance(rows):
def restore_recovery(session, recovery, rows): def restore_recovery(session, recovery, rows):
session.restore_task_deferrals(rows)
for row in rows: for row in rows:
if row.get('kind') == 'joint_zero_recovery': if row.get('kind') == 'joint_zero_recovery':
recovery.restore(row['task_name'], row['joints']) recovery.restore(row['task_name'], row['joints'])
@@ -171,9 +203,21 @@ def visit_rows(resume, task_key, cycle):
rows = [r for r in resume.rows if record_scan_identity(r) in units rows = [r for r in resume.rows if record_scan_identity(r) in units
and r.get('kind') not in {'joint_zero_reference', 'joint_zero_sample'}] and r.get('kind') not in {'joint_zero_reference', 'joint_zero_sample'}]
if units: if units:
from ..core.domain.capture_plan import TASKWISE_STAGED
from .engine import CalibrationEngine from .engine import CalibrationEngine
if resume.profile.quality.training_policy == TASKWISE_STAGED and cycle in {1, 2}:
# Persist the causal decision beside the visit that produced it.
# This lets a later restart reconstruct an optional cycle 2 from
# the new journal without depending on in-memory profile state.
assessed = list(range(cycle + 1))
decisions = [r for r in resume.rows if r.get('kind') == 'training_decision'
and r.get('task_name') == task_key and r.get('assessed_cycles') == assessed]
if len(decisions) != 1:
raise ValueError('taskwise_resume_training_decision_missing')
rows.extend(decisions)
stage = [u for u in CalibrationEngine(resume.profile).scan_units() if u.cycle == cycle] stage = [u for u in CalibrationEngine(resume.profile).scan_units() if u.cycle == cycle]
if cycle in {1, 2} and stage[-1].task_key == task_key: if (resume.profile.quality.training_policy != TASKWISE_STAGED
and cycle in {1, 2} and stage[-1].task_key == task_key):
decisions = [r for r in resume.rows if r.get('kind') == 'staged_training_decision'] decisions = [r for r in resume.rows if r.get('kind') == 'staged_training_decision']
candidates = decisions[:1] if cycle == 1 else decisions[1:] candidates = decisions[:1] if cycle == 1 else decisions[1:]
rows.extend(candidates) rows.extend(candidates)
@@ -144,6 +144,8 @@ def render_status_zh(value: Mapping[str, Any], estimator=None) -> str:
stage_line = f"阶段:{phase_label}(第 {task.get('cycle') or '-'}/{task.get('cycle_count', 4)} 轮)" stage_line = f"阶段:{phase_label}(第 {task.get('cycle') or '-'}/{task.get('cycle_count', 4)} 轮)"
if capture_stage in {"command_training", "command_validation"}: if capture_stage in {"command_training", "command_validation"}:
stage_line = "阶段:" + ("稳态指令映射训练" if capture_stage == "command_training" else "稳态指令映射独立验证") stage_line = "阶段:" + ("稳态指令映射训练" if capture_stage == "command_training" else "稳态指令映射独立验证")
elif capture_stage == "training_supplement":
stage_line = f"阶段:{phase_label}(当前任务训练补采 1/1)"
lines = [ lines = [
f"[{value.get('model', '未知型号')} / {value.get('side', '?')} / {value.get('serial_number', '?')}] {progress:6.1%} 预计剩余 {eta_text}", f"[{value.get('model', '未知型号')} / {value.get('side', '?')} / {value.get('serial_number', '?')}] {progress:6.1%} 预计剩余 {eta_text}",
stage_line, stage_line,
@@ -0,0 +1,237 @@
"""Bounded task deferral for recoverable visual observability failures.
The queue only reorders uncompleted scan units inside one staged cycle. It
never changes accepted units, reference models, quality limits or holdout
semantics. Safety failures are deliberately outside this module.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
POLICY = "dependency_ordered_observation_deferral_v1"
RECOVERABLE_REASONS = frozenset({
"rigid_pair_candidates_unresolved",
"rigid_pair_pose_uncertain",
"rigid_pair_viewpoint_span_insufficient",
"rigid_pair_source_missing",
"observer_images_missing",
"observer_preparation_candidates_missing",
"motion_candidates_missing",
"motion_candidate_assignment_ambiguous",
"motion_candidate_family_discontinuous",
"motion_candidate_family_interrupted",
"motion_geometry_ambiguous",
"image_motion_alternative_angles_unresolved",
"image_motion_families_not_distinguishable",
"image_motion_frame_angles_ambiguous",
"image_motion_frame_angles_uncertain",
"image_motion_frame_angles_unobservable",
"image_motion_insufficient_independent_frames",
"image_motion_no_observable_image_model",
"image_motion_shared_pose_unresolved",
"parent_reference_samples_missing",
"image_reference_current_image_moved",
"pose_image_motion_root_missing",
})
class TaskObservationState(str, Enum):
PASSED = "passed"
PENDING = "pending"
DEPENDENCY_PENDING = "dependency_pending"
UNOBSERVABLE = "unobservable"
def task_dependencies(profile):
"""Compile parent task -> descendant tasks from observed Tag roles."""
task_by_joint = {joint: task.key for task in profile.motion.tasks for joint in task.joints}
owners = {}
for joint, spec in profile.measurement.measurements.items():
task = task_by_joint.get(joint)
if task is not None and spec.child_role:
owners.setdefault((spec.view, spec.child_role), set()).add(task)
graph = {task.key: set() for task in profile.motion.tasks}
for joint, spec in profile.measurement.measurements.items():
child_task = task_by_joint.get(joint)
if child_task is None or not spec.parent_role:
continue
for parent_task in owners.get((spec.view, spec.parent_role), ()):
if parent_task != child_task:
graph[parent_task].add(child_task)
return {key: frozenset(value) for key, value in graph.items()}
def dependency_closure(profile, task_key):
graph = task_dependencies(profile)
pending, result = [task_key], set()
while pending:
current = pending.pop()
if current in result:
continue
result.add(current)
pending.extend(graph.get(current, ()))
return frozenset(result)
def recoverable_observation_reason(reason):
text = str(reason)
return any(code in text for code in RECOVERABLE_REASONS)
@dataclass(frozen=True)
class DeferralDecision:
task_name: str
cycle: int
attempt: int
state: TaskObservationState
affected_tasks: tuple[str, ...]
reason: str
moved_units: int
def as_record(self, *, stamp_ns, corner_evidence_sha256=""):
dependency_state = (TaskObservationState.DEPENDENCY_PENDING.value
if self.state == TaskObservationState.PENDING
else TaskObservationState.UNOBSERVABLE.value)
return {
"kind": "task_observation_state",
"schema_version": 1,
"policy": POLICY,
"task_name": self.task_name,
"cycle": self.cycle,
"attempt": self.attempt,
"state": self.state.value,
"affected_tasks": list(self.affected_tasks),
"dependency_states": ({task: dependency_state
for task in self.affected_tasks if task != self.task_name}
if self.state != TaskObservationState.PASSED else {}),
"reason": self.reason,
"moved_units": self.moved_units,
"corner_evidence_sha256": corner_evidence_sha256,
"stamp_ns": int(stamp_ns),
}
class TaskDeferralQueue:
"""Own the finite retry state independently of motion execution."""
def __init__(self, profile, *, revisit_limit=1):
if revisit_limit != 1:
raise ValueError("task_deferral_revisit_limit_must_be_one")
self.profile = profile
self.revisit_limit = revisit_limit
self.attempts = {}
self.unobservable = {}
self.resolved = set()
self.inferred_resolutions = set()
def decide(self, task_name, cycle, reason, moved_units):
key = (str(task_name), int(cycle))
if key in self.resolved:
raise ValueError("task_deferral_already_resolved")
attempt = self.attempts.get(key, 0) + 1
if attempt > self.revisit_limit + 1:
raise ValueError("task_deferral_budget_exhausted")
self.attempts[key] = attempt
affected = tuple(task.key for task in self.profile.motion.tasks
if task.key in dependency_closure(self.profile, task_name))
state = (TaskObservationState.PENDING if attempt <= self.revisit_limit
else TaskObservationState.UNOBSERVABLE)
if state == TaskObservationState.UNOBSERVABLE:
self.unobservable[key] = str(reason)
return DeferralDecision(str(task_name), int(cycle), attempt, state,
affected, str(reason), int(moved_units))
def resolve(self, task_name, cycle):
key = (str(task_name), int(cycle))
attempt = self.attempts.get(key)
if attempt is None or key in self.resolved:
return None
self.resolved.add(key)
self.unobservable.pop(key, None)
return DeferralDecision(key[0], key[1], attempt, TaskObservationState.PASSED,
(key[0],), "trajectory_observation_resolved", 0)
def restore(self, records):
records = tuple(records)
state_positions = {}
for position, row in enumerate(records):
if row.get("kind") != "task_observation_state":
continue
if row.get("policy") != POLICY or row.get("state") not in {
state.value for state in TaskObservationState}:
raise ValueError("task_deferral_checkpoint_policy_changed")
key = (str(row["task_name"]), int(row["cycle"]))
state_positions[key] = position
attempt = int(row["attempt"])
expected_affected = tuple(task.key for task in self.profile.motion.tasks
if task.key in dependency_closure(self.profile, key[0]))
if tuple(row.get("affected_tasks", ())) != expected_affected:
raise ValueError("task_deferral_checkpoint_dependencies_changed")
if row["state"] == TaskObservationState.PASSED.value:
if self.attempts.get(key) != attempt or key in self.resolved:
raise ValueError("task_deferral_checkpoint_resolution_changed")
self.resolved.add(key)
self.unobservable.pop(key, None)
continue
if attempt != self.attempts.get(key, 0)+1 or attempt not in (1, 2):
raise ValueError("task_deferral_checkpoint_budget_changed")
self.attempts[key] = attempt
if row["state"] == TaskObservationState.UNOBSERVABLE.value:
self.unobservable[key] = str(row.get("reason", ""))
# If a crash happened after the final scan-unit commit but before the
# adjacent PASSED state record, the complete visit itself closes the
# pending state. A partial direction can never do so.
from .engine import CalibrationEngine
from ..core.domain.motion_path import record_scan_identity
completion_positions = {}
for position, row in enumerate(records):
if row.get("kind") == "scan_unit_complete" and row.get("passed") is True:
completion_positions[record_scan_identity(row)] = position
units = CalibrationEngine(self.profile).scan_units()
for key in set(self.attempts) - self.resolved - set(self.unobservable):
required = {unit.identity for unit in units
if (unit.task_key, unit.cycle) == key}
boundary = state_positions[key]
if required and all(completion_positions.get(identity, -1) > boundary
for identity in required):
self.resolved.add(key)
self.inferred_resolutions.add(key)
def deferred_visit_keys(profile, records):
"""Return incomplete visits that may be followed by durable later visits.
A non-contiguous resume prefix is legal only when a versioned deferral
record explains the gap. Replaying the queue also validates retry order,
dependency closure and terminal state before the gap is trusted.
"""
rows = [row for row in records if row.get("kind") == "task_observation_state"]
if not rows:
return frozenset()
queue = TaskDeferralQueue(profile)
queue.restore(records)
latest = {}
for row in rows:
latest[(str(row["task_name"]), int(row["cycle"]))] = row
blocked = set()
for key, row in latest.items():
if row["state"] == TaskObservationState.PASSED.value or key in queue.resolved:
continue
blocked.update((task, key[1]) for task in row["affected_tasks"])
return frozenset(blocked)
def unresolved_observation_records(profile, records):
"""Return latest states that are not closed by a full durable visit."""
rows = [row for row in records if row.get("kind") == "task_observation_state"]
if not rows:
return ()
queue = TaskDeferralQueue(profile)
queue.restore(records)
latest = {}
for row in rows:
latest[(str(row["task_name"]), int(row["cycle"]))] = row
return tuple(row for key, row in latest.items() if key not in queue.resolved)
@@ -6,8 +6,8 @@ import threading
import time import time
import traceback import traceback
from ..core.domain.capture_plan import (ADAPTIVE, STAGED, CapturePlan, configure_training, from ..core.domain.capture_plan import (ADAPTIVE, STAGED, TASKWISE_STAGED,
evidence_digest, select_task_training) CapturePlan, configure_training, evidence_digest, select_task_training)
from ..core.domain.reference import JointZeroReference from ..core.domain.reference import JointZeroReference
from ..core.fitting.training_quality import (TRAINING_DECISION_VERSION, assess_training, from ..core.fitting.training_quality import (TRAINING_DECISION_VERSION, assess_training,
training_snapshot) training_snapshot)
@@ -22,7 +22,7 @@ def resolve_capture_plan(profile, records, *, source_urdf=None, verify_decisions
base = configure_training(profile, profile.quality.training_policy) base = configure_training(profile, profile.quality.training_policy)
headers = [row for row in records if row.get("kind") == "session_start"] headers = [row for row in records if row.get("kind") == "session_start"]
plan = CapturePlan.from_profile(base) plan = CapturePlan.from_profile(base)
if base.quality.training_policy != ADAPTIVE: if base.quality.training_policy not in {ADAPTIVE, TASKWISE_STAGED}:
if any(row.get("kind") in {"training_decision", "staged_training_decision"} for row in records): if any(row.get("kind") in {"training_decision", "staged_training_decision"} for row in records):
raise ValueError("adaptive_decisions_in_fixed_capture") raise ValueError("adaptive_decisions_in_fixed_capture")
if headers and "capture_plan" in headers[0] and headers[0]["capture_plan"] != plan.as_dict(): if headers and "capture_plan" in headers[0] and headers[0]["capture_plan"] != plan.as_dict():
@@ -63,6 +63,22 @@ class ZeroRecovery:
def restore(self, task_key, joints): def restore(self, task_key, joints):
self._used.add((task_key, tuple(joints))) self._used.add((task_key, tuple(joints)))
def take_parent_model_refresh(self, motion, *, stale_image_anchor):
"""Spend one fresh-model allowance for a resumed parent reference.
This does not reacquire its joint zero or scan. It only permits the
already declared safe approach to rebuild the image-domain model in
the current session's stationary camera coordinate frame.
"""
if (not motion.reference_reuse or not stale_image_anchor
or len(motion.zero_joints) != 1):
return None
key = (motion.task_key, tuple(motion.zero_joints))
if key in self._used:
return None
self._used.add(key)
return "refresh_parent_model"
def take(self, motion, *, geometry_error, reports, referenced_joints): def take(self, motion, *, geometry_error, reports, referenced_joints):
if motion.reference_reuse: if motion.reference_reuse:
return None return None
@@ -11,7 +11,7 @@ from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.profiles.export import dump_hand_profile from linkerhand_calibration.profiles.export import dump_hand_profile
from linkerhand_calibration.profiles.observer_policy import configure_observer_policy from linkerhand_calibration.profiles.observer_policy import configure_observer_policy
from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract
from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade, _copy_indexed_evidence
from linkerhand_calibration.runtime.joint_resume import _index_motion_provenance from linkerhand_calibration.runtime.joint_resume import _index_motion_provenance
@@ -81,3 +81,18 @@ def test_active_observation_migration_only_adds_unmeasured_bounded_paths(migrati
checkpoint.joints.references['index_dip']=object() checkpoint.joints.references['index_dip']=object()
with pytest.raises(ValueError,match='changes_existing_contract'): with pytest.raises(ValueError,match='changes_existing_contract'):
upgrade.metadata(checkpoint,inputs,'source.jsonl','c'*64) upgrade.metadata(checkpoint,inputs,'source.jsonl','c'*64)
def test_indexed_checkpoint_migration_copies_content_bound_archive(tmp_path):
from linkerhand_calibration.core.artifacts.evidence_journal import (
initialize_evidence_journal, append_indexed_records, archive_path, manifest_path,
)
source = tmp_path/'source/raw_samples.jsonl'
source.parent.mkdir()
initialize_evidence_journal(source, threshold_bytes=128)
append_indexed_records(source, ({'kind': 'pnp_candidate_frame', 'view': 'side',
'image_stamp_ns': 1, 'pixels': 'x'*2048},), durable=True)
target = tmp_path/'target'; target.mkdir()
_copy_indexed_evidence(source, target)
assert (target/archive_path(source).name).read_bytes() == archive_path(source).read_bytes()
assert (target/manifest_path(source).name).read_bytes() == manifest_path(source).read_bytes()
@@ -117,7 +117,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile):
segment = task.segments[0] segment = task.segments[0]
start = build_calibration_motion_command(task, segment.start, profile=profile, segment_key=segment.key) start = build_calibration_motion_command(task, segment.start, profile=profile, segment_key=segment.key)
end = build_calibration_motion_command(task, segment.end, profile=profile, segment_key=segment.key) end = build_calibration_motion_command(task, segment.end, profile=profile, segment_key=segment.key)
execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200), execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200,
measured_channels=segment.moving_channels),
initial_command=start, initial_feedback=start, now=0., identity='linked', initial_command=start, initial_feedback=start, now=0., identity='linked',
visual_observer=VisualMotionObserver(profile)) visual_observer=VisualMotionObserver(profile))
previous = execution.sample(0) previous = execution.sample(0)
@@ -127,8 +128,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile):
assert actual[3] == pytest.approx(80+175*progress, abs=1.) assert actual[3] == pytest.approx(80+175*progress, abs=1.)
assert actual[2] == actual[4] == actual[5] assert actual[2] == actual[4] == actual[5]
# Integer bytes carry at most one unit of quantization error. # Integer bytes carry at most one unit of quantization error.
assert max(abs(a-b) for a,b in zip(actual,previous)) <= 30*.01+1. assert max(abs(a-b) for a,b in zip(actual,previous)) <= 20*.01+1.
assert math.pi*255/(2*execution.duration) <= 30.000001 assert math.pi*255/(2*execution.duration) <= 20.000001
previous = actual previous = actual
@@ -22,6 +22,8 @@ from linkerhand_calibration.runtime.parent_reference import ParentReferenceRegis
from linkerhand_calibration.runtime.capture import ObservationCapture from linkerhand_calibration.runtime.capture import ObservationCapture
from linkerhand_calibration.runtime.reference_lock import ReferenceLock from linkerhand_calibration.runtime.reference_lock import ReferenceLock
from linkerhand_calibration.runtime.motion_execution import MotionCommand from linkerhand_calibration.runtime.motion_execution import MotionCommand
from linkerhand_calibration.runtime.execution import SessionExecution
from linkerhand_calibration.runtime.session import CalibrationPhase as Phase
from linkerhand_calibration.runtime.zero_recovery import ZeroRecovery from linkerhand_calibration.runtime.zero_recovery import ZeroRecovery
from o30_capture_fixture import SOURCE from o30_capture_fixture import SOURCE
@@ -101,6 +103,11 @@ def test_registry_needs_both_sources_and_fresh_check_and_preserves_zeros(setup):
with pytest.raises(ValueError, match="source_missing"): owner.sources(task, 2) with pytest.raises(ValueError, match="source_missing"): owner.sources(task, 2)
with pytest.raises(ValueError, match="unverified"): owner.geometry(task, 1) with pytest.raises(ValueError, match="unverified"): owner.geometry(task, 1)
refresh = owner.authorize_refreshed_pose(task, 1, 9, reference)
assert refresh["passed_zero_preserved"] is True
assert refresh["image_model_sha256"] == roll_report["image_model_sha256"]
assert owner.pose_source(task, 1) == (roll, roll_report)
for row in rows: for row in rows:
row["pnp_observation_evidence"] = {"child": dict(tag_role="thumb_mcp", row["pnp_observation_evidence"] = {"child": dict(tag_role="thumb_mcp",
image_model_sha256=roll_report["image_model_sha256"], image_model_sha256=roll_report["image_model_sha256"],
@@ -168,11 +175,42 @@ def test_stationary_reference_cannot_trigger_a_new_solver_or_extra_motion_retry(
assert owner.request("front", motion, session_epoch=1, motion_version=10) is None assert owner.request("front", motion, session_epoch=1, motion_version=10) is None
recovery = ZeroRecovery() recovery = ZeroRecovery()
assert recovery.take(motion, geometry_error="", reports={}, referenced_joints=()) is None assert recovery.take(motion, geometry_error="", reports={}, referenced_joints=()) is None
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=False) is None
ordinary = replace(motion, reference_only=False, reference_reuse=False, zero_joints=("thumb_ip",)) ordinary = replace(motion, reference_only=False, reference_reuse=False, zero_joints=("thumb_ip",))
assert recovery.take(ordinary, geometry_error="ambiguous", reports={"front": { assert recovery.take(ordinary, geometry_error="ambiguous", reports={"front": {
"status": "unresolved", "reason": "image_motion_families_not_distinguishable"}}, referenced_joints=()) is None "status": "unresolved", "reason": "image_motion_families_not_distinguishable"}}, referenced_joints=()) is None
def test_stale_cross_session_anchor_gets_one_model_refresh_without_replacing_zero(setup):
profile, task, _ = setup
parent = task.parent_reference.pose_joint
motion = MotionCommand("joint_zero",
profile.motion.joint_zero_references[task.joints[0]].command, 200,
task_key=task.key, cycle=0, zero_joints=(parent,),
reference_only=True, reference_reuse=True)
recovery = ZeroRecovery()
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) == "refresh_parent_model"
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) is None
execution = SessionExecution(profile)
execution.session.phase = Phase.PREPARE
frozen = object()
execution.zero_references[parent] = frozen
execution._effects = [motion]
execution.refresh_parent_model(motion.target)
assert execution.zero_references[parent] is frozen
assert execution._effects[-1] is motion
refresh = execution._effects[:-1]
assert refresh and refresh[-1].phase == "joint_zero"
assert refresh[-1].zero_joints == (parent,)
assert all(item.parent_model_refresh_for == task.key for item in refresh)
assert all(item.reference_only and not item.reference_reuse for item in refresh)
moving = [item for item in refresh if item.phase == "zero_approach"]
from linkerhand_calibration.core.fitting.motion_fit import channel_for_joint
channel = channel_for_joint(profile, parent)
assert max(item.target[channel] for item in moving) != min(item.target[channel] for item in moving)
def test_coordinator_waits_for_current_parent_images_and_keeps_both_joint_zeros(setup, tmp_path, monkeypatch): def test_coordinator_waits_for_current_parent_images_and_keeps_both_joint_zeros(setup, tmp_path, monkeypatch):
from linkerhand_calibration.runtime.coordinator import CalibrationCoordinator from linkerhand_calibration.runtime.coordinator import CalibrationCoordinator
from linkerhand_calibration.runtime import coordinator as module from linkerhand_calibration.runtime import coordinator as module
@@ -44,6 +44,73 @@ def test_passed_parent_model_keeps_original_source_in_new_observation_epoch():
assert dict(pose_source[0].evidence_ids)['moving'] == 'a' * 64 assert dict(pose_source[0].evidence_ids)['moving'] == 'a' * 64
def test_passed_resume_does_not_restore_incomplete_task_preparation(monkeypatch):
profile = load_bundled_hand_profile('o30_right_18')
engine, resume = CalibrationEngine(profile), JointResume(profile)
passed, incomplete = profile.motion.tasks[:2]
rows = [*passed_records(engine, passed),
dict(kind='motion_branch_observation', task_name=passed.key,
view=passed.view, image_stamp_ns=1, session_epoch=1),
dict(kind='motion_branch_observation', task_name=incomplete.key,
view=incomplete.view, image_stamp_ns=2, session_epoch=1),
dict(kind='motion_branch_initialization', task_name=incomplete.key,
view=incomplete.view, status='resolved', session_epoch=1)]
monkeypatch.setattr(passed_resume, 'read_joint_zero_references',
lambda *args: stub_references([passed, incomplete]))
passed_resume.stage_passed_tasks(resume, engine, rows)
pending = passed_resume.reference_import(resume)
assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key}
assert not pending.model_records
def test_taskwise_passed_resume_does_not_restore_unpassed_visit_model():
from linkerhand_calibration.runtime import staged_resume
profile = load_bundled_hand_profile('o30_right_18')
passed, incomplete = profile.motion.tasks[:2]
references = stub_references([passed, incomplete])
rows = [
dict(kind='motion_branch_observation', task_name=passed.key,
view=passed.view, image_stamp_ns=1, session_epoch=1),
dict(kind='motion_branch_observation', task_name=incomplete.key,
view=incomplete.view, image_stamp_ns=2, session_epoch=1),
dict(kind='motion_branch_initialization', task_name=incomplete.key,
view=incomplete.view, status='resolved', session_epoch=1),
]
accepted = next(unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == passed.key)
resume = JointResume(profile)
resume.rows, resume.references, resume.units = tuple(rows), references, {accepted.identity}
pending = staged_resume.reference_import(resume, reuse_passed_visits=True)
assert set(pending.references) == set(passed.joints)
assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key}
assert not pending.model_records
def test_passed_resume_omits_orphaned_observer_cancellation():
from linkerhand_calibration.runtime import staged_resume
profile = load_bundled_hand_profile('o30_right_18')
task = profile.motion.tasks[0]
accepted = next(unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == task.key)
orphan = dict(kind='observer_preparation_cancelled_before_excursion',
joint=task.joints[0], observer_preparation_id='a'*64,
session_epoch=1, stamp_ns=1)
resume = JointResume(profile)
resume.rows = (orphan,)
resume.references = stub_references([task])
resume.units = {accepted.identity}
pending = staged_resume.reference_import(resume, reuse_passed_visits=True)
assert orphan not in pending.rows
@pytest.mark.parametrize('failure', [None, 'missing', 'failed', 'support', 'new_attempt']) @pytest.mark.parametrize('failure', [None, 'missing', 'failed', 'support', 'new_attempt'])
def test_last_group_failure_keeps_128_prior_passes_without_recalculating(monkeypatch, failure): def test_last_group_failure_keeps_128_prior_passes_without_recalculating(monkeypatch, failure):
profile = load_bundled_hand_profile('o30_right_18') profile = load_bundled_hand_profile('o30_right_18')
@@ -81,7 +148,10 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon
expected_units = sum(unit.task_key == task.key for unit in host.execution.session.engine.scan_units()) expected_units = sum(unit.task_key == task.key for unit in host.execution.session.engine.scan_units())
rows = [dict(kind='motion_branch_observation', view=task.view, image_stamp_ns=i, rows = [dict(kind='motion_branch_observation', view=task.view, image_stamp_ns=i,
session_epoch=1, task_name=task.key) session_epoch=1, task_name=task.key)
for i in range(257)] + passed_records(host.execution.session.engine, task) for i in range(257)] + [dict(kind='observer_preparation_decision',
task_name=task.key, joint=task.joints[0], action='excite',
action_index=0, session_epoch=1, stamp_ns=1)] + passed_records(
host.execution.session.engine, task)
references = stub_references([task]) references = stub_references([task])
monkeypatch.setattr(passed_resume, 'read_joint_zero_references', lambda *args: references) monkeypatch.setattr(passed_resume, 'read_joint_zero_references', lambda *args: references)
resume = JointResume(host.profile) resume = JointResume(host.profile)
@@ -123,13 +193,19 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon
assert host.state in {'ABORTED', 'PAUSED'} assert host.state in {'ABORTED', 'PAUSED'}
assert not host.execution.zero_references and host._resumed_count == 0 assert not host.execution.zero_references and host._resumed_count == 0
else: else:
for _ in range(20): # Reference restoration is intentionally sliced across callbacks
# and journal barriers run on a worker. Give that worker real
# scheduling opportunities instead of spinning only the fake clock.
import time
for _ in range(400):
host.receive_feedback(host.profile.command.names, host.profile.command.baseline_values, host.ports.clock_ns()) host.receive_feedback(host.profile.command.names, host.profile.command.baseline_values, host.ports.clock_ns())
host.tick() host.tick()
if host.execution.session.current_unit.task_key != task.key: if host.execution.session.current_unit.task_key != task.key:
break break
time.sleep(.001)
assert host._resumed_count == expected_units assert host._resumed_count == expected_units
assert host.execution.zero_references == references assert host.execution.zero_references == references
assert not host.branch_initialization.observer_preparation.decisions
assert host.segment is None assert host.segment is None
import json import json
imported = [json.loads(line) for line in host.raw_path.read_text().splitlines()] imported = [json.loads(line) for line in host.raw_path.read_text().splitlines()]
@@ -10,6 +10,8 @@ from linkerhand_calibration.runtime.resume_storage import (
DeferredCaptureRecord, load_passed_capture, DeferredCaptureRecord, load_passed_capture,
) )
from linkerhand_calibration.runtime.joint_resume import ResumeJournalImport, _index_motion_provenance from linkerhand_calibration.runtime.joint_resume import ResumeJournalImport, _index_motion_provenance
from linkerhand_calibration.runtime.staged_resume import _essential_indexed_provenance
from linkerhand_calibration.runtime.motion_provenance import source_frame_sha256
from linkerhand_calibration.core.domain.motion_path import record_scan_identity from linkerhand_calibration.core.domain.motion_path import record_scan_identity
from linkerhand_calibration.runtime.diagnostic_capture import reject_diagnostic_capture from linkerhand_calibration.runtime.diagnostic_capture import reject_diagnostic_capture
from linkerhand_calibration.core.artifacts.evidence_journal import ( from linkerhand_calibration.core.artifacts.evidence_journal import (
@@ -18,6 +20,8 @@ from linkerhand_calibration.core.artifacts.evidence_journal import (
) )
from linkerhand_calibration.storage import append_jsonl, append_jsonl_many from linkerhand_calibration.storage import append_jsonl, append_jsonl_many
from linkerhand_calibration.runtime.acquisition import load_capture from linkerhand_calibration.runtime.acquisition import load_capture
from linkerhand_calibration.runtime.capture_index import CaptureRecordIndex
from linkerhand_calibration.runtime.resume import discover_resume_candidate
def capture(tmp_path, rows): def capture(tmp_path, rows):
@@ -49,7 +53,7 @@ def test_scalar_index_never_reads_image_payload_and_import_is_bounded(tmp_path,
assert len(batch) == size and len(calls)-before == size assert len(batch) == size and len(calls)-before == size
assert all(type(r) is dict for r in batch) assert all(type(r) is dict for r in batch)
imported.extend(batch) imported.extend(batch)
assert pending.complete and imported == original assert pending.transfer_complete and not pending.complete and imported == original
assert all("pnp_observation_evidence" not in r.metadata for r in rows) assert all("pnp_observation_evidence" not in r.metadata for r in rows)
@@ -101,6 +105,7 @@ def test_indexed_journal_keeps_resume_fields_inline_and_restores_full_evidence(t
deferred = load_passed_capture(path) deferred = load_passed_capture(path)
assert all(isinstance(row, DeferredCaptureRecord) for row in deferred[1:]) assert all(isinstance(row, DeferredCaptureRecord) for row in deferred[1:])
assert deferred[1]["relative_quaternion_xyzw"] == [0., 0., 0., 1.] assert deferred[1]["relative_quaternion_xyzw"] == [0., 0., 0., 1.]
assert "relative_quaternion_xyzw" not in deferred[1].metadata
assert deferred[1].materialize() == samples[0] assert deferred[1].materialize() == samples[0]
@@ -162,3 +167,90 @@ def test_resume_import_hard_links_compressed_evidence_without_copying_records(tm
archive_path(source).unlink() archive_path(source).unlink()
source.unlink() source.unlink()
assert load_capture(target) == original assert load_capture(target) == original
def test_linked_joint_samples_use_bounded_batches_and_concrete_training_digest(tmp_path):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
leading = [dict(kind="image_observation_frame", task_name="index_dip_side",
cycle=0, direction="increasing", image_stamp_ns=-stamp-1,
pixels=list(range(100))) for stamp in range(10)]
samples = [dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip",
cycle=0, direction="increasing", image_stamp_ns=stamp,
command_u8=stamp, feedback_u8=stamp, relative_quaternion_xyzw=[0., 0., 0., 1.])
for stamp in range(100)]
append_jsonl_many(path, (*leading, *samples))
pending = ResumeJournalImport(tuple(load_passed_capture(path)))
assert len(pending.take_batch()) == len(leading)
first = pending.take_batch()
assert len(first) == len(samples) and all(row.get("kind") == "joint_sample" for row in first)
index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True)
index.extend(first)
assert len(index) == len(samples)
assert all(len(row["source_record_sha256"]) == 64 for row in index)
def test_training_index_materializes_each_deferred_sample_once(tmp_path, monkeypatch):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
sample = dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip",
cycle=0, direction="increasing", image_stamp_ns=1, command_u8=20,
feedback_u8=19, relative_quaternion_xyzw=[0., 0., 0., 1.],
relative_translation_xyz_m=[0., 0., .01])
append_jsonl(path, sample)
row = load_passed_capture(path)[0]
original = DeferredCaptureRecord.materialize_plain
calls = []
def observed(record):
calls.append(record.reference.record_sha256)
return original(record)
monkeypatch.setattr(DeferredCaptureRecord, "materialize_plain", observed)
index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True)
index.append(row)
assert len(calls) == 1
assert index[0]["relative_translation_xyz_m"] == [0., 0., .01]
def test_compact_passed_resume_keeps_only_digest_referenced_motion_frames(tmp_path):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
used = {"kind": "motion_branch_observation", "task_name": "index_dip_side",
"view": "side", "image_stamp_ns": 10, "pixels": list(range(100))}
unused = {**used, "image_stamp_ns": 20, "pixels": list(range(101))}
accepted = {"kind": "motion_branch_initialization", "status": "resolved",
"task_name": "index_dip_side", "view": "side",
"source_frame_hashes": {"10": source_frame_sha256(used)}}
append_jsonl_many(path, (used, unused, accepted))
indexed = tuple(_index_motion_provenance(load_passed_capture(path)).values())
essential = _essential_indexed_provenance(indexed)
observations = [row for row in essential if row.get("kind") == "motion_branch_observation"]
assert len(observations) == 1
assert observations[0].canonical_sha256 == source_frame_sha256(used)
def test_discovery_reads_large_compressed_session_header(tmp_path):
from linkerhand_calibration.runtime.engine import ACQUISITION_POLICY_VERSION
from linkerhand_calibration.core.geometry.pnp import POSE_TRACKING_POLICY_VERSION
session = tmp_path / "SERIAL" / "20260101_000000"
session.mkdir(parents=True)
path = session / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=128)
hashes = {"profile_config_sha256": "a" * 64}
contract = {"large": list(range(100))}
append_jsonl_many(path, [{"kind": "session_start", "profile_id": "O30/right/layout/v1",
"serial_number": "SERIAL", "acquisition_policy_version": ACQUISITION_POLICY_VERSION,
"capture_schedule_version": "unified_schedule_v13_staged_training",
"pose_tracking_policy_version": POSE_TRACKING_POLICY_VERSION, "protected_hashes": hashes,
"capture_plan": {"policy": "staged_2_to_3"}, "preparation_contract": contract},
{"kind": "fixed_base_reference_locked"},
{"kind": "scan_unit_complete", "task_name": "index_dip_side", "cycle": 0,
"direction": "increasing", "passed": True}])
assert discover_resume_candidate(tmp_path / "SERIAL", profile_id="O30/right/layout/v1",
serial_number="SERIAL", protected_hashes=hashes, training_policy="staged_2_to_3",
preparation_contract=contract) == session
@@ -10,6 +10,7 @@ from linkerhand_calibration.profiles import load_hand_profile
from linkerhand_calibration.profiles.export import write_hand_profile from linkerhand_calibration.profiles.export import write_hand_profile
from linkerhand_calibration.profiles.validator import validate_executable_profile from linkerhand_calibration.profiles.validator import validate_executable_profile
from linkerhand_calibration.profiles.preparation import preparation_contract from linkerhand_calibration.profiles.preparation import preparation_contract
from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER
from linkerhand_calibration.runtime.observer_reference import ObserverBridgeInput,build_observer_bridge,validate_observer_references from linkerhand_calibration.runtime.observer_reference import ObserverBridgeInput,build_observer_bridge,validate_observer_references
from linkerhand_calibration.runtime.shared_tag_reference import bridge_frames from linkerhand_calibration.runtime.shared_tag_reference import bridge_frames
from linkerhand_calibration.runtime.branch_initialization import BranchInitializationRequest,solve_initialization,initialization_artifacts from linkerhand_calibration.runtime.branch_initialization import BranchInitializationRequest,solve_initialization,initialization_artifacts
@@ -72,6 +73,17 @@ def test_independent_parent_branches_converge_on_real_bounded_probe(tmp_path):
assert bridge.translation_uncertainty_m<.001 assert bridge.translation_uncertainty_m<.001
assert all(p==0 for _,p in record['pose']['comparison_adjusted_p_values']) assert all(p==0 for _,p in record['pose']['comparison_adjusted_p_values'])
assert set(record['pose']['training_stamps']).isdisjoint(record['pose']['validation_stamps']) assert set(record['pose']['training_stamps']).isdisjoint(record['pose']['validation_stamps'])
measurements=dict(profile.measurement.measurements)
measurements['index_dip']=replace(measurements['index_dip'],
observer_pose_policy=TRAJECTORY_OBSERVER)
trajectory_profile=replace(profile,measurement=replace(profile.measurement,
measurements=measurements))
trajectory_bridge,trajectory_record=build_observer_bridge(trajectory_profile,
ObserverBridgeInput('index_dip',model,parent,(),endpoints,(16,),evidence))
assert trajectory_record['version']=='observer_pose_bridge_v4'
assert trajectory_record['trajectory_branch_decision']['holdout_role']=='veto_only'
assert trajectory_record['trajectory_branch_decision']['corner_evidence_sha256']
assert trajectory_bridge.rotation_uncertainty_rad<.017453293
@pytest.fixture(scope='module') @pytest.fixture(scope='module')
@@ -7,7 +7,9 @@ from pathlib import Path
import pytest import pytest
from linkerhand_calibration.core.domain.capture_plan import STAGED, CapturePlan, configure_training, select_task_training from linkerhand_calibration.core.domain.capture_plan import (
STAGED, TASKWISE_STAGED, CapturePlan, configure_training, select_task_training,
)
from linkerhand_calibration.core.domain.reference import read_joint_zero_references from linkerhand_calibration.core.domain.reference import read_joint_zero_references
from linkerhand_calibration.profiles import load_bundled_hand_profile from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.runtime.engine import CalibrationEngine from linkerhand_calibration.runtime.engine import CalibrationEngine
@@ -41,6 +43,52 @@ def test_all_first_visits_precede_second_and_validation_keeps_original_motion():
assert fixed.quality.training_policy == 'fixed' assert fixed.quality.training_policy == 'fixed'
def test_taskwise_schedule_keeps_first_pass_and_validation_global_but_supplements_locally():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED)
plan = CapturePlan.from_profile(profile).as_dict()
assert plan['version'] == 'capture_plan_v3'
assert plan['schedule'] == (
'first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation'
)
first_task = profile.motion.tasks[0]
profile = select_task_training(profile, first_task.key, (0, 1, 2))
units = CalibrationEngine(profile).scan_units()
visits = []
for unit in units:
visit = unit.task_key, unit.cycle
if not visits or visits[-1] != visit:
visits.append(visit)
task_keys = [task.key for task in profile.motion.tasks]
assert visits[:len(task_keys)] == [(key, 0) for key in task_keys]
training_visits = visits[len(task_keys):-len(task_keys)]
assert training_visits[:2] == [(first_task.key, 1), (first_task.key, 2)]
assert training_visits[2:] == [(key, 1) for key in task_keys[1:]]
assert visits[-len(task_keys):] == [(key, 3) for key in task_keys]
def test_taskwise_resume_carries_the_decision_that_created_optional_round():
from linkerhand_calibration.runtime.joint_resume import JointResume
from linkerhand_calibration.runtime.staged_resume import visit_rows
profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED)
task = profile.motion.tasks[0]
profile = select_task_training(profile, task.key, (0, 1, 2))
units = [unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == task.key and unit.cycle == 2]
decision = {'kind': 'training_decision', 'task_name': task.key,
'assessed_cycles': [0, 1, 2], 'decision': 'freeze'}
rows = [dict(kind='scan_unit_complete', task_name=unit.task_key,
cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key,
passed=True) for unit in units]
resume = JointResume(profile)
resume.rows = (*rows, decision)
resume.units = {unit.identity for unit in units}
resume.verified_visits.update((joint, 2) for joint in task.joints)
used, imported = visit_rows(resume, task.key, 2)
assert used == tuple(sorted(unit.identity for unit in units))
assert [row for row in imported if row.get('kind') == 'training_decision'] == [decision]
def test_revisit_reuses_model_but_checks_zero_via_existing_path(): def test_revisit_reuses_model_but_checks_zero_via_existing_path():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED) profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
execution = SessionExecution(profile) execution = SessionExecution(profile)
@@ -0,0 +1,207 @@
"""v3 trajectory observation, bounded deferral and publication refusal."""
import hashlib
from types import SimpleNamespace
import pytest
from linkerhand_calibration.core.domain.capture_plan import (
STAGED, TASKWISE_STAGED, configure_training, select_task_training,
)
from linkerhand_calibration.core.urdf.kinematics import UrdfKinematicModel
from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER
from linkerhand_calibration.profiles.observer_policy import configure_observer_policy
from linkerhand_calibration.profiles.observer_preparation import configure_observer_preparations
from linkerhand_calibration.profiles.export import dump_hand_profile
from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract
from linkerhand_calibration.runtime.artifacts.capture_validation import validate_capture_units
from linkerhand_calibration.runtime.session import CalibrationPhase, CalibrationSession
from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade
from linkerhand_calibration.runtime.task_deferral import (
POLICY, TaskDeferralQueue, TaskObservationState, deferred_visit_keys,
dependency_closure, recoverable_observation_reason,
)
from o30_capture_fixture import SOURCE
@pytest.fixture
def trajectory_profile():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
profile = configure_preparation(profile, 'occlusion_aware_trajectory_v3')
return configure_observer_preparations(profile, UrdfKinematicModel(SOURCE))
def test_v3_is_a_versioned_profile_contract_with_two_action_bound(trajectory_profile):
policy = preparation_contract(trajectory_profile)['observer_preparations']
assert policy['policy'] == 'occlusion_aware_trajectory_v3'
assert policy['trajectory_solver']['holdout_selects_model'] is False
assert policy['trajectory_solver']['maximum_actions_per_task'] == 2
assert all(1 <= len(plan.actions) <= 2
for plan in trajectory_profile.motion.observer_preparations.values())
assert trajectory_profile.measurement.measurements['index_dip'].observer_pose_policy \
== TRAJECTORY_OBSERVER
def test_visual_ambiguity_is_deferred_but_hardware_failure_is_not():
assert recoverable_observation_reason('image_motion_frame_angles_ambiguous')
assert recoverable_observation_reason('motion_candidates_missing')
assert not recoverable_observation_reason('sdk_disconnected')
assert not recoverable_observation_reason('feedback_stalled')
assert not recoverable_observation_reason('over_temperature')
def test_parent_deferral_moves_dependency_closure_to_stage_end(trajectory_profile):
assert 'index_dip_side' in dependency_closure(trajectory_profile, 'index_pip_side')
session = CalibrationSession(trajectory_profile)
session.phase = CalibrationPhase.PREPARE
session._unit_index = next(i for i, unit in enumerate(session._units)
if unit.task_key == 'index_pip_side' and unit.cycle == 0)
original_prefix = tuple(unit.identity for unit in session._units[:session._unit_index])
decision = session.defer_current_task('rigid_pair_candidates_unresolved')
assert decision.state == TaskObservationState.PENDING
assert {'index_pip_side', 'index_dip_side'} <= set(decision.affected_tasks)
record = decision.as_record(stamp_ns=1)
assert record['dependency_states']['index_dip_side'] == 'dependency_pending'
assert tuple(unit.identity for unit in session._units[:session._unit_index]) == original_prefix
same_cycle = [unit.task_key for unit in session._units if unit.cycle == 0]
assert same_cycle[-1] == 'index_dip_side'
def test_staged_assessment_waits_for_deferred_visit(trajectory_profile):
from linkerhand_calibration.runtime.execution import SessionExecution
execution = SessionExecution(trajectory_profile)
session = execution.session
session.phase = CalibrationPhase.PREPARE
session._unit_index = next(i for i, unit in enumerate(session._units)
if unit.task_key == 'index_pip_side' and unit.cycle == 1)
session.defer_current_task('rigid_pair_candidates_unresolved')
cycle = [unit for unit in session.scheduled_units if unit.cycle == 1]
assert cycle[-1].task_key == 'index_dip_side'
session._unit_index = session._units.index(cycle[-2])
assert not execution.training_check_due()
session._unit_index = session._units.index(cycle[-1])
assert execution.training_check_due()
def test_second_failure_is_terminal_and_budget_survives_restart(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2)
records = [first.as_record(stamp_ns=1, corner_evidence_sha256='a'*64)]
restored = TaskDeferralQueue(trajectory_profile)
restored.restore(records)
second = restored.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2)
assert second.state == TaskObservationState.UNOBSERVABLE
records.append(second.as_record(stamp_ns=2, corner_evidence_sha256='b'*64))
again = TaskDeferralQueue(trajectory_profile)
again.restore(records)
assert again.attempts[('index_dip_side', 0)] == 2
assert ('index_dip_side', 0) in again.unobservable
with pytest.raises(ValueError, match='budget_exhausted'):
again.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 1)
with pytest.raises(ValueError, match='budget_changed'):
again.restore([first.as_record(stamp_ns=3)])
def test_restart_restores_pending_visit_at_end_of_taskwise_stage(trajectory_profile):
profile = configure_training(trajectory_profile, TASKWISE_STAGED)
profile = select_task_training(profile, profile.motion.tasks[0].key, (0, 1, 2))
queue = TaskDeferralQueue(profile)
pending = queue.decide('index_dip_side', 0,
'image_motion_no_observable_image_model', 2).as_record(stamp_ns=1)
session = CalibrationSession(profile)
session.restore_task_deferrals([pending])
first_stage = [unit for unit in session.scheduled_units if unit.cycle == 0]
assert first_stage[-1].task_key == 'index_dip_side'
assert session.task_deferrals.attempts[('index_dip_side', 0)] == 1
# The same rule treats cycle 1 and optional cycle 2 as one training stage,
# so a restored revisit cannot jump ahead of later training or into holdout.
queue = TaskDeferralQueue(profile)
pending = queue.decide('index_dip_side', 1,
'image_motion_no_observable_image_model', 2).as_record(stamp_ns=2)
session = CalibrationSession(profile)
session.restore_task_deferrals([pending])
training_stage = [unit for unit in session.scheduled_units if unit.cycle in {1, 2}]
assert training_stage[-1].task_key == 'index_dip_side'
assert session.scheduled_units[session.scheduled_units.index(training_stage[-1]) + 1].cycle == 3
def test_deferral_audit_allows_only_declared_noncontiguous_resume_gap(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
pending = queue.decide('index_pip_side', 1,
'rigid_pair_candidates_unresolved', 4).as_record(stamp_ns=1)
keys = deferred_visit_keys(trajectory_profile, [pending])
assert ('index_pip_side', 1) in keys
assert ('index_dip_side', 1) in keys
changed = {**pending, 'affected_tasks': ['index_pip_side']}
with pytest.raises(ValueError, match='dependencies_changed'):
deferred_visit_keys(trajectory_profile, [changed])
def test_full_visit_commit_closes_crash_window_but_partial_visit_does_not(trajectory_profile):
from linkerhand_calibration.runtime.engine import CalibrationEngine
queue = TaskDeferralQueue(trajectory_profile)
pending = queue.decide('index_dip_side', 1,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1)
units = [unit for unit in CalibrationEngine(trajectory_profile).scan_units()
if (unit.task_key, unit.cycle) == ('index_dip_side', 1)]
completed = [dict(kind='scan_unit_complete', task_name=unit.task_key,
cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key,
passed=True) for unit in units]
partial = TaskDeferralQueue(trajectory_profile)
partial.restore([pending, *completed[:-1]])
assert ('index_dip_side', 1) not in partial.resolved
restored = TaskDeferralQueue(trajectory_profile)
restored.restore([pending, *completed])
assert ('index_dip_side', 1) in restored.inferred_resolutions
def test_unobservable_record_blocks_release_before_fitting(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1)
row = queue.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=2)
with pytest.raises(ValueError, match='capture_unobservable_tasks:index_dip_side:0:unobservable'):
validate_capture_units(trajectory_profile, [first, row])
def test_resolved_revisit_clears_release_blocker_state(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2)
passed = queue.resolve('index_dip_side', 0)
assert passed is not None and passed.state == TaskObservationState.PASSED
restored = TaskDeferralQueue(trajectory_profile)
restored.restore([first.as_record(stamp_ns=1), passed.as_record(stamp_ns=2)])
assert ('index_dip_side', 0) in restored.resolved
assert not restored.unobservable
def test_v3_checkpoint_upgrade_preserves_accepted_joint_semantics():
base = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
source = configure_preparation(base, 'occlusion_aware_witness_v2')
source = configure_observer_preparations(source, UrdfKinematicModel(SOURCE))
preserved = set(source.measurement.measurements)-{'index_dip'}
target = configure_observer_policy(source, preserved_joints=preserved,
policy=TRAJECTORY_OBSERVER)
source_yaml, target_yaml = (dump_hand_profile(profile).encode()
for profile in (source, target))
upgrade = ObserverPolicyUpgrade(source, target, source_yaml, target_yaml,
source_urdf=SOURCE)
inputs = {'profile_config_sha256': hashlib.sha256(target_yaml).hexdigest(),
'source_urdf_sha256': 'a'*64}
old_inputs = {**inputs, 'profile_config_sha256': hashlib.sha256(source_yaml).hexdigest()}
header = {'kind': 'session_start', **old_inputs,
'preparation_contract': preparation_contract(source)}
reference = {'kind': 'fixed_base_reference_locked',
'protected_hashes': {**old_inputs, 'intrinsics_sha256': 'b'*64}}
checkpoint = SimpleNamespace(header=header, fixed_reference=reference,
rows=(header, reference), joints=SimpleNamespace(
references={joint: object() for joint in preserved}, units=set()))
_, _, audit = upgrade.metadata(checkpoint, inputs, 'source.jsonl', 'c'*64)
assert audit['policy'] == 'unmeasured_observer_upgrade_v3'
assert audit['preserved_joints'] == sorted(preserved)
assert audit['observer_policy_changes'] == {
'index_dip': {'previous': 'rigid_pair_v1', 'current': TRAJECTORY_OBSERVER}}