O30标定修改

This commit is contained in:
lxp
2026-09-21 18:16:44 +08:00
parent 54b8f66b2f
commit 8456aa7f2d
47 changed files with 2043 additions and 245 deletions
@@ -22,19 +22,10 @@ import zlib
FORMAT = "linkerhand_indexed_evidence_v1"
REFERENCE_KEY = "$evidence_record"
FIELDS_KEY = "$record_fields"
SCHEMA_KEY = "$record_schema"
CANONICAL_SHA256_KEY = "$record_canonical_sha256"
DEFAULT_THRESHOLD_BYTES = 16 * 1024
MAXIMUM_BLOCK_BYTES = 4 * 1024 * 1024
INDEXED_SEQUENCE_FIELDS = {
"relative_quaternion_xyzw", "relative_translation_xyz_m",
"parent_pose_common", "child_pose_common", "view_normal_common_xyz",
"camera_center_common_xyz_m", "state_u8", "state_rad",
"command_vector_u8", "command_vector_rad", "steady_training_nodes",
"command_direction_by_index",
}
MOTION_INDEX_FIELDS = {
"zero_joints", "command_vector", "feedback_vector",
}
INDEXED_SCALAR_FIELDS = {
"kind", "profile_id", "serial_number", "task_name", "joint",
"observation_joint", "view", "image_stamp_ns", "cycle", "direction",
@@ -60,7 +51,7 @@ CANONICAL_INDEX_KINDS = {
"capture_scope_extension", "capture_observation_revision",
"passed_tasks_reused", "passed_pose_models_restored", "staged_pose_models_restored",
"resume_verification", "joint_zero_recovery", "visual_motion_observation",
"visual_motion_settled",
"visual_motion_settled", "task_observation_state",
}
@@ -113,7 +104,8 @@ def archive_path(journal_path: str | Path) -> Path:
return path.with_name(path.stem + ".evidence.zlib")
def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEFAULT_THRESHOLD_BYTES) -> None:
def initialize_evidence_journal(journal_path: str | Path, *,
threshold_bytes=DEFAULT_THRESHOLD_BYTES, compression_level=1) -> None:
"""Enable indexed storage for a new journal without changing old captures."""
journal = Path(journal_path)
marker = manifest_path(journal)
@@ -125,11 +117,16 @@ def initialize_evidence_journal(journal_path: str | Path, *, threshold_bytes=DEF
if journal.exists() and journal.stat().st_size:
raise ValueError("indexed_evidence_requires_new_or_migrated_journal")
journal.parent.mkdir(parents=True, exist_ok=True)
if type(compression_level) is not int or not 1 <= compression_level <= 9:
raise ValueError("invalid_evidence_compression_level")
from .storage import atomic_write_json
atomic_write_json(marker, {
"format": FORMAT,
"archive": archive_path(journal).name,
"codec": "zlib_level_1_independent_blocks",
"codec": f"zlib_level_{compression_level}_independent_blocks",
"compression_level": compression_level,
"envelope_fields": "archive_bound_v2",
"field_schemas": {},
"large_record_threshold_bytes": int(threshold_bytes),
"maximum_uncompressed_block_bytes": MAXIMUM_BLOCK_BYTES,
})
@@ -153,24 +150,33 @@ def is_indexed_journal(journal_path: str | Path) -> bool:
def _resume_index(record: Mapping) -> dict:
indexed = {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record
return {key: record[key] for key in INDEXED_SCALAR_FIELDS if key in record
and (record[key] is None or isinstance(record[key], (str, int, float, bool)))}
if record.get("kind") in {"joint_sample", "secondary_joint_sample"}:
for key in INDEXED_SEQUENCE_FIELDS:
if key in record:
indexed[key] = record[key]
if record.get("kind") == "motion_branch_observation":
for key in MOTION_INDEX_FIELDS:
if key in record:
indexed[key] = record[key]
return indexed
def _envelope(record: Mapping, fields, reference) -> dict:
def _schema_identity(fields) -> str:
return hashlib.sha256(json.dumps(list(fields), separators=(",", ":")).encode()).hexdigest()[:16]
@lru_cache(maxsize=512)
def _schema_fields(journal_path: str | Path, identity: str):
marker = _journal_configuration(str(Path(journal_path).resolve())) or {}
fields = marker.get("field_schemas", {}).get(identity)
if not isinstance(fields, list) or not all(isinstance(field, str) for field in fields):
raise ValueError("compressed_evidence_schema_missing")
if _schema_identity(fields) != identity:
raise ValueError("compressed_evidence_schema_changed")
return tuple(fields)
def _envelope(record: Mapping, reference, schema_identity) -> dict:
indexed = _resume_index(record)
if REFERENCE_KEY in indexed or FIELDS_KEY in indexed:
if REFERENCE_KEY in indexed or FIELDS_KEY in indexed or SCHEMA_KEY in indexed:
raise ValueError("calibration_record_uses_reserved_storage_field")
result = {REFERENCE_KEY: reference, FIELDS_KEY: list(fields), **indexed}
# record_sha256 binds the complete archived object. Repeating its field
# names in every one-line resume index added tens of MB without adding an
# independent integrity check.
result = {REFERENCE_KEY: reference, SCHEMA_KEY: schema_identity, **indexed}
if record.get("kind") in CANONICAL_INDEX_KINDS:
result[CANONICAL_SHA256_KEY] = hashlib.sha256(json.dumps(record, sort_keys=True,
separators=(",", ":"), allow_nan=False).encode()).hexdigest()
@@ -190,42 +196,64 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
if marker is None:
raise ValueError("unsupported_evidence_journal_format")
threshold = int(marker["large_record_threshold_bytes"])
compression_level = int(marker.get("compression_level", 1))
if not 1 <= compression_level <= 9:
raise ValueError("invalid_evidence_compression_level")
prepared = []
imported = {}
for record in records:
if REFERENCE_KEY in record or FIELDS_KEY in record:
if REFERENCE_KEY in record or FIELDS_KEY in record or SCHEMA_KEY in record:
raise ValueError("calibration_record_uses_reserved_storage_field")
reference = getattr(record, "reference", None)
field_names = getattr(record, "field_names", None)
fields = tuple(record) if field_names is None else tuple(field_names)
schema_identity = _schema_identity(fields)
if isinstance(reference, EvidenceRecordReference):
imported[len(prepared)] = _linked_envelope(journal, record)
imported[len(prepared)] = (record, schema_identity)
line = None
else:
line = json.dumps(record, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
prepared.append((record, line))
prepared.append((record, line, schema_identity, fields))
if not prepared:
return
schemas = dict(marker.get("field_schemas", {}))
changed = False
for _, _, identity, fields in prepared:
existing = schemas.get(identity)
if existing is not None and tuple(existing) != fields:
raise ValueError("compressed_evidence_schema_collision")
if existing is None:
schemas[identity] = list(fields)
changed = True
if changed:
marker = {**marker, "field_schemas": schemas}
from .storage import atomic_write_json
atomic_write_json(manifest_path(journal), marker)
_journal_configuration.cache_clear()
evidence = archive_path(journal)
evidence.parent.mkdir(parents=True, exist_ok=True)
output: list[bytes | None] = [None] * len(prepared)
for index, envelope in imported.items():
for index, (record, schema_identity) in imported.items():
envelope = _linked_envelope(journal, record, schema_identity)
output[index] = json.dumps(envelope, ensure_ascii=False,
separators=(",", ":")).encode("utf-8")
with evidence.open("ab") as archive:
large: list[tuple[int, Mapping, bytes]] = []
large: list[tuple[int, Mapping, bytes, str]] = []
size = 0
def flush_block():
nonlocal large, size
if not large:
return
block = b"".join(line for _, _, line in large)
compressed = zlib.compress(block, 1)
block = b"".join(line for _, _, line, _ in large)
compressed = zlib.compress(block, compression_level)
offset = archive.tell()
_write_all(archive, compressed)
block_digest = hashlib.sha256(compressed).hexdigest()
local = 0
for index, record, line in large:
for index, record, line, schema_identity in large:
reference = {
"version": 1, "offset": offset,
"compressed_length": len(compressed),
@@ -233,12 +261,12 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
"record_offset": local, "record_length": len(line),
"record_sha256": hashlib.sha256(line).hexdigest(),
}
output[index] = json.dumps(_envelope(record, record.keys(), reference),
output[index] = json.dumps(_envelope(record, reference, schema_identity),
ensure_ascii=False, separators=(",", ":")).encode("utf-8")
local += len(line)
large, size = [], 0
for index, (record, line) in enumerate(prepared):
for index, (record, line, schema_identity, _) in enumerate(prepared):
if line is None:
continue
if len(line) < threshold and record.get("kind") not in DETAILED_RECORD_KINDS:
@@ -246,7 +274,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
continue
if large and size + len(line) > MAXIMUM_BLOCK_BYTES:
flush_block()
large.append((index, record, line))
large.append((index, record, line, schema_identity))
size += len(line)
flush_block()
archive.flush()
@@ -260,7 +288,7 @@ def append_indexed_records(journal_path: str | Path, records: Iterable[Mapping],
os.fsync(stream.fileno())
def _linked_envelope(destination: Path, record) -> dict:
def _linked_envelope(destination: Path, record, schema_identity) -> dict:
"""Hard-link an immutable source archive and retain its indexed row."""
reference = record.reference
source_archive = (record.path.parent / reference.archive
@@ -287,8 +315,7 @@ def _linked_envelope(destination: Path, record) -> dict:
linked = EvidenceRecordReference(reference.offset, reference.compressed_length,
reference.block_sha256, reference.record_offset, reference.record_length,
reference.record_sha256, target.name)
result = {REFERENCE_KEY: linked.as_mapping(), FIELDS_KEY: list(record.field_names),
**record.metadata}
result = {REFERENCE_KEY: linked.as_mapping(), SCHEMA_KEY: schema_identity, **record.metadata}
if record.canonical_sha256:
result[CANONICAL_SHA256_KEY] = record.canonical_sha256
return result
@@ -336,7 +363,10 @@ def decode_envelope(journal_path: str | Path, envelope: Mapping, decoder):
if REFERENCE_KEY not in envelope:
return envelope
value = decoder.loads(evidence_record_bytes(journal_path, envelope[REFERENCE_KEY]))
if not isinstance(value, dict) or tuple(value) != tuple(envelope.get(FIELDS_KEY, ())):
expected_fields = (envelope.get(FIELDS_KEY) if FIELDS_KEY in envelope
else _schema_fields(journal_path, envelope[SCHEMA_KEY]) if SCHEMA_KEY in envelope else None)
if (not isinstance(value, dict) or expected_fields is not None
and tuple(value) != tuple(expected_fields)):
raise ValueError("compressed_evidence_record_shape_changed")
return value
@@ -359,7 +389,9 @@ def compact_existing_journal(source_path: str | Path, *, replace=False) -> dict:
for path in (compact, compact_archive, manifest_path(compact)):
if path.exists():
path.unlink()
initialize_evidence_journal(compact)
# Closed checkpoints favor disk footprint. Live capture remains level 1 so
# evidence persistence cannot stall the control loop.
initialize_evidence_journal(compact, compression_level=6)
source_digest = hashlib.sha256()
record_count = 0
@@ -13,6 +13,8 @@ PLAN_VERSION = "capture_plan_v1"
FIXED = "fixed"
ADAPTIVE = "adaptive_2_to_3"
STAGED = "staged_2_to_3"
TASKWISE_STAGED = "taskwise_staged_2_to_3"
STAGED_POLICIES = frozenset((STAGED, TASKWISE_STAGED))
LEGACY_TRAINING = (0, 1, 2)
LEGACY_HOLDOUT = 3
@@ -67,19 +69,25 @@ class CapturePlan:
return next(task for task in self.tasks if task.task_key == key)
def as_dict(self):
result = {"version": "capture_plan_v2" if self.policy == STAGED else PLAN_VERSION, "policy": self.policy,
version = ("capture_plan_v3" if self.policy == TASKWISE_STAGED else
"capture_plan_v2" if self.policy == STAGED else PLAN_VERSION)
result = {"version": version, "policy": self.policy,
"command_capture_mode": self.command_capture_mode,
"motion_contract": json.loads(self.motion_contract_json),
"tasks": {t.task_key: {"training_cycles": list(t.training),
"holdout_cycle": t.holdout,
"command_training_cycles": list(t.command_training),
"command_holdout_cycle": t.command_holdout} for t in self.tasks}}
if self.policy == STAGED:
if self.policy in STAGED_POLICIES:
result["stages"] = [
{"name": name, "cycle": cycle, "tasks": [t.task_key for t in self.tasks
if cycle in t.motion_cycles]}
for name, cycle in (("first_training", 0), ("second_training", 1),
("supplement", 2), ("validation", 3))]
if self.policy == TASKWISE_STAGED:
result["schedule"] = (
"first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation"
)
return result
@property
@@ -101,7 +109,8 @@ def task_partition(profile, task=None, *, joint=None):
if joint is not None:
task = task_for_joint(profile, joint)
key = getattr(task, "key", task)
default = LEGACY_TRAINING[:2] if profile.quality.training_policy == STAGED else profile.quality.training_cycles
default = (LEGACY_TRAINING[:2] if profile.quality.training_policy in STAGED_POLICIES
else profile.quality.training_cycles)
training = tuple(profile.quality.task_training_cycles.get(key, default))
holdout = profile.quality.holdout_cycle
separate = profile.acquisition.command_capture_mode == "separate"
@@ -119,20 +128,21 @@ def select_task_training(profile, task_key, cycles):
raise ValueError("capture_plan_unknown_task")
if cycles not in (LEGACY_TRAINING, LEGACY_TRAINING[:2]):
raise ValueError("capture_plan_invalid_training_partition")
if cycles != tuple(profile.quality.training_cycles) and profile.quality.training_policy not in {ADAPTIVE, STAGED}:
if (cycles != tuple(profile.quality.training_cycles)
and profile.quality.training_policy not in {ADAPTIVE, *STAGED_POLICIES}):
raise ValueError("capture_plan_fixed_training_cannot_change")
selections = {**profile.quality.task_training_cycles, task_key: cycles}
return replace(profile, quality=replace(profile.quality, task_training_cycles=selections))
def configure_training(profile, policy):
if policy not in {FIXED, ADAPTIVE, STAGED}:
if policy not in {FIXED, ADAPTIVE, *STAGED_POLICIES}:
raise ValueError("unknown_training_policy")
if profile.artifacts.output_schema_version in {2, 3} and (
tuple(profile.quality.training_cycles) != LEGACY_TRAINING
or profile.quality.holdout_cycle != LEGACY_HOLDOUT):
raise ValueError("production_partition_requires_versioned_training_policy")
if policy in {ADAPTIVE, STAGED} and (profile.artifacts.output_schema_version < 3
if policy in {ADAPTIVE, *STAGED_POLICIES} and (profile.artifacts.output_schema_version < 3
or not profile.command_based_release
or profile.acquisition.command_capture_mode != "interleaved"
or tuple(profile.quality.training_cycles) != LEGACY_TRAINING
@@ -9,7 +9,8 @@ from typing import Mapping
from ..domain.profile import CalibrationProfile, validate_profile
from ..domain.result import CalibrationResult, JointMapping
from ..domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, training_cycles
from ..domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED, TASKWISE_STAGED,
training_cycles)
from ..urdf.kinematics import UrdfKinematicModel
from .motion_fit import channel_for_joint, fit_profile_motion, joint_input_direction
from .spatial import ZeroCalibrationProfile, fit_joint_axis_measurement, solve_urdf_zero_offsets, with_depth_free_axis_projection
@@ -98,7 +99,8 @@ def build_axis_observations(profile, model, geometry_records, motion, zero_profi
rows = [dict(row, command_u8=int(round(float(row[profile.curve_input_domain])))) for row in rows]
parent = zero_profile.phase_parent_joint.get(joint)
if parent is not None and parent not in by_joint:
if cycle != plan.task(task.key).holdout and profile.quality.training_policy in {ADAPTIVE, STAGED}:
if (cycle != plan.task(task.key).holdout
and profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}):
continue # No same-cycle parent observation: do not fabricate a geometric sample.
raise ValueError("spatial axis order must place phase parents first")
task = next(task for task in profile.motion.tasks if joint in task.joints)
@@ -182,7 +184,7 @@ def fit_profile_calibration(profile: CalibrationProfile, source_urdf: Path, reco
zero = solve_urdf_zero_offsets(source_urdf=source_urdf, measurements=observations,
curves=spatial_curves, motor_by_joint={joint: channel_for_joint(profile, joint) for joint in curves},
zero_profile=zero_profile, training_cycles=geometry_training, validation_cycle=geometry_holdout,
allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED},
allow_partial_training_cycles=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED},
frozen_training=None if frozen_training is None else frozen_training["zero"],
maximum_offset_rad=math.radians(20), finger_maximum_offset_rad=math.radians(20),
maximum_validation_mae_rad=math.radians(1), maximum_validation_p95_rad=math.radians(2),
@@ -9,7 +9,8 @@ from pathlib import Path
import numpy as np
from scipy.stats import t as student_t
from ..domain.capture_plan import CapturePlan, evidence_digest, select_task_training, training_cycles
from ..domain.capture_plan import (CapturePlan, TASKWISE_STAGED, evidence_digest,
select_task_training, training_cycles)
from ..domain.motion_path import record_scan_identity
from ..domain.sampling import command_nodes
from ..urdf.kinematics import UrdfKinematicModel
@@ -94,16 +95,22 @@ def _curve_repeatability(profile, task, motion, steady, cycles):
def assess_training(profile, task_key, cycles, rows, references, source_urdf):
"""Assess an immutable training snapshot. Missing future geometry adds a round.
"""Assess one immutable task snapshot without reading validation data.
The local angle confidence is NOT a zero-offset certificate. When the full
spatial dependency graph is available, the production training-only solver
also checks real zero confidence. Otherwise the conservative third round is
retained and the final spatial gate remains mandatory.
"""
Adaptive collection keeps the older conservative behavior: missing
full-hand geometry requests a third round. Taskwise staged collection uses
local two-round repeatability to decide immediately; its full-hand spatial
fit remains a separate mandatory gate after every training task is frozen.
"""
cycles = tuple(cycles)
chosen = select_task_training(profile, task_key, cycles)
task = next(t for t in profile.motion.tasks if t.key == task_key)
taskwise = profile.quality.training_policy == TASKWISE_STAGED
if taskwise:
# A task decision is deliberately local and therefore remains stable
# when later fingers are acquired. Full-hand zero geometry is checked
# by the final training-only fit before any artifact can be published.
rows = tuple(row for row in rows if row.get("task_name") == task_key)
if any(row["cycle"] not in training_cycles(chosen, row["task_name"]) for row in rows):
raise ValueError("training_assessment_received_nontraining_data")
sweep, steady = _by_joint(rows, "sweep"), _by_joint(rows, "steady")
@@ -125,8 +132,8 @@ retained and the final spatial gate remains mandatory.
fitted_model = {"curves": {name: asdict(curve) for name, curve in local.coordinate_curves.items()},
"mappings": {name: asdict(mapping) for name, mapping in local.mappings.items()}}
failures, metrics = _curve_repeatability(chosen, task, local, steady, cycles)
required = set(profile.zero.axis_joints)
ready = required <= sweep.keys() & steady.keys() & references.keys()
required = set() if taskwise else set(profile.zero.axis_joints)
ready = bool(required) and required <= sweep.keys() & steady.keys() & references.keys()
if ready:
from .session import compile_spatial_profile, build_axis_observations
from .spatial_solver.solve import fit_urdf_zero_training
@@ -154,7 +161,8 @@ retained and the final spatial gate remains mandatory.
failures.extend(zero.failures)
except (ValueError, KeyError, np.linalg.LinAlgError) as error:
failures.append(str(error))
decision = ("add_training" if len(cycles) == 2 and (failures or geometry["status"] == "deferred")
decision = ("add_training" if len(cycles) == 2 and (failures
or (geometry["status"] == "deferred" and not taskwise))
else "fail" if failures else "freeze")
record = {"kind": "training_decision", "version": TRAINING_DECISION_VERSION,
"task_name": task_key, "assessed_cycles": list(cycles), "decision": decision,
@@ -95,4 +95,11 @@ class EvidenceDecoder:
return row
def loads(self, line):
# Most evidence rows contain observations only. Avoid recursively
# invoking the model interning hook for every nested pose/candidate
# dictionary when neither interned object can be present.
frozen_key = b'"frozen_image_model"' if isinstance(line, bytes) else '"frozen_image_model"'
reference_key = b'"reference_frame"' if isinstance(line, bytes) else '"reference_frame"'
if frozen_key not in line and reference_key not in line:
return json.loads(line)
return json.loads(line, object_hook=self.object_hook)
@@ -19,6 +19,7 @@ from .observer_pose import ObserverPose, OBSERVER_POSE_LIMITS
from .image_model_selection import compare_image_losses, holm_adjusted_probabilities
RIGID_PAIR_CONDITIONING = 'rigid_tag_pair_pose_v1'
RIGID_PAIR_TRAJECTORY_POLICY = 'rigid_pair_full_trajectory_v3'
def _pose_vector(pose):
@@ -144,12 +145,23 @@ def _pose_bounds(fit, count):
def fit_rigid_tag_pair(parents, candidates, corners, matrices, size, stamps, *, parent_corners, parent_size,
parent_candidates=None):
parent_candidates=None, trajectory_states=None):
count = len(stamps)
if count < 24 or tuple(stamps) != tuple(sorted(set(stamps))) or any(not c for c in candidates):
raise ValueError('rigid_pair_independent_images_missing')
if parent_candidates is not None and (len(parent_candidates) != count or any(not p for p in parent_candidates)):
raise ValueError('rigid_pair_parent_candidates_missing')
if trajectory_states is not None:
states = tuple(trajectory_states)
if len(states) != count or any(len(value) != 2 for value in states):
raise ValueError('rigid_pair_trajectory_state_missing')
phases = np.asarray([value[0] for value in states], dtype=int)
commands = np.asarray([value[1] for value in states], dtype=float)
differences = np.diff(commands)
if (not np.all(np.isfinite(commands)) or np.ptp(commands) <= 0
or not {1, 2} <= set(phases)
or not np.any(differences > 0) or not np.any(differences < 0)):
raise ValueError('rigid_pair_trajectory_direction_insufficient')
images = _PairImages(np.asarray(parent_corners), np.asarray(corners), np.asarray(matrices),
square_object_points(parent_size), square_object_points(size), np.asarray([_pose_vector(p) for p in parents]))
if (images.parent_pixels.shape != (count, 4, 2) or images.child_pixels.shape != (count, 4, 2)
@@ -9,11 +9,18 @@ from dataclasses import replace
LEGACY_OBSERVER = 'frozen_parent_v1'
RIGID_OBSERVER = 'rigid_pair_v1'
TRAJECTORY_OBSERVER = 'rigid_pair_trajectory_v3'
def is_rigid_observer(policy):
"""Return whether a policy requires independent rigid-pair evidence."""
return policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}
def rigid_observer_dependencies(profile):
"""Return recipient -> independently measured parent hinge dependencies."""
if profile.vision_motion or profile.quality.training_policy != 'staged_2_to_3':
from ..core.domain.capture_plan import STAGED_POLICIES
if profile.vision_motion or profile.quality.training_policy not in STAGED_POLICIES:
return {}
measurements = profile.measurement.measurements
tasks = {joint: task for task in profile.motion.tasks for joint in task.joints}
@@ -34,13 +41,15 @@ def rigid_observer_dependencies(profile):
return dependencies
def configure_observer_policy(profile, *, preserved_joints=()):
def configure_observer_policy(profile, *, preserved_joints=(), policy=RIGID_OBSERVER):
"""Upgrade every supported unmeasured recipient in a single graph pass."""
if policy not in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}:
raise ValueError('observer_policy_unknown_strategy')
preserved = frozenset(preserved_joints)
measurements = dict(profile.measurement.measurements)
if not preserved <= measurements.keys():
raise ValueError('observer_policy_unknown_preserved_joint')
for joint in rigid_observer_dependencies(profile):
if joint not in preserved:
measurements[joint] = replace(measurements[joint], observer_pose_policy=RIGID_OBSERVER)
measurements[joint] = replace(measurements[joint], observer_pose_policy=policy)
return replace(profile, measurement=replace(profile.measurement, measurements=measurements))
@@ -35,7 +35,12 @@ def _bounded_actions(profile, first):
def configure_observer_preparations(profile, model, *, preserved_joints=()):
profile = configure_observer_policy(profile, preserved_joints=preserved_joints)
from .observer_policy import TRAJECTORY_OBSERVER
strategy = (TRAJECTORY_OBSERVER if any(
spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in profile.measurement.measurements.values()) else None)
profile = configure_observer_policy(profile, preserved_joints=preserved_joints,
**({"policy": strategy} if strategy else {}))
channels = compile_tag_feedback_channels(profile, model)
by_channel = {channel_for_joint(profile, name): name for name in profile.measurement.measurements}
plans = dict(profile.motion.observer_preparations)
@@ -88,7 +93,9 @@ def validate_observer_preparations(profile, model):
def _validate_action(profile, joint, plan, channels, dependencies, index):
if joint not in dependencies or profile.measurement.measurements[joint].observer_pose_policy != 'rigid_pair_v1':
from .observer_policy import is_rigid_observer
if joint not in dependencies or not is_rigid_observer(
profile.measurement.measurements[joint].observer_pose_policy):
raise ValueError('observer_preparation_dependency_invalid')
spec = profile.measurement.measurements[joint]
channel = channel_for_joint(profile, plan.joint)
@@ -15,8 +15,9 @@ def preparation_image_budget(profile, *, joint_count=1):
from ..core.geometry.tag_pose.production_image_motion import (
DENSE_PREPARATION_SAMPLING, image_parameters_for_sampling,
)
from ..core.domain.capture_plan import STAGED_POLICIES
dense = (profile.key.model == 'O30'
and profile.quality.training_policy == 'staged_2_to_3'
and profile.quality.training_policy in STAGED_POLICIES
and bool(profile.motion.preparation_witnesses) and joint_count == 1)
return ((256, image_parameters_for_sampling(DENSE_PREPARATION_SAMPLING)) if dense
else (128, image_parameters_for_sampling()))
@@ -50,9 +51,20 @@ def preparation_contract(profile):
value['observer_pose_policies'] = observers
if profile.motion.observer_preparations:
from ..core.domain.profile import ObserverPreparationPlan
from .observer_policy import TRAJECTORY_OBSERVER
sequence = any(isinstance(p, ObserverPreparationPlan) for p in profile.motion.observer_preparations.values())
value['observer_preparations'] = dict(policy='bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1',
trajectory = any(spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in profile.measurement.measurements.values())
value['observer_preparations'] = dict(policy='occlusion_aware_trajectory_v3' if trajectory
else 'bounded_rigid_observation_v2' if sequence else 'bounded_rigid_observation_v1',
paths={name: asdict(spec) for name, spec in profile.motion.observer_preparations.items()})
if trajectory:
value['observer_preparations']['trajectory_solver'] = {
'policy': 'rigid_pair_full_trajectory_v3',
'training_validation_split': 'chronological_even_odd',
'maximum_actions_per_task': 2,
'holdout_selects_model': False,
}
# The live header and decoded JSON must compare identically (including
# tuple-valued poses), without relying on incidental Python container types.
return json.loads(json.dumps(value, allow_nan=False))
@@ -72,11 +84,18 @@ def validate_preparation_contract(profile, records):
if len(declared) != 1 or declared[0] != preparation_contract(profile):
raise ValueError('preparation_contract_profile_changed')
tasks = {task.key: set(task.joints) for task in profile.motion.tasks}
refreshed_parents = {(row.get('task_name'), row.get('pose_joint'),
row.get('image_model_sha256')) for row in records
if row.get('kind') == 'parent_reference_model_refreshed'
and row.get('passed_zero_preserved') is True}
for row in records:
if row.get('kind') != 'motion_branch_initialization' or row.get('status') != 'resolved':
continue
joints = row.get('zero_joints', ())
if not joints or not set(joints) <= tasks.get(row.get('task_name'), set()):
ordinary = bool(joints) and set(joints) <= tasks.get(row.get('task_name'), set())
refreshed = (len(joints) == 1 and (row.get('task_name'), joints[0],
row.get('image_model_sha256')) in refreshed_parents)
if not ordinary and not refreshed:
raise ValueError('preparation_contract_model_dependency_changed')
_, expected = preparation_image_budget(profile, joint_count=len(joints))
if image_parameters_for_sampling(row.get('image_sampling_policy')) != expected:
@@ -84,9 +103,12 @@ def validate_preparation_contract(profile, records):
def configure_preparation(profile, policy='occlusion_aware_witness_v2'):
if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2'}:
if policy not in {'occlusion_aware_witness_v1', 'occlusion_aware_witness_v2',
'occlusion_aware_trajectory_v3'}:
raise ValueError('unknown_preparation_policy')
if profile.key.profile_id != 'O30/right/o30_right_18/v1' or profile.quality.training_policy != 'staged_2_to_3':
from ..core.domain.capture_plan import STAGED_POLICIES
if (profile.key.profile_id != 'O30/right/o30_right_18/v1'
or profile.quality.training_policy not in STAGED_POLICIES):
raise ValueError('preparation_witness_requires_o30_staged_session')
from ..core.fitting.motion_fit import channel_for_joint
witnesses = {}
@@ -102,9 +124,10 @@ def configure_preparation(profile, policy='occlusion_aware_witness_v2'):
excursion[channel] = command[channel]-step if command[channel]-step >= lower else command[channel]+step
witnesses[target] = ModelPreparation(donor, command, (command, tuple(excursion)))
prepared = replace(profile, motion=replace(profile.motion, preparation_witnesses=witnesses))
if policy == 'occlusion_aware_witness_v2':
from .observer_policy import configure_observer_policy
prepared = configure_observer_policy(prepared)
if policy in {'occlusion_aware_witness_v2', 'occlusion_aware_trajectory_v3'}:
from .observer_policy import configure_observer_policy, RIGID_OBSERVER, TRAJECTORY_OBSERVER
strategy = TRAJECTORY_OBSERVER if policy == 'occlusion_aware_trajectory_v3' else RIGID_OBSERVER
prepared = configure_observer_policy(prepared, policy=strategy)
return prepared
@@ -36,18 +36,20 @@ def validate_executable_profile(profile, source_urdf):
if joint == observer:
raise ValueError(f"absolute_zero_unobservable:{joint}:a free Tag on its own hinge is not a CAD datum")
measured = {joint for task in profile.motion.tasks for joint in task.joints}
from .observer_policy import rigid_observer_dependencies, RIGID_OBSERVER
from .observer_policy import rigid_observer_dependencies, is_rigid_observer
observer_dependencies = rigid_observer_dependencies(profile)
if not measured <= model.joints.keys() or not (profile.zero.active_joints | profile.zero.passive_joints) <= model.joints.keys():
raise ValueError("Profile joint keys must name URDF joints; SDK names belong in command.names")
for name, spec in profile.measurement.measurements.items():
if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1"}:
if spec.observer_pose_policy not in {"frozen_parent_v1", "rigid_pair_v1",
"rigid_pair_trajectory_v3"}:
raise ValueError(f"unknown_observer_pose_policy:{name}")
if spec.observer_pose_policy == "rigid_pair_v1" and (
from ..core.domain.capture_plan import STAGED_POLICIES
if is_rigid_observer(spec.observer_pose_policy) and (
spec.kind != "relative_rotation" or profile.vision_motion
or profile.quality.training_policy != "staged_2_to_3"):
or profile.quality.training_policy not in STAGED_POLICIES):
raise ValueError(f"rigid_pair_requires_staged_relative_measurement:{name}")
if spec.observer_pose_policy == RIGID_OBSERVER and name not in observer_dependencies:
if is_rigid_observer(spec.observer_pose_policy) and name not in observer_dependencies:
raise ValueError(f"rigid_pair_requires_independent_parent_hinge:{name}")
if spec.kind not in {"relative_rotation", "urdf_axis_chain", "curve", "axis_cross_view_validation"}:
raise ValueError(f"unsupported measurement primitive:{name}:{spec.kind}")
@@ -53,6 +53,11 @@ def validate_capture_provenance(profile, serial_number, hashes, records):
def validate_capture_units(profile, records):
"""Offline and resumed data must satisfy the actual post-sweep policy."""
from ..task_deferral import unresolved_observation_records
blocked = unresolved_observation_records(profile, records)
if blocked:
raise ValueError("capture_unobservable_tasks:" + ",".join(sorted(
f"{row.get('task_name')}:{row.get('cycle')}:{row.get('state')}" for row in blocked)))
from ...core.fitting.command_sampling import validate_sampling_plans
from ..training import resolve_capture_plan
profile = resolve_capture_plan(profile, records, require_complete=True)
@@ -8,6 +8,7 @@ training/holdout observations to revise an existing reference.
from dataclasses import asdict, dataclass, replace
from collections import deque
from collections.abc import Mapping
import hashlib
import json
import math
@@ -89,7 +90,11 @@ def _image_references(record, roles):
def _finite_report(value):
if isinstance(value, float) and not math.isfinite(value):
return None
if isinstance(value, dict):
# Resume keeps large evidence rows as lazy Mapping objects. Solver reports
# are durable journal records, so materialize any nested mapping at this
# single ownership boundary instead of leaking storage-layer objects into
# the JSON encoder.
if isinstance(value, Mapping):
return {key: _finite_report(item) for key, item in value.items()}
if isinstance(value, (list, tuple)):
return [_finite_report(item) for item in value]
@@ -125,8 +130,9 @@ def solve_initialization(request: BranchInitializationRequest):
return replace(result, preparation_transitions=(record,))
bridges, records, rejections = [], [], []
if request.observer_profile is not None:
from ..profiles.observer_policy import is_rigid_observer
required = {joint for joint, spec in request.observer_profile.measurement.measurements.items()
if joint in request.zero_joints and spec.observer_pose_policy == 'rigid_pair_v1'}
if joint in request.zero_joints and is_rigid_observer(spec.observer_pose_policy)}
if not required <= {source.joint for source in request.observer_sources}:
return ImageMotionResolution(False, 'rigid_pair_source_missing')
if request.observer_sources:
@@ -137,7 +143,7 @@ def solve_initialization(request: BranchInitializationRequest):
bridges.append(bridge)
records.append(record)
except (ValueError, KeyError, TypeError, IndexError) as error:
if request.observer_profile.measurement.measurements[source.joint].observer_pose_policy == 'rigid_pair_v1':
if is_rigid_observer(request.observer_profile.measurement.measurements[source.joint].observer_pose_policy):
return ImageMotionResolution(False, str(error))
rejections.append(str(error))
witness_bridges, witness_records = [], []
@@ -228,7 +234,8 @@ def initialization_artifacts(request, resolution):
if record.get("observer_pose_bridges"):
from ..core.geometry.tag_pose.pose_bridge import observer_pose_selection_policy, RIGID_OBSERVER_SELECTION_POLICY
record["image_model_selection_policy"] = (RIGID_OBSERVER_SELECTION_POLICY
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges'])
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'} for r in record['observer_pose_bridges'])
else observer_pose_selection_policy(request.pose_bridges))
if record.get("preparation_transitions"):
from ..core.geometry.tag_pose.transition_image_solver import TRANSITION_SELECTION_POLICY
@@ -241,6 +248,7 @@ def initialization_artifacts(request, resolution):
if any(r.get('conditioning') for r in record['preparation_witnesses']) else WITNESS_POLICY)
record["image_geometry_constraints"] = [asdict(item) for item in request.geometry_constraints]
record["source_urdf_sha256"] = request.source_urdf_sha256
record = _finite_report(record)
if not resolution.resolved:
return None, record
existing = dict(request.existing_evidence_ids)
@@ -272,7 +280,8 @@ def initialization_artifacts(request, resolution):
payload['image_model_selection_policy'] = record['image_model_selection_policy']
if record.get("observer_pose_bridges"):
payload["observer_pose_bridges"] = record["observer_pose_bridges"]
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'} for r in record['observer_pose_bridges']):
if any(r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'} for r in record['observer_pose_bridges']):
payload['image_model_selection_policy'] = record['image_model_selection_policy']
if record.get("preparation_transitions"):
payload["preparation_transitions"] = record["preparation_transitions"]
@@ -340,6 +349,7 @@ class BranchInitialization:
self._invalid_reason = ""
self._requested = set()
self.reference_spec = None
self.parent_model_refresh_for = ""
self.reports = {}
self.path_index = None
@@ -367,6 +377,7 @@ class BranchInitialization:
self._invalid_reason = "motion_approach_path_discontinuous"
self.path_index = index
self.reference_spec = (motion.reference_command, motion.reference_approach) if motion.reference_only else None
self.parent_model_refresh_for = motion.parent_model_refresh_for
self.task_name = motion.task_key
self.zero_joints = motion.reference_joints
self.motion_version = motion_version
@@ -378,6 +389,7 @@ class BranchInitialization:
self.task_name, self.zero_joints = motion.task_key, motion.zero_joints
self.session_epoch = session_epoch
self._invalid_reason = "motion_approach_evidence_missing"
self.parent_model_refresh_for = motion.parent_model_refresh_for
elif self.path_index is not None:
spec = self.profile.motion.joint_zero_references[motion.zero_joints[0]]
if spec.evidence_scope != "path" or self.path_index != len(spec.approach_commands):
@@ -539,7 +551,7 @@ class BranchInitialization:
reason = reason or 'shared_pose_bridge_invalid:' + str(error)
source_hinges, parent_reference_sha256 = (), ""
task = next(task for task in self.profile.motion.tasks if task.key == self.task_name)
if task.parent_reference is not None:
if task.parent_reference is not None and not self.parent_model_refresh_for:
try:
source_hinges, parent_reference_sha256 = self.parent_references.geometry(task, session_epoch)
except ValueError as error:
@@ -550,15 +562,17 @@ class BranchInitialization:
tuple(item.geometry.relation for item in source_hinges) + relations)
from .observer_reference import observer_inputs
observer_sources = observer_inputs(self, view, self._endpoint_records.get(view, ()),
{f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else ()
{f.stamp_ns for f in frames}, session_epoch) if (
self.image_geometry and not self.parent_model_refresh_for) else ()
transition_source = None
if self.image_geometry:
if self.image_geometry and not self.parent_model_refresh_for:
try:
transition_source = self.transition_capture.snapshot(view)
except ValueError as error:
reason = reason or str(error)
witness_sources = self.witness_capture.inputs(view, self.zero_joints,
self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if self.image_geometry else ()
self._endpoint_records.get(view, ()), {f.stamp_ns for f in frames}, session_epoch) if (
self.image_geometry and not self.parent_model_refresh_for) else ()
return BranchInitializationRequest(self.task_name, view, self.zero_joints,
session_epoch, self.motion_version if self.motion_version is not None else motion_version, self.generation,
frames,
@@ -157,6 +157,11 @@ class ObservationCapture:
self._preparation_identity = None
self.install_motion_model(model)
def current_image_reference_stable(self, view):
"""Check the independent current-session anchor, not an active old model."""
tracker = self._image_references.get(view)
return tracker is not None and tracker.tracking_current_image()
def install_motion_model(self, model: MotionBranchModel):
"""Add evidence before zero capture without replacing existing roles."""
key = (model.task_name, model.view)
@@ -64,14 +64,27 @@ class CaptureRecordIndex(Sequence):
source = getattr(row, "metadata", row)
indexed = {key: value for key, value in source.items()
if value is None or isinstance(value, (str, int, float, bool))}
for key in ('relative_quaternion_xyzw', 'steady_training_nodes', 'command_direction_by_index'):
if key in row:
indexed[key] = tuple(row[key])
if self.retain_training_geometry:
from ..core.fitting.training_quality import TRAINING_FIELDS
from ..core.domain.capture_plan import evidence_digest
indexed.update({key: row[key] for key in TRAINING_FIELDS if key in row})
indexed['source_record_sha256'] = evidence_digest(row)
from .resume_storage import materialize_plain_record
# A deferred row decompresses and parses its evidence when a
# non-indexed field is accessed. Materialize it exactly once;
# probing every training field through Mapping.__contains__
# used to decode the same sample dozens of times.
materialized = materialize_plain_record(row)
indexed.update({key: materialized[key] for key in TRAINING_FIELDS
if key in materialized})
# DeferredCaptureRecord implements Mapping for indexed reads,
# but json.dumps intentionally accepts only concrete JSON
# containers. Hash the immutable materialized record exactly
# as an eager capture would.
indexed['source_record_sha256'] = evidence_digest(materialized)
else:
for key in ('relative_quaternion_xyzw', 'steady_training_nodes',
'command_direction_by_index'):
if key in row:
indexed[key] = tuple(row[key])
self._store(indexed)
elif kind in {'command_sampling_plan', 'training_decision', 'staged_training_decision'}:
self._store(row)
@@ -8,10 +8,12 @@ from dataclasses import dataclass, replace
import hashlib
import json
from pathlib import Path
import shutil
from ..core.domain.profile import CalibrationProfile
from ..profiles import load_hand_profile
from ..profiles.observer_policy import configure_observer_policy
from ..profiles.observer_policy import (configure_observer_policy,
RIGID_OBSERVER, TRAJECTORY_OBSERVER)
from ..profiles.preparation import preparation_contract
from ..profiles.validator import validate_executable_profile
from .engine import CalibrationEngine
@@ -19,6 +21,23 @@ from .joint_resume import PreparedResume, JointResume
from .staged_resume import stage_visits
def _copy_indexed_evidence(source_path, output_dir):
"""Copy content-addressed evidence before exposing the migrated index."""
from ..core.artifacts.evidence_journal import (
is_indexed_journal, manifest_path, archive_path,
)
if not is_indexed_journal(source_path):
return
for source_evidence in (manifest_path(source_path), archive_path(source_path)):
target_evidence = output_dir/source_evidence.name
shutil.copy2(source_evidence, target_evidence)
with source_evidence.open('rb') as source_stream, target_evidence.open('rb') as target_stream:
source_hash = hashlib.file_digest(source_stream, 'sha256').hexdigest()
target_hash = hashlib.file_digest(target_stream, 'sha256').hexdigest()
if source_hash != target_hash:
raise ValueError('observer_upgrade_evidence_archive_changed')
def preserved_observer_joints(checkpoint):
if any(r.get('kind') == 'staged_training_decision' for r in checkpoint.rows):
raise ValueError('observer_upgrade_training_already_decided')
@@ -45,14 +64,23 @@ class ObserverPolicyUpgrade:
return cls(source, target, Path(source_path).read_bytes(), Path(target_path).read_bytes())
def metadata(self, checkpoint, protected_inputs, source_path, source_digest):
from .resume_storage import materialize_record
source_header = materialize_record(checkpoint.header)
source_reference = materialize_record(checkpoint.fixed_reference)
preserved = preserved_observer_joints(checkpoint)
expected = configure_observer_policy(self.source, preserved_joints=preserved)
target_strategies = {spec.observer_pose_policy
for spec in self.target.measurement.measurements.values()
if spec.observer_pose_policy in {RIGID_OBSERVER, TRAJECTORY_OBSERVER}}
strategy = (TRAJECTORY_OBSERVER if TRAJECTORY_OBSERVER in target_strategies
else RIGID_OBSERVER)
expected = configure_observer_policy(self.source, preserved_joints=preserved,
policy=strategy)
if self.target.motion.observer_preparations != self.source.motion.observer_preparations:
if self.source_urdf is None:
raise ValueError('observer_upgrade_source_geometry_missing')
from ..core.urdf.kinematics import UrdfKinematicModel
from ..profiles.observer_preparation import configure_observer_preparations
expected = configure_observer_preparations(self.source, UrdfKinematicModel(self.source_urdf),
expected = configure_observer_preparations(expected, UrdfKinematicModel(self.source_urdf),
preserved_joints=preserved)
if self.target != expected:
raise ValueError('observer_upgrade_changes_existing_contract')
@@ -63,19 +91,19 @@ class ObserverPolicyUpgrade:
if protected_inputs.get('profile_config_sha256') != new_hash:
raise ValueError('observer_upgrade_target_hash_invalid')
old_inputs = {**protected_inputs, 'profile_config_sha256': old_hash}
for row in (checkpoint.header, checkpoint.fixed_reference):
for row in (source_header, source_reference):
actual = row.get('protected_hashes', row)
if any(actual.get(key) != value for key, value in old_inputs.items()):
raise ValueError('observer_upgrade_protected_inputs_changed')
if checkpoint.header.get('preparation_contract') != preparation_contract(self.source):
if source_header.get('preparation_contract') != preparation_contract(self.source):
raise ValueError('observer_upgrade_source_preparation_changed')
header = {**checkpoint.header, **protected_inputs,
header = {**source_header, **protected_inputs,
'preparation_contract': preparation_contract(self.target),
'resume_checkpoint_requested': True}
if 'protected_hashes' in header:
header['protected_hashes'] = {**header['protected_hashes'], **protected_inputs}
reference = {**checkpoint.fixed_reference, 'protected_hashes': {
**checkpoint.fixed_reference['protected_hashes'], **protected_inputs}}
reference = {**source_reference, 'protected_hashes': {
**source_reference['protected_hashes'], **protected_inputs}}
changes = {joint: dict(previous=spec.observer_pose_policy,
current=self.target.measurement.measurements[joint].observer_pose_policy)
for joint, spec in self.source.measurement.measurements.items()
@@ -83,7 +111,7 @@ class ObserverPolicyUpgrade:
audit = dict(kind='capture_observation_revision', policy='unmeasured_observer_upgrade_v1',
source_raw_samples=str(Path(source_path).resolve()), source_raw_sha256=source_digest,
source_profile_sha256=old_hash, target_profile_sha256=new_hash,
original_header=checkpoint.header, original_fixed_reference=checkpoint.fixed_reference,
original_header=source_header, original_fixed_reference=source_reference,
preserved_joints=sorted(preserved), observer_policy_changes=changes,
completed_units=[list(unit) for unit in sorted(checkpoint.joints.units)],
physical_zero_reverification_performed=False)
@@ -92,6 +120,11 @@ class ObserverPolicyUpgrade:
audit.update(policy='unmeasured_observer_upgrade_v2', observer_preparations={
name: asdict(plan) for name, plan in self.target.motion.observer_preparations.items()
if self.source.motion.observer_preparations.get(name) != plan})
if strategy == TRAJECTORY_OBSERVER:
audit.update(policy='unmeasured_observer_upgrade_v3',
target_observation_policy='occlusion_aware_trajectory_v3',
accepted_evidence_semantics='preserved_without_reinterpretation',
pending_evidence_semantics='full_trajectory_with_bounded_observation')
return header, reference, audit
@@ -117,26 +150,34 @@ def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number,
or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans):
raise ValueError('observer_upgrade_changes_reusable_evidence')
output_dir.mkdir(parents=True, exist_ok=False)
temporary = output_dir/'raw_samples.jsonl.tmp'
digest = hashlib.sha256()
with source_path.open('rb') as source, temporary.open('xb') as target:
for line in source:
digest.update(line)
row = json.loads(line) if line.strip() else {}
if row.get('kind') in replacement:
target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode())
else:
target.write(line)
target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode())
if digest.hexdigest() != source_digest:
temporary.unlink()
raise ValueError('observer_upgrade_source_changed_during_replay')
(output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml)
(output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml)
report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'}
(output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
temporary.rename(output_dir/'raw_samples.jsonl')
return report
try:
_copy_indexed_evidence(source_path, output_dir)
temporary = output_dir/'raw_samples.jsonl.tmp'
digest = hashlib.sha256()
with source_path.open('rb') as source, temporary.open('xb') as target:
for line in source:
digest.update(line)
row = json.loads(line) if line.strip() else {}
if row.get('kind') in replacement:
target.write((json.dumps(replacement[row['kind']], ensure_ascii=False)+'\n').encode())
else:
target.write(line)
target.write(('\n'+json.dumps(audit, ensure_ascii=False)+'\n').encode())
if digest.hexdigest() != source_digest:
raise ValueError('observer_upgrade_source_changed_during_replay')
(output_dir/'source_profile.yaml').write_bytes(upgrade.source_yaml)
(output_dir/'target_profile.yaml').write_bytes(upgrade.target_yaml)
report = {**audit, 'reusable_units': len(verified.units), 'result': 'STATIC_REPLAY_PASSED'}
installed = output_dir/'raw_samples.jsonl'
temporary.rename(installed)
if (output_dir/'raw_samples.storage.json').is_file():
from ..core.artifacts.evidence_journal import compact_existing_journal
report['storage_compaction'] = compact_existing_journal(installed, replace=True)
(output_dir/'observer_upgrade.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except Exception:
shutil.rmtree(output_dir, ignore_errors=True)
raise
def main():
@@ -0,0 +1,173 @@
"""Audited migration from whole-hand staged training to taskwise staged training.
The measurement contract, motion paths, accepted scan identities and raw image
evidence stay unchanged. Only the ordering of future training visits and the
time at which a task-local repeatability decision is made are revised. The
source journal is immutable and the target journal links its content-addressed
evidence blocks.
"""
from dataclasses import dataclass
import hashlib
import json
from pathlib import Path
import shutil
from ..core.domain.capture_plan import (CapturePlan, STAGED, TASKWISE_STAGED,
configure_training)
from ..profiles import load_hand_profile
from ..profiles.export import dump_hand_profile
from ..profiles.preparation import preparation_contract
from ..profiles.validator import validate_executable_profile
from .engine import CalibrationEngine, capture_schedule_version
from .joint_resume import PreparedResume, JointResume
from .resume_storage import materialize_record
from .staged_resume import stage_passed_visits
@dataclass(frozen=True)
class TaskwiseTrainingUpgrade:
source: object
target: object
source_yaml: bytes
target_yaml: bytes
@classmethod
def load(cls, source_profile):
source = load_hand_profile(source_profile)
if source.quality.training_policy != STAGED:
raise ValueError("taskwise_upgrade_requires_staged_source")
target = configure_training(source, TASKWISE_STAGED)
return cls(source, target, Path(source_profile).read_bytes(),
dump_hand_profile(target).encode())
def replacement_records(self, checkpoint, *, source_path, source_digest):
if any(row.get("kind") in {"staged_training_decision", "training_decision"}
for row in checkpoint.rows):
raise ValueError("taskwise_upgrade_training_decision_already_recorded")
source_header = materialize_record(checkpoint.header)
source_reference = materialize_record(checkpoint.fixed_reference)
old_hash = hashlib.sha256(self.source_yaml).hexdigest()
new_hash = hashlib.sha256(self.target_yaml).hexdigest()
actual = source_header.get("protected_hashes", source_header)
if actual.get("profile_config_sha256") != old_hash:
raise ValueError("taskwise_upgrade_source_profile_hash_changed")
if source_header.get("preparation_contract") != preparation_contract(self.source):
raise ValueError("taskwise_upgrade_source_preparation_changed")
if preparation_contract(self.target) != preparation_contract(self.source):
raise ValueError("taskwise_upgrade_changes_preparation_contract")
header = {**source_header,
"capture_schedule_version": capture_schedule_version(self.target),
"capture_plan": CapturePlan.from_profile(self.target).as_dict(),
"profile_config_sha256": new_hash,
"resume_checkpoint_requested": True}
if "protected_hashes" in header:
header["protected_hashes"] = {**header["protected_hashes"],
"profile_config_sha256": new_hash}
reference = {**source_reference, "protected_hashes": {
**source_reference["protected_hashes"], "profile_config_sha256": new_hash}}
audit = {
"kind": "capture_schedule_revision",
"version": "taskwise_staged_training_upgrade_v1",
"source_raw_samples": str(Path(source_path).resolve()),
"source_raw_sha256": source_digest,
"source_profile_sha256": old_hash,
"target_profile_sha256": new_hash,
"source_policy": STAGED,
"target_policy": TASKWISE_STAGED,
"accepted_evidence_semantics": "preserved_without_reinterpretation",
"future_schedule": (
"first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation"
),
"completed_units": [list(unit) for unit in sorted(checkpoint.joints.units)],
}
return header, reference, audit
def upgrade_checkpoint(upgrade, source_path, output_dir, *, serial_number,
source_urdf, image_replay_workers=1):
source_path, output_dir = Path(source_path).resolve(), Path(output_dir).resolve()
if output_dir.exists():
raise FileExistsError(output_dir)
for profile in (upgrade.source, upgrade.target):
validate_executable_profile(profile, source_urdf)
with source_path.open("rb") as stream:
source_digest = hashlib.file_digest(stream, "sha256").hexdigest()
checkpoint = PreparedResume.load(upgrade.source, CalibrationEngine(upgrade.source),
source_path, serial_number, image_replay_workers=image_replay_workers,
resume_mode="passed", source_urdf=source_urdf)
header, reference, audit = upgrade.replacement_records(checkpoint,
source_path=source_path, source_digest=source_digest)
replacement = {"session_start": header, "fixed_base_reference_locked": reference}
rows = tuple(replacement.get(row.get("kind"), row) for row in checkpoint.rows) + (audit,)
# The source is itself an unchanged-installation ``passed`` checkpoint.
# Its compact journal intentionally omits historical witness windows that
# strict replay would require. Preserve and compare the same durable pass
# decisions here; finalization still performs the complete evidence replay
# before it may publish JSON/URDF artifacts.
verified = JointResume(upgrade.target, image_replay_workers=image_replay_workers)
stage_passed_visits(verified, CalibrationEngine(upgrade.target), rows)
if (verified.units != checkpoint.joints.units
or verified.references != checkpoint.joints.references
or verified.first_cycle_spans != checkpoint.joints.first_cycle_spans):
raise ValueError("taskwise_upgrade_changes_reusable_evidence")
output_dir.mkdir(parents=True, exist_ok=False)
try:
raw = output_dir / "raw_samples.jsonl"
from ..core.artifacts.evidence_journal import initialize_evidence_journal
from ..core.artifacts.storage import append_jsonl_many, sync_jsonl
initialize_evidence_journal(raw)
for offset in range(0, len(rows), 4096):
append_jsonl_many(raw, rows[offset:offset + 4096], durable=False)
sync_jsonl(raw)
(output_dir / "source_profile.yaml").write_bytes(upgrade.source_yaml)
target_profile = output_dir / "target_profile.yaml"
target_profile.write_bytes(upgrade.target_yaml)
prepared = PreparedResume.load(upgrade.target, CalibrationEngine(upgrade.target),
raw, serial_number, image_replay_workers=image_replay_workers,
resume_mode="passed", source_urdf=source_urdf)
if (prepared.completed_units != frozenset(verified.units)
or prepared.joints.references != verified.references
or prepared.joints.first_cycle_spans != verified.first_cycle_spans):
raise ValueError("taskwise_upgrade_written_checkpoint_changed")
report = {**audit, "reusable_units": len(verified.units),
"zero_references": len(verified.references),
"target_profile": str(target_profile),
"resume_validation": "passed_mode_loaded",
"historical_replay_performed": False,
"full_replay_required_before_publication": True,
"result": "PASSED_EVIDENCE_PRESERVATION_VERIFIED"}
(output_dir / "training_upgrade.json").write_text(
json.dumps(report, ensure_ascii=False, indent=2) + "\n")
return report
except Exception:
shutil.rmtree(output_dir, ignore_errors=True)
raise
def main():
import argparse
from ..product import load_product_config
from .image_replay import checkpoint_replay_workers
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--source-profile", required=True)
parser.add_argument("--target-product", required=True)
parser.add_argument("--source-checkpoint", required=True)
parser.add_argument("--output-dir", required=True)
args = parser.parse_args()
product = load_product_config(args.target_product, check_can=False)
upgrade = TaskwiseTrainingUpgrade.load(args.source_profile)
report = upgrade_checkpoint(upgrade, args.source_checkpoint, args.output_dir,
serial_number=product.serial_number, source_urdf=product.source_urdf,
image_replay_workers=checkpoint_replay_workers())
print(json.dumps({key: report[key] for key in (
"result", "reusable_units", "zero_references", "target_profile"
)}, ensure_ascii=False), flush=True)
if __name__ == "__main__":
main()
@@ -42,7 +42,8 @@ from .resume import ResumeDecision, ResumeVerifier, fingerprint_from_mapping
from .safety import SafetyPolicy, SafetySample
from .session import CalibrationPhase as Phase
from .snapshot import DeviceReadiness, CameraReadiness, RuntimeSnapshot, build_snapshot, legacy_state
from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED
from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED,
TASKWISE_STAGED, STAGED_POLICIES)
_TERMINAL_STATES = frozenset({"PASSED", "DIAGNOSTIC_COMPLETE", "PAUSED", "ABORTED", "FAILED"})
@@ -73,7 +74,7 @@ class CalibrationCoordinator:
self.training_worker = None
self._training_request = None
self._frozen_training_sha256 = None
if profile.quality.training_policy in {ADAPTIVE, STAGED}:
if profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED}:
if self.diagnostic_capture is not None:
raise ValueError("adaptive_training_cannot_run_as_diagnostic")
from .training import TrainingWorker
@@ -91,10 +92,12 @@ class CalibrationCoordinator:
self.command_direction_history = deque(maxlen=2000)
self._command_directions = [""]*self.command_count
self.finalization = finalization or FinalizationController(profile, parameters.session_dir)
from .journal_durability import JournalDurabilityWorker
self.journal_durability = JournalDurabilityWorker()
from .capture_index import CaptureRecordIndex
self.capture_index = CaptureRecordIndex(retain_complete=(
self.diagnostic_capture is not None or not getattr(self.finalization, "uses_journal", False)),
retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED})
retain_training_geometry=profile.quality.training_policy in {ADAPTIVE, STAGED, TASKWISE_STAGED})
self._image_record_stamps = {}
self.last_command = None
self.commanded_speed = None
@@ -585,6 +588,7 @@ class CalibrationCoordinator:
self._observation_epoch += 1
self.branch_initialization.clear()
try:
self.journal_durability.close()
sync_jsonl(self.raw_path)
finally:
self.finalization.close()
@@ -640,28 +644,7 @@ class CalibrationCoordinator:
if self._passed_reference_import is not None:
self._publish_command(list(current))
pending = self._passed_reference_import
self._write_resume_batch(pending)
if pending.complete:
from .passed_resume import restore_parent_models, restored_model_record
binding = restored_model_record(pending.model_records,
self._observation_epoch, self.ports.clock_ns())
if self.profile.quality.training_policy == STAGED and self.parameters.resume_mode != "passed":
binding.update(kind="staged_pose_models_restored", resume_mode="verify",
physical_zero_reverification_required=True)
binding.pop("unchanged_installation_declared", None)
append_jsonl(self.raw_path, binding)
self.capture_index.append(binding)
restore_parent_models(self.branch_initialization.parent_references,
pending.model_records, self._observation_epoch)
from .observer_reference import restore_observer_records
restore_observer_records(self.branch_initialization, pending.rows)
self.branch_initialization.observer_preparation.restore(pending.rows)
for row in pending.rows:
if row.get("kind") == "motion_branch_observation":
self.branch_initialization.witness_capture.observe(row)
self.execution.zero_references.update(pending.references)
if self.visual_motion is not None:
self.visual_motion.freeze_zeros(pending.references)
if self._write_resume_batch(pending) and self._restore_passed_reference_batch(pending):
self._passed_reference_import = None
return
if self._reference_import is not None and not self._reference_import.complete:
@@ -674,7 +657,7 @@ class CalibrationCoordinator:
return
if phase in {Phase.PREPARE, Phase.MAPPING_PROBE} and self.segment is None and session.current_unit is not None:
unit = session.current_unit
if self.profile.quality.training_policy == STAGED:
if self.profile.quality.training_policy in STAGED_POLICIES:
used, imported = self.joint_resume.take_task(unit.task_key, cycle=unit.cycle)
else:
used, imported = self.joint_resume.take_task(unit.task_key)
@@ -751,6 +734,17 @@ class CalibrationCoordinator:
if self.execution.last_diagnostic_quality is not None else {})}
self.capture_index.append(record)
append_jsonl(self.raw_path, record)
next_unit = session.current_unit
visit_complete = (quality.passed and (next_unit is None
or (next_unit.task_key, next_unit.cycle) != (
action.scan_unit.task_key, action.scan_unit.cycle)))
if visit_complete:
resolved_deferral = session.task_deferrals.resolve(
action.scan_unit.task_key, action.scan_unit.cycle)
if resolved_deferral is not None:
resolved_record = resolved_deferral.as_record(stamp_ns=self.ports.clock_ns())
self.capture_index.append(resolved_record)
append_jsonl(self.raw_path, resolved_record)
if self.execution.last_training_decision is not None:
decision = self.execution.last_training_decision
append_jsonl(self.raw_path, decision)
@@ -903,6 +897,8 @@ class CalibrationCoordinator:
elif geometry_error:
if self._recover_joint_zero(motion, now, geometry_error):
return
if self._defer_unobservable_task(motion, geometry_error):
return
if self.diagnostic_capture is not None:
geometry_error += ":diagnostic_reference=" + self._zero_reference_error
self._pause("joint_zero_motion_unresolved:" + geometry_error)
@@ -910,7 +906,10 @@ class CalibrationCoordinator:
else:
if self._recover_joint_zero(motion, now, ""):
return
self._pause("joint_zero_unavailable:"+self._zero_reference_error+self._joint_zero_branch_details(motion))
unavailable = self._zero_reference_error+self._joint_zero_branch_details(motion)
if self._defer_unobservable_task(motion, unavailable):
return
self._pause("joint_zero_unavailable:"+unavailable)
return
if (self.segment.arrived(now) and motion.phase != "joint_zero") or steady_complete:
if motion.phase in {'observer_preflight', 'observer_probe'}:
@@ -1028,6 +1027,14 @@ class CalibrationCoordinator:
raise ValueError("parent_reference_zero_missing")
record = self.branch_initialization.parent_references.confirm(task,
self._observation_epoch, self._segment_number, reference, rows)
elif motion.parent_model_refresh_for:
task = next(task for task in self.profile.motion.tasks
if task.key == motion.parent_model_refresh_for)
reference = self.execution.zero_references.get(task.parent_reference.pose_joint)
if reference is None:
raise ValueError("parent_reference_zero_missing")
record = self.branch_initialization.parent_references.authorize_refreshed_pose(
task, self._observation_epoch, self._segment_number, reference)
except ValueError as error:
self._zero_reference_error = str(error)
return False
@@ -1172,6 +1179,38 @@ class CalibrationCoordinator:
def _recover_joint_zero(self, motion, now, geometry_error):
if self.diagnostic_capture is not None:
return False
branch_details = self._joint_zero_branch_details(motion)
task = next((task for task in self.profile.motion.tasks
if task.key == motion.task_key), None)
stale_parent_anchor = (motion.reference_reuse
and task is not None and self.capture.current_image_reference_stable(task.view)
and self._zero_reference_error.startswith("parent_reference_samples_missing:")
and "image_reference_current_image_moved" in branch_details)
action = self.zero_recovery.take_parent_model_refresh(
motion, stale_image_anchor=stale_parent_anchor)
if action is not None:
from .zero_recovery import FailureCategory
record = {"kind": "joint_zero_recovery", "task_name": motion.task_key,
"failure_category": FailureCategory.REFERENCE_CHANGED.value,
"capture_plan_sha256": CapturePlan.from_profile(self.execution.profile).sha256,
"joints": list(motion.zero_joints), "action": action,
"reason": self._zero_reference_error+branch_details,
"previous_session_epoch": self._observation_epoch,
"session_epoch": self._observation_epoch+1,
"attempt": 1, "stamp_ns": self.ports.clock_ns()}
append_jsonl(self.raw_path, record)
self.capture_index.append(record)
self._observation_epoch += 1
self.branch_initialization.parent_references.advance_recovery_epoch(
self._observation_epoch-1, self._observation_epoch)
self.execution.refresh_parent_model(self.last_command or motion.target)
self.branch_initialization.clear()
self._steady_rows.clear()
self._zero_capture_started_at = self._steady_capture_after_ns = None
self._motion = self.segment = None
self._zero_geometry_started_at = self._zero_geometry_completed_at = None
self.reason = "joint_zero_recovery:"+action+":"+str(motion.task_key)
return True
if geometry_error and self._recover_observer_preparation(motion):
return True
action = self.zero_recovery.take(motion, geometry_error=geometry_error,
@@ -1279,6 +1318,41 @@ class CalibrationCoordinator:
self.reason = 'observer_preparation:' + motion.zero_joints[0]
return True
def _defer_unobservable_task(self, motion, geometry_error):
"""Continue independent collection after bounded v3 observation fails."""
from .task_deferral import recoverable_observation_reason
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
refresh = bool(motion.parent_model_refresh_for)
if ((motion.reference_only and not (refresh or motion.reference_reuse))
or motion.reference_check or len(motion.zero_joints) != 1
or not recoverable_observation_reason(geometry_error)
or not any(spec.observer_pose_policy == TRAJECTORY_OBSERVER
for spec in self.profile.measurement.measurements.values())):
return False
import hashlib
import json
reports = tuple(self.branch_initialization.reports.values())
hashes = {report.get('view', ''): report.get('source_frame_hashes', {})
for report in reports}
corner_digest = hashlib.sha256(json.dumps(hashes, sort_keys=True,
separators=(',', ':'), allow_nan=False).encode()).hexdigest()
decision = self.execution.defer_current_task(geometry_error)
record = decision.as_record(stamp_ns=self.ports.clock_ns(),
corner_evidence_sha256=corner_digest)
append_jsonl(self.raw_path, record)
self.capture_index.append(record)
self._observation_epoch += 1
self.branch_initialization.parent_references.advance_recovery_epoch(
self._observation_epoch-1, self._observation_epoch)
self.branch_initialization.clear()
self._steady_rows.clear()
self._zero_capture_started_at = None
self._steady_capture_after_ns = None
self._zero_geometry_started_at = self._zero_geometry_completed_at = None
self.segment, self._motion = None, None
self.reason = f"task_observation_{decision.state.value}:{decision.task_name}"
return True
def _joint_zero_branch_details(self, motion):
task = next(task for task in self.profile.motion.tasks if task.key == motion.task_key)
if motion.reference_only:
@@ -1598,12 +1672,113 @@ class CalibrationCoordinator:
append_jsonl(self.raw_path, {"kind": "fixed_base_reference_locked", **self._current_fingerprint})
def _write_resume_batch(self, pending):
"""Keep both zero-evidence and scan imports outside an unbounded tick."""
rows = pending.take_batch()
append_jsonl_many(self.raw_path, rows, durable=False)
self.capture_index.extend(rows)
if pending.complete:
sync_jsonl(self.raw_path)
"""Transfer bounded rows and poll a non-blocking durability barrier."""
if not pending.transfer_complete:
rows = pending.take_batch()
append_jsonl_many(self.raw_path, rows, durable=False)
self.capture_index.extend(rows)
if not pending.transfer_complete:
return False
if pending.durability_future is None:
pending.durability_future = self.journal_durability.submit(
lambda: sync_jsonl(self.raw_path))
return False
if not pending.durability_future.done():
return False
pending.durability_future.result()
pending.durable = True
return True
def _restore_passed_reference_batch(self, pending):
"""Restore imported runtime indexes in bounded control-loop slices."""
if pending.restore_phase == "binding":
from .passed_resume import restored_model_record
binding = restored_model_record(pending.model_records,
self._observation_epoch, self.ports.clock_ns())
if self.profile.quality.training_policy in STAGED_POLICIES and self.parameters.resume_mode != "passed":
binding.update(kind="staged_pose_models_restored", resume_mode="verify",
physical_zero_reverification_required=True)
binding.pop("unchanged_installation_declared", None)
append_jsonl_many(self.raw_path, (binding,), durable=False)
self.capture_index.append(binding)
pending.durability_future = self.journal_durability.submit(
lambda: sync_jsonl(self.raw_path))
pending.restore_phase = "binding_sync"
return False
if pending.restore_phase == "binding_sync":
if not pending.durability_future.done():
return False
pending.durability_future.result()
pending.restore_phase = "models"
return False
if pending.restore_phase == "models":
from .passed_resume import restore_parent_models
restore_parent_models(self.branch_initialization.parent_references,
pending.model_records, self._observation_epoch, pending.rows)
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation")
pending.restore_offset = 0
pending.restore_phase = "observer_records"
return False
if pending.restore_phase == "observer_records":
from .observer_reference import restore_observer_records
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 512]
restore_observer_records(self.branch_initialization, batch)
pending.restore_offset += len(batch)
if pending.restore_offset == len(pending.restore_rows):
if self.parameters.resume_mode == "passed":
# Passed tasks contribute immutable zero/model evidence,
# never unfinished live actions. Replaying an old active
# observer decision can move an already accepted joint
# while the next task is preparing.
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation"
and row.get("preparation_witness_joint"))
pending.restore_offset = 0
pending.restore_phase = "witness"
return False
preparation_kinds = {"observer_preparation_decision",
"image_observation_frame", "observer_preparation_segment_complete",
"observer_preparation_excursion_started",
"observer_preparation_cancelled_before_excursion"}
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") in preparation_kinds)
pending.restore_offset = 0
pending.restore_phase = "observer_preparation"
return False
if pending.restore_phase == "observer_preparation":
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 256]
self.branch_initialization.observer_preparation.restore(batch)
pending.restore_offset += len(batch)
if pending.restore_offset == len(pending.restore_rows):
pending.restore_rows = tuple(row for row in pending.rows
if row.get("kind") == "motion_branch_observation"
and row.get("preparation_witness_joint"))
pending.restore_offset = 0
pending.restore_phase = "witness"
return False
if pending.restore_phase == "witness":
batch = pending.restore_rows[pending.restore_offset:pending.restore_offset + 64]
for row in batch:
self.branch_initialization.witness_capture.observe(row)
pending.restore_offset += len(batch)
if pending.restore_offset < len(pending.restore_rows):
return False
pending.restore_rows = ()
pending.restore_phase = "complete"
self.execution.zero_references.update(pending.references)
if self.visual_motion is not None:
self.visual_motion.freeze_zeros(pending.references)
return True
if pending.restore_phase == "complete":
return True
raise RuntimeError("unknown_passed_reference_restore_phase")
def _import_resume_batch(self):
pending = self._resume_import
@@ -1666,7 +1841,7 @@ class CalibrationCoordinator:
if self.profile.artifacts.output_schema_version >= 3:
self.joint_resume = checkpoint.joints
decision = replace(decision, reason="整场参考通过,等待逐关节验证原零位后复用")
if self.profile.quality.training_policy == STAGED:
if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import restore_recovery
self._passed_reference_import = checkpoint.passed_import
restore_recovery(self.execution.session, self.zero_recovery, rows)
@@ -1676,7 +1851,7 @@ class CalibrationCoordinator:
decision = replace(decision, completed_units=tuple(sorted(complete)))
except (OSError, ValueError, KeyError, TypeError) as error:
decision = ResumeDecision(False, "旧断点校验未通过,已重新采集", (str(error),))
if self.profile.quality.training_policy == STAGED and not decision.reuse:
if self.profile.quality.training_policy in STAGED_POLICIES and not decision.reuse:
self._pause("staged_resume_reference_or_evidence_changed:" + str(decision.incompatible_fields))
return
decision = self.execution.restore(decision, rows)
@@ -1701,7 +1876,7 @@ class CalibrationCoordinator:
return
self.joint_resume = checkpoint.joints
self._passed_reference_import = checkpoint.passed_import
if self.profile.quality.training_policy == STAGED:
if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import restore_recovery
restore_recovery(self.execution.session, self.zero_recovery, checkpoint.rows)
if checkpoint.frozen_training is not None:
@@ -1709,7 +1884,7 @@ class CalibrationCoordinator:
# The existing bounded importer skips each task only after its original
# rows and zero references are durable. No old task is moved or scanned.
self._resume_message = "按已通过记录续采,跳过历史回放和已完成关节零位复核"
if self.profile.quality.training_policy == STAGED:
if self.profile.quality.training_policy in STAGED_POLICIES:
self._resume_message = "断点证据校验通过,直接复用已完成访问;不重复运动或零位复核"
self.execution.restore(ResumeDecision(True, self._resume_message), ())
@@ -15,7 +15,8 @@ from .diagnostic_capture import DiagnosticCapturePlan
from ..core.domain.profile import ACQUISITION_POLICY_VERSION
from ..core.domain.motion_path import task_segments, scan_identity
from ..core.domain.capture_plan import CapturePlan, ADAPTIVE, STAGED, LEGACY_TRAINING, LEGACY_HOLDOUT
from ..core.domain.capture_plan import (CapturePlan, ADAPTIVE, STAGED,
TASKWISE_STAGED, STAGED_POLICIES, LEGACY_TRAINING, LEGACY_HOLDOUT)
TRAINING_CYCLES = LEGACY_TRAINING
HOLDOUT_CYCLE = LEGACY_HOLDOUT
CAPTURE_SCHEDULE_VERSION = "unified_schedule_v7_single_pass_endpoint_images"
@@ -27,8 +28,10 @@ PARENT_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v10_verified_parent_geomet
IMAGE_REFERENCE_SCHEDULE_VERSION = "unified_schedule_v11_stationary_image_reference"
ADAPTIVE_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v12_adaptive_training"
STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v13_staged_training"
TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION = "unified_schedule_v14_taskwise_staged_training"
SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION,
STAGED_CAPTURE_SCHEDULE_VERSION, ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}"
STAGED_CAPTURE_SCHEDULE_VERSION, TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION,
ADAPTIVE_CAPTURE_SCHEDULE_VERSION, SEPARATE_CAPTURE_SCHEDULE_VERSION, *(f"{SEGMENTED_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{PATH_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{VISUAL_CAPTURE_SCHEDULE_VERSION}_{mode}"
for mode in ("interleaved", "separate")), *(f"{PARENT_REFERENCE_SCHEDULE_VERSION}_{feedback}_{mode}"
@@ -40,6 +43,8 @@ SUPPORTED_CAPTURE_SCHEDULE_VERSIONS = frozenset((CAPTURE_SCHEDULE_VERSION,
def capture_schedule_version(profile):
if profile.quality.training_policy == STAGED:
return STAGED_CAPTURE_SCHEDULE_VERSION
if profile.quality.training_policy == TASKWISE_STAGED:
return TASKWISE_STAGED_CAPTURE_SCHEDULE_VERSION
if profile.quality.training_policy == ADAPTIVE:
return ADAPTIVE_CAPTURE_SCHEDULE_VERSION
if profile.acquisition.fixed_reference_mode == "stationary_image":
@@ -202,6 +207,17 @@ class CalibrationEngine:
segment.end, speed, "steady", segment.key) for segment in task_segments(task))
if self.profile.quality.training_policy == STAGED:
units.sort(key=lambda unit: unit.cycle) # Stable: declared task/segment order within a pass.
elif self.profile.quality.training_policy == TASKWISE_STAGED:
# Keep the already collected first whole-hand pass reusable. During
# the second pass each task's optional third training visit is
# placed immediately after that task. Validation remains a final,
# isolated whole-hand pass and can never precede future training.
task_order = {task.key: index for index, task in enumerate(self.profile.motion.tasks)}
units.sort(key=lambda unit: (
0 if unit.cycle == 0 else 1 if unit.cycle in {1, 2} else 2,
task_order[unit.task_key],
unit.cycle,
))
return tuple(units)
def mapping_probe_delta(self, task: TaskSpec) -> float | None:
@@ -287,7 +303,7 @@ class CalibrationEngine:
and session_start.get("acquisition_policy_version")
== ACQUISITION_POLICY_VERSION
and session_start.get("capture_schedule_version") in (
(capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, STAGED} or self.profile.acquisition.fixed_reference_mode == "stationary_image"
(capture_schedule_version(self.profile),) if (self.profile.quality.training_policy in {ADAPTIVE, *STAGED_POLICIES} or self.profile.acquisition.fixed_reference_mode == "stationary_image"
or self.profile.vision_motion or any(task.segments for task in self.profile.motion.tasks)
or any(task.parent_reference is not None for task in self.profile.motion.tasks)
or any(spec.evidence_scope == "path" for spec in self.profile.motion.joint_zero_references.values())
@@ -55,8 +55,8 @@ class SessionExecution:
def _make_effects(self, current):
action = self.action
from ..core.domain.capture_plan import STAGED
staged = self.profile.quality.training_policy == STAGED
from ..core.domain.capture_plan import STAGED_POLICIES
staged = self.profile.quality.training_policy in STAGED_POLICIES
visit = None if action.scan_unit is None else (action.scan_unit.task_key, action.scan_unit.cycle)
new_visit = staged and visit != self._entered_visit
previous_visit = self._entered_visit
@@ -298,6 +298,38 @@ class SessionExecution:
current = target
self._effects[:1] = [*effects, motion]
def refresh_parent_model(self, current):
"""Acquire a current image model without replacing a passed zero.
Parent-reference reuse deliberately starts with the immutable passed
model. If only its stationary image anchor is stale, run the same
reviewed approach used by that parent's original zero acquisition and
fit a session-local model before retrying the held-pose check.
"""
motion = self._effects[0] if self._effects else None
if (self.session.phase != Phase.PREPARE or motion is None
or motion.phase != "joint_zero" or not motion.reference_reuse
or len(motion.zero_joints) != 1):
raise ValueError("parent_model_refresh_outside_reference_reuse")
name = motion.zero_joints[0]
spec = self.profile.motion.joint_zero_references[name]
source_task = next(task for task in self.profile.motion.tasks
if task.key == spec.task_key)
metadata = dict(task_key=motion.task_key, command_index=motion.command_index,
cycle=motion.cycle, attempt=motion.attempt, reference_only=True,
reference_command=spec.command, reference_approach=spec.approach_commands,
parent_model_refresh_for=motion.task_key)
effects = []
for target in (*spec.approach_commands, spec.command):
effects.extend(MotionCommand("zero_approach", point, motion.speed,
reference_joints=(name,), **metadata)
for point in build_joint_reference_waypoints(
source_task, target, profile=self.profile, current_command=current))
current = target
effects.append(MotionCommand("joint_zero", spec.command, motion.speed,
zero_joints=(name,), **metadata))
self._effects[:0] = effects
def prepend_observer_effects(self, effects):
if (self.session.phase != Phase.PREPARE or not self._effects or not effects
or any(effect.phase != 'observer_probe' or len(effect.reference_joints) != 1
@@ -313,6 +345,12 @@ class SessionExecution:
zero_joints=(task.parent_reference.pose_joint,)))
self.prepend_observer_effects(effects)
def defer_current_task(self, reason):
decision = self.session.defer_current_task(reason)
self._effects.clear()
self._action_key = None
return decision
def record_diagnostic_zero_attempt(self, attempt: DiagnosticZeroAttempt):
"""A diagnostic observation closes an attempt, never creates a zero."""
motion = self._effects[0] if self._effects else None
@@ -325,13 +363,13 @@ class SessionExecution:
self.diagnostic_zero_attempts.update({name: attempt for name in attempt.joint_names})
def training_check_due(self):
from ..core.domain.capture_plan import ADAPTIVE, STAGED
from ..core.domain.capture_plan import ADAPTIVE, STAGED, TASKWISE_STAGED
unit = self.action.scan_unit
if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED} or self.diagnostic_capture is not None
if (self.profile.quality.training_policy not in {ADAPTIVE, STAGED, TASKWISE_STAGED} or self.diagnostic_capture is not None
or unit is None or unit.cycle not in (1, 2)
or (unit.task_key, unit.cycle) in self._training_checks):
return False
round_units = [u for u in self.session.engine.scan_units()
round_units = [u for u in self.session.scheduled_units
if (self.profile.quality.training_policy == STAGED or u.task_key == unit.task_key)
and u.cycle == unit.cycle]
return unit.identity == round_units[-1].identity
@@ -118,6 +118,13 @@ class ImageReferenceTracker:
return (snapshot is self._motion_snapshot and snapshot.epoch == self._epoch
and snapshot.reference is self.reference and snapshot.stamp_ns == self._last_stamp)
def tracking_current_image(self):
"""Whether the latest frame still matches this tracker's own anchor."""
with self._lock:
return (self.reference is not None and self._snapshot is not None
and self._snapshot[3].get("branch_status") == "tracking"
and self._snapshot[1] == self._last_stamp)
def reference_confirmed(self, role, revision):
with self._lock:
snapshot = self._motion_snapshot
@@ -19,22 +19,46 @@ class ResumeJournalImport:
rows: tuple
offset: int = field(default=0, init=False)
durability_future: object | None = field(default=None, init=False, repr=False)
durable: bool = field(default=False, init=False)
def take_batch(self):
from .resume_storage import DeferredCaptureRecord, materialize_record
linked = (self.offset < len(self.rows)
and isinstance(self.rows[self.offset], DeferredCaptureRecord)
and self.rows[self.offset].reference is not None)
limit = 512 if linked else 64
# Linked evidence can normally be copied without decompression. Scan
# samples additionally feed the live training index, which reads their
# geometry and digest. Find that boundary across the whole candidate
# batch so a cheap image row cannot hide dozens of expensive samples.
limit = 64
if linked:
geometry_kinds = {"joint_sample", "secondary_joint_sample"}
candidate = self.rows[self.offset:self.offset + 512]
first_geometry = next((index for index, row in enumerate(candidate)
if row.get("kind") in geometry_kinds), None)
if first_geometry is None:
limit = 512
elif first_geometry > 0:
limit = first_geometry
else:
# Training geometry is materialized once per row by the live
# index. A 512-row slice remains bounded while avoiding one
# control-loop turn for every 64 historical records.
limit = 512
batch = tuple(row if isinstance(row, DeferredCaptureRecord) and row.reference is not None
else materialize_record(row) for row in self.rows[self.offset:self.offset + limit])
self.offset += len(batch)
return batch
@property
def complete(self):
def transfer_complete(self):
return self.offset == len(self.rows)
@property
def complete(self):
return self.durable
@dataclass
class TaskResumeImport(ResumeJournalImport):
@@ -86,17 +110,18 @@ class PreparedResume:
raise ValueError("checkpoint identity/policy changed")
from .training import resolve_capture_plan
from .engine import CalibrationEngine
from ..core.domain.capture_plan import STAGED
from ..core.domain.capture_plan import STAGED, STAGED_POLICIES
profile = resolve_capture_plan(profile, rows, source_urdf=source_urdf,
verify_decisions=profile.quality.training_policy == STAGED and source_urdf is not None)
verify_decisions=profile.quality.training_policy in STAGED_POLICIES and source_urdf is not None)
engine = CalibrationEngine(profile)
complete = frozenset(record_scan_identity(r) for r in rows
if r.get("kind") == "scan_unit_complete" and r.get("passed") is True)
joints = JointResume(profile, image_replay_workers=image_replay_workers)
if profile.quality.training_policy == STAGED:
if profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import stage_visits, stage_passed_visits, load_frozen_checkpoint, reference_import
(stage_passed_visits if resume_mode == "passed" else stage_visits)(joints, engine, rows)
frozen = load_frozen_checkpoint(profile, rows, path, source_urdf)
frozen = (load_frozen_checkpoint(profile, rows, path, source_urdf)
if profile.quality.training_policy == STAGED else None)
# Index/hash historical provenance before live callbacks start.
# The control loop only authorizes and drains this bounded import.
pending = reference_import(joints, reuse_passed_visits=resume_mode == "passed")
@@ -134,10 +159,12 @@ def _index_motion_provenance(rows):
kinds = {"motion_branch_observation", "motion_branch_initialization",
"observer_preparation_decision", "observer_preparation_segment_complete",
"observer_preparation_excursion_started", "observer_preparation_cancelled_before_excursion",
"task_observation_state",
"capture_scope_extension", "capture_observation_revision",
"passed_tasks_reused", "passed_pose_models_restored",
"staged_pose_models_restored", "resume_verification",
"parent_reference_verified", "joint_zero_recovery",
"parent_reference_verified", "parent_reference_model_refreshed",
"joint_zero_recovery",
"joint_zero_motion_reference_verified", "joint_zero_sample",
"visual_motion_observation", "visual_motion_settled"}
selected = (row for row in rows
@@ -313,8 +340,8 @@ class JointResume:
return tuple(rows)
def take_task(self, task_key, cycle=None):
from ..core.domain.capture_plan import STAGED
if self.profile.quality.training_policy == STAGED:
from ..core.domain.capture_plan import STAGED_POLICIES
if self.profile.quality.training_policy in STAGED_POLICIES:
from .staged_resume import visit_rows
return visit_rows(self, task_key, cycle)
task = next(t for t in self.profile.motion.tasks if t.key == task_key)
@@ -0,0 +1,22 @@
"""Run journal persistence barriers without blocking the control callback."""
from concurrent.futures import ThreadPoolExecutor
class JournalDurabilityWorker:
"""Single ordered worker for resume-import fsync barriers.
Resume rows are appended by the coordinator in bounded batches. The
final fsync can still take seconds on a large journal, so it runs here
while the coordinator continues feedback, safety and hold commands.
"""
def __init__(self):
self._executor = ThreadPoolExecutor(
max_workers=1, thread_name_prefix="calibration-journal-sync")
def submit(self, operation):
return self._executor.submit(operation)
def close(self):
self._executor.shutdown(wait=True, cancel_futures=False)
@@ -59,6 +59,10 @@ class MotionCommand:
visual_return_joints: tuple[str, ...] = ()
defer_settling_to_hold: bool = False
observer_preparation_id: str = ''
# A session-local model acquisition for a parent whose historical image
# anchor no longer matches the current, stable fixed-Tag image. The
# accepted joint zero and completed scan stay immutable.
parent_model_refresh_for: str = ''
@property
def recording(self):
@@ -135,8 +139,21 @@ class MotionExecution:
if self.profile.command.unit == "u8":
duration_key = ("scan_trajectory_full_range_seconds"
if self.command.phase in {"sweep", "steady"} else "command_trajectory_full_range_seconds")
full_range_duration = float(params.get(duration_key,
params.get("command_trajectory_full_range_seconds", 6.0)))
if len(self.command.measured_channels) > 1:
# O30's reviewed range-calibration group motion pairs SDK
# speed 200 with a 20 u8/s command path. Preserve that rate
# for coupled calibration segments: a faster continuous ramp
# can let the middle finger's feedback lag behind the intended
# 80-unit clearance while the ring finger is already loaded.
coupled_rate = float(self.command.speed)/10.0
if coupled_rate <= 0.0:
raise ValueError("coupled u8 motion speed must be positive")
full_range_duration = max(full_range_duration,
math.pi*255.0/(2.0*coupled_rate))
travelled, phase, duration = cosine_position_trajectory_u8(0, distance,
elapsed, float(params.get(duration_key, params.get("command_trajectory_full_range_seconds", 6.0))))
elapsed, full_range_duration)
else:
speed = self.command.speed
caps = self.profile.command.maximum_velocity
@@ -385,8 +385,9 @@ def validate_motion_provenance(profile, records, *, required=False, require_comp
from .shared_tag_evidence import validate_shared_tag_references
if any(payload.get('shared_tag_pose_bridges') for payload in evidence.values()):
validate_shared_tag_references(profile, records, evidence)
from ..profiles.observer_policy import is_rigid_observer
if (any(payload.get('observer_pose_bridges') for payload in evidence.values())
or any(s.observer_pose_policy == 'rigid_pair_v1' for s in profile.measurement.measurements.values())):
or any(is_rigid_observer(s.observer_pose_policy) for s in profile.measurement.measurements.values())):
from .observer_reference import validate_observer_references
validate_observer_references(profile, records, evidence)
image_frames, image_samples = _image_frames(records), {}
@@ -4,6 +4,8 @@ Decisions and completed segments are journaled. A restart cannot replenish the
motion budget or relabel an incomplete excursion as usable evidence.
"""
from dataclasses import dataclass, replace
import hashlib
import json
import math
import numpy as np
from scipy.spatial.transform import Rotation
@@ -16,6 +18,7 @@ from ..profiles.observer_preparation import observer_action, observer_actions
POLICY = 'bounded_rigid_observation_v1'
SEQUENCE_POLICY = 'bounded_rigid_observation_v2'
TRAJECTORY_POLICY = 'occlusion_aware_trajectory_v3'
PHASES = frozenset({'observer_preflight', 'observer_probe'})
RECOVERABLE = frozenset({'rigid_pair_candidates_unresolved', 'rigid_pair_pose_uncertain',
'rigid_pair_viewpoint_span_insufficient', 'rigid_pair_source_missing', 'observer_images_missing'})
@@ -44,6 +47,66 @@ validation or a statement that a pose is observable.
needs_excitation=len(selected) < 24 or span < 15.)
def trajectory_observability(rows, parent_role, child_role):
"""Measure whether a monocular trajectory can distinguish planar branches.
This is an acquisition decision only. It never marks geometry as passed;
the independent rigid-pair fit and final holdout still own acceptance.
"""
rows = tuple(sorted(rows, key=lambda row: row['image_stamp_ns']))
coverage = normal_coverage(rows, parent_role)
paired = [row for row in rows if {parent_role, child_role} <= set(row.get('tags', {}))]
separations, diagonals = [], []
for row in paired[::max(1, len(paired)//36)]:
tag = row['tags'][parent_role]
poses = solve_square_tag_ippe(np.asarray(tag['corners_xy']), tag_size_m=tag['tag_size_m'],
camera_matrix=np.asarray(row['camera_matrix']))
if len(poses) > 1:
separations.append(math.degrees((Rotation.from_quat(poses[0].quaternion_xyzw).inv()
* Rotation.from_quat(poses[1].quaternion_xyzw)).magnitude()))
corners = np.asarray(tag['corners_xy'], dtype=float)
diagonals.append(float(np.mean((np.linalg.norm(corners[0]-corners[2]),
np.linalg.norm(corners[1]-corners[3])))))
completeness = len(paired)/max(1, len(rows))
branch_separation = float(np.median(separations)) if separations else 0.
tag_diagonal = float(np.median(diagonals)) if diagonals else 0.
needs = (coverage['needs_excitation'] or completeness < .9
or branch_separation < 1.)
return {**coverage, 'paired_frames': len(paired),
'pair_detection_fraction': completeness,
'median_planar_branch_separation_deg': branch_separation,
'median_tag_diagonal_px': tag_diagonal,
'needs_excitation': needs}
def observation_action_utility(profile, joint, plan, rows):
"""Rank a declared safe action by expected out-of-plane information."""
spec = profile.measurement.measurements[joint]
metrics = trajectory_observability(rows, spec.parent_role, spec.child_role)
channel = channel_for_joint(profile, plan.joint)
lower, upper = profile.command.minimum_values[channel], profile.command.maximum_values[channel]
excursion = max(abs(point[channel]-plan.command[channel]) for point in plan.approach_commands)
fraction = excursion/(upper-lower)
# The donor was selected from the URDF dependency graph for a transverse
# axis. This conservative estimate is only used to choose acquisition;
# it is never reported as a measured joint angle.
expected_normal_gain = 60.*fraction
expected_pixel_separation = metrics['median_tag_diagonal_px'] * math.sin(math.radians(expected_normal_gain))
reliability = metrics['pair_detection_fraction']
return {**metrics, 'donor': plan.joint, 'excursion_fraction': fraction,
'expected_normal_gain_deg': expected_normal_gain,
'expected_candidate_separation_px': expected_pixel_separation,
'utility': expected_pixel_separation*reliability}
def corner_evidence_sha256(rows, roles):
payload = [{"stamp": row['image_stamp_ns'], "camera_matrix": row.get('camera_matrix'),
"tags": {role: row.get('tags', {}).get(role, {}).get('corners_xy') for role in roles}}
for row in rows]
return hashlib.sha256(json.dumps(payload, sort_keys=True, separators=(',', ':'),
allow_nan=False).encode()).hexdigest()
def latest_source_rows(owner, joint, epoch):
spec = owner.profile.measurement.measurements[joint]
reports = [report for (source_epoch, view, _), (model, report) in owner.parent_references.models.items()
@@ -90,7 +153,8 @@ class ObserverPreparation:
previous = self.decisions.get(joint)
if previous is not None and previous != row:
index = previous.get('action_index', 0)+1
if (row.get('policy') != SEQUENCE_POLICY or row.get('action_index') != index
if (row.get('policy') not in {SEQUENCE_POLICY, TRAJECTORY_POLICY}
or row.get('action_index') != index
or index >= len(observer_actions(self.owner.profile, joint))
or row.get('previous_preparation_id') != source_frame_sha256(previous)
or self.source(joint) is None):
@@ -175,13 +239,20 @@ class ObserverPreparation:
plan = actions[index]
spec = self.owner.profile.measurement.measurements[joint]
rows = self.source(joint).rows if previous is not None else latest_source_rows(self.owner, joint, epoch)
coverage = normal_coverage(rows, spec.parent_role)
coverage = (trajectory_observability(rows, spec.parent_role, spec.child_role)
if spec.observer_pose_policy == 'rigid_pair_trajectory_v3'
else normal_coverage(rows, spec.parent_role))
utility = observation_action_utility(self.owner.profile, joint, plan, rows)
policy = (TRAJECTORY_POLICY if spec.observer_pose_policy == 'rigid_pair_trajectory_v3'
else POLICY if len(actions)==1 else SEQUENCE_POLICY)
record = dict(kind='observer_preparation_decision', policy=POLICY if len(actions)==1 else SEQUENCE_POLICY, joint=joint,
task_name=motion.task_key, session_epoch=epoch, stamp_ns=stamp_ns,
action='excite' if reason or coverage['needs_excitation'] else 'reuse_source',
reason=reason or 'normal_coverage_preflight', coverage=coverage,
source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in rows},
donor=plan.joint, path=[list(p) for p in (*plan.approach_commands, plan.command)])
record.update(policy=policy, observability=coverage, action_utility=utility,
corner_evidence_sha256=corner_evidence_sha256(rows, (spec.parent_role, spec.child_role)))
if len(actions)>1:
record.update(action_index=index, action_limit=len(actions),
previous_preparation_id=None if previous is None else source_frame_sha256(previous))
@@ -11,6 +11,7 @@ from .motion_execution import feedback_tolerances
from .motion_provenance import source_frame_sha256
from .shared_tag_reference import bridge_frames
from ..profiles.observer_preparation import observer_action, observer_actions
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
def stationary_probe_tail(profile, joint, rows, target):
@@ -30,16 +31,16 @@ def stationary_probe_tail(profile, joint, rows, target):
def validate_probe_evidence(profile, joint, evidence, *, admission=None):
from .observer_preparation import POLICY, SEQUENCE_POLICY
from .observer_preparation import POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY
decision, rows, segments = evidence.decision, evidence.rows, evidence.segments
plan = observer_action(profile, joint, decision)
identity = source_frame_sha256(decision)
if (decision.get('policy') == SEQUENCE_POLICY
if (decision.get('policy') in {SEQUENCE_POLICY, TRAJECTORY_POLICY}
and decision.get('action_limit') != len(observer_actions(profile, joint))):
raise ValueError('observer_preparation_action_budget_changed')
path = (*plan.approach_commands, plan.command)
spec = profile.measurement.measurements[joint]
if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY} or decision.get('action') != 'excite'
if (decision.get('policy') not in {POLICY, SEQUENCE_POLICY, TRAJECTORY_POLICY} or decision.get('action') != 'excite'
or decision.get('joint') != joint or decision.get('donor') != plan.joint
or decision.get('task_name') != profile.motion.joint_zero_references[joint].task_key
or decision.get('path') != [list(p) for p in path]
@@ -139,9 +140,14 @@ def build_probe_bridge(profile, source):
parents.append(pair[0]); children.append(pair[1]); matrices.append(row['camera_matrix'])
parent_pixels.append(row['tags'][spec.parent_role]['corners_xy'])
child_pixels.append(row['tags'][spec.child_role]['corners_xy'])
plan = observer_action(profile, source.joint, source.preparation.decision)
donor_channel = channel_for_joint(profile, plan.joint)
trajectory_states = (tuple((row['reference_path_index'], row['command_vector'][donor_channel])
for row in rows) if spec.observer_pose_policy == TRAJECTORY_OBSERVER else None)
pose = fit_rigid_tag_pair([p[0] for p in parents], children, child_pixels, matrices,
tags[spec.child_role].size_m, tuple(r['image_stamp_ns'] for r in rows),
parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m, parent_candidates=parents)
parent_corners=parent_pixels, parent_size=tags[spec.parent_role].size_m,
parent_candidates=parents, trajectory_states=trajectory_states)
relation = MotionRelation(source.joint, spec.parent_role, spec.child_role)
bridge = SharedTagPoseBridge(relation, pose.quaternion_xyzw, pose.translation_xyz_m,
bridge_frames(endpoints, relation), rotation_uncertainty_rad=pose.rotation_uncertainty_rad,
@@ -158,6 +164,18 @@ def build_probe_bridge(profile, source):
preparation_started=source.preparation.started,
source_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in source.preparation.rows},
endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints})
if spec.observer_pose_policy == TRAJECTORY_OBSERVER:
from .observer_preparation import trajectory_observability, corner_evidence_sha256
record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={
'policy': 'rigid_pair_full_trajectory_v3',
'selection_source': 'training_trajectory',
'holdout_role': 'veto_only',
'observability': trajectory_observability(rows, spec.parent_role, spec.child_role),
'adjusted_candidate_probabilities': [list(value)
for value in pose.comparison_adjusted_p_values],
'corner_evidence_sha256': corner_evidence_sha256(
source.preparation.rows, (spec.parent_role, spec.child_role)),
})
return bridge, record
@@ -12,6 +12,7 @@ from .branch_initialization import _pose
from .shared_tag_reference import bridge_frames
from .motion_execution import feedback_tolerances
from .motion_provenance import source_frame_sha256
from ..profiles.observer_policy import is_rigid_observer
def observer_roles(profile, motion):
@@ -43,7 +44,7 @@ def build_observer_bridge(profile, source):
raise ValueError('observer_parent_model_invalid')
uncertainty = {name: (angle, point) for name, angle, point in source.parent_report['geometry_uncertainty']}
inherited = uncertainty[parent_model.geometry[0].relation.joint]
rigid = spec.observer_pose_policy == 'rigid_pair_v1'
rigid = is_rigid_observer(spec.observer_pose_policy)
if not rigid:
require_observer_precision_budget(inherited)
target = profile.motion.joint_zero_references[source.joint].command
@@ -109,6 +110,18 @@ def build_observer_bridge(profile, source):
endpoint_frame_hashes={str(r['image_stamp_ns']): source_frame_sha256(r) for r in endpoints})
if rigid:
record['conditioning'] = RIGID_PAIR_CONDITIONING
from ..profiles.observer_policy import TRAJECTORY_OBSERVER
if spec.observer_pose_policy == TRAJECTORY_OBSERVER:
from .observer_preparation import trajectory_observability, corner_evidence_sha256
record.update(version='observer_pose_bridge_v4', trajectory_branch_decision={
'policy': 'rigid_pair_full_trajectory_v3',
'selection_source': 'training_trajectory', 'holdout_role': 'veto_only',
'observability': trajectory_observability(rows, spec.parent_role, spec.child_role),
'adjusted_candidate_probabilities': [list(value)
for value in pose.comparison_adjusted_p_values],
'corner_evidence_sha256': corner_evidence_sha256(
rows, (spec.parent_role, spec.child_role)),
})
return bridge, record
@@ -139,7 +152,7 @@ def observer_inputs(owner, view, endpoint_rows, arc_stamps, epoch):
continue
held = tuple(sorted(owner.tag_feedback_channels[spec.child_role]-owner.tag_feedback_channels[spec.parent_role]))
result.append(ObserverBridgeInput(joint, model.image_model, report, rows, endpoints, held))
if spec.observer_pose_policy == 'rigid_pair_v1':
if is_rigid_observer(spec.observer_pose_policy):
# Select the latest eligible source before fitting. A failed
# fit cannot search older sources for a more favorable result.
break
@@ -182,9 +195,10 @@ def validate_observer_references(profile, records, evidence):
checked = set()
for payload in evidence.values():
required = {joint for joint, spec in profile.measurement.measurements.items()
if joint in payload['zero_joints'] and spec.observer_pose_policy == 'rigid_pair_v1'}
if joint in payload['zero_joints'] and is_rigid_observer(spec.observer_pose_policy)}
declared = [r['joint'] for r in payload.get('observer_pose_bridges', ())
if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3'}]
if r.get('version') in {'observer_pose_bridge_v2', 'observer_pose_bridge_v3',
'observer_pose_bridge_v4'}]
if set(declared) != required or len(declared) != len(required):
raise ValueError('rigid_pair_required_evidence_missing')
for payload in evidence.values():
@@ -204,7 +218,7 @@ def validate_observer_references(profile, records, evidence):
if (evidence_id not in evidence or evidence[evidence_id]['tag_role'] != role
or evidence[evidence_id]['image_model_sha256'] != record['parent_model_sha256']):
raise ValueError('observer_parent_identity_changed')
if record.get('version') == 'observer_pose_bridge_v3':
if record.get('version') in {'observer_pose_bridge_v3', 'observer_pose_bridge_v4'}:
from .observer_preparation_evidence import validate_probe_reference
validate_probe_reference(profile, records, payload, record, parent, observations)
continue
@@ -5,6 +5,7 @@ upstream pose change invalidates the former, but not a rigid Tag's mounting.
No curve, joint zero or candidate selection is copied between joints.
"""
from dataclasses import dataclass
import math
import numpy as np
@@ -15,6 +16,14 @@ from ..core.geometry.tag_pose.source_hinge import SourceHinge
from .resume import ResumeVerifier
@dataclass(frozen=True)
class RefreshedPoseSource:
"""Bind a refreshed model to its immutable source epoch across restarts."""
image_model_sha256: str
source_epoch: int
def check_parent_pose(profile, task, reference, rows, channels, *,
verification='staged_revisit_v1', feedback_limits=None):
"""Same pure check for live execution and persisted evidence read-back."""
@@ -78,6 +87,7 @@ class ParentReferenceRegistry:
self.models = {}
self.verified = {}
self.zero_references = {}
self.refreshed_pose_sources = {}
def remember_zero(self, reference, epoch):
for view in {sample.view for sample in reference.samples}:
@@ -104,6 +114,8 @@ class ParentReferenceRegistry:
for (epoch, view, joint), value in tuple(self.models.items()) if epoch == previous})
self.verified.update({(current, task): value
for (epoch, task), value in tuple(self.verified.items()) if epoch == previous})
self.refreshed_pose_sources.update({(current, task): value
for (epoch, task), value in tuple(self.refreshed_pose_sources.items()) if epoch == previous})
self.zero_references.update({(current, view, joint): value
for (epoch, view, joint), value in tuple(self.zero_references.items()) if epoch == previous})
@@ -129,6 +141,12 @@ class ParentReferenceRegistry:
"""
from ..core.geometry.tag_pose.pose_bridge import is_handoff_policy
pose, geometry = self.sources(task, epoch)
refreshed = self.refreshed_pose_sources.get((epoch, task.key))
if refreshed is not None:
if (refreshed.image_model_sha256 != pose[1].get("image_model_sha256")
or refreshed.source_epoch != pose[1].get("session_epoch")):
raise ValueError("parent_reference_refreshed_pose_source_changed")
return pose
if not is_handoff_policy(geometry[1].get('image_model_selection_policy')):
return pose
bridges = geometry[1].get('shared_tag_pose_bridges', ())
@@ -138,6 +156,50 @@ class ParentReferenceRegistry:
raise ValueError('parent_reference_handoff_source_changed')
return geometry
def authorize_refreshed_pose(self, task, epoch, version, reference):
"""Prefer a freshly measured pose model while retaining passed geometry."""
joint = task.parent_reference.pose_joint
model, report = self.models.get((epoch, task.view, joint), (None, None))
if (model is None or report is None or report.get("session_epoch") != epoch
or model.image_model is None
or {item.relation.joint for item in model.image_model.geometry} != {joint}):
raise ValueError("parent_reference_refresh_model_missing_or_changed")
identity = report.get("image_model_sha256")
if not isinstance(identity, str) or not identity:
raise ValueError("parent_reference_refresh_model_missing_or_changed")
self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource(identity, epoch)
from .motion_provenance import source_frame_sha256
return {"kind": "parent_reference_model_refreshed", "schema_version": 1,
"task_name": task.key, "view": task.view, "pose_joint": joint,
"session_epoch": epoch, "motion_version": version,
"image_model_sha256": identity,
"source_zero_sha256": source_frame_sha256(reference.as_record()),
"passed_zero_preserved": True,
"source_frame_hashes": dict(report.get("source_frame_hashes", {}))}
def restore_refreshed_pose(self, record, epoch):
"""Restore an explicit refresh grant without relabelling its source."""
task = next((item for item in self.profile.motion.tasks
if item.key == record.get("task_name")), None)
if (task is None or task.parent_reference is None
or record.get("schema_version") != 1
or record.get("view") != task.view
or record.get("pose_joint") != task.parent_reference.pose_joint
or record.get("passed_zero_preserved") is not True
or type(record.get("session_epoch")) is not int):
raise ValueError("parent_reference_refresh_record_changed")
model, report = self.models.get((epoch, task.view,
task.parent_reference.pose_joint), (None, None))
identity = record.get("image_model_sha256")
if (model is None or report is None or model.image_model is None
or identity != report.get("image_model_sha256")
or record["session_epoch"] != report.get("session_epoch")
or dict(record.get("source_frame_hashes", {}))
!= dict(report.get("source_frame_hashes", {}))):
raise ValueError("parent_reference_refresh_model_missing_or_changed")
self.refreshed_pose_sources[(epoch, task.key)] = RefreshedPoseSource(
identity, record["session_epoch"])
def activate_joints(self, joints, epoch, capture):
installed = set()
for joint in joints:
@@ -5,7 +5,7 @@ reverification runs here. New acquisitions and final artifact validation keep
their normal gates. An incomplete task is restarted as a whole.
"""
from dataclasses import dataclass
from dataclasses import dataclass, field
from ..core.domain.motion_path import record_scan_identity
from ..core.domain.reference import read_joint_zero_references
@@ -16,6 +16,29 @@ from .joint_resume import ResumeJournalImport, _index_motion_provenance
class PassedReferenceImport(ResumeJournalImport):
references: dict
model_records: tuple
restore_phase: str = field(default="binding", init=False)
restore_rows: tuple = field(default=(), init=False, repr=False)
restore_offset: int = field(default=0, init=False)
def coherent_observer_preparation_rows(rows):
"""Discard transition fragments whose originating decision is absent.
Compact checkpoints may contain a cancellation after an earlier migration
already removed the decision it cancelled. Such a fragment represents no
live state and cannot be replayed on its own. Decisions and all transitions
still present together remain byte-for-byte unchanged.
"""
from .motion_provenance import source_frame_sha256
decisions = {source_frame_sha256(row) for row in rows
if row.get("kind") == "observer_preparation_decision"
and row.get("action") == "excite"}
bound_kinds = {"observer_preparation_segment_complete",
"observer_preparation_excursion_started",
"observer_preparation_cancelled_before_excursion"}
return tuple(row for row in rows if row.get("kind") not in bound_kinds
or row.get("observer_preparation_id") in decisions)
def stage_passed_tasks(resume, engine, rows):
@@ -48,15 +71,25 @@ def reference_import(resume):
"""Import prior evidence durably before skipping any completed task."""
provenance = _index_motion_provenance(resume.rows)
references = resume.references
tasks = {reference.task_key for reference in references.values()}
parent_images = {(row["view"], int(stamp)) for row in provenance.values()
if row.get("kind") == "parent_reference_verified" for stamp in row["source_frame_hashes"]}
if row.get("kind") == "parent_reference_verified"
and row.get("task_name") in tasks for stamp in row["source_frame_hashes"]}
parent_images.update((sample["view"], sample["image_stamp_ns"])
for row in provenance.values() if row.get("kind") == "parent_reference_verified"
and row.get("task_name") in tasks
for sample in row["source_reference"]["samples"])
zero_images = parent_images | {(row.get("view"), row.get("image_stamp_ns"))
for row in provenance.values() if row.get("kind") == "joint_zero_sample" and row.get("joint") in references}
selected = []
for row in provenance.values():
# Passed mode restarts every incomplete task as a whole. Its old
# branch models and preparation images belong to an abandoned visit;
# restoring them would bind the new session's current image anchor to
# a previous camera epoch. Task-independent audit rows remain useful.
task_name = row.get("task_name")
if task_name is not None and task_name not in tasks:
continue
image = row.get("view"), row.get("image_stamp_ns")
if (row.get("kind") in {"joint_zero_sample", "joint_zero_motion_reference_verified"}
and row.get("joint") not in references and image not in parent_images):
@@ -64,7 +97,7 @@ def reference_import(resume):
if row.get("kind") == "pnp_candidate_frame" and image not in zero_images:
continue
selected.append(row)
tasks = {reference.task_key for reference in references.values()}
selected = coherent_observer_preparation_rows(selected)
models = tuple(row for row in selected if row.get("kind") == "motion_branch_initialization"
and row.get("status") == "resolved" and row.get("task_name") in tasks)
audit = {"kind": "passed_tasks_reused", "resume_mode": "passed",
@@ -76,7 +109,7 @@ def reference_import(resume):
references=references, model_records=models)
def restore_parent_models(registry, records, epoch):
def restore_parent_models(registry, records, epoch, provenance_records=()):
"""Keep source identities while making passed installations available to new tasks."""
from ..core.geometry.image_motion_replay import frozen_image_model
from .branch_initialization import MotionBranchModel
@@ -87,6 +120,12 @@ def restore_parent_models(registry, records, epoch):
model = MotionBranchModel(row["task_name"], row["view"], (),
tuple(sorted(row["evidence_ids"].items())), frozen_image_model(row))
registry.remember(model, row, epoch=epoch)
latest_refreshes = {}
for row in provenance_records:
if row.get("kind") == "parent_reference_model_refreshed":
latest_refreshes[row.get("task_name")] = row
for record in latest_refreshes.values():
registry.restore_refreshed_pose(record, epoch)
def restored_model_record(records, epoch, stamp_ns):
@@ -93,6 +93,11 @@ def discover_resume_candidate(
reject_diagnostic_capture((row,))
kind = str(row.get("kind", ""))
if kind == "session_start":
from ..core.artifacts.evidence_journal import REFERENCE_KEY, decode_envelope
if REFERENCE_KEY in row:
from ..core.geometry.frozen_evidence import EvidenceDecoder
row = decode_envelope(candidate / "raw_samples.jsonl", row,
EvidenceDecoder(verify_hashes=False))
if start is not None:
raise ValueError("multiple checkpoint session headers")
start = row
@@ -35,7 +35,9 @@ class DeferredCaptureRecord(Mapping):
@classmethod
def from_envelope(cls, path, envelope, decoder):
from ..core.artifacts.evidence_journal import CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY
from ..core.artifacts.evidence_journal import (
CANONICAL_SHA256_KEY, FIELDS_KEY, REFERENCE_KEY, SCHEMA_KEY, _schema_fields,
)
value = object.__new__(cls)
value.path, value.offset, value.length = path, 0, 0
from ..core.artifacts.evidence_journal import EvidenceRecordReference
@@ -43,41 +45,54 @@ class DeferredCaptureRecord(Mapping):
value.digest = bytes.fromhex(value.reference.record_sha256)
value.canonical_sha256 = envelope.get(CANONICAL_SHA256_KEY)
value.metadata = {key: item for key, item in envelope.items()
if key not in {REFERENCE_KEY, FIELDS_KEY, CANONICAL_SHA256_KEY}}
value.field_names = _shared_field_names(tuple(envelope[FIELDS_KEY]))
if key not in {REFERENCE_KEY, FIELDS_KEY, SCHEMA_KEY, CANONICAL_SHA256_KEY}}
fields = envelope.get(FIELDS_KEY)
if fields is None and SCHEMA_KEY in envelope:
fields = _schema_fields(path, envelope[SCHEMA_KEY])
value.field_names = None if fields is None else _shared_field_names(tuple(fields))
value.decoder = decoder
return value
def materialize(self):
return self.decoder.loads(self._record_bytes())
def materialize_plain(self):
"""Decode one integrity-checked row without building shared models."""
return json.loads(self._record_bytes())
def _record_bytes(self):
if self.reference is not None:
from ..core.artifacts.evidence_journal import evidence_record_bytes
line = evidence_record_bytes(self.path, self.reference)
return self.decoder.loads(line)
return evidence_record_bytes(self.path, self.reference)
with self.path.open("rb") as stream:
stream.seek(self.offset)
line = stream.read(self.length)
if hashlib.sha256(line).digest() != self.digest:
raise ValueError("resume_source_record_changed")
return self.decoder.loads(line)
return line
def __getitem__(self, key):
if key in self.metadata:
return self.metadata[key]
if key not in self.field_names:
if self.field_names is not None and key not in self.field_names:
raise KeyError(key)
return self.materialize()[key]
def __iter__(self):
return iter(self.field_names)
return iter(self.materialize() if self.field_names is None else self.field_names)
def __len__(self):
return len(self.field_names)
return len(self.materialize()) if self.field_names is None else len(self.field_names)
def materialize_record(row):
return row.materialize() if isinstance(row, DeferredCaptureRecord) else row
def materialize_plain_record(row):
return row.materialize_plain() if isinstance(row, DeferredCaptureRecord) else row
def load_passed_capture(path):
"""Read once without retaining every scan's expanded image dictionaries.
@@ -101,7 +116,15 @@ def load_passed_capture(path):
if not line.strip():
continue
try:
row = decoder.loads(line)
# Indexed envelopes contain only scalar lookup fields and an
# evidence reference. Running the recursive evidence decoder
# over tens of thousands of these small rows dominated resume
# startup even though there is no model to intern. Preserve
# the decoder for the few inline records that carry shared
# frozen objects.
needs_evidence_decoder = (b'"frozen_image_model"' in line
or b'"reference_frame"' in line)
row = decoder.loads(line) if needs_evidence_decoder else json.loads(line)
except json.JSONDecodeError as error:
raise ValueError(f"invalid calibration JSONL at line {number}") from error
if not isinstance(row, dict):
@@ -7,6 +7,7 @@ hardware. Pending legacy status/artifact contracts are isolated in compat.
from __future__ import annotations
import argparse
import hashlib
import json
import math
import os
@@ -136,6 +137,21 @@ def run_online(
if diagnostic is not None and not commands_enabled:
raise ValueError("diagnostic_capture requires commands_enabled")
session = create_session_directory(config.session_root)
# A CLI-selected preparation policy is a real protected profile, not an
# in-process hint. Materialize its canonical bytes so the launched node,
# resume selector and journal all bind the identical effective contract.
from ..profiles.export import dump_hand_profile
effective_profile = dump_hand_profile(profile).encode()
effective_digest = hashlib.sha256(effective_profile).hexdigest()
from ..profiles import load_hand_profile
runtime_profile_changed = (config.profile_config is None
or load_hand_profile(config.profile_config) != profile)
if runtime_profile_changed:
effective_path = session / "effective_profile.yaml"
effective_path.write_bytes(effective_profile)
from dataclasses import replace
config = replace(config, profile_config=effective_path,
profile_config_sha256=effective_digest)
from ..profiles.validator import validate_executable_profile, require_release_geometry
contract_report = validate_executable_profile(profile, config.source_urdf)
atomic_write_json(session/"measurement_contract.json", contract_report)
@@ -307,7 +323,8 @@ def main(args: list[str] | None = None) -> None:
parser.add_argument("--record-bag", action="store_true")
parser.add_argument("--commands-disabled", action="store_true")
parser.add_argument("--no-resume", action="store_true")
parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3"), default=None,
parser.add_argument("--training-policy", choices=("fixed", "adaptive_2_to_3", "staged_2_to_3",
"taskwise_staged_2_to_3"), default=None,
help="试验采集策略:两轮训练,必要时补第三轮;保留独立验证,默认沿用配置")
parser.add_argument("--startup-timeout-seconds", type=float, default=None,
help="等待节点初始化及设备就绪的秒数(默认 120);大断点逐帧校验可显式延长,不改变运动保护")
@@ -322,9 +339,10 @@ def main(args: list[str] | None = None) -> None:
parser.add_argument("--offline-raw", default="")
parser.add_argument("--offline-output", default="")
parser.add_argument("--publish-offline", action="store_true")
parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_witness_v2',
choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2"),
help="新 O30 staged 准备策略;省略取值使用 v2,按观测依赖统一启用刚体双 Tag 求解;v1 仅保留历史兼容")
parser.add_argument("--preparation-policy", nargs='?', const='occlusion_aware_trajectory_v3',
choices=("occlusion_aware_witness_v1", "occlusion_aware_witness_v2",
"occlusion_aware_trajectory_v3"),
help="O30 staged 准备策略;省略取值使用 v3 整段轨迹消歧和有界自动观察;v1/v2 仅保留历史兼容")
selected = parser.parse_args(args)
if selected.startup_timeout_seconds is not None and (
not math.isfinite(selected.startup_timeout_seconds) or selected.startup_timeout_seconds <= 0
@@ -357,6 +375,10 @@ def main(args: list[str] | None = None) -> None:
from ..profiles.preparation import configure_preparation
try:
profile = configure_preparation(config.calibration_contract.typed_profile, selected.preparation_policy)
if selected.preparation_policy == 'occlusion_aware_trajectory_v3':
from ..core.urdf.kinematics import UrdfKinematicModel
from ..profiles.observer_preparation import configure_observer_preparations
profile = configure_observer_preparations(profile, UrdfKinematicModel(config.source_urdf))
config = replace(config, calibration_contract=ProductCalibrationContract(declarative=profile))
except ValueError as error:
parser.error(str(error))
@@ -62,7 +62,7 @@ class CalibrationSession:
self.required_observation_views = required_observation_views(profile, diagnostic_capture)
self.engine = CalibrationEngine(profile, diagnostic_capture=diagnostic_capture)
self.phase = CalibrationPhase.WAIT_DEVICE
self._units = self.engine.scan_units()
self._units = list(self.engine.scan_units())
self._unit_index = 0
self._retry_by_unit: dict[tuple[str, int, str], int] = {}
self._mapping_probed_channels: set[int] = set()
@@ -72,6 +72,8 @@ class CalibrationSession:
self._pause = PauseStatus()
self._reference_locked = False
self._rescan_pending = False
from .task_deferral import TaskDeferralQueue
self.task_deferrals = TaskDeferralQueue(profile)
@property
def current_unit(self) -> ScanUnit | None:
@@ -79,13 +81,17 @@ class CalibrationSession:
return None
return self._units[self._unit_index]
@property
def scheduled_units(self):
return tuple(self._units)
def select_training(self, task_key, cycles):
"""Replace only future optional units; evidence IDs never change."""
from ..core.domain.capture_plan import select_task_training
current = self.current_unit
self.profile = select_task_training(self.profile, task_key, cycles)
self.engine.profile = self.profile
self._units = self.engine.scan_units()
self._units = list(self.engine.scan_units())
if current is not None:
self._unit_index = next(i for i, unit in enumerate(self._units) if unit.identity == current.identity)
@@ -161,7 +167,7 @@ class CalibrationSession:
self._require(CalibrationPhase.PREPARE)
unit = self.current_unit
if unit is None:
self.phase = CalibrationPhase.RETURN_BASELINE
self._finish_capture_schedule()
return
task = self._task(unit.task_key)
if (
@@ -227,6 +233,79 @@ class CalibrationSession:
# The executor elides already-satisfied waypoints without motion.
self.phase = CalibrationPhase.PREPARE
return
self._finish_capture_schedule()
def defer_current_task(self, reason: str):
"""Move one recoverable visual visit to the end of its current stage.
A second failure records the visit as unobservable and removes only
its still-uncompleted units. Finalization is then blocked explicitly.
"""
self._require(CalibrationPhase.PREPARE)
unit = self.current_unit
if unit is None:
raise ValueError("task_deferral_without_current_unit")
from .task_deferral import dependency_closure, TaskObservationState
affected = dependency_closure(self.profile, unit.task_key)
moved = self._move_visit_to_stage_end(unit.cycle, affected)
attempt = self.task_deferrals.attempts.get((unit.task_key, unit.cycle), 0)+1
decision = self.task_deferrals.decide(unit.task_key, unit.cycle, reason, len(moved))
if decision.state == TaskObservationState.PENDING:
self._insert_at_stage_end(unit.cycle, moved)
else:
# Descendants cannot be accepted without the missing parent
# geometry. Remove their future visits from every later cycle.
self._units = self._units[:self._unit_index] + [candidate
for candidate in self._units[self._unit_index:]
if candidate.task_key not in set(decision.affected_tasks)]
self._rescan_pending = False
if self.current_unit is None:
self._finish_capture_schedule()
else:
self.phase = CalibrationPhase.PREPARE
return decision
def restore_task_deferrals(self, records):
"""Restore retry budgets and the exact stage-local pending order."""
self.task_deferrals.restore(records)
from .task_deferral import TaskObservationState, unresolved_observation_records
for row in unresolved_observation_records(self.profile, records):
if row.get("state") != TaskObservationState.PENDING.value:
continue
moved = self._move_visit_to_stage_end(
int(row["cycle"]), frozenset(row["affected_tasks"]))
self._insert_at_stage_end(int(row["cycle"]), moved)
def _stage_index(self, cycle):
from ..core.domain.capture_plan import TASKWISE_STAGED
if self.profile.quality.training_policy == TASKWISE_STAGED:
return 0 if cycle == 0 else 1 if cycle in {1, 2} else 2
return cycle
def _move_visit_to_stage_end(self, cycle, affected_tasks):
indices = [index for index in range(self._unit_index, len(self._units))
if self._units[index].cycle == cycle
and self._units[index].task_key in affected_tasks]
if not indices:
raise ValueError("task_deferral_has_no_pending_units")
moved = [self._units[index] for index in indices]
for index in reversed(indices):
del self._units[index]
return moved
def _insert_at_stage_end(self, cycle, moved):
stage = self._stage_index(cycle)
insertion = next((index for index in range(self._unit_index, len(self._units))
if self._stage_index(self._units[index].cycle) > stage), len(self._units))
self._units[insertion:insertion] = moved
def _finish_capture_schedule(self):
if self.task_deferrals.unobservable:
details = {f"{task}:{cycle}": reason
for (task, cycle), reason in self.task_deferrals.unobservable.items()}
self.fail("unobservable_tasks", "安全观察和一次阶段回访后仍不可观测",
details, suggestion="保留数据;调整 Tag 朝向、增加非共面 Tag 或改变机位后重试")
return
self.phase = CalibrationPhase.RETURN_BASELINE
def return_complete(self) -> None:
@@ -331,6 +410,21 @@ class CalibrationSession:
ids = tuple(tag.tag_id for tag in view.tags if tag.role in roles)
if ids:
required[view.name] = ids
from ..core.domain.capture_plan import TASKWISE_STAGED
taskwise = self.profile.quality.training_policy == TASKWISE_STAGED
displayed_cycle = (None if unit is None or unit.sample_phase == "steady" else
({0: 1, 1: 2, 2: 2, 3: 3}[unit.cycle] if taskwise else
list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key
and u.sample_phase == "sweep")).index(unit.cycle) + 1))
displayed_cycle_count = (3 if taskwise else
len(self.diagnostic_capture.cycles) if self.diagnostic_capture is not None else
len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key
and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1)
capture_stage = ("training_supplement" if taskwise and unit is not None and unit.cycle == 2
else "command_validation" if unit is not None and unit.sample_phase == "steady"
and unit.cycle == self.profile.quality.holdout_cycle + 2
else "command_training" if unit is not None and unit.sample_phase == "steady"
else "motion")
return CalibrationStatus(
state=self.phase.value,
phase=self.phase.value,
@@ -344,22 +438,17 @@ class CalibrationSession:
task=TaskStatus(
key="" if task is None else task.key,
label="" if task is None else " / ".join(task.joints),
cycle=(None if unit is None or unit.sample_phase == "steady" else
list(dict.fromkeys(u.cycle for u in self._units if u.task_key == unit.task_key
and u.sample_phase == "sweep")).index(unit.cycle) + 1),
cycle_count=(
len(self.diagnostic_capture.cycles)
if self.diagnostic_capture is not None
else len({u.cycle for u in self._units if unit is not None and u.task_key == unit.task_key
and u.sample_phase == "sweep"}) or len(self.profile.quality.training_cycles) + 1
),
cycle=displayed_cycle,
cycle_count=displayed_cycle_count,
direction="" if unit is None else unit.direction,
capture_stage=("command_validation" if unit is not None and unit.sample_phase == "steady"
and unit.cycle == self.profile.quality.holdout_cycle + 2 else "command_training" if unit is not None
and unit.sample_phase == "steady" else "motion"),
capture_stage=capture_stage,
required_tag_ids_by_view=required,
),
resume={"message": self._resume_message},
resume={"message": self._resume_message,
"deferred_tasks": {f"{task}:{cycle}": attempt
for (task, cycle), attempt in self.task_deferrals.attempts.items()},
"unobservable_tasks": [f"{task}:{cycle}"
for task, cycle in self.task_deferrals.unobservable]},
pause=self._pause,
)
@@ -18,7 +18,8 @@ SOURCE_PHASES = frozenset({'sweep', 'steady', 'joint_zero'})
def enabled(profile):
return (profile.quality.training_policy == 'staged_2_to_3'
from ..core.domain.capture_plan import STAGED_POLICIES
return (profile.quality.training_policy in STAGED_POLICIES
and bool(profile.motion.preparation_witnesses))
@@ -12,6 +12,8 @@ from .scan_quality import evaluate_capture_unit
def stage_visits(resume, engine, rows):
from .task_deferral import deferred_visit_keys
deferred = deferred_visit_keys(resume.profile, rows)
references = read_joint_zero_references(resume.profile, rows)
validate_motion_provenance(resume.profile, rows, references=references,
image_replay_workers=resume.image_replay_workers)
@@ -22,11 +24,15 @@ def stage_visits(resume, engine, rows):
if row.get('kind') == 'scan_unit_complete':
latest[record_scan_identity(row)] = row
spans, valid = {}, set()
for (key, _), units in visits.items():
for (key, cycle), units in visits.items():
task = next(t for t in resume.profile.motion.tasks if t.key == key)
if not set(task.joints) <= references.keys():
if (key, cycle) in deferred:
continue
break
if any(latest.get(u.identity, {}).get('passed') is not True for u in units):
if (key, cycle) in deferred:
continue
break
if not all(evaluate_capture_unit(resume.profile, u, latest[u.identity].get('attempt', 1), rows,
first_cycle_spans=spans).passed for u in units):
@@ -49,6 +55,8 @@ def stage_passed_visits(resume, engine, rows):
Every imported evidence record remains content-bound and finalization still
performs the complete image/provenance replay before publication.
"""
from .task_deferral import deferred_visit_keys
deferred = deferred_visit_keys(resume.profile, rows)
references = read_joint_zero_references(resume.profile, rows)
visits, latest = OrderedDict(), {}
for unit in engine.scan_units():
@@ -63,9 +71,13 @@ def stage_passed_visits(resume, engine, rows):
for (task_key, cycle), units in visits.items():
task = next(task for task in resume.profile.motion.tasks if task.key == task_key)
if not set(task.joints) <= references.keys():
if (task_key, cycle) in deferred:
continue
break
completions = [latest.get(unit.identity) for unit in units]
if any(row is None or row.get("passed") is not True for row in completions):
if (task_key, cycle) in deferred:
continue
break
valid.update(unit.identity for unit in units)
if cycle == 0:
@@ -98,10 +110,23 @@ def load_frozen_checkpoint(profile, rows, path, source):
def reference_import(resume, *, reuse_passed_visits=False):
from .joint_resume import _index_motion_provenance
from .passed_resume import PassedReferenceImport
from .passed_resume import PassedReferenceImport, coherent_observer_preparation_rows
provenance = tuple(_index_motion_provenance(resume.rows).values())
if reuse_passed_visits:
reused_tasks = {identity[0] for identity in resume.units}
reused_joints = {joint for task in resume.profile.motion.tasks
if task.key in reused_tasks for joint in task.joints}
references = {joint: reference for joint, reference in resume.references.items()
if joint in reused_joints}
# A visit that did not pass is restarted with a fresh preparation
# model in the current image epoch. Reusing its old candidate path is
# neither a passed-unit optimization nor valid cross-session evidence.
provenance = tuple(row for row in provenance
if row.get("task_name") is None or row.get("task_name") in reused_tasks)
provenance = _essential_indexed_provenance(provenance)
provenance = coherent_observer_preparation_rows(provenance)
else:
references = resume.references
models = tuple(r for r in provenance if r.get('kind') == 'motion_branch_initialization' and r.get('status') == 'resolved')
audit = ()
if reuse_passed_visits:
@@ -111,17 +136,19 @@ def reference_import(resume, *, reuse_passed_visits=False):
resume.verified_visits.update((name, cycle) for task in resume.profile.motion.tasks
for key, cycle in visits if key == task.key for name in task.joints)
audit = ({'kind': 'passed_tasks_reused', 'resume_mode': 'passed',
'unchanged_installation_declared': True, 'training_policy': 'staged_2_to_3',
'unchanged_installation_declared': True,
'training_policy': resume.profile.quality.training_policy,
'completed_units': [list(key) for key in sorted(resume.units)],
'historical_replay_performed': False, 'physical_zero_reverification_performed': False,
'full_replay_required_before_publication': True},)
return PassedReferenceImport(rows=(*(r.as_record() for r in resume.references.values()), *provenance, *audit),
references=resume.references, model_records=models)
return PassedReferenceImport(rows=(*(r.as_record() for r in references.values()), *provenance, *audit),
references=references, model_records=models)
def _essential_indexed_provenance(rows):
"""Keep source frames actually named by accepted models in compact journals."""
import re
from collections.abc import Mapping
from .resume_storage import DeferredCaptureRecord, materialize_record
if not any(isinstance(row, DeferredCaptureRecord) and row.canonical_sha256 for row in rows):
@@ -129,17 +156,21 @@ def _essential_indexed_provenance(rows):
hashes = set()
def collect(value, key=""):
if isinstance(value, dict):
if isinstance(value, Mapping):
for name, item in value.items():
collect(item, name)
# Digest maps are keyed by image stamp. Preserve the parent
# field name while descending into them; replacing it with the
# numeric stamp made every digest invisible to this filter.
collect(item, key if "hash" in key.lower() else str(name))
elif isinstance(value, (list, tuple)):
for item in value:
collect(item, key)
elif (isinstance(value, str) and "hash" in key
elif (isinstance(value, str) and "hash" in key.lower()
and re.fullmatch(r"[0-9a-f]{64}", value)):
hashes.add(value)
evidence_kinds = {"motion_branch_initialization", "parent_reference_verified",
"parent_reference_model_refreshed",
"observer_preparation_decision", "observer_preparation_segment_complete",
"joint_zero_motion_reference_verified"}
for row in rows:
@@ -152,6 +183,7 @@ def _essential_indexed_provenance(rows):
def restore_recovery(session, recovery, rows):
session.restore_task_deferrals(rows)
for row in rows:
if row.get('kind') == 'joint_zero_recovery':
recovery.restore(row['task_name'], row['joints'])
@@ -171,9 +203,21 @@ def visit_rows(resume, task_key, cycle):
rows = [r for r in resume.rows if record_scan_identity(r) in units
and r.get('kind') not in {'joint_zero_reference', 'joint_zero_sample'}]
if units:
from ..core.domain.capture_plan import TASKWISE_STAGED
from .engine import CalibrationEngine
if resume.profile.quality.training_policy == TASKWISE_STAGED and cycle in {1, 2}:
# Persist the causal decision beside the visit that produced it.
# This lets a later restart reconstruct an optional cycle 2 from
# the new journal without depending on in-memory profile state.
assessed = list(range(cycle + 1))
decisions = [r for r in resume.rows if r.get('kind') == 'training_decision'
and r.get('task_name') == task_key and r.get('assessed_cycles') == assessed]
if len(decisions) != 1:
raise ValueError('taskwise_resume_training_decision_missing')
rows.extend(decisions)
stage = [u for u in CalibrationEngine(resume.profile).scan_units() if u.cycle == cycle]
if cycle in {1, 2} and stage[-1].task_key == task_key:
if (resume.profile.quality.training_policy != TASKWISE_STAGED
and cycle in {1, 2} and stage[-1].task_key == task_key):
decisions = [r for r in resume.rows if r.get('kind') == 'staged_training_decision']
candidates = decisions[:1] if cycle == 1 else decisions[1:]
rows.extend(candidates)
@@ -144,6 +144,8 @@ def render_status_zh(value: Mapping[str, Any], estimator=None) -> str:
stage_line = f"阶段:{phase_label}(第 {task.get('cycle') or '-'}/{task.get('cycle_count', 4)} 轮)"
if capture_stage in {"command_training", "command_validation"}:
stage_line = "阶段:" + ("稳态指令映射训练" if capture_stage == "command_training" else "稳态指令映射独立验证")
elif capture_stage == "training_supplement":
stage_line = f"阶段:{phase_label}(当前任务训练补采 1/1)"
lines = [
f"[{value.get('model', '未知型号')} / {value.get('side', '?')} / {value.get('serial_number', '?')}] {progress:6.1%} 预计剩余 {eta_text}",
stage_line,
@@ -0,0 +1,237 @@
"""Bounded task deferral for recoverable visual observability failures.
The queue only reorders uncompleted scan units inside one staged cycle. It
never changes accepted units, reference models, quality limits or holdout
semantics. Safety failures are deliberately outside this module.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import Enum
POLICY = "dependency_ordered_observation_deferral_v1"
RECOVERABLE_REASONS = frozenset({
"rigid_pair_candidates_unresolved",
"rigid_pair_pose_uncertain",
"rigid_pair_viewpoint_span_insufficient",
"rigid_pair_source_missing",
"observer_images_missing",
"observer_preparation_candidates_missing",
"motion_candidates_missing",
"motion_candidate_assignment_ambiguous",
"motion_candidate_family_discontinuous",
"motion_candidate_family_interrupted",
"motion_geometry_ambiguous",
"image_motion_alternative_angles_unresolved",
"image_motion_families_not_distinguishable",
"image_motion_frame_angles_ambiguous",
"image_motion_frame_angles_uncertain",
"image_motion_frame_angles_unobservable",
"image_motion_insufficient_independent_frames",
"image_motion_no_observable_image_model",
"image_motion_shared_pose_unresolved",
"parent_reference_samples_missing",
"image_reference_current_image_moved",
"pose_image_motion_root_missing",
})
class TaskObservationState(str, Enum):
PASSED = "passed"
PENDING = "pending"
DEPENDENCY_PENDING = "dependency_pending"
UNOBSERVABLE = "unobservable"
def task_dependencies(profile):
"""Compile parent task -> descendant tasks from observed Tag roles."""
task_by_joint = {joint: task.key for task in profile.motion.tasks for joint in task.joints}
owners = {}
for joint, spec in profile.measurement.measurements.items():
task = task_by_joint.get(joint)
if task is not None and spec.child_role:
owners.setdefault((spec.view, spec.child_role), set()).add(task)
graph = {task.key: set() for task in profile.motion.tasks}
for joint, spec in profile.measurement.measurements.items():
child_task = task_by_joint.get(joint)
if child_task is None or not spec.parent_role:
continue
for parent_task in owners.get((spec.view, spec.parent_role), ()):
if parent_task != child_task:
graph[parent_task].add(child_task)
return {key: frozenset(value) for key, value in graph.items()}
def dependency_closure(profile, task_key):
graph = task_dependencies(profile)
pending, result = [task_key], set()
while pending:
current = pending.pop()
if current in result:
continue
result.add(current)
pending.extend(graph.get(current, ()))
return frozenset(result)
def recoverable_observation_reason(reason):
text = str(reason)
return any(code in text for code in RECOVERABLE_REASONS)
@dataclass(frozen=True)
class DeferralDecision:
task_name: str
cycle: int
attempt: int
state: TaskObservationState
affected_tasks: tuple[str, ...]
reason: str
moved_units: int
def as_record(self, *, stamp_ns, corner_evidence_sha256=""):
dependency_state = (TaskObservationState.DEPENDENCY_PENDING.value
if self.state == TaskObservationState.PENDING
else TaskObservationState.UNOBSERVABLE.value)
return {
"kind": "task_observation_state",
"schema_version": 1,
"policy": POLICY,
"task_name": self.task_name,
"cycle": self.cycle,
"attempt": self.attempt,
"state": self.state.value,
"affected_tasks": list(self.affected_tasks),
"dependency_states": ({task: dependency_state
for task in self.affected_tasks if task != self.task_name}
if self.state != TaskObservationState.PASSED else {}),
"reason": self.reason,
"moved_units": self.moved_units,
"corner_evidence_sha256": corner_evidence_sha256,
"stamp_ns": int(stamp_ns),
}
class TaskDeferralQueue:
"""Own the finite retry state independently of motion execution."""
def __init__(self, profile, *, revisit_limit=1):
if revisit_limit != 1:
raise ValueError("task_deferral_revisit_limit_must_be_one")
self.profile = profile
self.revisit_limit = revisit_limit
self.attempts = {}
self.unobservable = {}
self.resolved = set()
self.inferred_resolutions = set()
def decide(self, task_name, cycle, reason, moved_units):
key = (str(task_name), int(cycle))
if key in self.resolved:
raise ValueError("task_deferral_already_resolved")
attempt = self.attempts.get(key, 0) + 1
if attempt > self.revisit_limit + 1:
raise ValueError("task_deferral_budget_exhausted")
self.attempts[key] = attempt
affected = tuple(task.key for task in self.profile.motion.tasks
if task.key in dependency_closure(self.profile, task_name))
state = (TaskObservationState.PENDING if attempt <= self.revisit_limit
else TaskObservationState.UNOBSERVABLE)
if state == TaskObservationState.UNOBSERVABLE:
self.unobservable[key] = str(reason)
return DeferralDecision(str(task_name), int(cycle), attempt, state,
affected, str(reason), int(moved_units))
def resolve(self, task_name, cycle):
key = (str(task_name), int(cycle))
attempt = self.attempts.get(key)
if attempt is None or key in self.resolved:
return None
self.resolved.add(key)
self.unobservable.pop(key, None)
return DeferralDecision(key[0], key[1], attempt, TaskObservationState.PASSED,
(key[0],), "trajectory_observation_resolved", 0)
def restore(self, records):
records = tuple(records)
state_positions = {}
for position, row in enumerate(records):
if row.get("kind") != "task_observation_state":
continue
if row.get("policy") != POLICY or row.get("state") not in {
state.value for state in TaskObservationState}:
raise ValueError("task_deferral_checkpoint_policy_changed")
key = (str(row["task_name"]), int(row["cycle"]))
state_positions[key] = position
attempt = int(row["attempt"])
expected_affected = tuple(task.key for task in self.profile.motion.tasks
if task.key in dependency_closure(self.profile, key[0]))
if tuple(row.get("affected_tasks", ())) != expected_affected:
raise ValueError("task_deferral_checkpoint_dependencies_changed")
if row["state"] == TaskObservationState.PASSED.value:
if self.attempts.get(key) != attempt or key in self.resolved:
raise ValueError("task_deferral_checkpoint_resolution_changed")
self.resolved.add(key)
self.unobservable.pop(key, None)
continue
if attempt != self.attempts.get(key, 0)+1 or attempt not in (1, 2):
raise ValueError("task_deferral_checkpoint_budget_changed")
self.attempts[key] = attempt
if row["state"] == TaskObservationState.UNOBSERVABLE.value:
self.unobservable[key] = str(row.get("reason", ""))
# If a crash happened after the final scan-unit commit but before the
# adjacent PASSED state record, the complete visit itself closes the
# pending state. A partial direction can never do so.
from .engine import CalibrationEngine
from ..core.domain.motion_path import record_scan_identity
completion_positions = {}
for position, row in enumerate(records):
if row.get("kind") == "scan_unit_complete" and row.get("passed") is True:
completion_positions[record_scan_identity(row)] = position
units = CalibrationEngine(self.profile).scan_units()
for key in set(self.attempts) - self.resolved - set(self.unobservable):
required = {unit.identity for unit in units
if (unit.task_key, unit.cycle) == key}
boundary = state_positions[key]
if required and all(completion_positions.get(identity, -1) > boundary
for identity in required):
self.resolved.add(key)
self.inferred_resolutions.add(key)
def deferred_visit_keys(profile, records):
"""Return incomplete visits that may be followed by durable later visits.
A non-contiguous resume prefix is legal only when a versioned deferral
record explains the gap. Replaying the queue also validates retry order,
dependency closure and terminal state before the gap is trusted.
"""
rows = [row for row in records if row.get("kind") == "task_observation_state"]
if not rows:
return frozenset()
queue = TaskDeferralQueue(profile)
queue.restore(records)
latest = {}
for row in rows:
latest[(str(row["task_name"]), int(row["cycle"]))] = row
blocked = set()
for key, row in latest.items():
if row["state"] == TaskObservationState.PASSED.value or key in queue.resolved:
continue
blocked.update((task, key[1]) for task in row["affected_tasks"])
return frozenset(blocked)
def unresolved_observation_records(profile, records):
"""Return latest states that are not closed by a full durable visit."""
rows = [row for row in records if row.get("kind") == "task_observation_state"]
if not rows:
return ()
queue = TaskDeferralQueue(profile)
queue.restore(records)
latest = {}
for row in rows:
latest[(str(row["task_name"]), int(row["cycle"]))] = row
return tuple(row for key, row in latest.items() if key not in queue.resolved)
@@ -6,8 +6,8 @@ import threading
import time
import traceback
from ..core.domain.capture_plan import (ADAPTIVE, STAGED, CapturePlan, configure_training,
evidence_digest, select_task_training)
from ..core.domain.capture_plan import (ADAPTIVE, STAGED, TASKWISE_STAGED,
CapturePlan, configure_training, evidence_digest, select_task_training)
from ..core.domain.reference import JointZeroReference
from ..core.fitting.training_quality import (TRAINING_DECISION_VERSION, assess_training,
training_snapshot)
@@ -22,7 +22,7 @@ def resolve_capture_plan(profile, records, *, source_urdf=None, verify_decisions
base = configure_training(profile, profile.quality.training_policy)
headers = [row for row in records if row.get("kind") == "session_start"]
plan = CapturePlan.from_profile(base)
if base.quality.training_policy != ADAPTIVE:
if base.quality.training_policy not in {ADAPTIVE, TASKWISE_STAGED}:
if any(row.get("kind") in {"training_decision", "staged_training_decision"} for row in records):
raise ValueError("adaptive_decisions_in_fixed_capture")
if headers and "capture_plan" in headers[0] and headers[0]["capture_plan"] != plan.as_dict():
@@ -63,6 +63,22 @@ class ZeroRecovery:
def restore(self, task_key, joints):
self._used.add((task_key, tuple(joints)))
def take_parent_model_refresh(self, motion, *, stale_image_anchor):
"""Spend one fresh-model allowance for a resumed parent reference.
This does not reacquire its joint zero or scan. It only permits the
already declared safe approach to rebuild the image-domain model in
the current session's stationary camera coordinate frame.
"""
if (not motion.reference_reuse or not stale_image_anchor
or len(motion.zero_joints) != 1):
return None
key = (motion.task_key, tuple(motion.zero_joints))
if key in self._used:
return None
self._used.add(key)
return "refresh_parent_model"
def take(self, motion, *, geometry_error, reports, referenced_joints):
if motion.reference_reuse:
return None
@@ -11,7 +11,7 @@ from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.profiles.export import dump_hand_profile
from linkerhand_calibration.profiles.observer_policy import configure_observer_policy
from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract
from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade
from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade, _copy_indexed_evidence
from linkerhand_calibration.runtime.joint_resume import _index_motion_provenance
@@ -81,3 +81,18 @@ def test_active_observation_migration_only_adds_unmeasured_bounded_paths(migrati
checkpoint.joints.references['index_dip']=object()
with pytest.raises(ValueError,match='changes_existing_contract'):
upgrade.metadata(checkpoint,inputs,'source.jsonl','c'*64)
def test_indexed_checkpoint_migration_copies_content_bound_archive(tmp_path):
from linkerhand_calibration.core.artifacts.evidence_journal import (
initialize_evidence_journal, append_indexed_records, archive_path, manifest_path,
)
source = tmp_path/'source/raw_samples.jsonl'
source.parent.mkdir()
initialize_evidence_journal(source, threshold_bytes=128)
append_indexed_records(source, ({'kind': 'pnp_candidate_frame', 'view': 'side',
'image_stamp_ns': 1, 'pixels': 'x'*2048},), durable=True)
target = tmp_path/'target'; target.mkdir()
_copy_indexed_evidence(source, target)
assert (target/archive_path(source).name).read_bytes() == archive_path(source).read_bytes()
assert (target/manifest_path(source).name).read_bytes() == manifest_path(source).read_bytes()
@@ -117,7 +117,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile):
segment = task.segments[0]
start = build_calibration_motion_command(task, segment.start, profile=profile, segment_key=segment.key)
end = build_calibration_motion_command(task, segment.end, profile=profile, segment_key=segment.key)
execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200),
execution = MotionExecution(profile, MotionCommand('sweep', tuple(end), 200,
measured_channels=segment.moving_channels),
initial_command=start, initial_feedback=start, now=0., identity='linked',
visual_observer=VisualMotionObserver(profile))
previous = execution.sample(0)
@@ -127,8 +128,8 @@ def test_group_schedule_native_coverage_nodes_and_peak_velocity(profile):
assert actual[3] == pytest.approx(80+175*progress, abs=1.)
assert actual[2] == actual[4] == actual[5]
# Integer bytes carry at most one unit of quantization error.
assert max(abs(a-b) for a,b in zip(actual,previous)) <= 30*.01+1.
assert math.pi*255/(2*execution.duration) <= 30.000001
assert max(abs(a-b) for a,b in zip(actual,previous)) <= 20*.01+1.
assert math.pi*255/(2*execution.duration) <= 20.000001
previous = actual
@@ -22,6 +22,8 @@ from linkerhand_calibration.runtime.parent_reference import ParentReferenceRegis
from linkerhand_calibration.runtime.capture import ObservationCapture
from linkerhand_calibration.runtime.reference_lock import ReferenceLock
from linkerhand_calibration.runtime.motion_execution import MotionCommand
from linkerhand_calibration.runtime.execution import SessionExecution
from linkerhand_calibration.runtime.session import CalibrationPhase as Phase
from linkerhand_calibration.runtime.zero_recovery import ZeroRecovery
from o30_capture_fixture import SOURCE
@@ -101,6 +103,11 @@ def test_registry_needs_both_sources_and_fresh_check_and_preserves_zeros(setup):
with pytest.raises(ValueError, match="source_missing"): owner.sources(task, 2)
with pytest.raises(ValueError, match="unverified"): owner.geometry(task, 1)
refresh = owner.authorize_refreshed_pose(task, 1, 9, reference)
assert refresh["passed_zero_preserved"] is True
assert refresh["image_model_sha256"] == roll_report["image_model_sha256"]
assert owner.pose_source(task, 1) == (roll, roll_report)
for row in rows:
row["pnp_observation_evidence"] = {"child": dict(tag_role="thumb_mcp",
image_model_sha256=roll_report["image_model_sha256"],
@@ -168,11 +175,42 @@ def test_stationary_reference_cannot_trigger_a_new_solver_or_extra_motion_retry(
assert owner.request("front", motion, session_epoch=1, motion_version=10) is None
recovery = ZeroRecovery()
assert recovery.take(motion, geometry_error="", reports={}, referenced_joints=()) is None
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=False) is None
ordinary = replace(motion, reference_only=False, reference_reuse=False, zero_joints=("thumb_ip",))
assert recovery.take(ordinary, geometry_error="ambiguous", reports={"front": {
"status": "unresolved", "reason": "image_motion_families_not_distinguishable"}}, referenced_joints=()) is None
def test_stale_cross_session_anchor_gets_one_model_refresh_without_replacing_zero(setup):
profile, task, _ = setup
parent = task.parent_reference.pose_joint
motion = MotionCommand("joint_zero",
profile.motion.joint_zero_references[task.joints[0]].command, 200,
task_key=task.key, cycle=0, zero_joints=(parent,),
reference_only=True, reference_reuse=True)
recovery = ZeroRecovery()
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) == "refresh_parent_model"
assert recovery.take_parent_model_refresh(motion, stale_image_anchor=True) is None
execution = SessionExecution(profile)
execution.session.phase = Phase.PREPARE
frozen = object()
execution.zero_references[parent] = frozen
execution._effects = [motion]
execution.refresh_parent_model(motion.target)
assert execution.zero_references[parent] is frozen
assert execution._effects[-1] is motion
refresh = execution._effects[:-1]
assert refresh and refresh[-1].phase == "joint_zero"
assert refresh[-1].zero_joints == (parent,)
assert all(item.parent_model_refresh_for == task.key for item in refresh)
assert all(item.reference_only and not item.reference_reuse for item in refresh)
moving = [item for item in refresh if item.phase == "zero_approach"]
from linkerhand_calibration.core.fitting.motion_fit import channel_for_joint
channel = channel_for_joint(profile, parent)
assert max(item.target[channel] for item in moving) != min(item.target[channel] for item in moving)
def test_coordinator_waits_for_current_parent_images_and_keeps_both_joint_zeros(setup, tmp_path, monkeypatch):
from linkerhand_calibration.runtime.coordinator import CalibrationCoordinator
from linkerhand_calibration.runtime import coordinator as module
@@ -44,6 +44,73 @@ def test_passed_parent_model_keeps_original_source_in_new_observation_epoch():
assert dict(pose_source[0].evidence_ids)['moving'] == 'a' * 64
def test_passed_resume_does_not_restore_incomplete_task_preparation(monkeypatch):
profile = load_bundled_hand_profile('o30_right_18')
engine, resume = CalibrationEngine(profile), JointResume(profile)
passed, incomplete = profile.motion.tasks[:2]
rows = [*passed_records(engine, passed),
dict(kind='motion_branch_observation', task_name=passed.key,
view=passed.view, image_stamp_ns=1, session_epoch=1),
dict(kind='motion_branch_observation', task_name=incomplete.key,
view=incomplete.view, image_stamp_ns=2, session_epoch=1),
dict(kind='motion_branch_initialization', task_name=incomplete.key,
view=incomplete.view, status='resolved', session_epoch=1)]
monkeypatch.setattr(passed_resume, 'read_joint_zero_references',
lambda *args: stub_references([passed, incomplete]))
passed_resume.stage_passed_tasks(resume, engine, rows)
pending = passed_resume.reference_import(resume)
assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key}
assert not pending.model_records
def test_taskwise_passed_resume_does_not_restore_unpassed_visit_model():
from linkerhand_calibration.runtime import staged_resume
profile = load_bundled_hand_profile('o30_right_18')
passed, incomplete = profile.motion.tasks[:2]
references = stub_references([passed, incomplete])
rows = [
dict(kind='motion_branch_observation', task_name=passed.key,
view=passed.view, image_stamp_ns=1, session_epoch=1),
dict(kind='motion_branch_observation', task_name=incomplete.key,
view=incomplete.view, image_stamp_ns=2, session_epoch=1),
dict(kind='motion_branch_initialization', task_name=incomplete.key,
view=incomplete.view, status='resolved', session_epoch=1),
]
accepted = next(unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == passed.key)
resume = JointResume(profile)
resume.rows, resume.references, resume.units = tuple(rows), references, {accepted.identity}
pending = staged_resume.reference_import(resume, reuse_passed_visits=True)
assert set(pending.references) == set(passed.joints)
assert {row.get('task_name') for row in pending.rows if row.get('task_name')} == {passed.key}
assert not pending.model_records
def test_passed_resume_omits_orphaned_observer_cancellation():
from linkerhand_calibration.runtime import staged_resume
profile = load_bundled_hand_profile('o30_right_18')
task = profile.motion.tasks[0]
accepted = next(unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == task.key)
orphan = dict(kind='observer_preparation_cancelled_before_excursion',
joint=task.joints[0], observer_preparation_id='a'*64,
session_epoch=1, stamp_ns=1)
resume = JointResume(profile)
resume.rows = (orphan,)
resume.references = stub_references([task])
resume.units = {accepted.identity}
pending = staged_resume.reference_import(resume, reuse_passed_visits=True)
assert orphan not in pending.rows
@pytest.mark.parametrize('failure', [None, 'missing', 'failed', 'support', 'new_attempt'])
def test_last_group_failure_keeps_128_prior_passes_without_recalculating(monkeypatch, failure):
profile = load_bundled_hand_profile('o30_right_18')
@@ -81,7 +148,10 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon
expected_units = sum(unit.task_key == task.key for unit in host.execution.session.engine.scan_units())
rows = [dict(kind='motion_branch_observation', view=task.view, image_stamp_ns=i,
session_epoch=1, task_name=task.key)
for i in range(257)] + passed_records(host.execution.session.engine, task)
for i in range(257)] + [dict(kind='observer_preparation_decision',
task_name=task.key, joint=task.joints[0], action='excite',
action_index=0, session_epoch=1, stamp_ns=1)] + passed_records(
host.execution.session.engine, task)
references = stub_references([task])
monkeypatch.setattr(passed_resume, 'read_joint_zero_references', lambda *args: references)
resume = JointResume(host.profile)
@@ -123,13 +193,19 @@ def test_direct_resume_imports_durably_then_skips_old_joint_motion(tmp_path, mon
assert host.state in {'ABORTED', 'PAUSED'}
assert not host.execution.zero_references and host._resumed_count == 0
else:
for _ in range(20):
# Reference restoration is intentionally sliced across callbacks
# and journal barriers run on a worker. Give that worker real
# scheduling opportunities instead of spinning only the fake clock.
import time
for _ in range(400):
host.receive_feedback(host.profile.command.names, host.profile.command.baseline_values, host.ports.clock_ns())
host.tick()
if host.execution.session.current_unit.task_key != task.key:
break
time.sleep(.001)
assert host._resumed_count == expected_units
assert host.execution.zero_references == references
assert not host.branch_initialization.observer_preparation.decisions
assert host.segment is None
import json
imported = [json.loads(line) for line in host.raw_path.read_text().splitlines()]
@@ -10,6 +10,8 @@ from linkerhand_calibration.runtime.resume_storage import (
DeferredCaptureRecord, load_passed_capture,
)
from linkerhand_calibration.runtime.joint_resume import ResumeJournalImport, _index_motion_provenance
from linkerhand_calibration.runtime.staged_resume import _essential_indexed_provenance
from linkerhand_calibration.runtime.motion_provenance import source_frame_sha256
from linkerhand_calibration.core.domain.motion_path import record_scan_identity
from linkerhand_calibration.runtime.diagnostic_capture import reject_diagnostic_capture
from linkerhand_calibration.core.artifacts.evidence_journal import (
@@ -18,6 +20,8 @@ from linkerhand_calibration.core.artifacts.evidence_journal import (
)
from linkerhand_calibration.storage import append_jsonl, append_jsonl_many
from linkerhand_calibration.runtime.acquisition import load_capture
from linkerhand_calibration.runtime.capture_index import CaptureRecordIndex
from linkerhand_calibration.runtime.resume import discover_resume_candidate
def capture(tmp_path, rows):
@@ -49,7 +53,7 @@ def test_scalar_index_never_reads_image_payload_and_import_is_bounded(tmp_path,
assert len(batch) == size and len(calls)-before == size
assert all(type(r) is dict for r in batch)
imported.extend(batch)
assert pending.complete and imported == original
assert pending.transfer_complete and not pending.complete and imported == original
assert all("pnp_observation_evidence" not in r.metadata for r in rows)
@@ -101,6 +105,7 @@ def test_indexed_journal_keeps_resume_fields_inline_and_restores_full_evidence(t
deferred = load_passed_capture(path)
assert all(isinstance(row, DeferredCaptureRecord) for row in deferred[1:])
assert deferred[1]["relative_quaternion_xyzw"] == [0., 0., 0., 1.]
assert "relative_quaternion_xyzw" not in deferred[1].metadata
assert deferred[1].materialize() == samples[0]
@@ -162,3 +167,90 @@ def test_resume_import_hard_links_compressed_evidence_without_copying_records(tm
archive_path(source).unlink()
source.unlink()
assert load_capture(target) == original
def test_linked_joint_samples_use_bounded_batches_and_concrete_training_digest(tmp_path):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
leading = [dict(kind="image_observation_frame", task_name="index_dip_side",
cycle=0, direction="increasing", image_stamp_ns=-stamp-1,
pixels=list(range(100))) for stamp in range(10)]
samples = [dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip",
cycle=0, direction="increasing", image_stamp_ns=stamp,
command_u8=stamp, feedback_u8=stamp, relative_quaternion_xyzw=[0., 0., 0., 1.])
for stamp in range(100)]
append_jsonl_many(path, (*leading, *samples))
pending = ResumeJournalImport(tuple(load_passed_capture(path)))
assert len(pending.take_batch()) == len(leading)
first = pending.take_batch()
assert len(first) == len(samples) and all(row.get("kind") == "joint_sample" for row in first)
index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True)
index.extend(first)
assert len(index) == len(samples)
assert all(len(row["source_record_sha256"]) == 64 for row in index)
def test_training_index_materializes_each_deferred_sample_once(tmp_path, monkeypatch):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
sample = dict(kind="joint_sample", task_name="index_dip_side", joint="index_dip",
cycle=0, direction="increasing", image_stamp_ns=1, command_u8=20,
feedback_u8=19, relative_quaternion_xyzw=[0., 0., 0., 1.],
relative_translation_xyz_m=[0., 0., .01])
append_jsonl(path, sample)
row = load_passed_capture(path)[0]
original = DeferredCaptureRecord.materialize_plain
calls = []
def observed(record):
calls.append(record.reference.record_sha256)
return original(record)
monkeypatch.setattr(DeferredCaptureRecord, "materialize_plain", observed)
index = CaptureRecordIndex(retain_complete=False, retain_training_geometry=True)
index.append(row)
assert len(calls) == 1
assert index[0]["relative_translation_xyz_m"] == [0., 0., .01]
def test_compact_passed_resume_keeps_only_digest_referenced_motion_frames(tmp_path):
path = tmp_path / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=1)
used = {"kind": "motion_branch_observation", "task_name": "index_dip_side",
"view": "side", "image_stamp_ns": 10, "pixels": list(range(100))}
unused = {**used, "image_stamp_ns": 20, "pixels": list(range(101))}
accepted = {"kind": "motion_branch_initialization", "status": "resolved",
"task_name": "index_dip_side", "view": "side",
"source_frame_hashes": {"10": source_frame_sha256(used)}}
append_jsonl_many(path, (used, unused, accepted))
indexed = tuple(_index_motion_provenance(load_passed_capture(path)).values())
essential = _essential_indexed_provenance(indexed)
observations = [row for row in essential if row.get("kind") == "motion_branch_observation"]
assert len(observations) == 1
assert observations[0].canonical_sha256 == source_frame_sha256(used)
def test_discovery_reads_large_compressed_session_header(tmp_path):
from linkerhand_calibration.runtime.engine import ACQUISITION_POLICY_VERSION
from linkerhand_calibration.core.geometry.pnp import POSE_TRACKING_POLICY_VERSION
session = tmp_path / "SERIAL" / "20260101_000000"
session.mkdir(parents=True)
path = session / "raw_samples.jsonl"
initialize_evidence_journal(path, threshold_bytes=128)
hashes = {"profile_config_sha256": "a" * 64}
contract = {"large": list(range(100))}
append_jsonl_many(path, [{"kind": "session_start", "profile_id": "O30/right/layout/v1",
"serial_number": "SERIAL", "acquisition_policy_version": ACQUISITION_POLICY_VERSION,
"capture_schedule_version": "unified_schedule_v13_staged_training",
"pose_tracking_policy_version": POSE_TRACKING_POLICY_VERSION, "protected_hashes": hashes,
"capture_plan": {"policy": "staged_2_to_3"}, "preparation_contract": contract},
{"kind": "fixed_base_reference_locked"},
{"kind": "scan_unit_complete", "task_name": "index_dip_side", "cycle": 0,
"direction": "increasing", "passed": True}])
assert discover_resume_candidate(tmp_path / "SERIAL", profile_id="O30/right/layout/v1",
serial_number="SERIAL", protected_hashes=hashes, training_policy="staged_2_to_3",
preparation_contract=contract) == session
@@ -10,6 +10,7 @@ from linkerhand_calibration.profiles import load_hand_profile
from linkerhand_calibration.profiles.export import write_hand_profile
from linkerhand_calibration.profiles.validator import validate_executable_profile
from linkerhand_calibration.profiles.preparation import preparation_contract
from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER
from linkerhand_calibration.runtime.observer_reference import ObserverBridgeInput,build_observer_bridge,validate_observer_references
from linkerhand_calibration.runtime.shared_tag_reference import bridge_frames
from linkerhand_calibration.runtime.branch_initialization import BranchInitializationRequest,solve_initialization,initialization_artifacts
@@ -72,6 +73,17 @@ def test_independent_parent_branches_converge_on_real_bounded_probe(tmp_path):
assert bridge.translation_uncertainty_m<.001
assert all(p==0 for _,p in record['pose']['comparison_adjusted_p_values'])
assert set(record['pose']['training_stamps']).isdisjoint(record['pose']['validation_stamps'])
measurements=dict(profile.measurement.measurements)
measurements['index_dip']=replace(measurements['index_dip'],
observer_pose_policy=TRAJECTORY_OBSERVER)
trajectory_profile=replace(profile,measurement=replace(profile.measurement,
measurements=measurements))
trajectory_bridge,trajectory_record=build_observer_bridge(trajectory_profile,
ObserverBridgeInput('index_dip',model,parent,(),endpoints,(16,),evidence))
assert trajectory_record['version']=='observer_pose_bridge_v4'
assert trajectory_record['trajectory_branch_decision']['holdout_role']=='veto_only'
assert trajectory_record['trajectory_branch_decision']['corner_evidence_sha256']
assert trajectory_bridge.rotation_uncertainty_rad<.017453293
@pytest.fixture(scope='module')
@@ -7,7 +7,9 @@ from pathlib import Path
import pytest
from linkerhand_calibration.core.domain.capture_plan import STAGED, CapturePlan, configure_training, select_task_training
from linkerhand_calibration.core.domain.capture_plan import (
STAGED, TASKWISE_STAGED, CapturePlan, configure_training, select_task_training,
)
from linkerhand_calibration.core.domain.reference import read_joint_zero_references
from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.runtime.engine import CalibrationEngine
@@ -41,6 +43,52 @@ def test_all_first_visits_precede_second_and_validation_keeps_original_motion():
assert fixed.quality.training_policy == 'fixed'
def test_taskwise_schedule_keeps_first_pass_and_validation_global_but_supplements_locally():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED)
plan = CapturePlan.from_profile(profile).as_dict()
assert plan['version'] == 'capture_plan_v3'
assert plan['schedule'] == (
'first_whole_hand_pass_then_taskwise_second_pass_with_immediate_supplement_then_validation'
)
first_task = profile.motion.tasks[0]
profile = select_task_training(profile, first_task.key, (0, 1, 2))
units = CalibrationEngine(profile).scan_units()
visits = []
for unit in units:
visit = unit.task_key, unit.cycle
if not visits or visits[-1] != visit:
visits.append(visit)
task_keys = [task.key for task in profile.motion.tasks]
assert visits[:len(task_keys)] == [(key, 0) for key in task_keys]
training_visits = visits[len(task_keys):-len(task_keys)]
assert training_visits[:2] == [(first_task.key, 1), (first_task.key, 2)]
assert training_visits[2:] == [(key, 1) for key in task_keys[1:]]
assert visits[-len(task_keys):] == [(key, 3) for key in task_keys]
def test_taskwise_resume_carries_the_decision_that_created_optional_round():
from linkerhand_calibration.runtime.joint_resume import JointResume
from linkerhand_calibration.runtime.staged_resume import visit_rows
profile = configure_training(load_bundled_hand_profile('o30_right_18'), TASKWISE_STAGED)
task = profile.motion.tasks[0]
profile = select_task_training(profile, task.key, (0, 1, 2))
units = [unit for unit in CalibrationEngine(profile).scan_units()
if unit.task_key == task.key and unit.cycle == 2]
decision = {'kind': 'training_decision', 'task_name': task.key,
'assessed_cycles': [0, 1, 2], 'decision': 'freeze'}
rows = [dict(kind='scan_unit_complete', task_name=unit.task_key,
cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key,
passed=True) for unit in units]
resume = JointResume(profile)
resume.rows = (*rows, decision)
resume.units = {unit.identity for unit in units}
resume.verified_visits.update((joint, 2) for joint in task.joints)
used, imported = visit_rows(resume, task.key, 2)
assert used == tuple(sorted(unit.identity for unit in units))
assert [row for row in imported if row.get('kind') == 'training_decision'] == [decision]
def test_revisit_reuses_model_but_checks_zero_via_existing_path():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
execution = SessionExecution(profile)
@@ -0,0 +1,207 @@
"""v3 trajectory observation, bounded deferral and publication refusal."""
import hashlib
from types import SimpleNamespace
import pytest
from linkerhand_calibration.core.domain.capture_plan import (
STAGED, TASKWISE_STAGED, configure_training, select_task_training,
)
from linkerhand_calibration.core.urdf.kinematics import UrdfKinematicModel
from linkerhand_calibration.profiles import load_bundled_hand_profile
from linkerhand_calibration.profiles.observer_policy import TRAJECTORY_OBSERVER
from linkerhand_calibration.profiles.observer_policy import configure_observer_policy
from linkerhand_calibration.profiles.observer_preparation import configure_observer_preparations
from linkerhand_calibration.profiles.export import dump_hand_profile
from linkerhand_calibration.profiles.preparation import configure_preparation, preparation_contract
from linkerhand_calibration.runtime.artifacts.capture_validation import validate_capture_units
from linkerhand_calibration.runtime.session import CalibrationPhase, CalibrationSession
from linkerhand_calibration.runtime.checkpoint_observer_upgrade import ObserverPolicyUpgrade
from linkerhand_calibration.runtime.task_deferral import (
POLICY, TaskDeferralQueue, TaskObservationState, deferred_visit_keys,
dependency_closure, recoverable_observation_reason,
)
from o30_capture_fixture import SOURCE
@pytest.fixture
def trajectory_profile():
profile = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
profile = configure_preparation(profile, 'occlusion_aware_trajectory_v3')
return configure_observer_preparations(profile, UrdfKinematicModel(SOURCE))
def test_v3_is_a_versioned_profile_contract_with_two_action_bound(trajectory_profile):
policy = preparation_contract(trajectory_profile)['observer_preparations']
assert policy['policy'] == 'occlusion_aware_trajectory_v3'
assert policy['trajectory_solver']['holdout_selects_model'] is False
assert policy['trajectory_solver']['maximum_actions_per_task'] == 2
assert all(1 <= len(plan.actions) <= 2
for plan in trajectory_profile.motion.observer_preparations.values())
assert trajectory_profile.measurement.measurements['index_dip'].observer_pose_policy \
== TRAJECTORY_OBSERVER
def test_visual_ambiguity_is_deferred_but_hardware_failure_is_not():
assert recoverable_observation_reason('image_motion_frame_angles_ambiguous')
assert recoverable_observation_reason('motion_candidates_missing')
assert not recoverable_observation_reason('sdk_disconnected')
assert not recoverable_observation_reason('feedback_stalled')
assert not recoverable_observation_reason('over_temperature')
def test_parent_deferral_moves_dependency_closure_to_stage_end(trajectory_profile):
assert 'index_dip_side' in dependency_closure(trajectory_profile, 'index_pip_side')
session = CalibrationSession(trajectory_profile)
session.phase = CalibrationPhase.PREPARE
session._unit_index = next(i for i, unit in enumerate(session._units)
if unit.task_key == 'index_pip_side' and unit.cycle == 0)
original_prefix = tuple(unit.identity for unit in session._units[:session._unit_index])
decision = session.defer_current_task('rigid_pair_candidates_unresolved')
assert decision.state == TaskObservationState.PENDING
assert {'index_pip_side', 'index_dip_side'} <= set(decision.affected_tasks)
record = decision.as_record(stamp_ns=1)
assert record['dependency_states']['index_dip_side'] == 'dependency_pending'
assert tuple(unit.identity for unit in session._units[:session._unit_index]) == original_prefix
same_cycle = [unit.task_key for unit in session._units if unit.cycle == 0]
assert same_cycle[-1] == 'index_dip_side'
def test_staged_assessment_waits_for_deferred_visit(trajectory_profile):
from linkerhand_calibration.runtime.execution import SessionExecution
execution = SessionExecution(trajectory_profile)
session = execution.session
session.phase = CalibrationPhase.PREPARE
session._unit_index = next(i for i, unit in enumerate(session._units)
if unit.task_key == 'index_pip_side' and unit.cycle == 1)
session.defer_current_task('rigid_pair_candidates_unresolved')
cycle = [unit for unit in session.scheduled_units if unit.cycle == 1]
assert cycle[-1].task_key == 'index_dip_side'
session._unit_index = session._units.index(cycle[-2])
assert not execution.training_check_due()
session._unit_index = session._units.index(cycle[-1])
assert execution.training_check_due()
def test_second_failure_is_terminal_and_budget_survives_restart(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2)
records = [first.as_record(stamp_ns=1, corner_evidence_sha256='a'*64)]
restored = TaskDeferralQueue(trajectory_profile)
restored.restore(records)
second = restored.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2)
assert second.state == TaskObservationState.UNOBSERVABLE
records.append(second.as_record(stamp_ns=2, corner_evidence_sha256='b'*64))
again = TaskDeferralQueue(trajectory_profile)
again.restore(records)
assert again.attempts[('index_dip_side', 0)] == 2
assert ('index_dip_side', 0) in again.unobservable
with pytest.raises(ValueError, match='budget_exhausted'):
again.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 1)
with pytest.raises(ValueError, match='budget_changed'):
again.restore([first.as_record(stamp_ns=3)])
def test_restart_restores_pending_visit_at_end_of_taskwise_stage(trajectory_profile):
profile = configure_training(trajectory_profile, TASKWISE_STAGED)
profile = select_task_training(profile, profile.motion.tasks[0].key, (0, 1, 2))
queue = TaskDeferralQueue(profile)
pending = queue.decide('index_dip_side', 0,
'image_motion_no_observable_image_model', 2).as_record(stamp_ns=1)
session = CalibrationSession(profile)
session.restore_task_deferrals([pending])
first_stage = [unit for unit in session.scheduled_units if unit.cycle == 0]
assert first_stage[-1].task_key == 'index_dip_side'
assert session.task_deferrals.attempts[('index_dip_side', 0)] == 1
# The same rule treats cycle 1 and optional cycle 2 as one training stage,
# so a restored revisit cannot jump ahead of later training or into holdout.
queue = TaskDeferralQueue(profile)
pending = queue.decide('index_dip_side', 1,
'image_motion_no_observable_image_model', 2).as_record(stamp_ns=2)
session = CalibrationSession(profile)
session.restore_task_deferrals([pending])
training_stage = [unit for unit in session.scheduled_units if unit.cycle in {1, 2}]
assert training_stage[-1].task_key == 'index_dip_side'
assert session.scheduled_units[session.scheduled_units.index(training_stage[-1]) + 1].cycle == 3
def test_deferral_audit_allows_only_declared_noncontiguous_resume_gap(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
pending = queue.decide('index_pip_side', 1,
'rigid_pair_candidates_unresolved', 4).as_record(stamp_ns=1)
keys = deferred_visit_keys(trajectory_profile, [pending])
assert ('index_pip_side', 1) in keys
assert ('index_dip_side', 1) in keys
changed = {**pending, 'affected_tasks': ['index_pip_side']}
with pytest.raises(ValueError, match='dependencies_changed'):
deferred_visit_keys(trajectory_profile, [changed])
def test_full_visit_commit_closes_crash_window_but_partial_visit_does_not(trajectory_profile):
from linkerhand_calibration.runtime.engine import CalibrationEngine
queue = TaskDeferralQueue(trajectory_profile)
pending = queue.decide('index_dip_side', 1,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1)
units = [unit for unit in CalibrationEngine(trajectory_profile).scan_units()
if (unit.task_key, unit.cycle) == ('index_dip_side', 1)]
completed = [dict(kind='scan_unit_complete', task_name=unit.task_key,
cycle=unit.cycle, direction=unit.direction, segment_key=unit.segment_key,
passed=True) for unit in units]
partial = TaskDeferralQueue(trajectory_profile)
partial.restore([pending, *completed[:-1]])
assert ('index_dip_side', 1) not in partial.resolved
restored = TaskDeferralQueue(trajectory_profile)
restored.restore([pending, *completed])
assert ('index_dip_side', 1) in restored.inferred_resolutions
def test_unobservable_record_blocks_release_before_fitting(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=1)
row = queue.decide('index_dip_side', 0,
'rigid_pair_candidates_unresolved', 2).as_record(stamp_ns=2)
with pytest.raises(ValueError, match='capture_unobservable_tasks:index_dip_side:0:unobservable'):
validate_capture_units(trajectory_profile, [first, row])
def test_resolved_revisit_clears_release_blocker_state(trajectory_profile):
queue = TaskDeferralQueue(trajectory_profile)
first = queue.decide('index_dip_side', 0, 'rigid_pair_candidates_unresolved', 2)
passed = queue.resolve('index_dip_side', 0)
assert passed is not None and passed.state == TaskObservationState.PASSED
restored = TaskDeferralQueue(trajectory_profile)
restored.restore([first.as_record(stamp_ns=1), passed.as_record(stamp_ns=2)])
assert ('index_dip_side', 0) in restored.resolved
assert not restored.unobservable
def test_v3_checkpoint_upgrade_preserves_accepted_joint_semantics():
base = configure_training(load_bundled_hand_profile('o30_right_18'), STAGED)
source = configure_preparation(base, 'occlusion_aware_witness_v2')
source = configure_observer_preparations(source, UrdfKinematicModel(SOURCE))
preserved = set(source.measurement.measurements)-{'index_dip'}
target = configure_observer_policy(source, preserved_joints=preserved,
policy=TRAJECTORY_OBSERVER)
source_yaml, target_yaml = (dump_hand_profile(profile).encode()
for profile in (source, target))
upgrade = ObserverPolicyUpgrade(source, target, source_yaml, target_yaml,
source_urdf=SOURCE)
inputs = {'profile_config_sha256': hashlib.sha256(target_yaml).hexdigest(),
'source_urdf_sha256': 'a'*64}
old_inputs = {**inputs, 'profile_config_sha256': hashlib.sha256(source_yaml).hexdigest()}
header = {'kind': 'session_start', **old_inputs,
'preparation_contract': preparation_contract(source)}
reference = {'kind': 'fixed_base_reference_locked',
'protected_hashes': {**old_inputs, 'intrinsics_sha256': 'b'*64}}
checkpoint = SimpleNamespace(header=header, fixed_reference=reference,
rows=(header, reference), joints=SimpleNamespace(
references={joint: object() for joint in preserved}, units=set()))
_, _, audit = upgrade.metadata(checkpoint, inputs, 'source.jsonl', 'c'*64)
assert audit['policy'] == 'unmeasured_observer_upgrade_v3'
assert audit['preserved_joints'] == sorted(preserved)
assert audit['observer_policy_changes'] == {
'index_dip': {'previous': 'rigid_pair_v1', 'current': TRAJECTORY_OBSERVER}}