diff --git a/README.md b/README.md index 37e5fed..d2b1b22 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ completed. logging, manifests, validation, and executable H1--H4 adapters. - `code/analysis/`: independent endpoint reconstruction and paper source-data generation. -- `code/config/experiments/`: smoke, calibration, and locked-template study - specifications. +- `code/config/experiments/`: smoke, calibration, locked simulation, and + locked-template study specifications. - `docs/calibration/`: calibration policy, traceable audit, and machine-readable freeze decisions. - `paper/exoskeleton/IEEEtran/main2.tex`: canonical manuscript source. @@ -77,16 +77,28 @@ calibration paths: contact before any network study. - `bilateral_calibration_v3_energy_challenge.json` adds a synthetic, smooth upstream energy stress only for H4; it is not eligible for H3. +- `bilateral_network_v3_screening.json` pairs nominal, symmetric-delay, + asymmetric-delay, jitter, and loss profiles across free-space and contact + trajectories using common random numbers. -All three remain pre-prototype calibration evidence and are not manuscript +All four remain pre-prototype calibration evidence and are not manuscript Results or physical-system validation. +Disjoint-root locked protocols are provided for stable contact, the active H4 +energy challenge, and the proposed-loop Stage-B network study. The network +protocol is proposed-only: even if it passes, it cannot establish superiority +over mapping baselines. + The current calibration decision is recorded in `docs/calibration/CALIBRATION_AUDIT_2026-07-27.md`. It deliberately leaves H1 and the bilateral gain/energy settings unfrozen; calibration values are not manuscript Results. The companion formula-linked workbook is `outputs/calibration-20260727/calibration_audit_2026-07-27.xlsx`. +The expanded v3 confirmation, network endpoint semantics, Stage-B calibration +outcome, and frozen locked thresholds are recorded in +`docs/calibration/V3_CONFIRMATION_AND_NETWORK_STAGE_B_2026-07-27.md`. + ## Manuscript build Compile from `paper/exoskeleton` so the `assets/` paths resolve: diff --git a/code/analysis/make_paper_artifacts.py b/code/analysis/make_paper_artifacts.py index 3ad7e83..4ca9568 100644 --- a/code/analysis/make_paper_artifacts.py +++ b/code/analysis/make_paper_artifacts.py @@ -21,6 +21,16 @@ from experiments.validate import require_valid_batch from .metrics import derive_trial_metrics +PROVENANCE_IDENTITY_FIELDS = ( + "h2_data_group_id", + "h2_data_seed_record_hash", + "bilateral_data_group_id", + "bilateral_data_seed_record_hash", + "network_pair_group_id", + "network_profile_id", +) + + def _method_selected( family: str, family_configuration: Mapping[str, Any], @@ -99,10 +109,22 @@ def _identity_row( for name, value in sorted(trial.get("factors", {}).items()): row[f"factor_{name}"] = _scalar_cell(value) metadata = {} if trial_metadata is None else trial_metadata - for name in ("h2_data_group_id", "h2_data_seed_record_hash"): + for name in PROVENANCE_IDENTITY_FIELDS: value = metadata.get(name) if value is not None: row[name] = _scalar_cell(value) + if ( + "bilateral_data_seed_record_hash" not in row + and metadata.get("bilateral_data_seed_record") is not None + ): + row["bilateral_data_seed_record_hash"] = stable_hash( + metadata["bilateral_data_seed_record"], + prefix="bilateral-data-seed-record", + ) + if "network_profile_id" not in row: + profile = trial.get("factors", {}).get("network_profile") + if isinstance(profile, Mapping) and profile.get("profile_id") is not None: + row["network_profile_id"] = _scalar_cell(profile["profile_id"]) return row @@ -117,8 +139,7 @@ def _atomic_write_csv(path: Path, rows: Sequence[Mapping[str, Any]]) -> None: "trajectory_id", "trajectory_family", "replicate", - "h2_data_group_id", - "h2_data_seed_record_hash", + *PROVENANCE_IDENTITY_FIELDS, ] all_fields = {key for row in rows for key in row} fieldnames = [name for name in identity_order if name in all_fields] @@ -143,6 +164,245 @@ def _atomic_write_csv(path: Path, rows: Sequence[Mapping[str, Any]]) -> None: raise +def _finite_metric(row: Mapping[str, Any], name: str) -> float: + if name not in row: + raise ValueError( + f"network paired gates require metric {name!r}" + ) + try: + value = float(row[name]) + except (TypeError, ValueError) as error: + raise ValueError( + f"network paired metric {name!r} must be numeric" + ) from error + if not np.isfinite(value): + raise ValueError( + f"network paired metric {name!r} must be finite" + ) + return value + + +def _boolean_metric(row: Mapping[str, Any], name: str) -> bool: + if name not in row: + raise ValueError( + f"network paired gates require metric {name!r}" + ) + value = row[name] + if isinstance(value, (bool, np.bool_)): + return bool(value) + if isinstance(value, (int, float, np.integer, np.floating)): + numeric = float(value) + if np.isfinite(numeric) and numeric in (0.0, 1.0): + return bool(numeric) + raise ValueError( + f"network paired metric {name!r} must be boolean or 0/1" + ) + + +def _apply_network_paired_gates( + rows: list[dict[str, Any]], + metric_configuration: Mapping[str, Any], +) -> None: + network_configuration = metric_configuration.get("network") + if not isinstance(network_configuration, Mapping): + return + paired = network_configuration.get("paired_gates") + if paired is None: + return + if not isinstance(paired, Mapping): + raise ValueError("network.paired_gates must be a mapping") + nominal_profile_id = paired.get("nominal_profile_id", "nominal") + if not isinstance(nominal_profile_id, str) or not nominal_profile_id: + raise ValueError( + "network.paired_gates.nominal_profile_id must be non-empty" + ) + maximum_tracking_delta = paired.get( + "maximum_tracking_rmse_delta_vs_nominal_rad", + paired.get("max_tracking_delta"), + ) + maximum_contact_relative_change = paired.get( + "maximum_abs_contact_rms_relative_change_vs_nominal", + paired.get("max_abs_contact_relative_change"), + ) + if maximum_tracking_delta is None: + raise ValueError( + "network paired gates require " + "maximum_tracking_rmse_delta_vs_nominal_rad" + ) + if maximum_contact_relative_change is None: + raise ValueError( + "network paired gates require " + "maximum_abs_contact_rms_relative_change_vs_nominal" + ) + maximum_tracking_delta = float(maximum_tracking_delta) + maximum_contact_relative_change = float( + maximum_contact_relative_change + ) + contact_denominator_epsilon_N = float( + paired.get("contact_rms_denominator_epsilon_N", 1e-12) + ) + if ( + not np.isfinite(maximum_tracking_delta) + or maximum_tracking_delta < 0.0 + or not np.isfinite(maximum_contact_relative_change) + or maximum_contact_relative_change < 0.0 + or not np.isfinite(contact_denominator_epsilon_N) + or contact_denominator_epsilon_N <= 0.0 + ): + raise ValueError( + "network paired-gate thresholds must be finite/non-negative " + "and the contact denominator epsilon must be positive" + ) + + network_rows = [ + row for row in rows if "network_local_full_gate_pass" in row + ] + groups: dict[tuple[Any, ...], list[dict[str, Any]]] = {} + for row in network_rows: + missing_identity = [ + name + for name in ("network_pair_group_id", "network_profile_id") + if row.get(name) in (None, "") + ] + if missing_identity: + raise ValueError( + "network paired gates require identity fields " + f"{missing_identity} for trial {row.get('trial_id')}" + ) + group_key = ( + row["network_pair_group_id"], + row["method_id"], + row["trajectory_id"], + row["replicate"], + ) + groups.setdefault(group_key, []).append(row) + + for group_key, group_rows in groups.items(): + nominal_rows = [ + row + for row in group_rows + if row["network_profile_id"] == nominal_profile_id + ] + if len(nominal_rows) != 1: + reason = "missing" if not nominal_rows else "duplicate" + raise ValueError( + f"{reason} nominal network profile for group {group_key}; " + f"expected exactly one {nominal_profile_id!r}" + ) + nominal = nominal_rows[0] + nominal_tracking = _finite_metric( + nominal, + "network_slave_zero_delay_tracking_rmse_rad", + ) + nominal_feedback_lag = _finite_metric( + nominal, + "network_return_feedback_lag_rmse_Nm", + ) + contact_expected = _boolean_metric( + nominal, + "network_contact_expected", + ) + nominal_contact_rms: float | None = None + contact_denominator: float | None = None + if contact_expected: + nominal_contact_rms = _finite_metric( + nominal, + "network_contact_force_rms_N", + ) + contact_denominator = max( + abs(nominal_contact_rms), + contact_denominator_epsilon_N, + ) + for row in group_rows: + if ( + _boolean_metric(row, "network_contact_expected") + != contact_expected + ): + raise ValueError( + "network paired gates require a consistent " + f"contact condition for group {group_key}" + ) + if row is nominal: + tracking_delta = 0.0 + feedback_lag_delta = 0.0 + contact_relative_change = ( + 0.0 if contact_expected else None + ) + else: + tracking_delta = ( + _finite_metric( + row, + "network_slave_zero_delay_tracking_rmse_rad", + ) + - nominal_tracking + ) + feedback_lag_delta = ( + _finite_metric( + row, + "network_return_feedback_lag_rmse_Nm", + ) + - nominal_feedback_lag + ) + contact_relative_change = ( + ( + _finite_metric( + row, + "network_contact_force_rms_N", + ) + - nominal_contact_rms + ) + / contact_denominator + if contact_expected + else None + ) + tracking_gate = bool( + tracking_delta <= maximum_tracking_delta + ) + contact_gate = ( + bool( + abs(contact_relative_change) + <= maximum_contact_relative_change + ) + if contact_expected + else None + ) + paired_gate = bool( + tracking_gate + and (contact_gate if contact_expected else True) + ) + row.update( + { + ( + "network_slave_zero_delay_tracking_rmse_" + "delta_vs_nominal_rad" + ): tracking_delta, + ( + "network_zero_delay_tracking_rmse_" + "delta_vs_nominal_rad" + ): tracking_delta, + ( + "network_return_feedback_lag_rmse_" + "delta_vs_nominal_Nm" + ): feedback_lag_delta, + ( + "network_contact_rms_relative_change_" + "vs_nominal" + ): contact_relative_change, + ( + "network_contact_force_rms_relative_change_" + "vs_nominal" + ): contact_relative_change, + "network_paired_tracking_gate_pass": tracking_gate, + "network_paired_contact_gate_pass": contact_gate, + "network_paired_gate_pass": paired_gate, + "network_full_gate_pass": bool( + row["network_local_full_gate_pass"] + and paired_gate + ), + } + ) + + def generate_paper_source_data( batch_dir: Path, metric_configuration: Mapping[str, Any], @@ -194,6 +454,7 @@ def generate_paper_source_data( } ) + _apply_network_paired_gates(rows, metric_configuration) metric_path = derived_dir / "trial_metrics.jsonl" atomic_write_jsonl(metric_path, rows) table_files: dict[str, str] = {} @@ -202,10 +463,7 @@ def generate_paper_source_data( family_rows: list[dict[str, Any]] = [] prefix = f"{family}_" for row in rows: - provenance_identity_fields = { - "h2_data_group_id", - "h2_data_seed_record_hash", - } + provenance_identity_fields = set(PROVENANCE_IDENTITY_FIELDS) if not any( key.startswith(prefix) and key not in provenance_identity_fields @@ -224,8 +482,7 @@ def generate_paper_source_data( "trajectory_id", "trajectory_family", "replicate", - "h2_data_group_id", - "h2_data_seed_record_hash", + *PROVENANCE_IDENTITY_FIELDS, } or key.startswith("factor_") or key.startswith(prefix) diff --git a/code/analysis/metrics.py b/code/analysis/metrics.py index 2c2d4ab..6ebd00b 100644 --- a/code/analysis/metrics.py +++ b/code/analysis/metrics.py @@ -12,7 +12,7 @@ from typing import Any, Mapping import numpy as np -METRIC_SCHEMA_VERSION = "1.2.0" +METRIC_SCHEMA_VERSION = "1.3.0" class MetricError(ValueError): @@ -1271,6 +1271,602 @@ def compute_bilateral_diagnostics( } +_PACKET_EMPTY = 0 +_PACKET_ACTIVE = 1 +_PACKET_HELD = 2 +_PACKET_TIMED_OUT = 3 +_PACKET_RECOVERING = 4 + + +def _packet_state_vector(value: Any, name: str) -> np.ndarray: + raw = _vector(value, name) + _finite(raw, name) + if not np.all(raw == np.floor(raw)) or not np.all( + (raw >= _PACKET_EMPTY) & (raw <= _PACKET_RECOVERING) + ): + raise MetricError(f"{name} must contain only integer states 0..4") + return raw.astype(np.int64) + + +def _constant_sample_value( + value: Any, + name: str, + sample_count: int, +) -> tuple[np.ndarray, float]: + raw = np.asarray(value, dtype=float) + if raw.ndim == 0: + array = np.full(sample_count, float(raw)) + else: + array = _vector(raw, name) + if array.shape[0] != sample_count: + raise MetricError( + f"{name} has {array.shape[0]} samples; expected {sample_count}" + ) + _finite(array, name) + constant = float(array[0]) + if not np.allclose(array, constant, rtol=0.0, atol=1e-15): + raise MetricError(f"{name} must be constant within one trial") + return array, constant + + +def _packet_stream_diagnostics( + *, + direction: str, + state: Any, + active: Any, + fresh: Any, + age: Any, + sequence: Any, +) -> dict[str, Any]: + state_name = f"{direction}_packet_state" + active_name = f"{direction}_packet_active" + fresh_name = f"{direction}_packet_fresh" + age_name = f"{direction}_packet_age" + sequence_name = f"{direction}_packet_seq" + states = _packet_state_vector(state, state_name) + active_flags = _boolean_vector(active, active_name) + fresh_flags = _boolean_vector(fresh, fresh_name) + ages = _vector(age, age_name) + sequences = _vector(sequence, sequence_name) + _same_rows( + { + state_name: states, + active_name: active_flags, + fresh_name: fresh_flags, + age_name: ages, + sequence_name: sequences, + } + ) + + expected_active = np.isin( + states, + (_PACKET_ACTIVE, _PACKET_HELD, _PACKET_RECOVERING), + ) + expected_fresh = np.isin( + states, + (_PACKET_ACTIVE, _PACKET_RECOVERING), + ) + if not np.array_equal(active_flags, expected_active): + raise MetricError( + f"{active_name} disagrees with {state_name}; active states are " + "ACTIVE, HELD, and RECOVERING" + ) + if not np.array_equal(fresh_flags, expected_fresh): + raise MetricError( + f"{fresh_name} disagrees with {state_name}; fresh states are " + "ACTIVE and RECOVERING" + ) + + if np.any( + active_flags + & (~np.isfinite(ages) | (ages < 0.0)) + ): + raise MetricError( + f"{age_name} must be finite and non-negative while active" + ) + if np.any(~active_flags & ~np.isnan(ages)): + raise MetricError(f"{age_name} must be NaN while inactive") + + active_sequences = sequences[active_flags] + if np.any( + ~np.isfinite(active_sequences) + | (active_sequences < 0.0) + | (active_sequences != np.floor(active_sequences)) + ): + raise MetricError( + f"{sequence_name} must be a finite non-negative integer while active" + ) + inactive_sequences = sequences[~active_flags] + inactive_sequence_valid = np.isnan(inactive_sequences) | ( + inactive_sequences == -1.0 + ) + if not np.all(inactive_sequence_valid): + raise MetricError( + f"{sequence_name} must be -1 or NaN while inactive" + ) + + previous_sequence: int | None = None + for index in np.flatnonzero(active_flags): + current_sequence = int(sequences[index]) + if previous_sequence is None: + if states[index] == _PACKET_HELD: + raise MetricError( + f"{sequence_name} cannot start with a held packet" + ) + elif fresh_flags[index]: + if current_sequence <= previous_sequence: + raise MetricError( + f"{sequence_name} must strictly increase on fresh packets" + ) + elif current_sequence != previous_sequence: + raise MetricError( + f"{sequence_name} must remain constant while a packet is held" + ) + previous_sequence = current_sequence + + fresh_sequences = sequences[fresh_flags].astype(np.int64) + if fresh_sequences.size: + internal_span = int( + fresh_sequences[-1] - fresh_sequences[0] + 1 + ) + internal_missing_count = internal_span - int(fresh_sequences.size) + internal_missing_fraction = ( + float(internal_missing_count / internal_span) + if internal_span > 0 + else 0.0 + ) + else: + internal_span = 0 + internal_missing_count = 0 + internal_missing_fraction = 0.0 + + active_ages = ages[active_flags] + age_statistics = { + percentile: ( + float(np.percentile(active_ages, quantile)) + if active_ages.size + else None + ) + for percentile, quantile in ( + ("p50", 50), + ("p95", 95), + ("p99", 99), + ("max", 100), + ) + } + prefix = f"network_{direction}" + result = { + f"{prefix}_active_fraction": float(np.mean(active_flags)), + f"{prefix}_fresh_fraction": float(np.mean(fresh_flags)), + f"{prefix}_empty_fraction": float( + np.mean(states == _PACKET_EMPTY) + ), + f"{prefix}_held_fraction": float( + np.mean(states == _PACKET_HELD) + ), + f"{prefix}_timeout_fraction": float( + np.mean(states == _PACKET_TIMED_OUT) + ), + f"{prefix}_recovering_fraction": float( + np.mean(states == _PACKET_RECOVERING) + ), + f"{prefix}_fresh_packet_count": int(fresh_sequences.size), + f"{prefix}_internal_sequence_span": internal_span, + f"{prefix}_internal_missing_count": internal_missing_count, + f"{prefix}_internal_missing_fraction": ( + internal_missing_fraction + ), + } + for percentile, statistic in age_statistics.items(): + result[f"{prefix}_packet_age_{percentile}_s"] = statistic + # Keep the shorter spelling as an explicit alias for table consumers. + result[f"{prefix}_age_{percentile}_s"] = statistic + return result + + +def compute_network_diagnostics( + *, + forward_packet_state: Any, + return_packet_state: Any, + forward_packet_active: Any, + return_packet_active: Any, + forward_packet_fresh: Any, + return_packet_fresh: Any, + forward_packet_age: Any, + return_packet_age: Any, + forward_packet_seq: Any, + return_packet_seq: Any, + slave_zero_delay_tracking_error: Any, + slave_reference_lag_error: Any, + return_feedback_lag_error: Any, + contact_force_norm: Any, + contact_expected: Any, + configured_forward_delay_s: Any, + configured_return_delay_s: Any, + configured_forward_jitter_s: Any, + configured_return_jitter_s: Any, + configured_forward_packet_loss: Any, + configured_return_packet_loss: Any, + configured_forward_timeout_s: Any, + configured_return_timeout_s: Any, + contact_force_threshold_N: float = 1e-6, + maximum_zero_delay_tracking_rmse_rad: float | None = None, + minimum_active_fraction: float = 0.0, + minimum_forward_active_fraction: float | None = None, + minimum_return_active_fraction: float | None = None, + minimum_forward_fresh_fraction: float = 0.0, + minimum_return_fresh_fraction: float = 0.0, + maximum_forward_internal_missing_fraction: float = 1.0, + maximum_return_internal_missing_fraction: float = 1.0, + maximum_timeout_fraction: float = 1.0, + maximum_forward_timeout_fraction: float | None = None, + maximum_return_timeout_fraction: float | None = None, + maximum_age_s: float | None = None, + maximum_forward_age_s: float | None = None, + maximum_return_age_s: float | None = None, + minimum_contact_fraction: float = 0.0, + minimum_contact_rms_N: float = 0.0, + maximum_free_space_contact_peak_N: float | None = None, +) -> dict[str, Any]: + """Audit packet-state evidence and compute independent Stage-B endpoints.""" + zero_delay_error = _vector( + slave_zero_delay_tracking_error, + "slave_zero_delay_tracking_error", + ) + reference_lag_error = _vector( + slave_reference_lag_error, + "slave_reference_lag_error", + ) + feedback_lag_error = _vector( + return_feedback_lag_error, + "return_feedback_lag_error", + ) + contact_force = _vector(contact_force_norm, "contact_force_norm") + expected_contact = _boolean_vector( + contact_expected, "contact_expected" + ) + sample_count = _same_rows( + { + "slave_zero_delay_tracking_error": zero_delay_error, + "slave_reference_lag_error": reference_lag_error, + "return_feedback_lag_error": feedback_lag_error, + "contact_force_norm": contact_force, + "contact_expected": expected_contact, + } + ) + if not np.all(expected_contact == expected_contact[0]): + raise MetricError("contact_expected must be constant within one trial") + for array, name in ( + (zero_delay_error, "slave_zero_delay_tracking_error"), + (reference_lag_error, "slave_reference_lag_error"), + (feedback_lag_error, "return_feedback_lag_error"), + (contact_force, "contact_force_norm"), + ): + _finite(array, name) + if np.any(array < 0.0): + raise MetricError(f"{name} must be non-negative") + + forward_metrics = _packet_stream_diagnostics( + direction="forward", + state=forward_packet_state, + active=forward_packet_active, + fresh=forward_packet_fresh, + age=forward_packet_age, + sequence=forward_packet_seq, + ) + return_metrics = _packet_stream_diagnostics( + direction="return", + state=return_packet_state, + active=return_packet_active, + fresh=return_packet_fresh, + age=return_packet_age, + sequence=return_packet_seq, + ) + packet_lengths = { + "forward_packet_state": np.asarray(forward_packet_state), + "return_packet_state": np.asarray(return_packet_state), + "forward_packet_active": np.asarray(forward_packet_active), + "return_packet_active": np.asarray(return_packet_active), + } + if any(array.shape[0] != sample_count for array in packet_lengths.values()): + raise MetricError( + "network packet-state arrays must match tracking sample count" + ) + + configured_inputs = { + "forward_delay_s": configured_forward_delay_s, + "return_delay_s": configured_return_delay_s, + "forward_jitter_s": configured_forward_jitter_s, + "return_jitter_s": configured_return_jitter_s, + "forward_packet_loss": configured_forward_packet_loss, + "return_packet_loss": configured_return_packet_loss, + "forward_timeout_s": configured_forward_timeout_s, + "return_timeout_s": configured_return_timeout_s, + } + configured: dict[str, float] = {} + for name, value in configured_inputs.items(): + _, configured[name] = _constant_sample_value( + value, + f"configured_{name}", + sample_count, + ) + for name in ( + "forward_delay_s", + "return_delay_s", + "forward_jitter_s", + "return_jitter_s", + ): + if configured[name] < 0.0: + raise MetricError(f"configured_{name} must be non-negative") + for name in ("forward_packet_loss", "return_packet_loss"): + if not 0.0 <= configured[name] <= 1.0: + raise MetricError(f"configured_{name} must lie in [0, 1]") + for name in ("forward_timeout_s", "return_timeout_s"): + if configured[name] <= 0.0: + raise MetricError(f"configured_{name} must be positive") + + contact_force_threshold_N = float(contact_force_threshold_N) + if ( + not np.isfinite(contact_force_threshold_N) + or contact_force_threshold_N < 0.0 + ): + raise MetricError( + "contact_force_threshold_N must be finite and non-negative" + ) + + minimum_forward_active_fraction = float( + minimum_active_fraction + if minimum_forward_active_fraction is None + else minimum_forward_active_fraction + ) + minimum_return_active_fraction = float( + minimum_active_fraction + if minimum_return_active_fraction is None + else minimum_return_active_fraction + ) + maximum_forward_timeout_fraction = float( + maximum_timeout_fraction + if maximum_forward_timeout_fraction is None + else maximum_forward_timeout_fraction + ) + maximum_return_timeout_fraction = float( + maximum_timeout_fraction + if maximum_return_timeout_fraction is None + else maximum_return_timeout_fraction + ) + maximum_forward_age_s = ( + maximum_age_s + if maximum_forward_age_s is None + else maximum_forward_age_s + ) + maximum_return_age_s = ( + maximum_age_s + if maximum_return_age_s is None + else maximum_return_age_s + ) + fraction_thresholds = { + "minimum_forward_active_fraction": ( + minimum_forward_active_fraction + ), + "minimum_return_active_fraction": ( + minimum_return_active_fraction + ), + "minimum_forward_fresh_fraction": float( + minimum_forward_fresh_fraction + ), + "minimum_return_fresh_fraction": float( + minimum_return_fresh_fraction + ), + "maximum_forward_internal_missing_fraction": float( + maximum_forward_internal_missing_fraction + ), + "maximum_return_internal_missing_fraction": float( + maximum_return_internal_missing_fraction + ), + "maximum_forward_timeout_fraction": ( + maximum_forward_timeout_fraction + ), + "maximum_return_timeout_fraction": ( + maximum_return_timeout_fraction + ), + "minimum_contact_fraction": float(minimum_contact_fraction), + } + if any( + not np.isfinite(value) or not 0.0 <= value <= 1.0 + for value in fraction_thresholds.values() + ): + raise MetricError("network fraction gates must lie in [0, 1]") + nonnegative_optional_thresholds = { + "maximum_zero_delay_tracking_rmse_rad": ( + maximum_zero_delay_tracking_rmse_rad + ), + "maximum_forward_age_s": maximum_forward_age_s, + "maximum_return_age_s": maximum_return_age_s, + "maximum_free_space_contact_peak_N": ( + maximum_free_space_contact_peak_N + ), + } + for name, value in nonnegative_optional_thresholds.items(): + if value is not None and ( + not np.isfinite(float(value)) or float(value) < 0.0 + ): + raise MetricError(f"{name} must be finite and non-negative") + minimum_contact_rms_N = float(minimum_contact_rms_N) + if ( + not np.isfinite(minimum_contact_rms_N) + or minimum_contact_rms_N < 0.0 + ): + raise MetricError( + "minimum_contact_rms_N must be finite and non-negative" + ) + + zero_delay_rmse = float( + np.sqrt(np.mean(np.square(zero_delay_error))) + ) + reference_lag_rmse = float( + np.sqrt(np.mean(np.square(reference_lag_error))) + ) + feedback_lag_rmse = float( + np.sqrt(np.mean(np.square(feedback_lag_error))) + ) + contact_mask = contact_force > contact_force_threshold_N + contact_fraction = float(np.mean(contact_mask)) + contact_rms_all = float( + np.sqrt(np.mean(np.square(contact_force))) + ) + contact_rms = ( + float(np.sqrt(np.mean(np.square(contact_force[contact_mask])))) + if np.any(contact_mask) + else 0.0 + ) + contact_peak = float(np.max(contact_force)) + contact_is_expected = bool(expected_contact[0]) + + forward_active_gate = bool( + forward_metrics["network_forward_active_fraction"] + >= minimum_forward_active_fraction + ) + return_active_gate = bool( + return_metrics["network_return_active_fraction"] + >= minimum_return_active_fraction + ) + forward_fresh_gate = bool( + forward_metrics["network_forward_fresh_fraction"] + >= float(minimum_forward_fresh_fraction) + ) + return_fresh_gate = bool( + return_metrics["network_return_fresh_fraction"] + >= float(minimum_return_fresh_fraction) + ) + forward_internal_missing_gate = bool( + forward_metrics["network_forward_internal_missing_fraction"] + <= float(maximum_forward_internal_missing_fraction) + ) + return_internal_missing_gate = bool( + return_metrics["network_return_internal_missing_fraction"] + <= float(maximum_return_internal_missing_fraction) + ) + forward_timeout_gate = bool( + forward_metrics["network_forward_timeout_fraction"] + <= maximum_forward_timeout_fraction + ) + return_timeout_gate = bool( + return_metrics["network_return_timeout_fraction"] + <= maximum_return_timeout_fraction + ) + + def age_gate(direction: str, maximum: float | None) -> bool: + if maximum is None: + return True + age_max = forward_metrics[ + f"network_{direction}_packet_age_max_s" + ] if direction == "forward" else return_metrics[ + f"network_{direction}_packet_age_max_s" + ] + return age_max is not None and age_max <= float(maximum) + + forward_age_gate = age_gate("forward", maximum_forward_age_s) + return_age_gate = age_gate("return", maximum_return_age_s) + zero_delay_gate = bool( + maximum_zero_delay_tracking_rmse_rad is None + or zero_delay_rmse + <= float(maximum_zero_delay_tracking_rmse_rad) + ) + contact_fraction_gate = bool( + not contact_is_expected + or contact_fraction >= float(minimum_contact_fraction) + ) + contact_rms_gate = bool( + not contact_is_expected + or contact_rms >= minimum_contact_rms_N + ) + free_space_peak_gate = bool( + contact_is_expected + or maximum_free_space_contact_peak_N is None + or contact_peak <= float(maximum_free_space_contact_peak_N) + ) + contact_condition_gate = bool( + contact_fraction_gate + and contact_rms_gate + and free_space_peak_gate + ) + local_gate = bool( + zero_delay_gate + and forward_active_gate + and return_active_gate + and forward_fresh_gate + and return_fresh_gate + and forward_internal_missing_gate + and return_internal_missing_gate + and forward_timeout_gate + and return_timeout_gate + and forward_age_gate + and return_age_gate + and contact_condition_gate + ) + return { + **forward_metrics, + **return_metrics, + **{ + f"network_configured_{name}": value + for name, value in configured.items() + }, + "network_contact_expected": contact_is_expected, + "network_slave_zero_delay_tracking_rmse_rad": zero_delay_rmse, + "network_slave_zero_delay_tracking_p95_rad": float( + np.percentile(zero_delay_error, 95) + ), + "network_slave_zero_delay_tracking_max_rad": float( + np.max(zero_delay_error) + ), + "network_zero_delay_tracking_rmse_rad": zero_delay_rmse, + "network_zero_delay_tracking_p95_rad": float( + np.percentile(zero_delay_error, 95) + ), + "network_zero_delay_tracking_max_rad": float( + np.max(zero_delay_error) + ), + "network_slave_reference_lag_rmse_rad": reference_lag_rmse, + "network_slave_reference_lag_p95_rad": float( + np.percentile(reference_lag_error, 95) + ), + "network_slave_reference_lag_max_rad": float( + np.max(reference_lag_error) + ), + "network_return_feedback_lag_rmse_Nm": feedback_lag_rmse, + "network_return_feedback_lag_p95_Nm": float( + np.percentile(feedback_lag_error, 95) + ), + "network_return_feedback_lag_max_Nm": float( + np.max(feedback_lag_error) + ), + "network_contact_fraction": contact_fraction, + "network_contact_force_rms_N": contact_rms, + "network_contact_force_rms_all_samples_N": contact_rms_all, + "network_contact_force_peak_N": contact_peak, + "network_zero_delay_tracking_gate_pass": zero_delay_gate, + "network_forward_active_gate_pass": forward_active_gate, + "network_return_active_gate_pass": return_active_gate, + "network_forward_fresh_gate_pass": forward_fresh_gate, + "network_return_fresh_gate_pass": return_fresh_gate, + "network_forward_internal_missing_gate_pass": ( + forward_internal_missing_gate + ), + "network_return_internal_missing_gate_pass": ( + return_internal_missing_gate + ), + "network_forward_timeout_gate_pass": forward_timeout_gate, + "network_return_timeout_gate_pass": return_timeout_gate, + "network_forward_age_gate_pass": forward_age_gate, + "network_return_age_gate_pass": return_age_gate, + "network_contact_fraction_gate_pass": contact_fraction_gate, + "network_contact_rms_gate_pass": contact_rms_gate, + "network_free_space_peak_gate_pass": free_space_peak_gate, + "network_contact_condition_gate_pass": contact_condition_gate, + "network_local_gate_pass": local_gate, + } + + def _field( samples: Mapping[str, Any], fields: Mapping[str, str], @@ -2042,6 +2638,246 @@ def derive_trial_metrics( ), ) ) + elif family == "network": + required_upstream_metrics = ( + "h4_energy_audit_pass", + "bilateral_force_limit_gate_pass", + "bilateral_force_headroom_gate_pass", + "bilateral_master_tracking_gate_pass", + "bilateral_slave_tracking_gate_pass", + "bilateral_projection_gate_pass", + "bilateral_limit_active_gate_pass", + ) + missing_upstream_metrics = [ + name + for name in required_upstream_metrics + if name not in result + ] + if missing_upstream_metrics: + raise MetricError( + "network metrics must follow H4 and bilateral metrics; " + f"missing={missing_upstream_metrics}" + ) + gate_config = family_config.get("gates", {}) + if not isinstance(gate_config, Mapping): + raise MetricError("network gates must be a mapping") + network_metrics = compute_network_diagnostics( + forward_packet_state=_field( + samples, + fields, + "forward_packet_state", + "forward_packet_state", + ), + return_packet_state=_field( + samples, + fields, + "return_packet_state", + "return_packet_state", + ), + forward_packet_active=_field( + samples, + fields, + "forward_packet_active", + "forward_packet_active", + ), + return_packet_active=_field( + samples, + fields, + "return_packet_active", + "return_packet_active", + ), + forward_packet_fresh=_field( + samples, + fields, + "forward_packet_fresh", + "forward_packet_fresh", + ), + return_packet_fresh=_field( + samples, + fields, + "return_packet_fresh", + "return_packet_fresh", + ), + forward_packet_age=_field( + samples, + fields, + "forward_packet_age", + "forward_packet_age", + ), + return_packet_age=_field( + samples, + fields, + "return_packet_age", + "return_packet_age", + ), + forward_packet_seq=_field( + samples, + fields, + "forward_packet_seq", + "forward_packet_seq", + ), + return_packet_seq=_field( + samples, + fields, + "return_packet_seq", + "return_packet_seq", + ), + slave_zero_delay_tracking_error=_field( + samples, + fields, + "slave_zero_delay_tracking_error", + "slave_zero_delay_tracking_error", + ), + slave_reference_lag_error=_field( + samples, + fields, + "slave_reference_lag_error", + "slave_reference_lag_error", + ), + return_feedback_lag_error=_field( + samples, + fields, + "return_feedback_lag_error", + "return_feedback_lag_error", + ), + contact_force_norm=_field( + samples, + fields, + "contact_force_norm", + "contact_force_norm", + ), + contact_expected=_field( + samples, + fields, + "contact_expected", + "contact_expected", + ), + configured_forward_delay_s=_field( + samples, + fields, + "configured_forward_delay_s", + "configured_forward_delay_s", + ), + configured_return_delay_s=_field( + samples, + fields, + "configured_return_delay_s", + "configured_return_delay_s", + ), + configured_forward_jitter_s=_field( + samples, + fields, + "configured_forward_jitter_s", + "configured_forward_jitter_s", + ), + configured_return_jitter_s=_field( + samples, + fields, + "configured_return_jitter_s", + "configured_return_jitter_s", + ), + configured_forward_packet_loss=_field( + samples, + fields, + "configured_forward_packet_loss", + "configured_forward_packet_loss", + ), + configured_return_packet_loss=_field( + samples, + fields, + "configured_return_packet_loss", + "configured_return_packet_loss", + ), + configured_forward_timeout_s=_field( + samples, + fields, + "configured_forward_timeout_s", + "configured_forward_timeout_s", + ), + configured_return_timeout_s=_field( + samples, + fields, + "configured_return_timeout_s", + "configured_return_timeout_s", + ), + contact_force_threshold_N=family_config.get( + "contact_force_threshold_N", 1e-6 + ), + maximum_zero_delay_tracking_rmse_rad=gate_config.get( + "maximum_slave_zero_delay_tracking_rmse_rad", + gate_config.get( + "maximum_zero_delay_tracking_rmse_rad" + ), + ), + minimum_active_fraction=gate_config.get( + "minimum_active_fraction", 0.0 + ), + minimum_forward_active_fraction=gate_config.get( + "minimum_forward_active_fraction" + ), + minimum_return_active_fraction=gate_config.get( + "minimum_return_active_fraction" + ), + minimum_forward_fresh_fraction=gate_config.get( + "minimum_forward_fresh_fraction", 0.0 + ), + minimum_return_fresh_fraction=gate_config.get( + "minimum_return_fresh_fraction", 0.0 + ), + maximum_forward_internal_missing_fraction=gate_config.get( + "maximum_forward_internal_missing_fraction", 1.0 + ), + maximum_return_internal_missing_fraction=gate_config.get( + "maximum_return_internal_missing_fraction", 1.0 + ), + maximum_timeout_fraction=gate_config.get( + "maximum_timeout_fraction", 1.0 + ), + maximum_forward_timeout_fraction=gate_config.get( + "maximum_forward_timeout_fraction" + ), + maximum_return_timeout_fraction=gate_config.get( + "maximum_return_timeout_fraction" + ), + maximum_age_s=gate_config.get("maximum_age_s"), + maximum_forward_age_s=gate_config.get( + "maximum_forward_packet_age_s", + gate_config.get("maximum_forward_age_s"), + ), + maximum_return_age_s=gate_config.get( + "maximum_return_packet_age_s", + gate_config.get("maximum_return_age_s"), + ), + minimum_contact_fraction=gate_config.get( + "minimum_contact_fraction", 0.0 + ), + minimum_contact_rms_N=gate_config.get( + "minimum_contact_rms_N", 0.0 + ), + maximum_free_space_contact_peak_N=gate_config.get( + "maximum_free_space_contact_force_N", + gate_config.get( + "maximum_free_space_contact_peak_N" + ), + ), + ) + upstream_gate_pass = all( + bool(result[name]) for name in required_upstream_metrics + ) + network_metrics.update( + { + "network_h4_bilateral_gate_pass": ( + upstream_gate_pass + ), + "network_local_full_gate_pass": bool( + upstream_gate_pass + and network_metrics[ + "network_local_gate_pass" + ] + ), + } + ) + result.update(network_metrics) elif family == "energy_challenge": required_metrics = ( "h4_energy_audit_pass", diff --git a/code/config/experiments/bilateral_locked_v3_energy_challenge.json b/code/config/experiments/bilateral_locked_v3_energy_challenge.json new file mode 100644 index 0000000..1a12eef --- /dev/null +++ b/code/config/experiments/bilateral_locked_v3_energy_challenge.json @@ -0,0 +1,81 @@ +{ + "study_id": "g0c_bilateral_locked_v3_energy_challenge", + "split": "locked", + "root_seed": 2026072742, + "replicates": 20, + "methods": [ + "proposed_energy" + ], + "trajectories": [ + { + "id": "slow_contact_supervisor_stress_locked", + "family": "contact_roundtrip", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + } + ], + "factors": { + "bilateral_data_root_seed": [ + 2026072743 + ], + "stable_contact_selection": [ + { + "source_study_id": "g0c_bilateral_locked_v3_stable_contact", + "environment_profile_id": "locked_k3200", + "haptic_profile_id": "locked_gain_035_wide_tank", + "required_gate": "bilateral_stable_contact_gate_pass", + "selection_status": "locked_protocol" + } + ], + "map_policy": [ + "source_stamped" + ], + "environment_profile": [ + { + "profile_id": "locked_mechanics_k3200", + "duration": 6.0, + "mapping_hz": 50.0, + "wall_fraction": 0.5, + "stiffness": 3200.0, + "damping": 25.0, + "force_limit": 20.0, + "transition_depth": 0.001, + "probe_fraction": 0.0, + "contact_probe_cycles": 0.0 + } + ], + "haptic_profile": [ + { + "profile_id": "locked_velocity_aligned_generalized_stress", + "feedback_strength": 0.35, + "energy_min": 0.0, + "energy_max": 0.002, + "energy_initial": 0.002, + "energy_probe_mode": "velocity_aligned_generalized", + "energy_probe_torque_Nm": 0.0395, + "energy_probe_start_fraction": 0.2, + "energy_probe_end_fraction": 0.4 + } + ], + "forward_delay_s": [ + 0.0 + ], + "return_delay_s": [ + 0.0 + ], + "forward_jitter_s": [ + 0.0 + ], + "return_jitter_s": [ + 0.0 + ], + "forward_packet_loss": [ + 0.0 + ], + "return_packet_loss": [ + 0.0 + ] + }, + "h3_eligible": false, + "status": "Locked 20-replicate synthetic H4 stress with a disjoint data root; the upstream generalized probe remains ineligible for H3 and is not a physical environment input" +} diff --git a/code/config/experiments/bilateral_locked_v3_stable_contact.json b/code/config/experiments/bilateral_locked_v3_stable_contact.json new file mode 100644 index 0000000..0c582e2 --- /dev/null +++ b/code/config/experiments/bilateral_locked_v3_stable_contact.json @@ -0,0 +1,69 @@ +{ + "study_id": "g0c_bilateral_locked_v3_stable_contact", + "split": "locked", + "root_seed": 2026072741, + "replicates": 20, + "methods": [ + "proposed_energy" + ], + "trajectories": [ + { + "id": "slow_contact_approach_locked", + "family": "contact_roundtrip", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + } + ], + "factors": { + "bilateral_data_root_seed": [ + 2026072743 + ], + "map_policy": [ + "source_stamped" + ], + "environment_profile": [ + { + "profile_id": "locked_k3200", + "duration": 6.0, + "mapping_hz": 50.0, + "wall_fraction": 0.5, + "stiffness": 3200.0, + "damping": 25.0, + "force_limit": 20.0, + "transition_depth": 0.001, + "probe_fraction": 0.0, + "contact_probe_cycles": 0.0 + } + ], + "haptic_profile": [ + { + "profile_id": "locked_gain_035_wide_tank", + "feedback_strength": 0.35, + "energy_min": 0.0, + "energy_max": 2.0, + "energy_initial": 1.0, + "energy_probe_mode": "none", + "energy_probe_torque_Nm": 0.0 + } + ], + "forward_delay_s": [ + 0.0 + ], + "return_delay_s": [ + 0.0 + ], + "forward_jitter_s": [ + 0.0 + ], + "return_jitter_s": [ + 0.0 + ], + "forward_packet_loss": [ + 0.0 + ], + "return_packet_loss": [ + 0.0 + ] + }, + "status": "Locked 20-replicate simulation confirmation with a data root disjoint from the v3 calibration grid; physical and human-subject claims remain out of scope" +} diff --git a/code/config/experiments/bilateral_network_v3_locked.json b/code/config/experiments/bilateral_network_v3_locked.json new file mode 100644 index 0000000..4e85d06 --- /dev/null +++ b/code/config/experiments/bilateral_network_v3_locked.json @@ -0,0 +1,129 @@ +{ + "study_id": "g0c_bilateral_network_v3_locked", + "split": "locked", + "root_seed": 2026072761, + "replicates": 20, + "methods": [ + "proposed_energy" + ], + "trajectories": [ + { + "id": "free_space_network_roundtrip_locked", + "family": "free_space", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + }, + { + "id": "slow_contact_network_roundtrip_locked", + "family": "contact_roundtrip", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + } + ], + "factors": { + "bilateral_data_root_seed": [ + 2026072763 + ], + "bilateral_pair_network_profiles": [ + true + ], + "network_common_random_numbers": [ + true + ], + "stable_contact_selection": [ + { + "source_study_id": "g0c_bilateral_locked_v3_stable_contact", + "environment_profile_id": "locked_k3200", + "haptic_profile_id": "locked_gain_035_wide_tank", + "required_gate": "bilateral_stable_contact_gate_pass", + "selection_status": "requires_locked_stage_a_pass" + } + ], + "map_policy": [ + "source_stamped" + ], + "environment_profile": [ + { + "profile_id": "network_locked_k3200", + "duration": 6.0, + "mapping_hz": 50.0, + "wall_fraction": 0.5, + "stiffness": 3200.0, + "damping": 25.0, + "force_limit": 20.0, + "transition_depth": 0.001, + "probe_fraction": 0.0, + "contact_probe_cycles": 0.0 + } + ], + "haptic_profile": [ + { + "profile_id": "network_locked_gain_035_wide_tank", + "feedback_strength": 0.35, + "energy_min": 0.0, + "energy_max": 2.0, + "energy_initial": 1.0, + "energy_probe_mode": "none", + "energy_probe_torque_Nm": 0.0 + } + ], + "network_profile": [ + { + "profile_id": "nominal", + "forward_delay_s": 0.0, + "return_delay_s": 0.0, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_delay_40ms", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "asymmetric_delay_20_60ms", + "forward_delay_s": 0.02, + "return_delay_s": 0.06, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_40ms_jitter_4ms", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.004, + "return_jitter_s": 0.004, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_40ms_loss_2pct", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.02, + "return_packet_loss": 0.02, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + } + ] + }, + "status": "Locked proposed-method Stage-B simulation confirmation with disjoint seeds; it supports no comparative superiority claim and emulates only IID delay, uniform jitter, and IID loss" +} diff --git a/code/config/experiments/bilateral_network_v3_screening.json b/code/config/experiments/bilateral_network_v3_screening.json new file mode 100644 index 0000000..a06d18c --- /dev/null +++ b/code/config/experiments/bilateral_network_v3_screening.json @@ -0,0 +1,129 @@ +{ + "study_id": "g0c_bilateral_network_v3_screening", + "split": "calibration", + "root_seed": 2026072751, + "replicates": 3, + "methods": [ + "proposed_energy" + ], + "trajectories": [ + { + "id": "free_space_network_roundtrip", + "family": "free_space", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + }, + { + "id": "slow_contact_network_roundtrip", + "family": "contact_roundtrip", + "duration_s": 6.0, + "contact_probe_fraction": 0.0 + } + ], + "factors": { + "bilateral_data_root_seed": [ + 2026072753 + ], + "bilateral_pair_network_profiles": [ + true + ], + "network_common_random_numbers": [ + true + ], + "stable_contact_selection": [ + { + "source_study_id": "g0c_bilateral_calibration_v3_stable_contact", + "environment_profile_id": "stable_candidate_k3200", + "haptic_profile_id": "gain_035_wide_tank", + "required_gate": "bilateral_stable_contact_gate_pass", + "selection_status": "expanded_20_seed_calibration_pass" + } + ], + "map_policy": [ + "source_stamped" + ], + "environment_profile": [ + { + "profile_id": "network_screen_k3200", + "duration": 6.0, + "mapping_hz": 50.0, + "wall_fraction": 0.5, + "stiffness": 3200.0, + "damping": 25.0, + "force_limit": 20.0, + "transition_depth": 0.001, + "probe_fraction": 0.0, + "contact_probe_cycles": 0.0 + } + ], + "haptic_profile": [ + { + "profile_id": "network_screen_gain_035_wide_tank", + "feedback_strength": 0.35, + "energy_min": 0.0, + "energy_max": 2.0, + "energy_initial": 1.0, + "energy_probe_mode": "none", + "energy_probe_torque_Nm": 0.0 + } + ], + "network_profile": [ + { + "profile_id": "nominal", + "forward_delay_s": 0.0, + "return_delay_s": 0.0, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_delay_40ms", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "asymmetric_delay_20_60ms", + "forward_delay_s": 0.02, + "return_delay_s": 0.06, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_40ms_jitter_4ms", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.004, + "return_jitter_s": 0.004, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + }, + { + "profile_id": "symmetric_40ms_loss_2pct", + "forward_delay_s": 0.04, + "return_delay_s": 0.04, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.02, + "return_packet_loss": 0.02, + "forward_timeout_s": 0.2, + "return_timeout_s": 0.2 + } + ] + }, + "status": "Three-seed paired Stage-B calibration only: deterministic IID delay/jitter/loss emulation with common random numbers; not a real-network or locked statistical claim" +} diff --git a/code/config/experiments/metrics_bilateral_network_v3_locked.json b/code/config/experiments/metrics_bilateral_network_v3_locked.json new file mode 100644 index 0000000..3272fd5 --- /dev/null +++ b/code/config/experiments/metrics_bilateral_network_v3_locked.json @@ -0,0 +1,87 @@ +{ + "enabled": [ + "h3", + "h4", + "bilateral", + "network" + ], + "h3": { + "include_methods": [ + "proposed_energy" + ], + "force_scale": 1.0, + "epsilon_energy_J": 1e-12, + "minimum_power_activity_J": 0.001 + }, + "h4": { + "include_methods": [ + "proposed_energy" + ], + "epsilon_torque_impulse_Nms": 1e-12, + "audit_tolerance_J": 1e-10 + }, + "bilateral": { + "include_methods": [ + "proposed_energy" + ], + "projection_tolerance": 1e-12, + "contact_force_threshold_N": 0.000001, + "required_audit_fields": [ + "wall_force_raw_N", + "wall_force_applied_N", + "wall_force_saturation_active", + "configured_wall_force_limit_N", + "master_joint_limit_active", + "slave_joint_limit_active", + "master_velocity_limit_active", + "slave_velocity_limit_active", + "master_acceleration_limit_active", + "slave_acceleration_limit_active", + "master_torque_saturation_active", + "slave_torque_saturation_active", + "haptic_rate_limit_active", + "haptic_torque_saturation_active", + "energy_probe_raw_work_J", + "h3_eligible" + ], + "gates": { + "minimum_contact_fraction": 0.0, + "minimum_contact_rms_N": 0.0, + "maximum_force_limit_hit_fraction": 0.0, + "minimum_force_headroom_N": 1.0, + "maximum_master_tracking_rmse_rad": 0.08, + "maximum_slave_tracking_rmse_rad": 0.08, + "minimum_projection_intervention_fraction": 0.0, + "maximum_projection_intervention_fraction": 0.05, + "maximum_limit_active_fraction": 0.0, + "minimum_energy_probe_raw_work_J": 0.0 + } + }, + "network": { + "include_methods": [ + "proposed_energy" + ], + "gates": { + "maximum_slave_zero_delay_tracking_rmse_rad": 0.06, + "minimum_forward_active_fraction": 0.98, + "minimum_return_active_fraction": 0.98, + "minimum_forward_fresh_fraction": 0.09, + "minimum_return_fresh_fraction": 0.5, + "maximum_forward_internal_missing_fraction": 0.05, + "maximum_return_internal_missing_fraction": 0.5, + "maximum_forward_timeout_fraction": 0.0, + "maximum_return_timeout_fraction": 0.0, + "maximum_forward_packet_age_s": 0.1, + "maximum_return_packet_age_s": 0.08, + "minimum_contact_fraction": 0.3, + "minimum_contact_rms_N": 0.1, + "maximum_free_space_contact_force_N": 0.000001 + }, + "paired_gates": { + "nominal_profile_id": "nominal", + "maximum_tracking_rmse_delta_vs_nominal_rad": 0.005, + "maximum_abs_contact_rms_relative_change_vs_nominal": 0.1 + } + }, + "status": "Frozen Stage-B v3 proposed-method simulation thresholds selected before the disjoint-seed locked run; H3 remains descriptive and free-space H3 may be activity-ineligible" +} diff --git a/code/config/experiments/metrics_bilateral_network_v3_screening.json b/code/config/experiments/metrics_bilateral_network_v3_screening.json new file mode 100644 index 0000000..e7112c8 --- /dev/null +++ b/code/config/experiments/metrics_bilateral_network_v3_screening.json @@ -0,0 +1,87 @@ +{ + "enabled": [ + "h3", + "h4", + "bilateral", + "network" + ], + "h3": { + "include_methods": [ + "proposed_energy" + ], + "force_scale": 1.0, + "epsilon_energy_J": 1e-12, + "minimum_power_activity_J": 0.001 + }, + "h4": { + "include_methods": [ + "proposed_energy" + ], + "epsilon_torque_impulse_Nms": 1e-12, + "audit_tolerance_J": 1e-10 + }, + "bilateral": { + "include_methods": [ + "proposed_energy" + ], + "projection_tolerance": 1e-12, + "contact_force_threshold_N": 0.000001, + "required_audit_fields": [ + "wall_force_raw_N", + "wall_force_applied_N", + "wall_force_saturation_active", + "configured_wall_force_limit_N", + "master_joint_limit_active", + "slave_joint_limit_active", + "master_velocity_limit_active", + "slave_velocity_limit_active", + "master_acceleration_limit_active", + "slave_acceleration_limit_active", + "master_torque_saturation_active", + "slave_torque_saturation_active", + "haptic_rate_limit_active", + "haptic_torque_saturation_active", + "energy_probe_raw_work_J", + "h3_eligible" + ], + "gates": { + "minimum_contact_fraction": 0.0, + "minimum_contact_rms_N": 0.0, + "maximum_force_limit_hit_fraction": 0.0, + "minimum_force_headroom_N": 1.0, + "maximum_master_tracking_rmse_rad": 0.08, + "maximum_slave_tracking_rmse_rad": 0.08, + "minimum_projection_intervention_fraction": 0.0, + "maximum_projection_intervention_fraction": 0.05, + "maximum_limit_active_fraction": 0.0, + "minimum_energy_probe_raw_work_J": 0.0 + } + }, + "network": { + "include_methods": [ + "proposed_energy" + ], + "gates": { + "maximum_slave_zero_delay_tracking_rmse_rad": 0.06, + "minimum_forward_active_fraction": 0.98, + "minimum_return_active_fraction": 0.98, + "minimum_forward_fresh_fraction": 0.09, + "minimum_return_fresh_fraction": 0.5, + "maximum_forward_internal_missing_fraction": 0.05, + "maximum_return_internal_missing_fraction": 0.5, + "maximum_forward_timeout_fraction": 0.0, + "maximum_return_timeout_fraction": 0.0, + "maximum_forward_packet_age_s": 0.1, + "maximum_return_packet_age_s": 0.08, + "minimum_contact_fraction": 0.3, + "minimum_contact_rms_N": 0.1, + "maximum_free_space_contact_force_N": 0.000001 + }, + "paired_gates": { + "nominal_profile_id": "nominal", + "maximum_tracking_rmse_delta_vs_nominal_rad": 0.005, + "maximum_abs_contact_rms_relative_change_vs_nominal": 0.1 + } + }, + "status": "Complete-grid Stage-B calibration thresholds; H3 is descriptive, freshness is distinct from active hold, and the frozen values require a disjoint-seed locked confirmation" +} diff --git a/code/core/network_emulator.py b/code/core/network_emulator.py index 0262ac4..77db01f 100644 --- a/code/core/network_emulator.py +++ b/code/core/network_emulator.py @@ -43,8 +43,16 @@ def generate_network_trace( duplicate_probability: float = 0.0, corrupt_probability: float = 0.0, seed: int = 0, + common_random_numbers: bool = False, ) -> tuple[NetworkTraceEntry, ...]: - """Generate a reusable trace without coupling other random streams.""" + """Generate a reusable trace without coupling other random streams. + + The default branch retains the historical conditional draw order exactly. + With common random numbers enabled, every packet consumes four uniforms in + the fixed order jitter/loss/duplicate/corrupt, even when a corresponding + magnitude or probability is zero. Network profiles can then threshold and + scale the same latent draws without shifting later packet decisions. + """ if count < 0: raise ValueError("count must be non-negative") probabilities = ( @@ -56,17 +64,28 @@ def generate_network_trace( raise ValueError("network probabilities must lie in [0, 1]") if base_delay_s < 0.0 or jitter_s < 0.0: raise ValueError("delay and jitter must be non-negative") + if not isinstance(common_random_numbers, (bool, np.bool_)): + raise ValueError("common_random_numbers must be boolean") rng = np.random.default_rng(seed) entries = [] for seq in range(count): - jitter = rng.uniform(-jitter_s, jitter_s) if jitter_s else 0.0 + if common_random_numbers: + jitter_draw, loss_draw, duplicate_draw, corrupt_draw = ( + rng.random(4) + ) + jitter = jitter_s * (2.0 * jitter_draw - 1.0) + else: + jitter = rng.uniform(-jitter_s, jitter_s) if jitter_s else 0.0 + loss_draw = rng.random() + duplicate_draw = rng.random() + corrupt_draw = rng.random() entries.append( NetworkTraceEntry( seq=seq, delay_s=max(0.0, base_delay_s + jitter), - lost=bool(rng.random() < loss_probability), - duplicate=bool(rng.random() < duplicate_probability), - corrupt=bool(rng.random() < corrupt_probability), + lost=bool(loss_draw < loss_probability), + duplicate=bool(duplicate_draw < duplicate_probability), + corrupt=bool(corrupt_draw < corrupt_probability), ) ) return tuple(entries) diff --git a/code/experiments/README.md b/code/experiments/README.md index d80ae1c..c9baac6 100644 --- a/code/experiments/README.md +++ b/code/experiments/README.md @@ -42,6 +42,7 @@ The v3 redesign separates branch-crossing evidence from contact/energy stress: execute_h1_retargeting h1_calibration_v3.json execute_bilateral_simulation bilateral_calibration_v3_stable_contact.json execute_bilateral_simulation bilateral_calibration_v3_energy_challenge.json +execute_bilateral_simulation bilateral_network_v3_screening.json ``` `h1_calibration_v3.json` is a deterministic branch-regression fixture, not @@ -51,10 +52,28 @@ challenge shares the corresponding `k=3200 N/m` groups, is explicitly ineligible for H3, and must be analyzed only with its H4/challenge metric configuration. -The bilateral network specification is a gated Stage B template. Its -`requires_stage_a_selection` flag means the haptic parameters are placeholders; -do not execute it as a locked study until the energy/gain Stage A acceptance -gate has passed. +The v3 Stage-B network screen freezes the selected Stage-A mechanics and haptic +settings. Within every trajectory/replicate block, it pairs nominal, +symmetric-delay, asymmetric-delay, jitter, and packet-loss profiles with common +random numbers. Its metrics distinguish packet availability from freshness and +compare delayed control references with within-trial transport shadows. Those +shadows retain the disturbed system state; the paired nominal trial remains the +causal network baseline. + +Disjoint-root locked v3 specifications are: + +```text +execute_bilateral_simulation bilateral_locked_v3_stable_contact.json +execute_bilateral_simulation bilateral_locked_v3_energy_challenge.json +execute_bilateral_simulation bilateral_network_v3_locked.json +``` + +The network locked study contains only the proposed method. It can test bounded +robustness under the registered emulator, but cannot establish superiority over +mapping baselines. + +The older `bilateral_calibration_v2_network.json` remains a historical +placeholder and must not be used as a confirmatory Stage-B protocol. An executor callable receives one immutable trial mapping and returns: @@ -98,6 +117,8 @@ bilateral_calibration_v3_stable_contact.json metrics_bilateral_v3_stable_contact.json bilateral_calibration_v3_energy_challenge.json metrics_bilateral_v3_energy_challenge.json +bilateral_network_v3_screening.json metrics_bilateral_network_v3_screening.json +bilateral_network_v3_locked.json metrics_bilateral_network_v3_locked.json ``` The bilateral configurations derive H3 for every mapping/supervisor condition, @@ -115,6 +136,11 @@ source-data. No Parquet dependency is required. streams are derived independently with NumPy `SeedSequence`; their serialized states are identical across methods in the same pair. +When `bilateral_pair_network_profiles=true`, `network_pair_group_id` excludes +the network treatment while retaining the trajectory, mechanics, control +settings, and replicate. This mode requires +`network_common_random_numbers=true`; otherwise planning is rejected. + Calibration, pilot, and locked studies must use separate specifications. A locked plan is immutable: changing a factor, method, trajectory, or seed invalidates its hashes. diff --git a/code/experiments/executors.py b/code/experiments/executors.py index 90126eb..b791ea2 100644 --- a/code/experiments/executors.py +++ b/code/experiments/executors.py @@ -93,6 +93,13 @@ def _profiled_factor( return profile.get(name, default) +def _boolean_factor(value: Any, name: str) -> bool: + """Return a strict experiment boolean without accepting truthy strings.""" + if not isinstance(value, (bool, np.bool_)): + raise ValueError(f"{name} must be boolean") + return bool(value) + + def _enum_code(member: Enum) -> int: return list(type(member)).index(member) @@ -1034,45 +1041,142 @@ def _bilateral_data_seed_group( factors = trial.get("factors", {}) if not isinstance(factors, Mapping): raise ValueError("trial factors must be a mapping") + pair_network_profiles = _boolean_factor( + factors.get("bilateral_pair_network_profiles", False), + "bilateral_pair_network_profiles", + ) + if pair_network_profiles and not config.network_common_random_numbers: + raise ValueError( + "bilateral_pair_network_profiles requires " + "network_common_random_numbers=true" + ) raw_root_seed = factors.get("bilateral_data_root_seed") if raw_root_seed is None: + if pair_network_profiles: + raise ValueError( + "bilateral_pair_network_profiles requires " + "bilateral_data_root_seed" + ) return None, None, None data_root_seed = int(raw_root_seed) if data_root_seed < 0: raise ValueError("bilateral_data_root_seed must be non-negative") - basis = { - "data_root_seed": data_root_seed, - "trajectory_family": str( - _trajectory_spec(trial).get("family", "") - ), - "replicate": int(trial.get("replicate", 0)), - "mechanical_and_network_inputs": { - "dt_s": config.dt, - "duration_s": config.duration, - "mapping_hz": config.mapping_hz, - "contact_probe_fraction": config.contact_probe_fraction, - "contact_probe_cycles": config.contact_probe_cycles, - "wall_fraction": config.wall_fraction, - "wall_stiffness_N_per_m": config.wall_stiffness, - "wall_damping_Ns_per_m": config.wall_damping, - "wall_force_limit_N": config.wall_force_limit, - "wall_transition_depth_m": config.wall_transition_depth, - "forward_delay_s": config.forward_delay_s, - "return_delay_s": config.feedback_delay_s, - "forward_jitter_s": config.forward_jitter_s, - "return_jitter_s": config.return_jitter_s, - "forward_packet_loss": config.forward_packet_loss, - "return_packet_loss": config.return_packet_loss, - "forward_timeout_s": config.forward_timeout_s, - "return_timeout_s": config.return_timeout_s, - "sensor_noise_std_Nm": config.sensor_noise_std, - "bias_calibration_samples": config.bias_calibration_samples, - "master_kp": list(config.master_kp), - "master_kd": list(config.master_kd), - "slave_kp": list(config.slave_kp), - "slave_kd": list(config.slave_kd), - }, - } + if pair_network_profiles: + basis = { + "data_root_seed": data_root_seed, + "trajectory": dict(_trajectory_spec(trial)), + "replicate": int(trial.get("replicate", 0)), + "network_pairing_strategy": { + "enabled": True, + "name": "common_simulation_seed_across_network_profiles", + "excluded_network_inputs": [ + "forward_delay_s", + "return_delay_s", + "forward_jitter_s", + "return_jitter_s", + "forward_packet_loss", + "return_packet_loss", + "forward_timeout_s", + "return_timeout_s", + ], + "network_common_random_numbers": ( + config.network_common_random_numbers + ), + }, + "mechanical_and_control_inputs": { + "dt_s": config.dt, + "duration_s": config.duration, + "mapping_hz": config.mapping_hz, + "slave_contact_frame": config.slave_contact_frame, + "contact_probe_fraction": config.contact_probe_fraction, + "contact_probe_cycles": config.contact_probe_cycles, + "wall_fraction": config.wall_fraction, + "wall_stiffness_N_per_m": config.wall_stiffness, + "wall_damping_Ns_per_m": config.wall_damping, + "wall_force_limit_N": config.wall_force_limit, + "wall_transition_depth_m": config.wall_transition_depth, + "master_kp": list(config.master_kp), + "master_kd": list(config.master_kd), + "slave_kp": list(config.slave_kp), + "slave_kd": list(config.slave_kd), + "master_acceleration_limits": list( + config.master_acceleration_limits + ), + "slave_acceleration_limits": list( + config.slave_acceleration_limits + ), + "master_tracking_effort_fraction": ( + config.master_tracking_effort_fraction + ), + "slave_tracking_effort_fraction": ( + config.slave_tracking_effort_fraction + ), + "velocity_limit_fraction": config.velocity_limit_fraction, + "soft_limit_buffer": config.soft_limit_buffer, + "feedback_strength": config.feedback_strength, + "haptic_filter_alpha": config.haptic_filter_alpha, + "haptic_torque_limits": list( + config.haptic_torque_limits + ), + "haptic_rate_limits": list(config.haptic_rate_limits), + "energy_min_J": config.energy_min, + "energy_max_J": config.energy_max, + "energy_initial_J": config.energy_initial, + "energy_probe_mode": config.energy_probe_mode, + "energy_probe_torque_Nm": config.energy_probe_torque_Nm, + "energy_probe_start_fraction": ( + config.energy_probe_start_fraction + ), + "energy_probe_end_fraction": ( + config.energy_probe_end_fraction + ), + "sensor_noise_std_Nm": config.sensor_noise_std, + "wrench_characteristic_length_m": ( + config.wrench_characteristic_length_m + ), + "wrench_scaled_damping": config.wrench_scaled_damping, + "sensor_bias_Nm": list(config.sensor_bias), + "bias_calibration_samples": config.bias_calibration_samples, + "joint_limit_margin": config.joint_limit_margin, + "differential_step": config.differential_step, + }, + } + else: + # Compatibility contract: do not add even constant keys here. Existing + # v3 plans depend on the exact historical seed-group hash. + basis = { + "data_root_seed": data_root_seed, + "trajectory_family": str( + _trajectory_spec(trial).get("family", "") + ), + "replicate": int(trial.get("replicate", 0)), + "mechanical_and_network_inputs": { + "dt_s": config.dt, + "duration_s": config.duration, + "mapping_hz": config.mapping_hz, + "contact_probe_fraction": config.contact_probe_fraction, + "contact_probe_cycles": config.contact_probe_cycles, + "wall_fraction": config.wall_fraction, + "wall_stiffness_N_per_m": config.wall_stiffness, + "wall_damping_Ns_per_m": config.wall_damping, + "wall_force_limit_N": config.wall_force_limit, + "wall_transition_depth_m": config.wall_transition_depth, + "forward_delay_s": config.forward_delay_s, + "return_delay_s": config.feedback_delay_s, + "forward_jitter_s": config.forward_jitter_s, + "return_jitter_s": config.return_jitter_s, + "forward_packet_loss": config.forward_packet_loss, + "return_packet_loss": config.return_packet_loss, + "forward_timeout_s": config.forward_timeout_s, + "return_timeout_s": config.return_timeout_s, + "sensor_noise_std_Nm": config.sensor_noise_std, + "bias_calibration_samples": config.bias_calibration_samples, + "master_kp": list(config.master_kp), + "master_kd": list(config.master_kd), + "slave_kp": list(config.slave_kp), + "slave_kd": list(config.slave_kd), + }, + } group_id = ( "bilateral-data-" + stable_hash(basis, prefix="bilateral-data-group")[:16] @@ -1085,6 +1189,30 @@ def _bilateral_data_seed_group( return group_id, basis, seed_record +def _bilateral_network_pair_group_id( + trial: Mapping[str, Any], + data_group_basis: Mapping[str, Any] | None, +) -> str | None: + """Identify the network-profile pairing block when explicitly enabled.""" + enabled = _boolean_factor( + _factor(trial, "bilateral_pair_network_profiles", False), + "bilateral_pair_network_profiles", + ) + if not enabled: + return None + if data_group_basis is None: + raise ValueError( + "network pairing requires a bilateral data-group basis" + ) + return ( + "network-pair-" + + stable_hash( + data_group_basis, + prefix="bilateral-network-pair-group", + )[:16] + ) + + def _bilateral_scenario_and_config( trial: Mapping[str, Any], ) -> tuple[Any, SimulationConfig]: @@ -1183,6 +1311,15 @@ def _bilateral_scenario_and_config( profile_name="network_profile", ) ), + network_common_random_numbers=_boolean_factor( + _profiled_factor( + trial, + "network_common_random_numbers", + False, + profile_name="network_profile", + ), + "network_common_random_numbers", + ), forward_timeout_s=float( _profiled_factor( trial, @@ -1323,6 +1460,10 @@ def execute_bilateral_simulation(trial: Mapping[str, Any]) -> TrialPayload: data_group_id, data_group_basis, data_seed_record = ( _bilateral_data_seed_group(trial, config) ) + network_pair_group_id = _bilateral_network_pair_group_id( + trial, data_group_basis + ) + network_pairing_enabled = network_pair_group_id is not None trajectory = _trajectory_spec(trial) models = load_models(add_simulated_tcp=True) mapper = build_mapper(models) @@ -1368,6 +1509,48 @@ def execute_bilateral_simulation(trial: Mapping[str, Any]) -> TrialPayload: samples["configured_wall_force_limit_N"] = np.full( sample_count, config.wall_force_limit ) + samples["configured_forward_delay_s"] = np.full( + sample_count, config.forward_delay_s + ) + samples["configured_return_delay_s"] = np.full( + sample_count, config.feedback_delay_s + ) + samples["configured_forward_jitter_s"] = np.full( + sample_count, config.forward_jitter_s + ) + samples["configured_return_jitter_s"] = np.full( + sample_count, config.return_jitter_s + ) + samples["configured_forward_packet_loss"] = np.full( + sample_count, config.forward_packet_loss + ) + samples["configured_return_packet_loss"] = np.full( + sample_count, config.return_packet_loss + ) + samples["configured_forward_timeout_s"] = np.full( + sample_count, config.forward_timeout_s + ) + samples["configured_return_timeout_s"] = np.full( + sample_count, config.return_timeout_s + ) + samples["configured_mapping_hz"] = np.full( + sample_count, config.mapping_hz + ) + raw_contact_expected = trajectory.get( + "contact_expected", + trajectory.get("family") != "free_space", + ) + contact_expected = _boolean_factor( + raw_contact_expected, "trajectory.contact_expected" + ) + samples["contact_expected"] = np.full( + sample_count, int(contact_expected), dtype=np.int8 + ) + samples["configured_network_common_random_numbers"] = np.full( + sample_count, + int(config.network_common_random_numbers), + dtype=np.int8, + ) h3_eligible = config.energy_probe_mode == "none" samples["h3_eligible"] = np.full( sample_count, int(h3_eligible), dtype=np.int8 @@ -1418,6 +1601,20 @@ def execute_bilateral_simulation(trial: Mapping[str, Any]) -> TrialPayload: "bilateral_data_group_id": data_group_id, "bilateral_data_group_basis": data_group_basis, "bilateral_data_seed_record": data_seed_record, + "bilateral_data_seed_record_hash": ( + stable_hash( + data_seed_record, + prefix="bilateral-data-seed-record", + ) + if data_seed_record is not None + else None + ), + "network_pairing_enabled": network_pairing_enabled, + **( + {"network_pair_group_id": network_pair_group_id} + if network_pairing_enabled + else {} + ), "stable_contact_selection": _factor( trial, "stable_contact_selection", None ), @@ -1428,6 +1625,9 @@ def execute_bilateral_simulation(trial: Mapping[str, Any]) -> TrialPayload: "return_jitter_s": config.return_jitter_s, "forward_packet_loss": config.forward_packet_loss, "return_packet_loss": config.return_packet_loss, + "common_random_numbers": ( + config.network_common_random_numbers + ), "forward_timeout_s": config.forward_timeout_s, "return_timeout_s": config.return_timeout_s, }, diff --git a/code/simulate_closed_loop.py b/code/simulate_closed_loop.py index 33a5f27..84740b2 100644 --- a/code/simulate_closed_loop.py +++ b/code/simulate_closed_loop.py @@ -90,6 +90,7 @@ class SimulationConfig: return_jitter_s: float = 0.0 forward_packet_loss: float = 0.0 return_packet_loss: float = 0.0 + network_common_random_numbers: bool = False forward_timeout_s: float = 0.20 return_timeout_s: float = 0.20 contact_probe_fraction: float = 0.03 @@ -238,6 +239,10 @@ class SimulationConfig: raise ValueError("forward_packet_loss must lie in [0, 1]") if not 0.0 <= self.return_packet_loss <= 1.0: raise ValueError("return_packet_loss must lie in [0, 1]") + if not isinstance( + self.network_common_random_numbers, (bool, np.bool_) + ): + raise ValueError("network_common_random_numbers must be boolean") if self.contact_probe_fraction < 0.0 or self.contact_probe_cycles < 0.0: raise ValueError("contact probe parameters cannot be negative") if not ( @@ -866,6 +871,7 @@ def simulate_scenario( f"pose={map_debug['events']}, " f"A={map_debug['differential']['events']}" ) + q_s_zero_delay_ref = q_s_ref.copy() differential_feedback_valid = True qd_s_ref_hold = np.zeros(models.slave.nv, dtype=float) map_registry = MapRegistry(capacity=2048) @@ -900,6 +906,7 @@ def simulate_scenario( jitter_s=config.forward_jitter_s, loss_probability=config.forward_packet_loss, seed=config.seed + 1001, + common_random_numbers=config.network_common_random_numbers, ) return_trace = generate_network_trace( step_count, @@ -907,6 +914,7 @@ def simulate_scenario( jitter_s=config.return_jitter_s, loss_probability=config.return_packet_loss, seed=config.seed + 1002, + common_random_numbers=config.network_common_random_numbers, ) forward_channel: DeterministicChannel[ForwardPacket] = ( DeterministicChannel(forward_trace) @@ -963,11 +971,20 @@ def simulate_scenario( "source_map_id", "return_source_index", "return_packet_age", + "forward_packet_active", + "forward_packet_fresh", + "return_packet_fresh", + "forward_packet_age", + "forward_packet_seq", + "return_packet_seq", "forward_packet_state", "return_packet_state", "return_packet_active", "master_tracking_error", "slave_tracking_error", + "slave_zero_delay_tracking_error", + "slave_reference_lag_error", + "return_feedback_lag_error", "feedback_torque_norm", "mapped_torque_norm", "energy_probe_raw_power_W", @@ -991,12 +1008,14 @@ def simulate_scenario( "q_slave", "qd_slave", "q_slave_ref", + "q_slave_zero_delay_ref", "tau_slave_external", "tau_slave_estimated", "tau_slave_residual_source", "tau_slave_matched_wrench", "qd_slave_source", "tau_master_mapped", + "tau_master_zero_return_delay", "tau_master_candidate", "tau_master_applied", "tau_master_accepted", @@ -1071,6 +1090,7 @@ def simulate_scenario( differential_valid = bool(update_debug["differential_valid"]) if pose_valid: map_pose_success_count += 1 + q_s_zero_delay_ref = q_s_candidate.copy() if update_debug["differential_valid"]: A = A_candidate differential_valid_count += 1 @@ -1124,6 +1144,17 @@ def simulate_scenario( else: forward_packets_rejected += 1 held_forward = forward_receiver.sample(t) + forward_packet_active = held_forward.packet is not None + forward_packet_age = ( + max(0.0, t - held_forward.packet.source_time) + if held_forward.packet is not None + else math.nan + ) + forward_packet_seq = ( + held_forward.packet.seq + if held_forward.packet is not None + else -1 + ) if held_forward.packet is not None: q_s_ref = held_forward.packet.q_slave_ref.copy() qd_s_ref_hold = held_forward.packet.qd_slave_ctrl.copy() @@ -1223,19 +1254,17 @@ def simulate_scenario( tau_slave_measured, ) tau_slave_matched_wrench = J_slave_chest.T @ wrench_estimated - return_channel.send( - ReturnPacket( - seq=step, - source_index=step, - source_time=t, - echoed_map_id=active_forward_map_id, - residual=tau_slave_estimated, - wrench=wrench_estimated, - js_t_wrench=tau_slave_matched_wrench, - qd_slave_actual=qd_s, - ), - now=t, + current_return_packet = ReturnPacket( + seq=step, + source_index=step, + source_time=t, + echoed_map_id=active_forward_map_id, + residual=tau_slave_estimated, + wrench=wrench_estimated, + js_t_wrench=tau_slave_matched_wrench, + qd_slave_actual=qd_s, ) + return_channel.send(current_return_packet, now=t) for delivery in return_channel.poll(t): reception = return_receiver.accept(delivery) if reception.accepted: @@ -1245,10 +1274,25 @@ def simulate_scenario( held_return = return_receiver.sample(t) delayed_packet = held_return.packet return_packet_active = delayed_packet is not None + return_packet_seq = ( + delayed_packet.seq if delayed_packet is not None else -1 + ) if held_return.state is PacketState.TIMED_OUT: return_timeout_steps += 1 master_jacobian = renderer.CJ_master.chest_jacobian(q_m, qd_m) + zero_return_feedback = map_return_feedback( + kind=MappingKind(scenario.mapping), + packet=current_return_packet, + master_jacobian=master_jacobian, + maps=map_registry, + map_policy=MapPolicy(scenario.map_policy), + ) + tau_master_zero_return_delay = ( + zero_return_feedback.tau_master_raw.copy() + if zero_return_feedback.valid + else np.zeros(models.master.nv, dtype=float) + ) if delayed_packet is None: tau_master_mapped = np.zeros(models.master.nv, dtype=float) delayed_tau_slave = np.zeros(models.slave.nv, dtype=float) @@ -1287,6 +1331,11 @@ def simulate_scenario( if not feedback.valid: tau_master_mapped = np.zeros(models.master.nv, dtype=float) + return_feedback_lag_error = float( + np.linalg.norm( + tau_master_zero_return_delay - tau_master_mapped + ) + ) energy_probe_torque = np.zeros(models.master.nv, dtype=float) energy_probe_envelope = 0.0 normalized_time = t / config.duration @@ -1513,6 +1562,12 @@ def simulate_scenario( "source_map_id": source_map_id, "return_source_index": return_source_index, "return_packet_age": return_packet_age, + "forward_packet_active": int(forward_packet_active), + "forward_packet_fresh": int(held_forward.fresh), + "return_packet_fresh": int(held_return.fresh), + "forward_packet_age": forward_packet_age, + "forward_packet_seq": forward_packet_seq, + "return_packet_seq": return_packet_seq, "forward_packet_state": int(held_forward.state), "return_packet_state": int(held_return.state), "return_packet_active": int(return_packet_active), @@ -1522,6 +1577,25 @@ def simulate_scenario( "slave_tracking_error": float( np.linalg.norm(pin.difference(models.slave, q_s, q_s_ref)) ), + "slave_zero_delay_tracking_error": float( + np.linalg.norm( + pin.difference( + models.slave, + q_s, + q_s_zero_delay_ref, + ) + ) + ), + "slave_reference_lag_error": float( + np.linalg.norm( + pin.difference( + models.slave, + q_s_ref, + q_s_zero_delay_ref, + ) + ) + ), + "return_feedback_lag_error": return_feedback_lag_error, "feedback_torque_norm": float(np.linalg.norm(tau_master_applied)), "mapped_torque_norm": float(np.linalg.norm(tau_master_mapped)), "energy_probe_raw_power_W": energy_probe_raw_power_W, @@ -1551,12 +1625,16 @@ def simulate_scenario( "q_slave": q_s.copy(), "qd_slave": qd_s.copy(), "q_slave_ref": q_s_ref.copy(), + "q_slave_zero_delay_ref": q_s_zero_delay_ref.copy(), "tau_slave_external": tau_slave_external.copy(), "tau_slave_estimated": tau_slave_estimated.copy(), "tau_slave_residual_source": delayed_tau_slave.copy(), "tau_slave_matched_wrench": delayed_tau_matched.copy(), "qd_slave_source": qd_slave_source.copy(), "tau_master_mapped": tau_master_mapped.copy(), + "tau_master_zero_return_delay": ( + tau_master_zero_return_delay.copy() + ), "tau_master_candidate": tau_master_candidate.copy(), "tau_master_applied": tau_master_applied.copy(), "tau_master_accepted": tau_master_accepted.copy(), diff --git a/code/test/test_bilateral_calibration_v3.py b/code/test/test_bilateral_calibration_v3.py index 9454be2..5480285 100644 --- a/code/test/test_bilateral_calibration_v3.py +++ b/code/test/test_bilateral_calibration_v3.py @@ -2,6 +2,7 @@ """Bilateral v3 environment, force-audit, and stable-contact contracts.""" from pathlib import Path +import copy import sys import unittest @@ -13,9 +14,12 @@ sys.path.insert(0, str(CODE_ROOT)) from analysis.metrics import MetricError, derive_trial_metrics # noqa: E402 from experiments.executors import ( # noqa: E402 + _bilateral_data_seed_group, + _bilateral_network_pair_group_id, _bilateral_scenario_and_config, execute_bilateral_simulation, ) +from experiments.hashing import stable_hash # noqa: E402 from experiments.plan import build_trial_plan, load_document # noqa: E402 from experiments.rng import named_seed_record # noqa: E402 from simulate_closed_loop import Wall # noqa: E402 @@ -61,6 +65,21 @@ def propagation_trial(): } +def paired_network_trial(profile, *, replicate=0): + trial = copy.deepcopy(propagation_trial()) + trial["replicate"] = replicate + trial["factors"].update( + { + "bilateral_data_root_seed": 8127, + "bilateral_pair_network_profiles": True, + "network_common_random_numbers": True, + "duration_s": 0.12, + "network_profile": profile, + } + ) + return trial + + class BilateralCalibrationV3Test(unittest.TestCase): def test_environment_profile_and_direct_factors_propagate(self): _, config = _bilateral_scenario_and_config(propagation_trial()) @@ -176,6 +195,273 @@ class BilateralCalibrationV3Test(unittest.TestCase): challenge_seed, ) + def test_default_v3_data_seed_and_hash_remain_exactly_compatible(self): + plan = build_trial_plan( + load_document( + CONFIG_ROOT + / "bilateral_calibration_v3_stable_contact.json" + ) + ) + trial = next( + entry + for entry in plan["trials"] + if entry["replicate"] == 0 + and entry["factors"]["environment_profile"]["profile_id"] + == "stable_candidate_k3200" + and entry["factors"]["haptic_profile"]["profile_id"] + == "gain_035_wide_tank" + ) + _, config = _bilateral_scenario_and_config(trial) + group_id, basis, seed_record = _bilateral_data_seed_group( + trial, config + ) + + self.assertEqual(trial["trial_id"], "trial-e8342e4017a6d7c5") + self.assertEqual(config.seed, 1920787158) + self.assertEqual( + group_id, "bilateral-data-271d823f09a704f7" + ) + self.assertEqual( + stable_hash(basis, prefix="bilateral-data-group"), + "271d823f09a704f78dde74a3c3bb3c3e" + "0a5f3b3d5bd288a9c67996f4a4e1ce46", + ) + self.assertEqual( + stable_hash( + seed_record, prefix="bilateral-data-seed-record" + ), + "6e5e05b8ac7922c4679464a658a053be" + "c10da966e43b4d09f2fbaed990da2001", + ) + self.assertEqual( + seed_record, + { + "simulation": [ + 3977025919, + 244326538, + 539806858, + 605936004, + ] + }, + ) + + def test_network_profiles_can_share_seed_and_pair_group(self): + clean = { + "profile_id": "clean", + "forward_delay_s": 0.0, + "return_delay_s": 0.004, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.05, + "return_timeout_s": 0.05, + } + impaired = { + "profile_id": "impaired", + "forward_delay_s": 0.012, + "return_delay_s": 0.024, + "forward_jitter_s": 0.003, + "return_jitter_s": 0.005, + "forward_packet_loss": 0.1, + "return_packet_loss": 0.2, + "forward_timeout_s": 0.08, + "return_timeout_s": 0.10, + } + clean_trial = paired_network_trial(clean) + impaired_trial = paired_network_trial(impaired) + _, clean_config = _bilateral_scenario_and_config(clean_trial) + _, impaired_config = _bilateral_scenario_and_config(impaired_trial) + clean_group = _bilateral_data_seed_group( + clean_trial, clean_config + ) + impaired_group = _bilateral_data_seed_group( + impaired_trial, impaired_config + ) + + self.assertNotEqual( + clean_config.feedback_delay_s, + impaired_config.feedback_delay_s, + ) + self.assertEqual(clean_config.seed, impaired_config.seed) + self.assertEqual(clean_group, impaired_group) + clean_pair_id = _bilateral_network_pair_group_id( + clean_trial, clean_group[1] + ) + impaired_pair_id = _bilateral_network_pair_group_id( + impaired_trial, impaired_group[1] + ) + self.assertEqual(clean_pair_id, impaired_pair_id) + self.assertTrue( + clean_group[1]["network_pairing_strategy"][ + "network_common_random_numbers" + ] + ) + self.assertNotIn( + "forward_delay_s", + clean_group[1]["mechanical_and_control_inputs"], + ) + + replicate_trial = paired_network_trial(clean, replicate=1) + _, replicate_config = _bilateral_scenario_and_config( + replicate_trial + ) + replicate_group = _bilateral_data_seed_group( + replicate_trial, replicate_config + ) + self.assertNotEqual(clean_config.seed, replicate_config.seed) + self.assertNotEqual(clean_group[0], replicate_group[0]) + self.assertNotEqual( + clean_pair_id, + _bilateral_network_pair_group_id( + replicate_trial, replicate_group[1] + ), + ) + + def test_network_pairing_requires_common_random_numbers(self): + trial = paired_network_trial({"profile_id": "invalid_pairing"}) + trial["factors"]["network_common_random_numbers"] = False + with self.assertRaisesRegex( + ValueError, + "bilateral_pair_network_profiles requires " + "network_common_random_numbers=true", + ): + _bilateral_scenario_and_config(trial) + + def test_nominal_transport_matches_within_trial_shadows(self): + nominal = { + "profile_id": "nominal", + "forward_delay_s": 0.0, + "return_delay_s": 0.0, + "forward_jitter_s": 0.0, + "return_jitter_s": 0.0, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.04, + "return_timeout_s": 0.04, + } + samples = execute_bilateral_simulation( + paired_network_trial(nominal) + ).samples + np.testing.assert_array_equal( + samples["q_slave_ref"], + samples["q_slave_zero_delay_ref"], + ) + np.testing.assert_array_equal( + samples["tau_master_mapped"], + samples["tau_master_zero_return_delay"], + ) + np.testing.assert_array_equal( + samples["slave_reference_lag_error"], + np.zeros(samples["time"].shape[0]), + ) + np.testing.assert_array_equal( + samples["return_feedback_lag_error"], + np.zeros(samples["time"].shape[0]), + ) + + def test_short_network_run_exports_pairing_and_shadow_diagnostics(self): + profile = { + "profile_id": "short_delay", + "forward_delay_s": 0.006, + "return_delay_s": 0.008, + "forward_jitter_s": 0.001, + "return_jitter_s": 0.001, + "forward_packet_loss": 0.0, + "return_packet_loss": 0.0, + "forward_timeout_s": 0.04, + "return_timeout_s": 0.04, + } + payload = execute_bilateral_simulation( + paired_network_trial(profile) + ) + samples = payload.samples + sample_count = samples["time"].shape[0] + self.assertEqual(sample_count, 60) + + for field in ( + "forward_packet_active", + "forward_packet_fresh", + "return_packet_fresh", + "forward_packet_age", + "forward_packet_seq", + "return_packet_seq", + "slave_zero_delay_tracking_error", + "slave_reference_lag_error", + "return_feedback_lag_error", + ): + self.assertEqual(samples[field].shape, (sample_count,)) + for field in ( + "q_slave_zero_delay_ref", + "tau_master_zero_return_delay", + ): + self.assertEqual(samples[field].shape, (sample_count, 7)) + self.assertTrue(np.all(np.isfinite(samples[field]))) + + for direction in ("forward", "return"): + active = samples[f"{direction}_packet_active"].astype(bool) + fresh = samples[f"{direction}_packet_fresh"].astype(bool) + age = samples[f"{direction}_packet_age"] + seq = samples[f"{direction}_packet_seq"] + self.assertTrue(np.any(active)) + self.assertTrue(np.any(~active)) + self.assertTrue(np.all(~fresh | active)) + self.assertTrue(np.all(np.isnan(age[~active]))) + self.assertTrue(np.all(np.isfinite(age[active]))) + self.assertTrue(np.all(age[active] >= 0.0)) + self.assertTrue(np.all(seq[~active] == -1)) + self.assertTrue(np.all(seq[active] >= 0)) + + for field in ( + "slave_zero_delay_tracking_error", + "slave_reference_lag_error", + "return_feedback_lag_error", + ): + self.assertTrue(np.all(np.isfinite(samples[field]))) + self.assertTrue(np.all(samples[field] >= 0.0)) + np.testing.assert_allclose( + samples["return_feedback_lag_error"], + np.linalg.norm( + samples["tau_master_zero_return_delay"] + - samples["tau_master_mapped"], + axis=1, + ), + ) + + configured = { + "configured_forward_delay_s": 0.006, + "configured_return_delay_s": 0.008, + "configured_forward_jitter_s": 0.001, + "configured_return_jitter_s": 0.001, + "configured_forward_packet_loss": 0.0, + "configured_return_packet_loss": 0.0, + "configured_forward_timeout_s": 0.04, + "configured_return_timeout_s": 0.04, + "configured_mapping_hz": 40.0, + } + for field, expected in configured.items(): + np.testing.assert_array_equal( + samples[field], np.full(sample_count, expected) + ) + np.testing.assert_array_equal( + samples["contact_expected"], + np.ones(sample_count, dtype=np.int8), + ) + np.testing.assert_array_equal( + samples["configured_network_common_random_numbers"], + np.ones(sample_count, dtype=np.int8), + ) + + self.assertTrue(payload.metadata["network_pairing_enabled"]) + self.assertIn("network_pair_group_id", payload.metadata) + self.assertEqual( + payload.metadata["bilateral_data_seed_record_hash"], + stable_hash( + payload.metadata["bilateral_data_seed_record"], + prefix="bilateral-data-seed-record", + ), + ) + def test_stable_six_second_smoke_has_contact_without_force_limiting(self): plan = build_trial_plan( load_document( diff --git a/code/test/test_bilateral_network_v3_config.py b/code/test/test_bilateral_network_v3_config.py new file mode 100644 index 0000000..a69e772 --- /dev/null +++ b/code/test/test_bilateral_network_v3_config.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Versioned Stage-B v3 plan and locked-confirmation contracts.""" + +from pathlib import Path +import sys +import unittest + + +CODE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CODE_ROOT)) + +from experiments.plan import build_trial_plan, load_document # noqa: E402 + + +CONFIG_ROOT = CODE_ROOT / "config" / "experiments" + + +class BilateralNetworkV3ConfigTest(unittest.TestCase): + def test_screening_grid_is_directional_paired_and_bounded(self): + plan = build_trial_plan( + load_document( + CONFIG_ROOT / "bilateral_network_v3_screening.json" + ) + ) + + self.assertEqual(plan["pair_count"], 30) + self.assertEqual(plan["trial_count"], 30) + self.assertEqual( + {trial["method"]["method_id"] for trial in plan["trials"]}, + {"proposed_energy"}, + ) + self.assertEqual( + { + trial["trajectory"]["trajectory_id"] + for trial in plan["trials"] + }, + { + "free_space_network_roundtrip", + "slow_contact_network_roundtrip", + }, + ) + self.assertEqual( + { + trial["factors"]["network_profile"]["profile_id"] + for trial in plan["trials"] + }, + { + "nominal", + "symmetric_delay_40ms", + "asymmetric_delay_20_60ms", + "symmetric_40ms_jitter_4ms", + "symmetric_40ms_loss_2pct", + }, + ) + self.assertTrue( + all( + trial["factors"]["bilateral_pair_network_profiles"] + and trial["factors"]["network_common_random_numbers"] + for trial in plan["trials"] + ) + ) + + def test_locked_confirmation_uses_disjoint_data_root(self): + calibration = load_document( + CONFIG_ROOT + / "bilateral_calibration_v3_stable_contact.json" + ) + calibration_data_root = calibration["factors"][ + "bilateral_data_root_seed" + ][0] + locked_names = ( + "bilateral_locked_v3_stable_contact.json", + "bilateral_locked_v3_energy_challenge.json", + ) + locked_data_roots = set() + for name in locked_names: + specification = load_document(CONFIG_ROOT / name) + plan = build_trial_plan(specification) + self.assertEqual(plan["split"], "locked") + self.assertEqual(plan["pair_count"], 20) + self.assertEqual(plan["trial_count"], 20) + locked_data_roots.add( + specification["factors"]["bilateral_data_root_seed"][0] + ) + + self.assertEqual(len(locked_data_roots), 1) + self.assertNotIn(calibration_data_root, locked_data_roots) + + def test_network_metric_order_freezes_paired_gate_names(self): + configuration = load_document( + CONFIG_ROOT + / "metrics_bilateral_network_v3_screening.json" + ) + self.assertEqual( + configuration["enabled"], + ["h3", "h4", "bilateral", "network"], + ) + paired = configuration["network"]["paired_gates"] + self.assertEqual(paired["nominal_profile_id"], "nominal") + self.assertEqual( + paired[ + "maximum_tracking_rmse_delta_vs_nominal_rad" + ], + 0.005, + ) + self.assertEqual( + paired[ + "maximum_abs_contact_rms_relative_change_vs_nominal" + ], + 0.1, + ) + + def test_locked_network_grid_uses_disjoint_twenty_seed_protocol(self): + screening = load_document( + CONFIG_ROOT / "bilateral_network_v3_screening.json" + ) + locked = load_document( + CONFIG_ROOT / "bilateral_network_v3_locked.json" + ) + plan = build_trial_plan(locked) + + self.assertEqual(plan["split"], "locked") + self.assertEqual(plan["pair_count"], 200) + self.assertEqual(plan["trial_count"], 200) + self.assertEqual( + {trial["method"]["method_id"] for trial in plan["trials"]}, + {"proposed_energy"}, + ) + self.assertNotEqual( + screening["factors"]["bilateral_data_root_seed"][0], + locked["factors"]["bilateral_data_root_seed"][0], + ) + locked_metrics = load_document( + CONFIG_ROOT / "metrics_bilateral_network_v3_locked.json" + ) + screening_metrics = load_document( + CONFIG_ROOT + / "metrics_bilateral_network_v3_screening.json" + ) + self.assertEqual( + locked_metrics["network"]["paired_gates"], + screening_metrics["network"]["paired_gates"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/code/test/test_feedback_protocol.py b/code/test/test_feedback_protocol.py index ee6fded..1c5e966 100644 --- a/code/test/test_feedback_protocol.py +++ b/code/test/test_feedback_protocol.py @@ -27,6 +27,7 @@ from core.network_emulator import ( # noqa: E402 DeterministicChannel, NetworkTraceEntry, PacketReceiver, + generate_network_trace, ) @@ -140,6 +141,61 @@ class FeedbackProtocolTest(unittest.TestCase): self.assertIn(held.state, (PacketState.ACTIVE, PacketState.RECOVERING)) self.assertEqual(receiver.sample(0.2).state, PacketState.TIMED_OUT) + def test_default_network_trace_retains_historical_draw_sequence(self): + trace = generate_network_trace( + 4, + base_delay_s=0.05, + jitter_s=0.01, + loss_probability=0.25, + duplicate_probability=0.4, + corrupt_probability=0.1, + seed=123, + ) + expected = ( + (0.05364703726496287, True, True, False), + (0.04351811802170061, False, False, False), + (0.056395091231860046, False, False, False), + (0.05648483192194823, True, False, False), + ) + for entry, values in zip(trace, expected, strict=True): + self.assertEqual( + ( + entry.delay_s, + entry.lost, + entry.duplicate, + entry.corrupt, + ), + values, + ) + + def test_common_random_numbers_fix_packet_draw_alignment(self): + common = { + "count": 32, + "base_delay_s": 0.1, + "loss_probability": 0.35, + "duplicate_probability": 0.25, + "corrupt_probability": 0.15, + "seed": 91, + "common_random_numbers": True, + } + zero_jitter = generate_network_trace(jitter_s=0.0, **common) + small_jitter = generate_network_trace(jitter_s=0.01, **common) + large_jitter = generate_network_trace(jitter_s=0.03, **common) + + decisions = lambda trace: [ + (entry.lost, entry.duplicate, entry.corrupt) + for entry in trace + ] + self.assertEqual(decisions(zero_jitter), decisions(small_jitter)) + self.assertEqual(decisions(small_jitter), decisions(large_jitter)) + small_latent = np.array( + [(entry.delay_s - 0.1) / 0.01 for entry in small_jitter] + ) + large_latent = np.array( + [(entry.delay_s - 0.1) / 0.03 for entry in large_jitter] + ) + np.testing.assert_allclose(small_latent, large_latent) + if __name__ == "__main__": unittest.main() diff --git a/code/test/test_network_metrics_v3.py b/code/test/test_network_metrics_v3.py new file mode 100644 index 0000000..5332f64 --- /dev/null +++ b/code/test/test_network_metrics_v3.py @@ -0,0 +1,490 @@ +#!/usr/bin/env python3 +"""Strict Stage-B network metrics and paired-artifact tests.""" + +import csv +import json +from pathlib import Path +import sys +import tempfile +import unittest + +import numpy as np + + +CODE_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(CODE_ROOT)) + +from analysis.make_paper_artifacts import ( # noqa: E402 + _apply_network_paired_gates, + generate_paper_source_data, +) +from analysis.metrics import MetricError, derive_trial_metrics # noqa: E402 +from experiments.io import TrialPayload # noqa: E402 +from experiments.plan import build_trial_plan # noqa: E402 +from experiments.runner import run_trial_plan # noqa: E402 + + +LIMIT_FIELDS = ( + "master_joint_limit_active", + "slave_joint_limit_active", + "master_velocity_limit_active", + "slave_velocity_limit_active", + "master_acceleration_limit_active", + "slave_acceleration_limit_active", + "master_torque_saturation_active", + "slave_torque_saturation_active", + "haptic_rate_limit_active", + "haptic_torque_saturation_active", +) + + +def network_samples( + *, + contact_expected: bool = True, + contact_force_N: float = 0.2, + zero_delay_error_rad: float = 0.02, + reference_lag_error_rad: float = 0.01, + feedback_lag_error_Nm: float = 0.01, +) -> dict[str, np.ndarray]: + count = 5 + contact = np.full(count, contact_force_N, dtype=float) + zeros = np.zeros(count, dtype=np.int8) + samples: dict[str, np.ndarray] = { + "energy_before_J": np.ones(count), + "energy_after_J": np.ones(count), + "tau_master_candidate": np.zeros((count, 2)), + "tau_master_applied": np.zeros((count, 2)), + "qd_master": np.zeros((count, 2)), + "dt": np.full(count, 0.01), + "configured_energy_min_J": np.zeros(count), + "configured_energy_max_J": np.full(count, 2.0), + "master_tracking_error": np.full(count, 0.01), + "slave_tracking_error": np.full(count, 0.02), + "contact_force_norm": contact, + "rho": np.ones(count), + "wall_force_raw_N": contact.copy(), + "wall_force_applied_N": contact.copy(), + "wall_force_saturation_active": zeros.copy(), + "configured_wall_force_limit_N": np.full(count, 20.0), + "energy_probe_raw_work_J": np.zeros(count), + "forward_packet_state": np.array([4, 2, 1, 2, 1]), + "forward_packet_active": np.ones(count, dtype=np.int8), + "forward_packet_fresh": np.array([1, 0, 1, 0, 1]), + "forward_packet_age": np.array([0.0, 0.01, 0.0, 0.01, 0.0]), + "forward_packet_seq": np.array([0, 0, 1, 1, 2]), + "return_packet_state": np.array([0, 4, 2, 1, 2]), + "return_packet_active": np.array([0, 1, 1, 1, 1]), + "return_packet_fresh": np.array([0, 1, 0, 1, 0]), + "return_packet_age": np.array([np.nan, 0.0, 0.01, 0.0, 0.01]), + "return_packet_seq": np.array([-1, 0, 0, 1, 1]), + "slave_zero_delay_tracking_error": np.full( + count, zero_delay_error_rad + ), + "slave_reference_lag_error": np.full( + count, reference_lag_error_rad + ), + "return_feedback_lag_error": np.full( + count, feedback_lag_error_Nm + ), + "contact_expected": np.full( + count, int(contact_expected), dtype=np.int8 + ), + "configured_forward_delay_s": np.zeros(count), + "configured_return_delay_s": np.zeros(count), + "configured_forward_jitter_s": np.zeros(count), + "configured_return_jitter_s": np.zeros(count), + "configured_forward_packet_loss": np.zeros(count), + "configured_return_packet_loss": np.zeros(count), + "configured_forward_timeout_s": np.full(count, 0.2), + "configured_return_timeout_s": np.full(count, 0.2), + } + for name in LIMIT_FIELDS: + samples[name] = zeros.copy() + return samples + + +def network_metric_configuration(*, paired: bool = False) -> dict: + configuration = { + "enabled": ["h4", "bilateral", "network"], + "h4": {"audit_tolerance_J": 1e-12}, + "bilateral": { + "contact_force_threshold_N": 1e-6, + "gates": { + "minimum_contact_fraction": 0.0, + "minimum_contact_rms_N": 0.0, + "maximum_force_limit_hit_fraction": 0.0, + "minimum_force_headroom_N": 1.0, + "maximum_master_tracking_rmse_rad": 0.08, + "maximum_slave_tracking_rmse_rad": 0.08, + "minimum_projection_intervention_fraction": 0.0, + "maximum_projection_intervention_fraction": 0.05, + "maximum_limit_active_fraction": 0.0, + "minimum_energy_probe_raw_work_J": 0.0, + }, + }, + "network": { + "gates": { + "maximum_slave_zero_delay_tracking_rmse_rad": 0.1, + "minimum_forward_active_fraction": 0.8, + "minimum_return_active_fraction": 0.8, + "minimum_forward_fresh_fraction": 0.5, + "minimum_return_fresh_fraction": 0.4, + "maximum_forward_internal_missing_fraction": 0.0, + "maximum_return_internal_missing_fraction": 0.0, + "maximum_forward_timeout_fraction": 0.0, + "maximum_return_timeout_fraction": 0.0, + "maximum_forward_packet_age_s": 0.2, + "maximum_return_packet_age_s": 0.2, + "minimum_contact_fraction": 0.5, + "minimum_contact_rms_N": 0.1, + "maximum_free_space_contact_force_N": 1e-6, + } + }, + } + if paired: + configuration["network"]["paired_gates"] = { + "nominal_profile_id": "nominal", + "maximum_tracking_rmse_delta_vs_nominal_rad": 0.01, + "maximum_abs_contact_rms_relative_change_vs_nominal": 0.2, + } + return configuration + + +def paired_network_executor(trial): + profile = trial["factors"]["network_profile"]["profile_id"] + impaired = profile != "nominal" + return TrialPayload( + samples=network_samples( + zero_delay_error_rad=0.025 if impaired else 0.02, + feedback_lag_error_Nm=0.015 if impaired else 0.01, + contact_force_N=0.22 if impaired else 0.20, + ), + metadata={ + "bilateral_data_group_id": "bilateral-data-test", + "bilateral_data_seed_record_hash": "bilateral-seed-hash", + "network_pair_group_id": "network-pair-test", + }, + ) + + +class NetworkMetricsV3Test(unittest.TestCase): + def test_strict_packet_evidence_rejects_nan_state_and_sequence_forgery(self): + configuration = network_metric_configuration() + cases = [] + + active_nan = network_samples() + active_nan["forward_packet_age"][0] = np.nan + cases.append((active_nan, "finite and non-negative while active")) + + inactive_number = network_samples() + inactive_number["return_packet_age"][0] = 0.0 + cases.append((inactive_number, "must be NaN while inactive")) + + forged_state = network_samples() + forged_state["forward_packet_fresh"][1] = 1 + cases.append((forged_state, "disagrees with forward_packet_state")) + + stale_sequence = network_samples() + stale_sequence["forward_packet_seq"][2] = 0 + cases.append((stale_sequence, "strictly increase on fresh packets")) + + noninteger_state = network_samples() + noninteger_state["return_packet_state"] = ( + noninteger_state["return_packet_state"].astype(float) + ) + noninteger_state["return_packet_state"][2] = 2.5 + cases.append((noninteger_state, "integer states 0..4")) + + for samples, message in cases: + with self.subTest(message=message): + with self.assertRaisesRegex(MetricError, message): + derive_trial_metrics(samples, configuration) + + def test_freshness_and_internal_missing_gates_are_independent(self): + configuration = network_metric_configuration() + baseline = derive_trial_metrics( + network_samples(), + configuration, + ) + for name in ( + "network_forward_fresh_gate_pass", + "network_return_fresh_gate_pass", + "network_forward_internal_missing_gate_pass", + "network_return_internal_missing_gate_pass", + ): + self.assertTrue(baseline[name]) + + compatibility_configuration = network_metric_configuration() + for name in ( + "minimum_forward_fresh_fraction", + "minimum_return_fresh_fraction", + "maximum_forward_internal_missing_fraction", + "maximum_return_internal_missing_fraction", + ): + compatibility_configuration["network"]["gates"].pop(name) + compatibility = derive_trial_metrics( + network_samples(), + compatibility_configuration, + ) + self.assertTrue(compatibility["network_local_gate_pass"]) + + forward_stale = network_samples() + forward_stale["forward_packet_state"][-1] = 2 + forward_stale["forward_packet_fresh"][-1] = 0 + forward_stale["forward_packet_seq"][-1] = 1 + + return_stale = network_samples() + return_stale["return_packet_state"][3] = 2 + return_stale["return_packet_fresh"][3] = 0 + return_stale["return_packet_seq"][3:] = 0 + + forward_missing = network_samples() + forward_missing["forward_packet_seq"][2:] = [2, 2, 3] + + return_missing = network_samples() + return_missing["return_packet_seq"][3:] = 2 + + cases = ( + ( + forward_stale, + "network_forward_fresh_gate_pass", + ), + ( + return_stale, + "network_return_fresh_gate_pass", + ), + ( + forward_missing, + "network_forward_internal_missing_gate_pass", + ), + ( + return_missing, + "network_return_internal_missing_gate_pass", + ), + ) + for samples, failed_gate in cases: + with self.subTest(failed_gate=failed_gate): + metrics = derive_trial_metrics(samples, configuration) + self.assertFalse(metrics[failed_gate]) + self.assertFalse(metrics["network_local_gate_pass"]) + self.assertFalse(metrics["network_local_full_gate_pass"]) + + def test_contact_and_free_space_gates_are_condition_specific(self): + configuration = network_metric_configuration() + contact = derive_trial_metrics(network_samples(), configuration) + self.assertTrue(contact["network_contact_expected"]) + self.assertTrue(contact["network_contact_condition_gate_pass"]) + self.assertTrue(contact["network_local_full_gate_pass"]) + + missing_contact = derive_trial_metrics( + network_samples(contact_force_N=0.0), + configuration, + ) + self.assertFalse( + missing_contact["network_contact_fraction_gate_pass"] + ) + self.assertFalse(missing_contact["network_contact_rms_gate_pass"]) + self.assertFalse(missing_contact["network_local_full_gate_pass"]) + + free_space = derive_trial_metrics( + network_samples( + contact_expected=False, + contact_force_N=0.0, + ), + configuration, + ) + self.assertFalse(free_space["network_contact_expected"]) + self.assertTrue(free_space["network_free_space_peak_gate_pass"]) + self.assertTrue(free_space["network_local_full_gate_pass"]) + + false_contact = derive_trial_metrics( + network_samples( + contact_expected=False, + contact_force_N=0.01, + ), + configuration, + ) + self.assertFalse( + false_contact["network_free_space_peak_gate_pass"] + ) + self.assertFalse(false_contact["network_local_full_gate_pass"]) + + def test_network_family_requires_h4_and_bilateral_first(self): + with self.assertRaisesRegex( + MetricError, "must follow H4 and bilateral" + ): + derive_trial_metrics( + network_samples(), + {"enabled": ["network"], "network": {}}, + ) + + def test_paired_artifacts_preserve_ids_and_compute_nominal_deltas(self): + plan = build_trial_plan( + { + "study_id": "network_paired_source_data", + "split": "pilot", + "root_seed": 13, + "replicates": 1, + "methods": ["proposed_energy"], + "trajectories": [ + { + "trajectory_id": "contact_network", + "family": "contact_roundtrip", + } + ], + "factors": { + "network_profile": [ + {"profile_id": "nominal"}, + {"profile_id": "delay"}, + ] + }, + } + ) + configuration = network_metric_configuration(paired=True) + with tempfile.TemporaryDirectory() as temporary: + batch = Path(temporary) / "batch" + run_trial_plan(plan, batch, paired_network_executor) + manifest = generate_paper_source_data(batch, configuration) + self.assertEqual(manifest["family_row_counts"]["network"], 2) + rows = [ + json.loads(line) + for line in ( + batch / "derived" / "trial_metrics.jsonl" + ).read_text(encoding="utf-8").splitlines() + ] + nominal = next( + row + for row in rows + if row["network_profile_id"] == "nominal" + ) + delayed = next( + row + for row in rows + if row["network_profile_id"] == "delay" + ) + self.assertEqual(nominal["bilateral_data_group_id"], "bilateral-data-test") + self.assertEqual( + nominal["bilateral_data_seed_record_hash"], + "bilateral-seed-hash", + ) + self.assertEqual( + nominal["network_pair_group_id"], "network-pair-test" + ) + self.assertEqual( + nominal[ + "network_zero_delay_tracking_rmse_delta_vs_nominal_rad" + ], + 0.0, + ) + self.assertEqual( + nominal[ + "network_return_feedback_lag_rmse_delta_vs_nominal_Nm" + ], + 0.0, + ) + self.assertAlmostEqual( + delayed[ + "network_zero_delay_tracking_rmse_delta_vs_nominal_rad" + ], + 0.005, + ) + self.assertAlmostEqual( + delayed[ + "network_return_feedback_lag_rmse_delta_vs_nominal_Nm" + ], + 0.005, + ) + self.assertAlmostEqual( + delayed[ + "network_contact_rms_relative_change_vs_nominal" + ], + 0.1, + ) + self.assertTrue(delayed["network_paired_gate_pass"]) + self.assertTrue(delayed["network_full_gate_pass"]) + + with ( + batch / "paper" / "source_data" / "network.csv" + ).open(newline="", encoding="utf-8") as stream: + table = list(csv.DictReader(stream)) + self.assertEqual(len(table), 2) + for name in ( + "bilateral_data_group_id", + "bilateral_data_seed_record_hash", + "network_pair_group_id", + "network_profile_id", + ): + self.assertIn(name, table[0]) + + def test_free_space_paired_contact_metrics_are_not_applicable(self): + configuration = network_metric_configuration(paired=True) + + def row(profile, tracking_rmse): + return { + "trial_id": f"trial-{profile}", + "network_pair_group_id": "free-space-group", + "network_profile_id": profile, + "method_id": "proposed_energy", + "trajectory_id": "free-space", + "replicate": 0, + "network_local_full_gate_pass": True, + "network_contact_expected": False, + "network_slave_zero_delay_tracking_rmse_rad": tracking_rmse, + "network_return_feedback_lag_rmse_Nm": 0.01, + "network_contact_force_rms_N": 0.0, + } + + rows = [row("nominal", 0.02), row("delay", 0.04)] + _apply_network_paired_gates(rows, configuration) + + nominal, delayed = rows + for result in rows: + self.assertIsNone( + result[ + "network_contact_rms_relative_change_vs_nominal" + ] + ) + self.assertIsNone( + result[ + "network_contact_force_rms_relative_change_vs_nominal" + ] + ) + self.assertIsNone( + result["network_paired_contact_gate_pass"] + ) + self.assertEqual( + result["network_paired_gate_pass"], + result["network_paired_tracking_gate_pass"], + ) + self.assertTrue(nominal["network_paired_gate_pass"]) + self.assertFalse(delayed["network_paired_gate_pass"]) + + def test_paired_artifacts_reject_missing_or_duplicate_nominal(self): + configuration = network_metric_configuration(paired=True) + + def row(profile): + return { + "trial_id": f"trial-{profile}", + "network_pair_group_id": "group", + "network_profile_id": profile, + "method_id": "proposed_energy", + "trajectory_id": "contact", + "replicate": 0, + "network_local_full_gate_pass": True, + "network_contact_expected": True, + "network_slave_zero_delay_tracking_rmse_rad": 0.02, + "network_return_feedback_lag_rmse_Nm": 0.01, + "network_contact_force_rms_N": 0.2, + } + + with self.assertRaisesRegex(ValueError, "missing nominal"): + _apply_network_paired_gates([row("delay")], configuration) + with self.assertRaisesRegex(ValueError, "duplicate nominal"): + _apply_network_paired_gates( + [row("nominal"), row("nominal")], + configuration, + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/calibration/V3_CONFIRMATION_AND_NETWORK_STAGE_B_2026-07-27.md b/docs/calibration/V3_CONFIRMATION_AND_NETWORK_STAGE_B_2026-07-27.md new file mode 100644 index 0000000..907a1ec --- /dev/null +++ b/docs/calibration/V3_CONFIRMATION_AND_NETWORK_STAGE_B_2026-07-27.md @@ -0,0 +1,156 @@ +# V3 confirmation and Stage-B network audit — 2026-07-27 + +## Scope + +This record covers pre-prototype rigid-body simulation only. It does not +establish physical stability, real-network performance, wrench accuracy, or +human-subject benefit. The network backend emulates fixed delay, independent +uniform jitter, and independent packet loss; it does not emulate bandwidth +limits, burst loss, serialization, clock drift, or a measured network trace. + +## Expanded Stage-A calibration + +The selected contact condition is: + +- wall stiffness: `3200 N/m`; +- wall damping: `25 Ns/m`; +- haptic gain: `0.35`; +- wide-tank bounds: `[0, 2] J`, initial energy `1 J`; +- energy-challenge probe: `0.0395 Nm`. + +The original three calibration replicates were expanded to 20 replicates with +the same data-root convention. Therefore, these runs contain 17 additional +instances but are not an independent confirmation set. + +| Study | Gate result | Key range | +|---|---:|---| +| Stable contact | 20/20 pass | master RMSE `0.03585–0.03620 rad`; slave RMSE `0.04585–0.04622 rad`; contact RMS `0.10595–0.10762 N` | +| Synthetic H4 challenge | 20/20 pass | projection fraction `0.0483–0.2890`; shadow deficit `4.99e-5–1.51e-4 J` | + +Across 60,000 samples in each expanded study, all recorded wall, joint, +velocity, acceleration, actuator-torque, haptic-rate, and haptic-torque limit +flags remained zero. + +Local evidence: + +- `output/experiments/bilateral-v3-expanded-stable-20260727` + - plan hash: `34343a11dd0c807a9099bdb7c3442f77476f425ffed6ff97348aa3a72b2eac14`; + - row hash: `7653213736fa79fe03d82eef8d3f86cb17a8c7c0838e7461c76957d7c0a7ccc1`. +- `output/experiments/bilateral-v3-expanded-energy-20260727` + - plan hash: `8254f86ed97e872937405aea24d31c0d48a73c2f66d5a475f3767a15baedb0e0`; + - row hash: `483c31f2a584719fbd2db3113494b1e7599e9e7a44e1dd523cb9d17cd38c21bb`. + +The disjoint-root locked protocols are +`bilateral_locked_v3_stable_contact.json` and +`bilateral_locked_v3_energy_challenge.json`. + +## Stage-B calibration design + +The Stage-B screen fixes the selected mechanics and haptic settings, disables +the synthetic energy probe, and evaluates two 6 s trajectories: + +- free-space round trip; +- slow contact round trip. + +Five network profiles are paired with common random numbers inside each +trajectory/replicate block: + +1. nominal; +2. symmetric `40/40 ms` delay; +3. asymmetric `20/60 ms` delay with the same `80 ms` RTT; +4. symmetric `40/40 ms` delay plus independent uniform `±4 ms` jitter; +5. symmetric `40/40 ms` delay plus independent `2%` loss in each direction. + +The calibration grid contains `2 × 5 × 3 = 30` proposed-method trials. + +### Audited endpoints + +The legacy slave tracking error is relative to the delayed, held controller +reference and can hide forward-path lag. Stage-B therefore adds: + +- slave tracking error relative to the latest immediately available mapping + reference; +- lag between the delayed and latest mapping references; +- raw feedback-torque lag relative to an immediate-return shadow; +- packet source age, freshness, hold, timeout, recovery, and internal sequence + gaps; +- profile-minus-nominal paired degradation. + +The two within-trial shadows retain the current disturbed system state. They +remove one transport delay but are not independent zero-network +counterfactuals. The paired nominal trial is the causal network baseline. + +## Stage-B calibration outcome + +All 30 trials completed and validated: + +- local network gate: 30/30 pass; +- H4 plus bilateral safety gate: 30/30 pass; +- paired profile-minus-nominal gate: 30/30 pass; +- final Stage-B gate: 30/30 pass; +- H4 accounting audit: 30/30 pass; +- energy projection, force saturation, and all recorded state/actuator/haptic + limit fractions: zero. + +The most informative directional comparison holds RTT at `80 ms`: + +- changing `40/40 ms` to `20/60 ms` reduced contact reference-lag RMSE from + about `3.49 mrad` to `1.75 mrad`; +- the same change increased return-feedback-lag RMSE from about `15.2 mNm` to + `18.4 mNm`. + +The largest paired zero-delay tracking degradation was about `2.52 mrad` in +free space. The largest absolute contact-RMS change was about `0.53%`. +Free-space contact force remained exactly zero. + +The jitter condition needs explicit interpretation. Because the return channel +publishes at `500 Hz`, independent `±4 ms` jitter reorders many packets: + +- return internal sequence-gap fraction: approximately `44.1–45.3%`; +- return active fraction: approximately `99.3%`; +- timeout fraction: zero. + +Thus, the controller remained supplied by a held valid packet, but feedback +freshness was substantially lower. “No timeout” must not be reported as “no +information loss.” + +H3 is descriptive in this study. Free-space trials were below the registered +power-activity floor; contact trials were activity-valid. H3 is not a bounded +stability score and is not used as a Stage-B pass/fail gate. + +Local evidence: + +- `output/experiments/network-v3-screening-20260727`; +- plan hash: + `25adcf6bfce4c24dc0b13672258996e5216951479976fc9a9d8ef79f35944e79`; +- metric configuration hash: + `8c4a33203462ccaf0821ee89c3e288abd952dcf07dc7e776e7b770ecaac425da`; +- row hash: + `deeb8699deadebf2f551eef9f4ec7fb1b42c6364ae50d124d6b0f1296d68061a`. + +## Frozen locked thresholds + +The disjoint-seed locked protocol freezes the following before execution: + +- zero-delay slave tracking RMSE: `≤ 0.06 rad`; +- paired degradation relative to nominal: `≤ 0.005 rad`; +- absolute contact-RMS relative change: `≤ 10%`; +- contact fraction for contact trajectories: `≥ 0.30`; +- contact-active RMS: `≥ 0.10 N`; +- forward/return active fraction: `≥ 0.98`; +- forward/return fresh fraction: `≥ 0.09 / 0.50`; +- forward/return internal missing fraction: `≤ 0.05 / 0.50`; +- forward/return source age: `≤ 0.10 / 0.08 s`; +- forward/return timeout fraction: zero; +- free-space peak contact force: `≤ 1e-6 N`; +- energy projection fraction: `≤ 0.05`; +- all wall/state/actuator/haptic limit fractions: zero. + +Free-space contact relative change is reported as not applicable; its absolute +false-contact gate is used instead. + +The proposed-only locked protocol is +`bilateral_network_v3_locked.json`. Passing it can support a bounded statement +about the proposed loop under the registered emulator. It cannot support +superiority over mapping baselines; that requires a separate locked comparison +including `direct_energy` and `matched_wrench_energy`.