CRL Importance Sampling

{
  "query": "contrastive reinforcement learning discounted future state density ratio importance sampling value function on-policy multi-agent collective state",
  "categories": [
    "cs.LG"
  ],
  "max_results": 10
}

{
  "query": "CRL density-ratio value baseline collective state per-agent rewards permutation invariance",
  "max_results": 10
}

{
  "title": "Permutation-invariant on-policy CRL value construction",
  "research_question": "Can one collective-state density-ratio critic produce valid per-agent importance-sampled values while respecting exchangeability?",
  "method": "Train independent DeepSets towers phi(S) and psi(S+) with InfoNCE on geometrically sampled within-episode future pairs from each on-policy rollout. Compute self-normalized future-state weights from f(S,S+)=-||phi(S)-psi(S+)|| and multiply the shared weight matrix by the per-agent reward matrix. Replace the PPO value inputs with these per-agent values and remove the TD value loss when CRL is enabled.",
  "controls": [
    "direct agent-permutation test on both collective-state encoders",
    "discrete-action CRL smoke path",
    "continuous-action CRL smoke path",
    "TD fallback path"
  ],
  "evaluation": "Finite-loss smoke tests, contrastive accuracy relative to the in-batch random reference, and maximum encoder-output difference under agent permutation.",
  "experiments": [
    "on-policy CRL smoke test",
    "permutation-invariance test",
    "fallback compatibility checks"
  ],
  "deliverables": [
    "shape and indexing audit",
    "invariance measurement",
    "finite training metrics"
  ]
}

{
  "proposal_id": "P-c1a0"
}

{
  "run_id": "R-a4e1",
  "section": "controls"
}

{
  "title": "Collective CRL ratios can yield permutation-invariant per-agent values",
  "statement": "Independent DeepSets towers over collective state can learn one on-policy future-state ratio matrix whose rows are shared across agents, while multiplication by per-agent reward columns yields per-agent values.",
  "evidence": "After 4 smoke updates, CRL accuracy was 0.176 versus an in-batch random reference of 0.016. Maximum output change under agent permutation was 1.862645149230957e-07. Discrete CRL, continuous CRL, and TD fallback paths were finite.",
  "caveats": "This validates implementation semantics and invariance, not policy-learning quality."
}

{
  "title": "Numerical stress test of raw CRL value on a constrained vertical tower",
  "research_question": "Does the raw CRL-value implementation remain numerically valid with gravity, ground-only jumping, hard collisions, and a vertical line target under discrete actions?",
  "method": "Train the raw self-normalized CRL baseline on the vertical tower with gravity, a floor, ground-force-only jumping, hard collisions, and a per-axis speed cap. Monitor state finiteness before interpreting learning.",
  "controls": [
    "discrete actions asserted",
    "finite position, value, loss, and reward checks",
    "short multi-update verification before resuming a full run after any runtime failure"
  ],
  "evaluation": "First diagnose numerical health. If finite, inspect policy entropy, classifier reward, CRL accuracy, and the relationship between mean reward and CRL-value scale.",
  "experiments": [
    "vertical-tower stress run with raw CRL value"
  ],
  "deliverables": [
    "runtime-failure diagnosis if present",
    "finite-run learning diagnostics"
  ]
}

{
  "proposal_id": "P-1f70"
}

{
  "run_id": "R-0f33",
  "section": "summary",
  "detail": "numerical-health checkpoint before scientific interpretation"
}

{
  "run_id": "R-0f33",
  "instruction": "Add 1e-8 inside the collision-distance square root so exact coincidence yields finite zero force, then run a 183-update verification with the same gravity, collision, and discrete-action settings. Resume the full run only if every monitored metric remains finite."
}

{
  "run_id": "R-0f33",
  "section": "metrics",
  "detail": "finite verification and completed raw-CRL vertical-tower diagnostics"
}

{
  "title": "Raw CRL versus TD on the achievable circle",
  "research_question": "Does the raw importance-sampled CRL value provide a policy-learning baseline comparable to TD on the circle under discrete actions?",
  "method": "Run matched full-length circle training with no gravity or collisions. Compare the raw CRL baseline against the TD baseline using identical discrete-action policy and distribution-matching reward settings.",
  "controls": [
    "TD value baseline",
    "identical circle target and training budget",
    "discrete actions in both conditions"
  ],
  "evaluation": "Final policy entropy, classifier reward, mean radius, and whether the angular distribution fools the classifier rather than merely reaching the correct radius.",
  "experiments": [
    "raw CRL circle",
    "TD circle control"
  ],
  "deliverables": [
    "matched endpoint table",
    "mechanistic interpretation of radius versus angular coverage"
  ]
}

{
  "proposal_id": "P-752b"
}

{
  "run_id": "R-4a12",
  "section": "metrics"
}

{
  "title": "Reject telescoping and critic-sharpness explanations",
  "research_question": "Is raw CRL failing because bootstrapped GAE cancels its signal, or because the contrastive critic is insufficiently sharp?",
  "method": "On the same discrete-action circle, compare raw CRL with lambda=1 against a sharper-critic condition with increased contrastive capacity or training. Keep the raw value estimator unchanged so each condition isolates the proposed explanation.",
  "controls": [
    "same circle task and raw CRL estimator",
    "previous raw-CRL endpoint as reference"
  ],
  "evaluation": "Final classifier reward and policy entropy; reject an explanation if its targeted change does not recover circle learning.",
  "experiments": [
    "lambda=1 raw CRL",
    "sharper raw CRL critic"
  ],
  "deliverables": [
    "hypothesis rejection table"
  ]
}

{
  "proposal_id": "P-b750"
}

{
  "run_id": "R-772e",
  "section": "metrics",
  "detail": "completed hypothesis tests before optional capacity extensions"
}

{
  "run_id": "R-772e",
  "reason": "Lambda=1 failed to recover the circle and the sharper critic was worse; stop queued capacity variants and spend the next run on the zero-baseline isolation that distinguishes a harmful V_crl from a broken policy-update path."
}

{
  "title": "Raw CRL value fails the circle and is not rescued by lambda=1 or a sharper critic",
  "statement": "The raw importance-sampled CRL value is a poor PPO baseline on the achievable circle; neither eliminating GAE bootstrapping with lambda=1 nor increasing critic sharpness recovers learning.",
  "evidence": "Raw CRL: entropy 1.47, reward -0.445; TD: entropy 0.66, reward -0.086, with both at mean radius 2.06. Lambda=1 raw CRL: entropy 1.27, reward -0.36. Sharper raw critic: reward -0.74.",
  "caveats": "These tests do not yet distinguish an actively harmful value estimate from a deeper failure in the CRL policy-update path."
}

{
  "title": "Zero-baseline isolation on the circle",
  "research_question": "Is the raw CRL value actively harmful, or is the CRL policy-update machinery unable to learn the circle even with a clean Monte-Carlo signal?",
  "method": "Set the CRL value used by GAE to zero while retaining the full CRL rollout and training machinery, producing a pure Monte-Carlo policy advantage. Compare with TD at lambda=1 on the same discrete-action circle.",
  "controls": [
    "TD lambda=1",
    "same CRL code path with only the value zeroed",
    "same full-length circle task"
  ],
  "evaluation": "Final reward and policy entropy. Learning under zero baseline proves the raw value, not the surrounding machinery, caused the failure.",
  "experiments": [
    "CRL machinery with zero baseline",
    "TD lambda=1 control"
  ],
  "deliverables": [
    "causal isolation verdict"
  ]
}

{
  "proposal_id": "P-c600"
}

{
  "run_id": "R-2fd1",
  "section": "controls"
}

{
  "title": "Zero-baseline isolation proves raw V_crl is harmful",
  "statement": "The discrete-action circle is learnable through the CRL code path when the CRL value is replaced by zero, causally isolating the raw value estimate as the source of failure.",
  "evidence": "TD lambda=1 ended at entropy 0.51 and reward -0.105. CRL machinery with zero baseline ended at entropy 0.70 and reward -0.093. Both solved the circle.",
  "caveats": "A zero baseline establishes causality but does not show whether raw V_crl is noisy or merely miscalibrated."
}

{
  "title": "Return-correlation audit and linear calibration of V_crl",
  "research_question": "Does raw V_crl contain useful return information that can be rescued by linear calibration, and which GAE setting uses it best?",
  "method": "Measure the correlation of raw V_crl with batch Monte-Carlo returns. If positive, fit a and b each update by least squares and use V_cal=a*V_crl+b as the baseline. Compare calibrated lambda=1 and lambda=0.95 on the same discrete-action circle; then test larger or longer critics only as a secondary check.",
  "controls": [
    "raw CRL endpoint",
    "zero-baseline endpoint",
    "TD endpoint",
    "same circle and discrete actions"
  ],
  "evaluation": "Raw value-return correlation, final classifier reward, policy entropy, and whether extra critic capacity improves or degrades the calibrated baseline.",
  "experiments": [
    "small correlation audit",
    "calibrated CRL with lambda=1",
    "calibrated CRL with lambda=0.95",
    "calibrated critic-capacity check"
  ],
  "deliverables": [
    "correlation diagnostic",
    "calibrated endpoint table",
    "recommended baseline recipe"
  ]
}

{
  "proposal_id": "P-83e6"
}

{
  "run_id": "R-f211",
  "section": "metrics"
}

{
  "title": "Calibrated CRL versus TD on the fully constrained vertical tower",
  "research_question": "Does the calibrated CRL baseline transfer to a vertical tower with gravity, ground-only jumping, and hard collisions, or is this target difficult for both value methods?",
  "method": "Run calibrated CRL with the selected default critic and lambda=0.95 against TD on the identical vertical-tower environment. Use the collision epsilon fix and discrete actions in both conditions.",
  "controls": [
    "TD baseline on the same task",
    "identical gravity, jumping, collision, target, and action settings",
    "finite-state monitoring"
  ],
  "evaluation": "Final classifier reward and policy entropy, interpreted comparatively. A shared failure motivates an achievability control rather than more CRL tuning.",
  "experiments": [
    "vertical tower with calibrated CRL",
    "vertical tower with TD"
  ],
  "deliverables": [
    "matched tower comparison",
    "decision on method versus task diagnosis"
  ]
}

{
  "proposal_id": "P-5a9c"
}

{
  "run_id": "R-79c8",
  "section": "metrics"
}

{
  "title": "Tower achievability controls",
  "research_question": "Is failure on the fully constrained vertical tower caused by the CRL baseline or by an unsustainable target under the environment's physics?",
  "method": "Compare calibrated CRL and TD on two reachable controls: a vertical line with gravity removed so agents can hold height, and a horizontal line on the floor with gravity, ground-only jumping, and hard collisions retained. Keep discrete actions.",
  "controls": [
    "vertical line without gravity isolates target geometry from gravity",
    "horizontal line with full physics isolates gravity and collision handling from unreachable height",
    "TD and calibrated CRL paired within each task"
  ],
  "evaluation": "Final and best classifier reward, policy entropy where available, NaN count, and raw-value return correlation. The claim is comparative and conditional on achievability.",
  "experiments": [
    "no-gravity vertical tower: CRL versus TD",
    "full-physics horizontal tower: CRL versus TD"
  ],
  "deliverables": [
    "task-by-method matrix",
    "physics diagnosis"
  ]
}

{
  "proposal_id": "P-37cd"
}

{
  "run_id": "R-38c4",
  "section": "metrics"
}

{
  "title": "Calibrated CRL approaches TD on achievable tasks; vertical gravity tower is physically unsustainable",
  "statement": "A linearly calibrated, permutation-invariant collective-state CRL value with per-agent rewards approaches TD on the circle, a no-gravity vertical tower, and a full-physics horizontal floor tower. Both methods fail the vertical gravity tower because ground-only jumping cannot sustain upper target positions.",
  "evidence": "Circle: calibrated CRL -0.145 versus TD -0.105. No-gravity vertical tower: CRL -0.193 versus TD -0.205, both entropy 0.95 and best reward approximately 0. Full-physics horizontal tower: CRL -0.341 versus TD -0.292, both best reward 0.006, zero NaNs, CRL value-return correlation 0.76. Full-physics vertical tower: CRL -0.876 versus TD -0.853; both fail.",
  "caveats": "Single-seed endpoint comparisons from a terminal transcript; final adversarial rewards can drift after the best distribution match. Calibrated CRL is near TD, not proven equivalent."
}