{ "arms": { "baseline": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "baseline", "execution_error_rate": 0.525, "fallback_rate": 0.0, "mean_automated_score": -28.8, "mean_final_score": 4.3, "mean_score_auc": 9.55, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.2775401687666166, "pass_rate_ci95_low": 0.0, "sample_count": 10, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput", "iron_ore_throughput" ], "total_adapt_latency_seconds": 0.0, "total_model_latency_seconds": 1757.8781490810006, "total_tokens": 114465 }, "domain_only": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "domain_only", "execution_error_rate": 0.35, "fallback_rate": 0.0, "mean_automated_score": -29.1, "mean_final_score": 0.9, "mean_score_auc": 1.35, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.2775401687666166, "pass_rate_ci95_low": 0.0, "sample_count": 10, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput", "iron_ore_throughput" ], "total_adapt_latency_seconds": 17.814461480995305, "total_model_latency_seconds": 1926.3853546639966, "total_tokens": 128995 }, "memory_only": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "memory_only", "execution_error_rate": 0.55, "fallback_rate": 0.0, "mean_automated_score": -29.1, "mean_final_score": 0.9, "mean_score_auc": 1.95, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.2775401687666166, "pass_rate_ci95_low": 0.0, "sample_count": 10, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput", "iron_ore_throughput" ], "total_adapt_latency_seconds": 6.9264798930034885, "total_model_latency_seconds": 1983.483683827999, "total_tokens": 131199 }, "warm_frozen": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "warm_frozen", "execution_error_rate": 0.325, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.2775401687666166, "pass_rate_ci95_low": 0.0, "sample_count": 10, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput", "iron_ore_throughput" ], "total_adapt_latency_seconds": 23.06145454299258, "total_model_latency_seconds": 1702.3130158450003, "total_tokens": 138262 } }, "comparison_fingerprint": "6d9ef73080714dfbb12d5b9346b07ca62c69e96b8fae43e7e47352acbb8ba058", "duplicate_cells": [], "experiment_complete": true, "experiment_id": "experiment-20260811T041432Z-3ff8ac0a", "generated_from": ".fle/adapt1/local-research-v1", "missing_cells": [], "runs": [ { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 184.104069635001, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-baseline-iron-gear-wheel-throughput-20260811T043805Z-9dbb4d13", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11495 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 103.7113160079989, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-baseline-iron-gear-wheel-throughput-20260811T044153Z-9a6eb1a3", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11507 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 99.97653019400059, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-baseline-iron-gear-wheel-throughput-20260811T044421Z-11478b76", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11495 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 99.32107656200151, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-baseline-iron-gear-wheel-throughput-20260811T044644Z-f662d7b3", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11495 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 94.28871270400032, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-baseline-iron-gear-wheel-throughput-20260811T044908Z-68bb2c8b", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11495 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "baseline", "model_latency_seconds": 212.3591295439992, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-baseline-iron-ore-throughput-20260811T041432Z-93ba39d0", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 10784 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 4, "fallback_count": 0, "final_automated_score": -27.0, "final_score": 3.0, "mode": "baseline", "model_latency_seconds": 369.8995767309998, "normalized_reward_sum": -0.8125, "operational_failure": false, "run_id": "eval-baseline-iron-ore-throughput-20260811T041847Z-3acc6551", "score_auc": 7.5, "steps": 4, "success": false, "token_count": 12233 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 4, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 31.0, "mode": "baseline", "model_latency_seconds": 280.55051645599997, "normalized_reward_sum": 0.25, "operational_failure": false, "run_id": "eval-baseline-iron-ore-throughput-20260811T042547Z-7062a984", "score_auc": 77.5, "steps": 4, "success": false, "token_count": 11337 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -24.0, "final_score": 6.0, "mode": "baseline", "model_latency_seconds": 197.4558083309994, "normalized_reward_sum": 0.375, "operational_failure": false, "run_id": "eval-baseline-iron-ore-throughput-20260811T043113Z-a68f6de7", "score_auc": 6.0, "steps": 4, "success": false, "token_count": 12088 }, { "abstention_count": 4, "adapt_latency_seconds": 0.0, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -27.0, "final_score": 3.0, "mode": "baseline", "model_latency_seconds": 116.21141291599997, "normalized_reward_sum": -0.3125, "operational_failure": false, "run_id": "eval-baseline-iron-ore-throughput-20260811T043519Z-03ffbd8c", "score_auc": 4.5, "steps": 4, "success": false, "token_count": 10536 }, { "abstention_count": 0, "adapt_latency_seconds": 1.748698343999422, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 183.3295205059967, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-domain-only-iron-gear-wheel-throughput-20260811T055300Z-503192fe", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12762 }, { "abstention_count": 0, "adapt_latency_seconds": 1.725467223999658, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 104.19533724400026, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-domain-only-iron-gear-wheel-throughput-20260811T055650Z-56a74196", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12776 }, { "abstention_count": 0, "adapt_latency_seconds": 1.749128823999854, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 100.98425126600159, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-domain-only-iron-gear-wheel-throughput-20260811T055920Z-867a1d1a", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12776 }, { "abstention_count": 0, "adapt_latency_seconds": 1.8686770490003255, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 110.697940131, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-domain-only-iron-gear-wheel-throughput-20260811T060146Z-7eaebc2e", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12788 }, { "abstention_count": 0, "adapt_latency_seconds": 1.7458407780013658, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 111.96133488299893, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-domain-only-iron-gear-wheel-throughput-20260811T060423Z-5464cf81", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12788 }, { "abstention_count": 0, "adapt_latency_seconds": 1.7684618169987516, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 220.29862151100133, "normalized_reward_sum": 0.0, "operational_failure": false, "run_id": "eval-domain-only-iron-ore-throughput-20260811T052724Z-18fe1905", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 13140 }, { "abstention_count": 0, "adapt_latency_seconds": 1.701113772996905, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -27.0, "final_score": 3.0, "mode": "frozen", "model_latency_seconds": 197.37766378699962, "normalized_reward_sum": 0.1875, "operational_failure": false, "run_id": "eval-domain-only-iron-ore-throughput-20260811T053143Z-82ee189d", "score_auc": 4.5, "steps": 4, "success": false, "token_count": 13063 }, { "abstention_count": 0, "adapt_latency_seconds": 1.8012566169982165, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 192.86376461700092, "normalized_reward_sum": 0.0, "operational_failure": false, "run_id": "eval-domain-only-iron-ore-throughput-20260811T053546Z-4f08aeaf", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 13100 }, { "abstention_count": 0, "adapt_latency_seconds": 1.855793339000229, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 179.95120584100005, "normalized_reward_sum": 0.0, "operational_failure": false, "run_id": "eval-domain-only-iron-ore-throughput-20260811T053939Z-fc49e8bc", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12601 }, { "abstention_count": 0, "adapt_latency_seconds": 1.8500237160005781, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 4, "fallback_count": 0, "final_automated_score": -24.0, "final_score": 6.0, "mode": "frozen", "model_latency_seconds": 524.7257148779972, "normalized_reward_sum": -0.625, "operational_failure": false, "run_id": "eval-domain-only-iron-ore-throughput-20260811T054318Z-4c5a8c82", "score_auc": 9.0, "steps": 4, "success": false, "token_count": 13201 }, { "abstention_count": 4, "adapt_latency_seconds": 0.6505647300018609, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 3, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 172.81314948999716, "normalized_reward_sum": -0.75, "operational_failure": false, "run_id": "eval-memory-only-iron-gear-wheel-throughput-20260811T063515Z-c7ad1d97", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11490 }, { "abstention_count": 4, "adapt_latency_seconds": 0.7088123499979702, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 88.88181130799967, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-memory-only-iron-gear-wheel-throughput-20260811T063853Z-f31f68aa", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11507 }, { "abstention_count": 4, "adapt_latency_seconds": 0.6020742529981362, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 88.1305624029992, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-memory-only-iron-gear-wheel-throughput-20260811T064106Z-46af8192", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11507 }, { "abstention_count": 4, "adapt_latency_seconds": 0.6610415020004439, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 89.02740298599929, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-memory-only-iron-gear-wheel-throughput-20260811T064319Z-b8e542b1", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11504 }, { "abstention_count": 4, "adapt_latency_seconds": 0.6161615189994336, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 84.73087063599996, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-memory-only-iron-gear-wheel-throughput-20260811T064532Z-495221c9", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11495 }, { "abstention_count": 4, "adapt_latency_seconds": 0.7226878070014209, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 266.41544960800275, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-memory-only-iron-ore-throughput-20260811T060701Z-a7e9c777", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 14027 }, { "abstention_count": 4, "adapt_latency_seconds": 0.746186601001682, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 3, "fallback_count": 0, "final_automated_score": -27.0, "final_score": 3.0, "mode": "frozen", "model_latency_seconds": 282.62715138000203, "normalized_reward_sum": -0.5625, "operational_failure": false, "run_id": "eval-memory-only-iron-ore-throughput-20260811T061212Z-ed24ade5", "score_auc": 7.5, "steps": 4, "success": false, "token_count": 14937 }, { "abstention_count": 4, "adapt_latency_seconds": 0.7364082120020612, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 4, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 386.803294274001, "normalized_reward_sum": -1.0, "operational_failure": false, "run_id": "eval-memory-only-iron-ore-throughput-20260811T061738Z-15f0740f", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15455 }, { "abstention_count": 4, "adapt_latency_seconds": 0.763324677998753, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 216.2899796439997, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-memory-only-iron-ore-throughput-20260811T062450Z-43fffb4f", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 14009 }, { "abstention_count": 4, "adapt_latency_seconds": 0.7192182410017267, "adapt_selection_count": 0, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -24.0, "final_score": 6.0, "mode": "frozen", "model_latency_seconds": 307.76401209899814, "normalized_reward_sum": -0.125, "operational_failure": false, "run_id": "eval-memory-only-iron-ore-throughput-20260811T062911Z-1796daff", "score_auc": 12.0, "steps": 4, "success": false, "token_count": 15268 }, { "abstention_count": 0, "adapt_latency_seconds": 2.3891100529999676, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 206.36011696399873, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-warm-frozen-iron-gear-wheel-throughput-20260811T051308Z-3f6827ca", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12745 }, { "abstention_count": 0, "adapt_latency_seconds": 2.2224771580004017, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 106.6218733909991, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-warm-frozen-iron-gear-wheel-throughput-20260811T051721Z-604a43f4", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12776 }, { "abstention_count": 0, "adapt_latency_seconds": 2.2383537629957573, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 99.06111720000081, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-warm-frozen-iron-gear-wheel-throughput-20260811T051954Z-8aa7609f", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12762 }, { "abstention_count": 0, "adapt_latency_seconds": 2.16969051599699, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 2, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 100.98124689200085, "normalized_reward_sum": -0.5, "operational_failure": false, "run_id": "eval-warm-frozen-iron-gear-wheel-throughput-20260811T052219Z-d99c55e8", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 12762 }, { "abstention_count": 0, "adapt_latency_seconds": 2.2118679709983553, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 0, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 119.06327509500079, "normalized_reward_sum": 0.0, "operational_failure": false, "run_id": "eval-warm-frozen-iron-gear-wheel-throughput-20260811T052446Z-e8a54453", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 11866 }, { "abstention_count": 0, "adapt_latency_seconds": 2.4001992119983697, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 265.57117886000015, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-warm-frozen-iron-ore-throughput-20260811T045126Z-3aa0b172", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15031 }, { "abstention_count": 0, "adapt_latency_seconds": 2.282597641999928, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 232.29290490099993, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-warm-frozen-iron-ore-throughput-20260811T045638Z-343e61b7", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15059 }, { "abstention_count": 0, "adapt_latency_seconds": 2.4824934670014045, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 239.77889078299995, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-warm-frozen-iron-ore-throughput-20260811T050117Z-08f0016c", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15115 }, { "abstention_count": 0, "adapt_latency_seconds": 2.2888481259997207, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 93.67054061199997, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-warm-frozen-iron-ore-throughput-20260811T050603Z-a8c129ac", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15059 }, { "abstention_count": 0, "adapt_latency_seconds": 2.3758166350016836, "adapt_selection_count": 4, "ambiguous_write_count": 0, "completion_status": "trajectory_limit", "execution_error_count": 1, "fallback_count": 0, "final_automated_score": -30.0, "final_score": 0.0, "mode": "frozen", "model_latency_seconds": 238.91187114700006, "normalized_reward_sum": -0.25, "operational_failure": false, "run_id": "eval-warm-frozen-iron-ore-throughput-20260811T050823Z-6345d0a9", "score_auc": 0.0, "steps": 4, "success": false, "token_count": 15087 } ], "task_arms": { "iron_gear_wheel_throughput": { "baseline": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "baseline", "execution_error_rate": 0.5, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput" ], "total_adapt_latency_seconds": 0.0, "total_model_latency_seconds": 581.4017051030023, "total_tokens": 57487 }, "domain_only": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "domain_only", "execution_error_rate": 0.5, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput" ], "total_adapt_latency_seconds": 8.837812219000625, "total_model_latency_seconds": 611.1683840299975, "total_tokens": 63890 }, "memory_only": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "memory_only", "execution_error_rate": 0.55, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput" ], "total_adapt_latency_seconds": 3.2386543539978447, "total_model_latency_seconds": 523.5837968229953, "total_tokens": 57503 }, "warm_frozen": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "warm_frozen", "execution_error_rate": 0.4, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_gear_wheel_throughput" ], "total_adapt_latency_seconds": 11.231499460991472, "total_model_latency_seconds": 632.0876295420003, "total_tokens": 62911 } }, "iron_ore_throughput": { "baseline": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "baseline", "execution_error_rate": 0.55, "fallback_rate": 0.0, "mean_automated_score": -27.6, "mean_final_score": 8.6, "mean_score_auc": 19.1, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_ore_throughput" ], "total_adapt_latency_seconds": 0.0, "total_model_latency_seconds": 1176.4764439779983, "total_tokens": 56978 }, "domain_only": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "domain_only", "execution_error_rate": 0.2, "fallback_rate": 0.0, "mean_automated_score": -28.2, "mean_final_score": 1.8, "mean_score_auc": 2.7, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_ore_throughput" ], "total_adapt_latency_seconds": 8.97664926199468, "total_model_latency_seconds": 1315.2169706339992, "total_tokens": 65105 }, "memory_only": { "abstention_rate": 1.0, "adapt_selection_rate": 0.0, "ambiguous_write_count": 0, "arm": "memory_only", "execution_error_rate": 0.55, "fallback_rate": 0.0, "mean_automated_score": -28.2, "mean_final_score": 1.8, "mean_score_auc": 3.9, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_ore_throughput" ], "total_adapt_latency_seconds": 3.687825539005644, "total_model_latency_seconds": 1459.8998870050036, "total_tokens": 73696 }, "warm_frozen": { "abstention_rate": 0.0, "adapt_selection_rate": 1.0, "ambiguous_write_count": 0, "arm": "warm_frozen", "execution_error_rate": 0.25, "fallback_rate": 0.0, "mean_automated_score": -30.0, "mean_final_score": 0.0, "mean_score_auc": 0.0, "mean_steps": 4.0, "operational_failure_count": 0, "operational_failure_rate": 0.0, "pass_rate": 0.0, "pass_rate_ci95_high": 0.43449149475208104, "pass_rate_ci95_low": 0.0, "sample_count": 5, "success_count": 0, "tasks": [ "iron_ore_throughput" ], "total_adapt_latency_seconds": 11.829955082001106, "total_model_latency_seconds": 1070.225386303, "total_tokens": 75351 } } }, "total_runs": 40 }