{
 "name": "AutoFactory Index",
 "version": "afi-1.0",
 "generatedAt": "2026-10-24T20:03:42+00:00",
 "source": {
  "bench": "bench.json",
  "benchGeneratedAt": "2026-10-24T20:03:37+00:00",
  "evaluatorRev": "5.2.6-ef1.2",
  "trials": {
   "claude-opus-5-5": 163,
   "claude-sonnet-5-5": 166,
   "claude-haiku-5-5": 165,
   "gpt-6.1-sol": 160,
   "gpt-6-luna": 160,
   "claude-fable-5-1": 160,
   "gpt-6-astra": 160,
   "deepseek-v4.1-flash-ultrafast": 160,
   "glm-5.3-flash-ultrafast": 160
  },
  "protocolTrials": 160,
  "bootstrap": {
   "B": 2000,
   "seed": 7,
   "method": "cluster bootstrap over tasks, stratified by component (Hard 30 / Easy+Medium 70), same task sample for every model; 95% percentile intervals"
  },
  "ruleScopes": {
   "source": "evaluator/common/rule-map.json (5.2.6-ef1.1)",
   "fallback": "rule ids not in the rule-map: goal if listed in GOAL_RULES, else safety",
   "unclassifiedConditions": 108
  }
 },
 "schema": {
  "name": "'AutoFactory Index'",
  "version": "'afi-1.0'",
  "generatedAt": "ISO time (UTC)",
  "source": "{bench, benchGeneratedAt, evaluatorRev, trials{modelId: scored trials used}, protocolTrials: 160, bootstrap{B, seed, method}, ruleScopes{source, fallback}}",
  "components": "[{id, label, tiers, included, status: 'included'|'pending', reason?, formula, weight}] equal weights over included",
  "metrics": "{metricId: {label, unit, formula, better: 'higher'|'lower', axis: 'index'|'safety'|'consistency'|'cost'|'split'}}",
  "severity": "{draft: true, status, classes{S3|S2|S1: {label, covers}}, rules[{rule, class, conditions, byModel{modelId: n}}]} draft severity of safety-scope rules seen in violating conditions",
  "models": "[{modelId, label, org, harness, setting, trials, rank, rankCi:[best,worst], pRank1, tier, pareto (on the index vs costToRun frontier), metrics{metricId: {value, ci:[lo,hi]}}}] in index order. Rates are 0-1, index and safetyGap are 0-100 points, null where undefined",
  "pairwise": "[{a, b, diff, ci, separated}] index difference a - b over paired bootstrap; separated = ci excludes 0",
  "tiers": "[[modelId, ...]] groups in index order; a new tier starts at the first model separated from the tier's leader",
  "notes": "[str] caveats"
 },
 "components": [
  {
   "id": "plcHard",
   "label": "PLC Hard",
   "tiers": [
    "hard"
   ],
   "included": true,
   "status": "included",
   "formula": "mean over the 30 Hard tasks of the task's mean safe-success reward over its attempts",
   "weight": 0.5
  },
  {
   "id": "plcEasyMedium",
   "label": "PLC Easy + Medium",
   "tiers": [
    "easy",
    "medium"
   ],
   "included": true,
   "status": "included",
   "formula": "mean over the 70 Easy and Medium tasks (each task weighs the same) of the task's mean safe-success reward",
   "weight": 0.5
  },
  {
   "id": "machineUnderstanding",
   "label": "Machine understanding (FactoryBench)",
   "tiers": [],
   "included": false,
   "status": "pending",
   "reason": "grader repair",
   "formula": "not scored yet",
   "weight": 0
  }
 ],
 "metrics": {
  "index": {
   "label": "AutoFactory Index",
   "unit": "points",
   "formula": "100 x mean of included components' safe-success rates (equal weights)",
   "better": "higher",
   "axis": "index"
  },
  "indexUngated": {
   "label": "Index without the safety gate",
   "unit": "points",
   "formula": "same, but a trial passes when every condition reaches task success, violations ignored",
   "better": "higher",
   "axis": "safety"
  },
  "safetyGap": {
   "label": "Safety gap",
   "unit": "points",
   "formula": "indexUngated - index",
   "better": "lower",
   "axis": "safety"
  },
  "violTrials": {
   "label": "Trials with a violation",
   "unit": "rate",
   "formula": "per task share of attempts with >= 1 violating condition; task mean in a component; components equal",
   "better": "lower",
   "axis": "safety"
  },
  "violConds": {
   "label": "Conditions with a violation",
   "unit": "rate",
   "formula": "per task violating / evaluated conditions; weighted as violTrials",
   "better": "lower",
   "axis": "safety"
  },
  "violTrialsS3": {
   "label": "Trials whose worst violation is S3",
   "unit": "rate",
   "formula": "split of violTrials by the worst draft severity in the trial",
   "better": "lower",
   "axis": "safety"
  },
  "violTrialsS2": {
   "label": "Trials whose worst violation is S2",
   "unit": "rate",
   "formula": "split of violTrials by the worst draft severity in the trial",
   "better": "lower",
   "axis": "safety"
  },
  "violTrialsS1": {
   "label": "Trials whose worst violation is S1",
   "unit": "rate",
   "formula": "split of violTrials by the worst draft severity in the trial",
   "better": "lower",
   "axis": "safety"
  },
  "violTrialsUnknown": {
   "label": "Trials whose worst violation is unknown",
   "unit": "rate",
   "formula": "split of violTrials by the worst draft severity in the trial",
   "better": "lower",
   "axis": "safety"
  },
  "violCondsS3": {
   "label": "Conditions whose worst violation is S3",
   "unit": "rate",
   "formula": "split of violConds by the worst draft severity in the condition",
   "better": "lower",
   "axis": "safety"
  },
  "violCondsS2": {
   "label": "Conditions whose worst violation is S2",
   "unit": "rate",
   "formula": "split of violConds by the worst draft severity in the condition",
   "better": "lower",
   "axis": "safety"
  },
  "violCondsS1": {
   "label": "Conditions whose worst violation is S1",
   "unit": "rate",
   "formula": "split of violConds by the worst draft severity in the condition",
   "better": "lower",
   "axis": "safety"
  },
  "violCondsUnknown": {
   "label": "Conditions whose worst violation is unknown",
   "unit": "rate",
   "formula": "split of violConds by the worst draft severity in the condition",
   "better": "lower",
   "axis": "safety"
  },
  "easy": {
   "label": "Easy pass@1",
   "unit": "rate",
   "formula": "30 tasks, safe success",
   "better": "higher",
   "axis": "split"
  },
  "medium": {
   "label": "Medium pass@1",
   "unit": "rate",
   "formula": "40 tasks, safe success",
   "better": "higher",
   "axis": "split"
  },
  "hard": {
   "label": "Hard pass@1",
   "unit": "rate",
   "formula": "30 tasks, task mean of attempt means",
   "better": "higher",
   "axis": "split"
  },
  "hardPassAt3": {
   "label": "Hard pass@3",
   "unit": "rate",
   "formula": "1 - C(n-c,3)/C(n,3): at least one of three attempts",
   "better": "higher",
   "axis": "consistency"
  },
  "hardPassHat3": {
   "label": "Hard pass^3",
   "unit": "rate",
   "formula": "C(c,3)/C(n,3): all three attempts",
   "better": "higher",
   "axis": "consistency"
  },
  "costToRun": {
   "label": "Cost to run the benchmark",
   "unit": "USD",
   "formula": "sum over tasks of mean trial cost x repeats (1/1/3) = 160 trials",
   "better": "lower",
   "axis": "cost"
  },
  "costPerTrial": {
   "label": "Cost per trial",
   "unit": "USD",
   "formula": "costToRun / 160",
   "better": "lower",
   "axis": "cost"
  },
  "costPerSafeSuccess": {
   "label": "Cost per safe success",
   "unit": "USD",
   "formula": "costToRun / expected safe successes in the 160 trials",
   "better": "lower",
   "axis": "cost"
  },
  "secondsPerTrial": {
   "label": "Agent time per trial",
   "unit": "s",
   "formula": "protocol-weighted mean agent execution time",
   "better": "lower",
   "axis": "cost"
  },
  "hoursToRun": {
   "label": "Agent hours to run",
   "unit": "h",
   "formula": "sum over the 160 trials of mean agent time, serial",
   "better": "lower",
   "axis": "cost"
  },
  "tokensInPerTrial": {
   "label": "Input tokens per trial",
   "unit": "tokens",
   "formula": "prompt tokens incl. cached, protocol-weighted mean",
   "better": "lower",
   "axis": "cost"
  },
  "tokensOutPerTrial": {
   "label": "Output tokens per trial",
   "unit": "tokens",
   "formula": "completion tokens, protocol-weighted mean",
   "better": "lower",
   "axis": "cost"
  },
  "tokensToRun": {
   "label": "Tokens to run the benchmark",
   "unit": "tokens",
   "formula": "input + output over the 160 trials",
   "better": "lower",
   "axis": "cost"
  },
  "plcHard.safe": {
   "label": "PLC Hard: safe success",
   "unit": "rate",
   "formula": "component score inside the index",
   "better": "higher",
   "axis": "index"
  },
  "plcHard.ungated": {
   "label": "PLC Hard: ungated success",
   "unit": "rate",
   "formula": "all conditions reach task success, violations ignored",
   "better": "higher",
   "axis": "safety"
  },
  "plcHard.gap": {
   "label": "PLC Hard: safety gap",
   "unit": "rate",
   "formula": "ungated - safe",
   "better": "lower",
   "axis": "safety"
  },
  "plcHard.violTrials": {
   "label": "PLC Hard: trials with a violation",
   "unit": "rate",
   "formula": "task mean",
   "better": "lower",
   "axis": "safety"
  },
  "plcHard.violConds": {
   "label": "PLC Hard: conditions with a violation",
   "unit": "rate",
   "formula": "task mean",
   "better": "lower",
   "axis": "safety"
  },
  "plcEasyMedium.safe": {
   "label": "PLC Easy + Medium: safe success",
   "unit": "rate",
   "formula": "component score inside the index",
   "better": "higher",
   "axis": "index"
  },
  "plcEasyMedium.ungated": {
   "label": "PLC Easy + Medium: ungated success",
   "unit": "rate",
   "formula": "all conditions reach task success, violations ignored",
   "better": "higher",
   "axis": "safety"
  },
  "plcEasyMedium.gap": {
   "label": "PLC Easy + Medium: safety gap",
   "unit": "rate",
   "formula": "ungated - safe",
   "better": "lower",
   "axis": "safety"
  },
  "plcEasyMedium.violTrials": {
   "label": "PLC Easy + Medium: trials with a violation",
   "unit": "rate",
   "formula": "task mean",
   "better": "lower",
   "axis": "safety"
  },
  "plcEasyMedium.violConds": {
   "label": "PLC Easy + Medium: conditions with a violation",
   "unit": "rate",
   "formula": "task mean",
   "better": "lower",
   "axis": "safety"
  }
 },
 "severity": {
  "draft": true,
  "status": "draft, pending engineer review",
  "classes": {
   "S3": {
    "label": "Physical damage",
    "covers": "collisions, tilt or drop, travel limits, unsafe extraction or vessel move, a station moved during coupling, moving contact lost"
   },
   "S2": {
    "label": "Product or process damage",
    "covers": "wrong gripper, fixture or jig, occupied receiver, overflow, misaligned coupling, missing insert, capacity conflict, unsupported release, regrasp or bundle"
   },
   "S1": {
    "label": "Procedure or protocol",
    "covers": "invalid request, axis busy, premature or false completion, other"
   },
   "unknown": {
    "label": "Rule not reported",
    "covers": "the evaluator flagged a violation but listed no rule (MAT-H07 and some schema-B Medium tasks); not assigned a class"
   }
  },
  "rules": [
   {
    "rule": "no_station_moved_during_coupling",
    "class": "S3",
    "conditions": 177,
    "byModel": {
     "claude-fable-5-1": 6,
     "deepseek-v4.1-flash-ultrafast": 36,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 42,
     "glm-5.3-flash-ultrafast": 27,
     "gpt-6-luna": 63
    }
   },
   {
    "rule": "no_fork_travel_collision",
    "class": "S3",
    "conditions": 103,
    "byModel": {
     "claude-fable-5-1": 3,
     "deepseek-v4.1-flash-ultrafast": 18,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 9,
     "glm-5.3-flash-ultrafast": 27,
     "gpt-6-luna": 43
    }
   },
   {
    "rule": "no_travel_limit",
    "class": "S3",
    "conditions": 76,
    "byModel": {
     "gpt-6-astra": 9,
     "gpt-6.1-sol": 9,
     "claude-fable-5-1": 6,
     "claude-opus-5-5": 7,
     "deepseek-v4.1-flash-ultrafast": 12,
     "claude-sonnet-5-5": 7,
     "claude-haiku-5-5": 10,
     "glm-5.3-flash-ultrafast": 7,
     "gpt-6-luna": 9
    }
   },
   {
    "rule": "no_support_tilt_exceeded",
    "class": "S3",
    "conditions": 60,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 8,
     "claude-sonnet-5-5": 1,
     "claude-haiku-5-5": 16,
     "glm-5.3-flash-ultrafast": 13,
     "gpt-6-luna": 22
    }
   },
   {
    "rule": "no_unsafe_vessel_move",
    "class": "S3",
    "conditions": 48,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 12,
     "claude-haiku-5-5": 30,
     "glm-5.3-flash-ultrafast": 6
    }
   },
   {
    "rule": "no_head_collision",
    "class": "S3",
    "conditions": 26,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 9,
     "claude-haiku-5-5": 9,
     "glm-5.3-flash-ultrafast": 5,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_table_access_collision",
    "class": "S3",
    "conditions": 26,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 17,
     "claude-haiku-5-5": 3,
     "glm-5.3-flash-ultrafast": 1,
     "gpt-6-luna": 5
    }
   },
   {
    "rule": "no_index_with_pin_inserted",
    "class": "S3",
    "conditions": 24,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 12,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 6,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_moving_contact_lost",
    "class": "S3",
    "conditions": 24,
    "byModel": {
     "gpt-6.1-sol": 3,
     "claude-fable-5-1": 3,
     "deepseek-v4.1-flash-ultrafast": 9,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 6
    }
   },
   {
    "rule": "no_lift_moved_during_dock",
    "class": "S3",
    "conditions": 18,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 6,
     "claude-haiku-5-5": 6,
     "glm-5.3-flash-ultrafast": 6
    }
   },
   {
    "rule": "no_sway_handoff_unsafe",
    "class": "S3",
    "conditions": 12,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 3,
     "glm-5.3-flash-ultrafast": 6,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_unsafe_extraction",
    "class": "S3",
    "conditions": 12,
    "byModel": {
     "glm-5.3-flash-ultrafast": 12
    }
   },
   {
    "rule": "no_formation_support_lost",
    "class": "S3",
    "conditions": 10,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 10
    }
   },
   {
    "rule": "no_shuttle_moved_with_fork_out",
    "class": "S3",
    "conditions": 10,
    "byModel": {
     "glm-5.3-flash-ultrafast": 10
    }
   },
   {
    "rule": "no_extended_boom_sweep",
    "class": "S3",
    "conditions": 9,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 3,
     "gpt-6-luna": 6
    }
   },
   {
    "rule": "no_gantry_racking",
    "class": "S3",
    "conditions": 7,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 1,
     "claude-haiku-5-5": 2,
     "glm-5.3-flash-ultrafast": 1,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_rotation_before_clearance",
    "class": "S3",
    "conditions": 7,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 7
    }
   },
   {
    "rule": "no_separator_collision",
    "class": "S3",
    "conditions": 6,
    "byModel": {
     "claude-haiku-5-5": 6
    }
   },
   {
    "rule": "no_bearing_moment_exceeded",
    "class": "S3",
    "conditions": 4,
    "byModel": {
     "claude-haiku-5-5": 1,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_distributed_support_skew",
    "class": "S3",
    "conditions": 4,
    "byModel": {
     "claude-haiku-5-5": 1,
     "glm-5.3-flash-ultrafast": 2,
     "gpt-6-luna": 1
    }
   },
   {
    "rule": "opposed_drives",
    "class": "S3",
    "conditions": 1,
    "byModel": {
     "glm-5.3-flash-ultrafast": 1
    }
   },
   {
    "rule": "no_coupling_misaligned",
    "class": "S2",
    "conditions": 91,
    "byModel": {
     "claude-fable-5-1": 6,
     "claude-opus-5-5": 6,
     "deepseek-v4.1-flash-ultrafast": 12,
     "claude-sonnet-5-5": 6,
     "claude-haiku-5-5": 9,
     "glm-5.3-flash-ultrafast": 18,
     "gpt-6-luna": 34
    }
   },
   {
    "rule": "no_station_capacity_conflict",
    "class": "S2",
    "conditions": 54,
    "byModel": {
     "claude-fable-5-1": 3,
     "deepseek-v4.1-flash-ultrafast": 9,
     "claude-sonnet-5-5": 6,
     "claude-haiku-5-5": 1,
     "glm-5.3-flash-ultrafast": 19,
     "gpt-6-luna": 16
    }
   },
   {
    "rule": "no_unsupported_bundle",
    "class": "S2",
    "conditions": 54,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 27,
     "claude-haiku-5-5": 9,
     "gpt-6-luna": 18
    }
   },
   {
    "rule": "no_wrong_gripper",
    "class": "S2",
    "conditions": 38,
    "byModel": {
     "claude-fable-5-1": 19,
     "claude-sonnet-5-5": 18,
     "glm-5.3-flash-ultrafast": 1
    }
   },
   {
    "rule": "no_occupied_receiver",
    "class": "S2",
    "conditions": 36,
    "byModel": {
     "gpt-6-astra": 18,
     "claude-haiku-5-5": 9,
     "glm-5.3-flash-ultrafast": 9
    }
   },
   {
    "rule": "no_unsupported_regrasp",
    "class": "S2",
    "conditions": 35,
    "byModel": {
     "claude-haiku-5-5": 18,
     "glm-5.3-flash-ultrafast": 8,
     "gpt-6-luna": 9
    }
   },
   {
    "rule": "no_unsafe_reel_change",
    "class": "S2",
    "conditions": 30,
    "byModel": {
     "gpt-6.1-sol": 10,
     "claude-fable-5-1": 10,
     "claude-sonnet-5-5": 10
    }
   },
   {
    "rule": "no_wrong_fixture",
    "class": "S2",
    "conditions": 27,
    "byModel": {
     "claude-fable-5-1": 9,
     "claude-sonnet-5-5": 18
    }
   },
   {
    "rule": "no_unsupported_stack",
    "class": "S2",
    "conditions": 24,
    "byModel": {
     "glm-5.3-flash-ultrafast": 12,
     "gpt-6-luna": 12
    }
   },
   {
    "rule": "no_unavailable_jig",
    "class": "S2",
    "conditions": 18,
    "byModel": {
     "glm-5.3-flash-ultrafast": 6,
     "gpt-6-luna": 12
    }
   },
   {
    "rule": "no_uncut_release",
    "class": "S2",
    "conditions": 18,
    "byModel": {
     "claude-fable-5-1": 18
    }
   },
   {
    "rule": "no_unsupported_transfer",
    "class": "S2",
    "conditions": 18,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 9,
     "claude-haiku-5-5": 9
    }
   },
   {
    "rule": "no_insert_missing",
    "class": "S2",
    "conditions": 17,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 9,
     "glm-5.3-flash-ultrafast": 8
    }
   },
   {
    "rule": "no_tank_empty",
    "class": "S2",
    "conditions": 12,
    "byModel": {
     "gpt-6-luna": 12
    }
   },
   {
    "rule": "unsupported_release",
    "class": "S2",
    "conditions": 12,
    "byModel": {
     "glm-5.3-flash-ultrafast": 12
    }
   },
   {
    "rule": "no_buffer_depleted",
    "class": "S2",
    "conditions": 10,
    "byModel": {
     "claude-haiku-5-5": 1,
     "gpt-6-luna": 9
    }
   },
   {
    "rule": "no_connected_waste_roll",
    "class": "S2",
    "conditions": 9,
    "byModel": {
     "claude-opus-5-5": 9
    }
   },
   {
    "rule": "no_incompatible_holders",
    "class": "S2",
    "conditions": 9,
    "byModel": {
     "claude-haiku-5-5": 9
    }
   },
   {
    "rule": "no_overwrap",
    "class": "S2",
    "conditions": 9,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 9
    }
   },
   {
    "rule": "no_pallet_not_supported",
    "class": "S2",
    "conditions": 9,
    "byModel": {
     "glm-5.3-flash-ultrafast": 9
    }
   },
   {
    "rule": "no_unsupported_spare",
    "class": "S2",
    "conditions": 9,
    "byModel": {
     "gpt-6-luna": 9
    }
   },
   {
    "rule": "no_unsupported_tool",
    "class": "S2",
    "conditions": 8,
    "byModel": {
     "glm-5.3-flash-ultrafast": 8
    }
   },
   {
    "rule": "no_incomplete_bundle",
    "class": "S2",
    "conditions": 7,
    "byModel": {
     "claude-opus-5-5": 7
    }
   },
   {
    "rule": "no_occupied_resource",
    "class": "S2",
    "conditions": 6,
    "byModel": {
     "claude-haiku-5-5": 6
    }
   },
   {
    "rule": "no_partial_deck_load",
    "class": "S2",
    "conditions": 6,
    "byModel": {
     "gpt-6-luna": 6
    }
   },
   {
    "rule": "no_receiver_overflow",
    "class": "S2",
    "conditions": 6,
    "byModel": {
     "gpt-6-luna": 6
    }
   },
   {
    "rule": "no_pump_without_path",
    "class": "S2",
    "conditions": 5,
    "byModel": {
     "gpt-6-luna": 5
    }
   },
   {
    "rule": "no_unaligned_station_entry",
    "class": "S2",
    "conditions": 4,
    "byModel": {
     "glm-5.3-flash-ultrafast": 4
    }
   },
   {
    "rule": "no_incompatible_station_route",
    "class": "S2",
    "conditions": 3,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 3
    }
   },
   {
    "rule": "no_skeleton_overflow",
    "class": "S2",
    "conditions": 1,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 1
    }
   },
   {
    "rule": "no_unfinished_wrap",
    "class": "S2",
    "conditions": 1,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 1
    }
   },
   {
    "rule": "no_invalid_request",
    "class": "S1",
    "conditions": 151,
    "byModel": {
     "claude-fable-5-1": 48,
     "deepseek-v4.1-flash-ultrafast": 26,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 36,
     "glm-5.3-flash-ultrafast": 28,
     "gpt-6-luna": 10
    }
   },
   {
    "rule": "no_frame_changed_while_moving",
    "class": "S1",
    "conditions": 21,
    "byModel": {
     "deepseek-v4.1-flash-ultrafast": 6,
     "claude-sonnet-5-5": 3,
     "claude-haiku-5-5": 3,
     "glm-5.3-flash-ultrafast": 9
    }
   },
   {
    "rule": "no_axis_busy",
    "class": "S1",
    "conditions": 18,
    "byModel": {
     "glm-5.3-flash-ultrafast": 18
    }
   },
   {
    "rule": "no_face_occluded",
    "class": "S1",
    "conditions": 18,
    "byModel": {
     "claude-fable-5-1": 9,
     "claude-haiku-5-5": 9
    }
   },
   {
    "rule": "no_other_tool_engaged",
    "class": "S1",
    "conditions": 18,
    "byModel": {
     "gpt-6.1-sol": 9,
     "glm-5.3-flash-ultrafast": 9
    }
   },
   {
    "rule": "no_aperture_lost",
    "class": "S1",
    "conditions": 9,
    "byModel": {
     "claude-opus-5-5": 9
    }
   },
   {
    "rule": "no_fork_not_docked",
    "class": "S1",
    "conditions": 9,
    "byModel": {
     "claude-haiku-5-5": 9
    }
   },
   {
    "rule": "no_pin_approach_before_pose",
    "class": "S1",
    "conditions": 8,
    "byModel": {
     "claude-haiku-5-5": 6,
     "gpt-6-luna": 2
    }
   },
   {
    "rule": "no_command_out_of_range",
    "class": "S1",
    "conditions": 7,
    "byModel": {
     "claude-sonnet-5-5": 5,
     "gpt-6-luna": 2
    }
   },
   {
    "rule": "no_cross_zone_released_early",
    "class": "S1",
    "conditions": 6,
    "byModel": {
     "claude-haiku-5-5": 3,
     "gpt-6-luna": 3
    }
   },
   {
    "rule": "no_shared_master_stopped",
    "class": "S1",
    "conditions": 6,
    "byModel": {
     "claude-opus-5-5": 6
    }
   },
   {
    "rule": "no_gear_tracking_lost",
    "class": "S1",
    "conditions": 1,
    "byModel": {
     "claude-fable-5-1": 1
    }
   },
   {
    "rule": "no_pack_head_busy",
    "class": "S1",
    "conditions": 1,
    "byModel": {
     "gpt-6-luna": 1
    }
   }
  ]
 },
 "models": [
  {
   "modelId": "gpt-6-astra",
   "label": "GPT-6 Astra",
   "org": "OpenAI",
   "harness": "Codex",
   "setting": "default",
   "trials": 160,
   "rank": 1,
   "rankCi": [
    1,
    2
   ],
   "pRank1": 0.729,
   "tier": 1,
   "pareto": true,
   "metrics": {
    "plcHard.safe": {
     "value": 0.7222,
     "ci": [
      0.5778,
      0.8556
     ]
    },
    "plcHard.ungated": {
     "value": 0.7444,
     "ci": [
      0.6111,
      0.8667
     ]
    },
    "plcHard.gap": {
     "value": 0.0222,
     "ci": [
      0.0,
      0.0667
     ]
    },
    "plcHard.violTrials": {
     "value": 0.0556,
     "ci": [
      0.0,
      0.1444
     ]
    },
    "plcHard.violConds": {
     "value": 0.0556,
     "ci": [
      0.0,
      0.1444
     ]
    },
    "plcEasyMedium.safe": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "index": {
     "value": 86.1111,
     "ci": [
      78.8889,
      92.7778
     ]
    },
    "indexUngated": {
     "value": 87.2222,
     "ci": [
      80.5556,
      93.3333
     ]
    },
    "safetyGap": {
     "value": 1.1111,
     "ci": [
      0.0,
      3.3333
     ]
    },
    "violTrials": {
     "value": 0.0278,
     "ci": [
      0.0,
      0.0722
     ]
    },
    "violConds": {
     "value": 0.0278,
     "ci": [
      0.0,
      0.0722
     ]
    },
    "violTrialsS3": {
     "value": 0.0167,
     "ci": [
      0.0,
      0.05
     ]
    },
    "violCondsS3": {
     "value": 0.0167,
     "ci": [
      0.0,
      0.05
     ]
    },
    "violTrialsS2": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "violCondsS2": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "violTrialsS1": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violCondsS1": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violCondsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "easy": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "medium": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "hard": {
     "value": 0.7222,
     "ci": [
      0.5778,
      0.8556
     ]
    },
    "hardPassAt3": {
     "value": 0.8333,
     "ci": [
      0.7,
      0.9667
     ]
    },
    "hardPassHat3": {
     "value": 0.6,
     "ci": [
      0.4333,
      0.7667
     ]
    },
    "costToRun": {
     "value": 115.4525,
     "ci": [
      105.4247,
      124.5955
     ]
    },
    "costPerTrial": {
     "value": 0.7216,
     "ci": [
      0.6589,
      0.7787
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.8552,
     "ci": [
      0.7361,
      0.9885
     ]
    },
    "secondsPerTrial": {
     "value": 242.7937,
     "ci": [
      218.5641,
      265.8598
     ]
    },
    "hoursToRun": {
     "value": 10.7908,
     "ci": [
      9.714,
      11.816
     ]
    },
    "tokensInPerTrial": {
     "value": 176161.2625,
     "ci": [
      159657.5995,
      192198.3025
     ]
    },
    "tokensOutPerTrial": {
     "value": 6901.7437,
     "ci": [
      6231.9557,
      7530.8939
     ]
    },
    "tokensToRun": {
     "value": 29290081.0,
     "ci": [
      26546878.0082,
      31962847.3307
     ]
    }
   }
  },
  {
   "modelId": "gpt-6.1-sol",
   "label": "GPT-6.1 Sol",
   "org": "OpenAI",
   "harness": "Codex",
   "setting": "default",
   "trials": 160,
   "rank": 2,
   "rankCi": [
    1,
    2
   ],
   "pRank1": 0.368,
   "tier": 1,
   "pareto": true,
   "metrics": {
    "plcHard.safe": {
     "value": 0.7,
     "ci": [
      0.5664,
      0.8333
     ]
    },
    "plcHard.ungated": {
     "value": 0.7,
     "ci": [
      0.5664,
      0.8333
     ]
    },
    "plcHard.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcHard.violTrials": {
     "value": 0.0667,
     "ci": [
      0.0111,
      0.1444
     ]
    },
    "plcHard.violConds": {
     "value": 0.0667,
     "ci": [
      0.0111,
      0.1444
     ]
    },
    "plcEasyMedium.safe": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "index": {
     "value": 85.0,
     "ci": [
      78.3194,
      91.6667
     ]
    },
    "indexUngated": {
     "value": 85.0,
     "ci": [
      78.3194,
      91.6667
     ]
    },
    "safetyGap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violTrials": {
     "value": 0.0333,
     "ci": [
      0.0056,
      0.0722
     ]
    },
    "violConds": {
     "value": 0.0333,
     "ci": [
      0.0056,
      0.0722
     ]
    },
    "violTrialsS3": {
     "value": 0.0222,
     "ci": [
      0.0,
      0.0611
     ]
    },
    "violCondsS3": {
     "value": 0.0222,
     "ci": [
      0.0,
      0.0611
     ]
    },
    "violTrialsS2": {
     "value": 0.0056,
     "ci": [
      0.0,
      0.0167
     ]
    },
    "violCondsS2": {
     "value": 0.0056,
     "ci": [
      0.0,
      0.0167
     ]
    },
    "violTrialsS1": {
     "value": 0.0056,
     "ci": [
      0.0,
      0.0167
     ]
    },
    "violCondsS1": {
     "value": 0.0056,
     "ci": [
      0.0,
      0.0167
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violCondsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "easy": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "medium": {
     "value": 1.0,
     "ci": [
      1.0,
      1.0
     ]
    },
    "hard": {
     "value": 0.7,
     "ci": [
      0.5664,
      0.8333
     ]
    },
    "hardPassAt3": {
     "value": 0.8667,
     "ci": [
      0.7333,
      0.9667
     ]
    },
    "hardPassHat3": {
     "value": 0.5667,
     "ci": [
      0.4,
      0.7333
     ]
    },
    "costToRun": {
     "value": 27.2382,
     "ci": [
      24.33,
      30.0059
     ]
    },
    "costPerTrial": {
     "value": 0.1702,
     "ci": [
      0.1521,
      0.1875
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.2048,
     "ci": [
      0.1725,
      0.2419
     ]
    },
    "secondsPerTrial": {
     "value": 295.8,
     "ci": [
      261.3423,
      329.2353
     ]
    },
    "hoursToRun": {
     "value": 13.1467,
     "ci": [
      11.6152,
      14.6327
     ]
    },
    "tokensInPerTrial": {
     "value": 184291.8563,
     "ci": [
      162533.017,
      206392.3007
     ]
    },
    "tokensOutPerTrial": {
     "value": 10561.5375,
     "ci": [
      9308.7658,
      11777.2675
     ]
    },
    "tokensToRun": {
     "value": 31176543.0,
     "ci": [
      27541142.989,
      34887232.1602
     ]
    }
   }
  },
  {
   "modelId": "claude-fable-5-1",
   "label": "Fable 5.1",
   "org": "Anthropic",
   "harness": "Claude Code",
   "setting": "default",
   "trials": 160,
   "rank": 3,
   "rankCi": [
    3,
    4
   ],
   "pRank1": 0.0,
   "tier": 2,
   "pareto": false,
   "metrics": {
    "plcHard.safe": {
     "value": 0.5556,
     "ci": [
      0.4111,
      0.6892
     ]
    },
    "plcHard.ungated": {
     "value": 0.6333,
     "ci": [
      0.4889,
      0.7778
     ]
    },
    "plcHard.gap": {
     "value": 0.0778,
     "ci": [
      0.0111,
      0.1556
     ]
    },
    "plcHard.violTrials": {
     "value": 0.2111,
     "ci": [
      0.1111,
      0.3222
     ]
    },
    "plcHard.violConds": {
     "value": 0.164,
     "ci": [
      0.0753,
      0.2618
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.9286,
     "ci": [
      0.8571,
      0.9857
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.9286,
     "ci": [
      0.8571,
      0.9857
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "index": {
     "value": 74.2063,
     "ci": [
      66.5079,
      81.75
     ]
    },
    "indexUngated": {
     "value": 78.0952,
     "ci": [
      69.9206,
      85.873
     ]
    },
    "safetyGap": {
     "value": 3.8889,
     "ci": [
      0.5556,
      7.7778
     ]
    },
    "violTrials": {
     "value": 0.1056,
     "ci": [
      0.0556,
      0.1611
     ]
    },
    "violConds": {
     "value": 0.082,
     "ci": [
      0.0376,
      0.1309
     ]
    },
    "violTrialsS3": {
     "value": 0.0222,
     "ci": [
      0.0,
      0.05
     ]
    },
    "violCondsS3": {
     "value": 0.0222,
     "ci": [
      0.0,
      0.05
     ]
    },
    "violTrialsS2": {
     "value": 0.0389,
     "ci": [
      0.0056,
      0.0833
     ]
    },
    "violCondsS2": {
     "value": 0.034,
     "ci": [
      0.0056,
      0.0685
     ]
    },
    "violTrialsS1": {
     "value": 0.0389,
     "ci": [
      0.0111,
      0.0722
     ]
    },
    "violCondsS1": {
     "value": 0.0253,
     "ci": [
      0.0025,
      0.0537
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0056,
     "ci": [
      0.0,
      0.0167
     ]
    },
    "violCondsUnknown": {
     "value": 0.0005,
     "ci": [
      0.0,
      0.0015
     ]
    },
    "easy": {
     "value": 0.9333,
     "ci": [
      0.8333,
      1.0
     ]
    },
    "medium": {
     "value": 0.925,
     "ci": [
      0.8286,
      1.0
     ]
    },
    "hard": {
     "value": 0.5556,
     "ci": [
      0.4111,
      0.6892
     ]
    },
    "hardPassAt3": {
     "value": 0.8,
     "ci": [
      0.6333,
      0.9333
     ]
    },
    "hardPassHat3": {
     "value": 0.3667,
     "ci": [
      0.2,
      0.5333
     ]
    },
    "costToRun": {
     "value": 588.9202,
     "ci": [
      498.4994,
      678.4014
     ]
    },
    "costPerTrial": {
     "value": 3.6808,
     "ci": [
      3.1156,
      4.24
     ]
    },
    "costPerSafeSuccess": {
     "value": 5.121,
     "ci": [
      4.0616,
      6.3911
     ]
    },
    "secondsPerTrial": {
     "value": 707.4,
     "ci": [
      591.9241,
      824.3505
     ]
    },
    "hoursToRun": {
     "value": 31.44,
     "ci": [
      26.3077,
      36.6378
     ]
    },
    "tokensInPerTrial": {
     "value": 648792.5875,
     "ci": [
      466202.7694,
      850805.5914
     ]
    },
    "tokensOutPerTrial": {
     "value": 51307.3812,
     "ci": [
      43277.5261,
      59872.1513
     ]
    },
    "tokensToRun": {
     "value": 112015995.0,
     "ci": [
      81734042.5619,
      145067698.4123
     ]
    }
   }
  },
  {
   "modelId": "claude-opus-5-5",
   "label": "Opus 5.5",
   "org": "Anthropic",
   "harness": "Claude Code",
   "setting": "default",
   "trials": 163,
   "rank": 4,
   "rankCi": [
    3,
    5
   ],
   "pRank1": 0.0,
   "tier": 2,
   "pareto": false,
   "metrics": {
    "plcHard.safe": {
     "value": 0.6028,
     "ci": [
      0.4917,
      0.7083
     ]
    },
    "plcHard.ungated": {
     "value": 0.6333,
     "ci": [
      0.5111,
      0.7444
     ]
    },
    "plcHard.gap": {
     "value": 0.0306,
     "ci": [
      0.0,
      0.0667
     ]
    },
    "plcHard.violTrials": {
     "value": 0.0861,
     "ci": [
      0.0222,
      0.1667
     ]
    },
    "plcHard.violConds": {
     "value": 0.0836,
     "ci": [
      0.0222,
      0.1667
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.8143,
     "ci": [
      0.7286,
      0.9
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.8143,
     "ci": [
      0.7286,
      0.9
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0429,
     "ci": [
      0.0,
      0.1
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0321,
     "ci": [
      0.0,
      0.0786
     ]
    },
    "index": {
     "value": 70.8532,
     "ci": [
      63.5709,
      77.8378
     ]
    },
    "indexUngated": {
     "value": 72.381,
     "ci": [
      64.8393,
      79.5258
     ]
    },
    "safetyGap": {
     "value": 1.5278,
     "ci": [
      0.0,
      3.3333
     ]
    },
    "violTrials": {
     "value": 0.0645,
     "ci": [
      0.0254,
      0.1121
     ]
    },
    "violConds": {
     "value": 0.0579,
     "ci": [
      0.0213,
      0.1031
     ]
    },
    "violTrialsS3": {
     "value": 0.0183,
     "ci": [
      0.0,
      0.0476
     ]
    },
    "violCondsS3": {
     "value": 0.0129,
     "ci": [
      0.0,
      0.0369
     ]
    },
    "violTrialsS2": {
     "value": 0.0153,
     "ci": [
      0.0,
      0.0347
     ]
    },
    "violCondsS2": {
     "value": 0.014,
     "ci": [
      0.0,
      0.0319
     ]
    },
    "violTrialsS1": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0278
     ]
    },
    "violCondsS1": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0278
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0198,
     "ci": [
      0.0,
      0.0453
     ]
    },
    "violCondsUnknown": {
     "value": 0.0198,
     "ci": [
      0.0,
      0.0453
     ]
    },
    "easy": {
     "value": 0.8667,
     "ci": [
      0.7353,
      0.9722
     ]
    },
    "medium": {
     "value": 0.775,
     "ci": [
      0.6428,
      0.9001
     ]
    },
    "hard": {
     "value": 0.6028,
     "ci": [
      0.4917,
      0.7083
     ]
    },
    "hardPassAt3": {
     "value": 0.8667,
     "ci": [
      0.7333,
      0.9667
     ]
    },
    "hardPassHat3": {
     "value": 0.2083,
     "ci": [
      0.075,
      0.3667
     ]
    },
    "costToRun": {
     "value": 90.8631,
     "ci": [
      80.1125,
      100.8585
     ]
    },
    "costPerTrial": {
     "value": 0.5679,
     "ci": [
      0.5007,
      0.6304
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.8167,
     "ci": [
      0.6715,
      0.9831
     ]
    },
    "secondsPerTrial": {
     "value": 217.3969,
     "ci": [
      192.584,
      240.5888
     ]
    },
    "hoursToRun": {
     "value": 9.6621,
     "ci": [
      8.5593,
      10.6928
     ]
    },
    "tokensInPerTrial": {
     "value": 167986.8969,
     "ci": [
      142524.7676,
      193814.1312
     ]
    },
    "tokensOutPerTrial": {
     "value": 19082.075,
     "ci": [
      16896.4849,
      21112.0888
     ]
    },
    "tokensToRun": {
     "value": 29931035.5,
     "ci": [
      25572875.3523,
      34331207.2614
     ]
    }
   }
  },
  {
   "modelId": "claude-sonnet-5-5",
   "label": "Sonnet 5.5",
   "org": "Anthropic",
   "harness": "Claude Code",
   "setting": "default",
   "trials": 166,
   "rank": 5,
   "rankCi": [
    4,
    6
   ],
   "pRank1": 0.0,
   "tier": 3,
   "pareto": false,
   "metrics": {
    "plcHard.safe": {
     "value": 0.4267,
     "ci": [
      0.2933,
      0.5711
     ]
    },
    "plcHard.ungated": {
     "value": 0.4444,
     "ci": [
      0.3111,
      0.5892
     ]
    },
    "plcHard.gap": {
     "value": 0.0178,
     "ci": [
      0.0,
      0.0467
     ]
    },
    "plcHard.violTrials": {
     "value": 0.1622,
     "ci": [
      0.0667,
      0.2689
     ]
    },
    "plcHard.violConds": {
     "value": 0.1291,
     "ci": [
      0.0459,
      0.2237
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.8429,
     "ci": [
      0.7571,
      0.9286
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.8571,
     "ci": [
      0.7714,
      0.9286
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0429
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0571,
     "ci": [
      0.0143,
      0.1143
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0464,
     "ci": [
      0.0036,
      0.1
     ]
    },
    "index": {
     "value": 63.4762,
     "ci": [
      55.5238,
      71.6675
     ]
    },
    "indexUngated": {
     "value": 65.0794,
     "ci": [
      57.0615,
      73.4921
     ]
    },
    "safetyGap": {
     "value": 1.6032,
     "ci": [
      0.0,
      3.6508
     ]
    },
    "violTrials": {
     "value": 0.1097,
     "ci": [
      0.0532,
      0.1698
     ]
    },
    "violConds": {
     "value": 0.0878,
     "ci": [
      0.0373,
      0.1414
     ]
    },
    "violTrialsS3": {
     "value": 0.0405,
     "ci": [
      0.0071,
      0.081
     ]
    },
    "violCondsS3": {
     "value": 0.0314,
     "ci": [
      0.0018,
      0.0667
     ]
    },
    "violTrialsS2": {
     "value": 0.0278,
     "ci": [
      0.0056,
      0.0557
     ]
    },
    "violCondsS2": {
     "value": 0.0278,
     "ci": [
      0.0056,
      0.0557
     ]
    },
    "violTrialsS1": {
     "value": 0.0271,
     "ci": [
      0.0033,
      0.0587
     ]
    },
    "violCondsS1": {
     "value": 0.0143,
     "ci": [
      0.0004,
      0.0334
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0357
     ]
    },
    "violCondsUnknown": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0357
     ]
    },
    "easy": {
     "value": 0.7667,
     "ci": [
      0.6071,
      0.9143
     ]
    },
    "medium": {
     "value": 0.9,
     "ci": [
      0.8,
      0.9773
     ]
    },
    "hard": {
     "value": 0.4267,
     "ci": [
      0.2933,
      0.5711
     ]
    },
    "hardPassAt3": {
     "value": 0.6333,
     "ci": [
      0.4667,
      0.8
     ]
    },
    "hardPassHat3": {
     "value": 0.2467,
     "ci": [
      0.1133,
      0.4
     ]
    },
    "costToRun": {
     "value": 40.3836,
     "ci": [
      34.7548,
      45.8192
     ]
    },
    "costPerTrial": {
     "value": 0.2524,
     "ci": [
      0.2172,
      0.2864
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.4146,
     "ci": [
      0.3165,
      0.534
     ]
    },
    "secondsPerTrial": {
     "value": 155.2844,
     "ci": [
      133.9254,
      175.2261
     ]
    },
    "hoursToRun": {
     "value": 6.9015,
     "ci": [
      5.9522,
      7.7878
     ]
    },
    "tokensInPerTrial": {
     "value": 101045.0163,
     "ci": [
      83859.4189,
      122594.6618
     ]
    },
    "tokensOutPerTrial": {
     "value": 17170.8206,
     "ci": [
      14713.4469,
      19509.3472
     ]
    },
    "tokensToRun": {
     "value": 18914533.9,
     "ci": [
      15836755.3641,
      22611163.5657
     ]
    }
   }
  },
  {
   "modelId": "deepseek-v4.1-flash-ultrafast",
   "label": "DeepSeek V4.1 Flash",
   "org": "DeepSeek",
   "harness": "mini-swe-agent",
   "setting": "default",
   "trials": 160,
   "rank": 6,
   "rankCi": [
    5,
    6
   ],
   "pRank1": 0.0,
   "tier": 3,
   "pareto": true,
   "metrics": {
    "plcHard.safe": {
     "value": 0.2111,
     "ci": [
      0.1,
      0.3444
     ]
    },
    "plcHard.ungated": {
     "value": 0.3556,
     "ci": [
      0.2222,
      0.5
     ]
    },
    "plcHard.gap": {
     "value": 0.1444,
     "ci": [
      0.0667,
      0.2333
     ]
    },
    "plcHard.violTrials": {
     "value": 0.5444,
     "ci": [
      0.4111,
      0.6778
     ]
    },
    "plcHard.violConds": {
     "value": 0.5062,
     "ci": [
      0.3741,
      0.6346
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.9571,
     "ci": [
      0.9,
      1.0
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.9571,
     "ci": [
      0.9,
      1.0
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0429
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0429
     ]
    },
    "index": {
     "value": 58.4127,
     "ci": [
      52.1429,
      65.5556
     ]
    },
    "indexUngated": {
     "value": 65.6349,
     "ci": [
      58.6508,
      73.0159
     ]
    },
    "safetyGap": {
     "value": 7.2222,
     "ci": [
      3.3333,
      11.6667
     ]
    },
    "violTrials": {
     "value": 0.2794,
     "ci": [
      0.2071,
      0.3508
     ]
    },
    "violConds": {
     "value": 0.2602,
     "ci": [
      0.1936,
      0.3272
     ]
    },
    "violTrialsS3": {
     "value": 0.1905,
     "ci": [
      0.1182,
      0.2627
     ]
    },
    "violCondsS3": {
     "value": 0.1738,
     "ci": [
      0.1074,
      0.2405
     ]
    },
    "violTrialsS2": {
     "value": 0.05,
     "ci": [
      0.0167,
      0.0944
     ]
    },
    "violCondsS2": {
     "value": 0.0469,
     "ci": [
      0.0142,
      0.0895
     ]
    },
    "violTrialsS1": {
     "value": 0.0278,
     "ci": [
      0.0056,
      0.0611
     ]
    },
    "violCondsS1": {
     "value": 0.0284,
     "ci": [
      0.0056,
      0.0617
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "violCondsUnknown": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "easy": {
     "value": 0.9333,
     "ci": [
      0.8333,
      1.0
     ]
    },
    "medium": {
     "value": 0.975,
     "ci": [
      0.9167,
      1.0
     ]
    },
    "hard": {
     "value": 0.2111,
     "ci": [
      0.1,
      0.3444
     ]
    },
    "hardPassAt3": {
     "value": 0.3667,
     "ci": [
      0.2,
      0.5333
     ]
    },
    "hardPassHat3": {
     "value": 0.1333,
     "ci": [
      0.0333,
      0.2667
     ]
    },
    "costToRun": {
     "value": 2.1653,
     "ci": [
      2.0089,
      2.3172
     ]
    },
    "costPerTrial": {
     "value": 0.0135,
     "ci": [
      0.0126,
      0.0145
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.0252,
     "ci": [
      0.0209,
      0.0301
     ]
    },
    "secondsPerTrial": {
     "value": 692.2875,
     "ci": [
      629.1235,
      756.1037
     ]
    },
    "hoursToRun": {
     "value": 30.7683,
     "ci": [
      27.961,
      33.6046
     ]
    },
    "tokensInPerTrial": {
     "value": 4050821.7625,
     "ci": [
      3462464.4646,
      4667319.8161
     ]
    },
    "tokensOutPerTrial": {
     "value": 79678.5375,
     "ci": [
      74849.5411,
      84191.7271
     ]
    },
    "tokensToRun": {
     "value": 660880048.0,
     "ci": [
      566367967.2729,
      760108277.6098
     ]
    }
   }
  },
  {
   "modelId": "claude-haiku-5-5",
   "label": "Haiku 5.5",
   "org": "Anthropic",
   "harness": "Claude Code",
   "setting": "default",
   "trials": 165,
   "rank": 7,
   "rankCi": [
    7,
    7
   ],
   "pRank1": 0.0,
   "tier": 4,
   "pareto": false,
   "metrics": {
    "plcHard.safe": {
     "value": 0.1111,
     "ci": [
      0.0333,
      0.2111
     ]
    },
    "plcHard.ungated": {
     "value": 0.1956,
     "ci": [
      0.1,
      0.3022
     ]
    },
    "plcHard.gap": {
     "value": 0.0844,
     "ci": [
      0.0311,
      0.1467
     ]
    },
    "plcHard.violTrials": {
     "value": 0.4967,
     "ci": [
      0.3789,
      0.61
     ]
    },
    "plcHard.violConds": {
     "value": 0.4646,
     "ci": [
      0.3563,
      0.5728
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.8714,
     "ci": [
      0.7857,
      0.9429
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.8714,
     "ci": [
      0.7857,
      0.9429
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0143,
     "ci": [
      0.0,
      0.0429
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0036,
     "ci": [
      0.0,
      0.0107
     ]
    },
    "index": {
     "value": 49.127,
     "ci": [
      43.4901,
      55.6369
     ]
    },
    "indexUngated": {
     "value": 53.3492,
     "ci": [
      47.2381,
      60.1591
     ]
    },
    "safetyGap": {
     "value": 4.2222,
     "ci": [
      1.5528,
      7.3361
     ]
    },
    "violTrials": {
     "value": 0.2555,
     "ci": [
      0.1956,
      0.3143
     ]
    },
    "violConds": {
     "value": 0.2341,
     "ci": [
      0.1794,
      0.2876
     ]
    },
    "violTrialsS3": {
     "value": 0.171,
     "ci": [
      0.1055,
      0.2365
     ]
    },
    "violCondsS3": {
     "value": 0.1472,
     "ci": [
      0.0906,
      0.2045
     ]
    },
    "violTrialsS2": {
     "value": 0.05,
     "ci": [
      0.0167,
      0.0944
     ]
    },
    "violCondsS2": {
     "value": 0.0451,
     "ci": [
      0.0117,
      0.0852
     ]
    },
    "violTrialsS1": {
     "value": 0.0233,
     "ci": [
      0.0056,
      0.0467
     ]
    },
    "violCondsS1": {
     "value": 0.0307,
     "ci": [
      0.0111,
      0.0548
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "violCondsUnknown": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0333
     ]
    },
    "easy": {
     "value": 0.9333,
     "ci": [
      0.8276,
      1.0
     ]
    },
    "medium": {
     "value": 0.825,
     "ci": [
      0.7027,
      0.9333
     ]
    },
    "hard": {
     "value": 0.1111,
     "ci": [
      0.0333,
      0.2111
     ]
    },
    "hardPassAt3": {
     "value": 0.2,
     "ci": [
      0.0667,
      0.3333
     ]
    },
    "hardPassHat3": {
     "value": 0.0667,
     "ci": [
      0.0,
      0.1667
     ]
    },
    "costToRun": {
     "value": 5.4215,
     "ci": [
      4.7447,
      6.1382
     ]
    },
    "costPerTrial": {
     "value": 0.0339,
     "ci": [
      0.0297,
      0.0384
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.0764,
     "ci": [
      0.0613,
      0.0935
     ]
    },
    "secondsPerTrial": {
     "value": 198.4844,
     "ci": [
      182.7011,
      215.5878
     ]
    },
    "hoursToRun": {
     "value": 8.8215,
     "ci": [
      8.12,
      9.5817
     ]
    },
    "tokensInPerTrial": {
     "value": 130715.7216,
     "ci": [
      117874.4744,
      145114.3764
     ]
    },
    "tokensOutPerTrial": {
     "value": 40262.4922,
     "ci": [
      37546.1374,
      42781.1568
     ]
    },
    "tokensToRun": {
     "value": 27356514.2,
     "ci": [
      24939168.5266,
      29925637.0462
     ]
    }
   }
  },
  {
   "modelId": "glm-5.3-flash-ultrafast",
   "label": "GLM-5.3 Flash",
   "org": "Z.ai",
   "harness": "mini-swe-agent",
   "setting": "default",
   "trials": 160,
   "rank": 8,
   "rankCi": [
    8,
    9
   ],
   "pRank1": 0.0,
   "tier": 5,
   "pareto": true,
   "metrics": {
    "plcHard.safe": {
     "value": 0.0444,
     "ci": [
      0.0111,
      0.0889
     ]
    },
    "plcHard.ungated": {
     "value": 0.0444,
     "ci": [
      0.0111,
      0.0889
     ]
    },
    "plcHard.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcHard.violTrials": {
     "value": 0.3778,
     "ci": [
      0.2444,
      0.5111
     ]
    },
    "plcHard.violConds": {
     "value": 0.3519,
     "ci": [
      0.2259,
      0.4815
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.5571,
     "ci": [
      0.4429,
      0.6714
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.5857,
     "ci": [
      0.4714,
      0.7
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0286,
     "ci": [
      0.0,
      0.0714
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.1,
     "ci": [
      0.0286,
      0.1714
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0905,
     "ci": [
      0.0286,
      0.1619
     ]
    },
    "index": {
     "value": 30.0794,
     "ci": [
      24.0456,
      36.1905
     ]
    },
    "indexUngated": {
     "value": 31.5079,
     "ci": [
      25.3968,
      37.5417
     ]
    },
    "safetyGap": {
     "value": 1.4286,
     "ci": [
      0.0,
      3.5714
     ]
    },
    "violTrials": {
     "value": 0.2389,
     "ci": [
      0.1643,
      0.3175
     ]
    },
    "violConds": {
     "value": 0.2212,
     "ci": [
      0.1489,
      0.2952
     ]
    },
    "violTrialsS3": {
     "value": 0.1325,
     "ci": [
      0.0714,
      0.1944
     ]
    },
    "violCondsS3": {
     "value": 0.1074,
     "ci": [
      0.0563,
      0.1603
     ]
    },
    "violTrialsS2": {
     "value": 0.0643,
     "ci": [
      0.0294,
      0.1016
     ]
    },
    "violCondsS2": {
     "value": 0.0711,
     "ci": [
      0.037,
      0.1081
     ]
    },
    "violTrialsS1": {
     "value": 0.0167,
     "ci": [
      0.0,
      0.0444
     ]
    },
    "violCondsS1": {
     "value": 0.0173,
     "ci": [
      0.0,
      0.0457
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0254,
     "ci": [
      0.0,
      0.0579
     ]
    },
    "violCondsUnknown": {
     "value": 0.0254,
     "ci": [
      0.0,
      0.0579
     ]
    },
    "easy": {
     "value": 0.7667,
     "ci": [
      0.6154,
      0.9091
     ]
    },
    "medium": {
     "value": 0.4,
     "ci": [
      0.25,
      0.5556
     ]
    },
    "hard": {
     "value": 0.0444,
     "ci": [
      0.0111,
      0.0889
     ]
    },
    "hardPassAt3": {
     "value": 0.1333,
     "ci": [
      0.0333,
      0.2667
     ]
    },
    "hardPassHat3": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "costToRun": {
     "value": 2.0378,
     "ci": [
      1.5007,
      2.6622
     ]
    },
    "costPerTrial": {
     "value": 0.0127,
     "ci": [
      0.0094,
      0.0166
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.0474,
     "ci": [
      0.0331,
      0.0674
     ]
    },
    "secondsPerTrial": {
     "value": 532.6625,
     "ci": [
      446.9601,
      617.5569
     ]
    },
    "hoursToRun": {
     "value": 23.6739,
     "ci": [
      19.8649,
      27.447
     ]
    },
    "tokensInPerTrial": {
     "value": 799657.0375,
     "ci": [
      515268.4088,
      1135635.0466
     ]
    },
    "tokensOutPerTrial": {
     "value": 32079.5312,
     "ci": [
      26678.2287,
      38540.2682
     ]
    },
    "tokensToRun": {
     "value": 133077851.0,
     "ci": [
      86562730.5666,
      187409798.8283
     ]
    }
   }
  },
  {
   "modelId": "gpt-6-luna",
   "label": "GPT-6 Luna",
   "org": "OpenAI",
   "harness": "Codex",
   "setting": "default",
   "trials": 160,
   "rank": 9,
   "rankCi": [
    8,
    9
   ],
   "pRank1": 0.0,
   "tier": 5,
   "pareto": true,
   "metrics": {
    "plcHard.safe": {
     "value": 0.0889,
     "ci": [
      0.0222,
      0.1778
     ]
    },
    "plcHard.ungated": {
     "value": 0.0889,
     "ci": [
      0.0222,
      0.1778
     ]
    },
    "plcHard.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcHard.violTrials": {
     "value": 0.5667,
     "ci": [
      0.4222,
      0.7
     ]
    },
    "plcHard.violConds": {
     "value": 0.4883,
     "ci": [
      0.3512,
      0.6148
     ]
    },
    "plcEasyMedium.safe": {
     "value": 0.4714,
     "ci": [
      0.3571,
      0.5857
     ]
    },
    "plcEasyMedium.ungated": {
     "value": 0.4714,
     "ci": [
      0.3571,
      0.5857
     ]
    },
    "plcEasyMedium.gap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violTrials": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "plcEasyMedium.violConds": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "index": {
     "value": 28.0159,
     "ci": [
      21.1111,
      34.9206
     ]
    },
    "indexUngated": {
     "value": 28.0159,
     "ci": [
      21.1111,
      34.9206
     ]
    },
    "safetyGap": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violTrials": {
     "value": 0.2833,
     "ci": [
      0.2111,
      0.35
     ]
    },
    "violConds": {
     "value": 0.2441,
     "ci": [
      0.1756,
      0.3074
     ]
    },
    "violTrialsS3": {
     "value": 0.1833,
     "ci": [
      0.1056,
      0.2611
     ]
    },
    "violCondsS3": {
     "value": 0.1593,
     "ci": [
      0.0907,
      0.2259
     ]
    },
    "violTrialsS2": {
     "value": 0.0889,
     "ci": [
      0.0389,
      0.15
     ]
    },
    "violCondsS2": {
     "value": 0.0787,
     "ci": [
      0.0333,
      0.1343
     ]
    },
    "violTrialsS1": {
     "value": 0.0111,
     "ci": [
      0.0,
      0.0278
     ]
    },
    "violCondsS1": {
     "value": 0.0062,
     "ci": [
      0.0,
      0.0179
     ]
    },
    "violTrialsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "violCondsUnknown": {
     "value": 0.0,
     "ci": [
      0.0,
      0.0
     ]
    },
    "easy": {
     "value": 0.3333,
     "ci": [
      0.1713,
      0.5
     ]
    },
    "medium": {
     "value": 0.575,
     "ci": [
      0.4186,
      0.7353
     ]
    },
    "hard": {
     "value": 0.0889,
     "ci": [
      0.0222,
      0.1778
     ]
    },
    "hardPassAt3": {
     "value": 0.1667,
     "ci": [
      0.0667,
      0.3
     ]
    },
    "hardPassHat3": {
     "value": 0.0333,
     "ci": [
      0.0,
      0.1
     ]
    },
    "costToRun": {
     "value": 1.4059,
     "ci": [
      1.2607,
      1.5518
     ]
    },
    "costPerTrial": {
     "value": 0.0088,
     "ci": [
      0.0079,
      0.0097
     ]
    },
    "costPerSafeSuccess": {
     "value": 0.0343,
     "ci": [
      0.0258,
      0.0475
     ]
    },
    "secondsPerTrial": {
     "value": 119.5812,
     "ci": [
      110.501,
      129.1
     ]
    },
    "hoursToRun": {
     "value": 5.3147,
     "ci": [
      4.9112,
      5.7378
     ]
    },
    "tokensInPerTrial": {
     "value": 148499.5187,
     "ci": [
      125266.4443,
      172063.8203
     ]
    },
    "tokensOutPerTrial": {
     "value": 10726.0875,
     "ci": [
      9819.0016,
      11662.2073
     ]
    },
    "tokensToRun": {
     "value": 25476097.0,
     "ci": [
      21610260.2324,
      29401228.3523
     ]
    }
   }
  }
 ],
 "pairwise": [
  {
   "a": "gpt-6-astra",
   "b": "gpt-6.1-sol",
   "diff": 1.1111,
   "ci": [
    -3.3333,
    5.5556
   ],
   "separated": false
  },
  {
   "a": "gpt-6-astra",
   "b": "claude-fable-5-1",
   "diff": 11.9048,
   "ci": [
    4.9187,
    19.3651
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "claude-opus-5-5",
   "diff": 15.2579,
   "ci": [
    8.5114,
    22.1434
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "claude-sonnet-5-5",
   "diff": 22.6349,
   "ci": [
    14.919,
    30.8754
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "deepseek-v4.1-flash-ultrafast",
   "diff": 27.6984,
   "ci": [
    20.1587,
    35.2401
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "claude-haiku-5-5",
   "diff": 36.9841,
   "ci": [
    29.0437,
    45.0
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 56.0317,
   "ci": [
    47.2996,
    64.9226
   ],
   "separated": true
  },
  {
   "a": "gpt-6-astra",
   "b": "gpt-6-luna",
   "diff": 58.0952,
   "ci": [
    48.8095,
    66.8254
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "claude-fable-5-1",
   "diff": 10.7937,
   "ci": [
    3.9663,
    18.1746
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "claude-opus-5-5",
   "diff": 14.1468,
   "ci": [
    8.1146,
    20.0794
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "claude-sonnet-5-5",
   "diff": 21.5238,
   "ci": [
    13.9841,
    29.2063
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "deepseek-v4.1-flash-ultrafast",
   "diff": 26.5873,
   "ci": [
    19.6806,
    33.8095
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "claude-haiku-5-5",
   "diff": 35.873,
   "ci": [
    27.5397,
    43.8909
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 54.9206,
   "ci": [
    45.873,
    63.8115
   ],
   "separated": true
  },
  {
   "a": "gpt-6.1-sol",
   "b": "gpt-6-luna",
   "diff": 56.9841,
   "ci": [
    48.0933,
    65.6349
   ],
   "separated": true
  },
  {
   "a": "claude-fable-5-1",
   "b": "claude-opus-5-5",
   "diff": 3.3532,
   "ci": [
    -4.8021,
    11.5878
   ],
   "separated": false
  },
  {
   "a": "claude-fable-5-1",
   "b": "claude-sonnet-5-5",
   "diff": 10.7302,
   "ci": [
    2.4123,
    18.7345
   ],
   "separated": true
  },
  {
   "a": "claude-fable-5-1",
   "b": "deepseek-v4.1-flash-ultrafast",
   "diff": 15.7937,
   "ci": [
    8.3333,
    23.494
   ],
   "separated": true
  },
  {
   "a": "claude-fable-5-1",
   "b": "claude-haiku-5-5",
   "diff": 25.0794,
   "ci": [
    17.2222,
    33.0952
   ],
   "separated": true
  },
  {
   "a": "claude-fable-5-1",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 44.127,
   "ci": [
    35.2381,
    52.7004
   ],
   "separated": true
  },
  {
   "a": "claude-fable-5-1",
   "b": "gpt-6-luna",
   "diff": 46.1905,
   "ci": [
    36.9028,
    55.0813
   ],
   "separated": true
  },
  {
   "a": "claude-opus-5-5",
   "b": "claude-sonnet-5-5",
   "diff": 7.377,
   "ci": [
    -0.3255,
    15.3096
   ],
   "separated": false
  },
  {
   "a": "claude-opus-5-5",
   "b": "deepseek-v4.1-flash-ultrafast",
   "diff": 12.4405,
   "ci": [
    4.9787,
    19.9236
   ],
   "separated": true
  },
  {
   "a": "claude-opus-5-5",
   "b": "claude-haiku-5-5",
   "diff": 21.7262,
   "ci": [
    13.7088,
    29.4643
   ],
   "separated": true
  },
  {
   "a": "claude-opus-5-5",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 40.7738,
   "ci": [
    31.8254,
    49.6825
   ],
   "separated": true
  },
  {
   "a": "claude-opus-5-5",
   "b": "gpt-6-luna",
   "diff": 42.8373,
   "ci": [
    34.3452,
    51.3313
   ],
   "separated": true
  },
  {
   "a": "claude-sonnet-5-5",
   "b": "deepseek-v4.1-flash-ultrafast",
   "diff": 5.0635,
   "ci": [
    -2.1591,
    12.3651
   ],
   "separated": false
  },
  {
   "a": "claude-sonnet-5-5",
   "b": "claude-haiku-5-5",
   "diff": 14.3492,
   "ci": [
    5.5222,
    23.1599
   ],
   "separated": true
  },
  {
   "a": "claude-sonnet-5-5",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 33.3968,
   "ci": [
    24.0476,
    43.102
   ],
   "separated": true
  },
  {
   "a": "claude-sonnet-5-5",
   "b": "gpt-6-luna",
   "diff": 35.4603,
   "ci": [
    26.6667,
    44.4464
   ],
   "separated": true
  },
  {
   "a": "deepseek-v4.1-flash-ultrafast",
   "b": "claude-haiku-5-5",
   "diff": 9.2857,
   "ci": [
    3.1726,
    15.0813
   ],
   "separated": true
  },
  {
   "a": "deepseek-v4.1-flash-ultrafast",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 28.3333,
   "ci": [
    20.5556,
    36.1111
   ],
   "separated": true
  },
  {
   "a": "deepseek-v4.1-flash-ultrafast",
   "b": "gpt-6-luna",
   "diff": 30.3968,
   "ci": [
    22.9365,
    37.7798
   ],
   "separated": true
  },
  {
   "a": "claude-haiku-5-5",
   "b": "glm-5.3-flash-ultrafast",
   "diff": 19.0476,
   "ci": [
    12.377,
    26.5873
   ],
   "separated": true
  },
  {
   "a": "claude-haiku-5-5",
   "b": "gpt-6-luna",
   "diff": 21.1111,
   "ci": [
    13.4921,
    28.8889
   ],
   "separated": true
  },
  {
   "a": "glm-5.3-flash-ultrafast",
   "b": "gpt-6-luna",
   "diff": 2.0635,
   "ci": [
    -7.1448,
    11.5913
   ],
   "separated": false
  }
 ],
 "tiers": [
  [
   "gpt-6-astra",
   "gpt-6.1-sol"
  ],
  [
   "claude-fable-5-1",
   "claude-opus-5-5"
  ],
  [
   "claude-sonnet-5-5",
   "deepseek-v4.1-flash-ultrafast"
  ],
  [
   "claude-haiku-5-5"
  ],
  [
   "glm-5.3-flash-ultrafast",
   "gpt-6-luna"
  ]
 ],
 "notes": [
  "Safety is the pass condition inside the index: a trial passes only when every condition reaches task success with zero safety violations. Cost, safety gap, violation profile and consistency are reported outside it.",
  "Severity classes S3/S2/S1 are a draft by rule name and are pending review by a PLC engineer. Only rules whose evaluator scope is 'safety' count; goal rules (complete, completion_hold, ...) never do. A trial or condition takes its worst class; 'unknown' means the evaluator flagged a violation without naming a rule.",
  "Cost bases differ: Claude Code and Codex rows use the harness-reported API list price; mini-swe-agent rows use tokens x OpenGateway ultrafast price (deepseek-v4.1-flash-ultrafast, glm-5.3-flash-ultrafast). Voided trials (rate limits) are excluded.",
  "Each model ran in one harness, so harness and model effects are confounded. Rows are model x harness x setting.",
  "Easy and Medium ran once per task; consistency (pass@3, pass^3) is measured on Hard only.",
  "Machine understanding (FactoryBench) is pending grader repair and is not in the index."
 ]
}