{
  "workflow": {
    "16:fresh": {
      "passed": 253,
      "tasks": 256,
      "mean": 0.98828125,
      "template95": [
        0.9635627530364372,
        1.0
      ],
      "templates": 12,
      "whole_passed": 29,
      "sequences": 32,
      "whole_rate": 0.90625,
      "whole_exact95_fixed_bank": [
        0.7497730493162054,
        0.9802328198267152
      ],
      "independent_prediction": 0.9100064004664707,
      "first_passed": 201,
      "repeats": {
        "count": 0,
        "mean": 0.0,
        "template95": [
          0.0,
          0.0
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          20,
          20
        ],
        "backoff": [
          24,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          19,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          22,
          22
        ],
        "page_offset": [
          16,
          19
        ],
        "retry": [
          21,
          21
        ],
        "window_end": [
          23,
          23
        ],
        "wrap": [
          24,
          24
        ]
      }
    },
    "16:rolling": {
      "passed": 256,
      "tasks": 256,
      "mean": 1.0,
      "template95": [
        1.0,
        1.0
      ],
      "templates": 12,
      "whole_passed": 32,
      "sequences": 32,
      "whole_rate": 1.0,
      "whole_exact95_fixed_bank": [
        0.8911188393206472,
        1.0
      ],
      "independent_prediction": 1.0,
      "first_passed": 242,
      "repeats": {
        "count": 0,
        "mean": 0.0,
        "template95": [
          0.0,
          0.0
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          20,
          20
        ],
        "backoff": [
          24,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          19,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          22,
          22
        ],
        "page_offset": [
          19,
          19
        ],
        "retry": [
          21,
          21
        ],
        "window_end": [
          23,
          23
        ],
        "wrap": [
          24,
          24
        ]
      },
      "paired_history_effect": {
        "wins": 3,
        "losses": 0,
        "exact_mcnemar": 0.25,
        "template_label_swap_p": 1.0,
        "mean": 0.01171875,
        "template95": [
          0.0,
          0.03643724696356275
        ],
        "templates": 12
      }
    },
    "4:fresh": {
      "passed": 237,
      "tasks": 256,
      "mean": 0.92578125,
      "template95": [
        0.8464566929133859,
        1.0
      ],
      "templates": 12,
      "whole_passed": 17,
      "sequences": 32,
      "whole_rate": 0.53125,
      "whole_exact95_fixed_bank": [
        0.3474368092863929,
        0.7090601770939847
      ],
      "independent_prediction": 0.5395939176053899,
      "first_passed": 171,
      "repeats": {
        "count": 9,
        "mean": 0.03515625,
        "template95": [
          0.0,
          0.09125475285171103
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          13,
          20
        ],
        "backoff": [
          24,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          13,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          22,
          22
        ],
        "page_offset": [
          19,
          19
        ],
        "retry": [
          21,
          21
        ],
        "window_end": [
          17,
          23
        ],
        "wrap": [
          24,
          24
        ]
      },
      "paired_tasks_vs_bf16": {
        "wins": 3,
        "losses": 19,
        "exact_mcnemar": 0.0008554458618164062,
        "template_label_swap_p": 0.25,
        "mean": -0.0625,
        "template95": [
          -0.148,
          0.01171875
        ],
        "templates": 12
      },
      "paired_sequences_vs_bf16_fixed_bank": {
        "wins": 2,
        "losses": 14,
        "exact_mcnemar": 0.004180908203125
      }
    },
    "4:rolling": {
      "passed": 249,
      "tasks": 256,
      "mean": 0.97265625,
      "template95": [
        0.9325842696629213,
        1.0
      ],
      "templates": 12,
      "whole_passed": 25,
      "sequences": 32,
      "whole_rate": 0.78125,
      "whole_exact95_fixed_bank": [
        0.6002717362878924,
        0.9072284677152807
      ],
      "independent_prediction": 0.8010784600261767,
      "first_passed": 220,
      "repeats": {
        "count": 2,
        "mean": 0.0078125,
        "template95": [
          0.0,
          0.019455252918287938
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          20,
          20
        ],
        "backoff": [
          19,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          18,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          22,
          22
        ],
        "page_offset": [
          19,
          19
        ],
        "retry": [
          21,
          21
        ],
        "window_end": [
          22,
          23
        ],
        "wrap": [
          24,
          24
        ]
      },
      "paired_tasks_vs_bf16": {
        "wins": 0,
        "losses": 7,
        "exact_mcnemar": 0.015625,
        "template_label_swap_p": 0.25,
        "mean": -0.02734375,
        "template95": [
          -0.06741573033707865,
          0.0
        ],
        "templates": 12
      },
      "paired_sequences_vs_bf16_fixed_bank": {
        "wins": 0,
        "losses": 7,
        "exact_mcnemar": 0.015625
      },
      "paired_history_effect": {
        "wins": 17,
        "losses": 5,
        "exact_mcnemar": 0.01690053939819336,
        "template_label_swap_p": 0.5,
        "mean": 0.046875,
        "template95": [
          -0.0311284046692607,
          0.13253012048192772
        ],
        "templates": 12
      }
    },
    "3:fresh": {
      "passed": 221,
      "tasks": 256,
      "mean": 0.86328125,
      "template95": [
        0.696292735042735,
        1.0
      ],
      "templates": 12,
      "whole_passed": 7,
      "sequences": 32,
      "whole_rate": 0.21875,
      "whole_exact95_fixed_bank": [
        0.09277153228471935,
        0.39972826371210757
      ],
      "independent_prediction": 0.3084739303657366,
      "first_passed": 171,
      "repeats": {
        "count": 28,
        "mean": 0.109375,
        "template95": [
          0.0,
          0.23863636363636365
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          20,
          20
        ],
        "backoff": [
          24,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          13,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          22,
          22
        ],
        "page_offset": [
          19,
          19
        ],
        "retry": [
          21,
          21
        ],
        "window_end": [
          14,
          23
        ],
        "wrap": [
          4,
          24
        ]
      },
      "paired_tasks_vs_bf16": {
        "wins": 3,
        "losses": 35,
        "exact_mcnemar": 6.677873898297548e-08,
        "template_label_swap_p": 0.25,
        "mean": -0.125,
        "template95": [
          -0.2958801498127341,
          0.011811023622047244
        ],
        "templates": 12
      },
      "paired_sequences_vs_bf16_fixed_bank": {
        "wins": 1,
        "losses": 23,
        "exact_mcnemar": 2.9802322387695312e-06
      }
    },
    "3:rolling": {
      "passed": 222,
      "tasks": 256,
      "mean": 0.8671875,
      "template95": [
        0.7421875,
        0.9641434262948207
      ],
      "templates": 12,
      "whole_passed": 10,
      "sequences": 32,
      "whole_rate": 0.3125,
      "whole_exact95_fixed_bank": [
        0.16118472408820056,
        0.5000775816321696
      ],
      "independent_prediction": 0.31981886163234385,
      "first_passed": 174,
      "repeats": {
        "count": 29,
        "mean": 0.11328125,
        "template95": [
          0.03529411764705882,
          0.20703125
        ],
        "templates": 12
      },
      "per_template": {
        "align_up": [
          20,
          20
        ],
        "backoff": [
          24,
          24
        ],
        "batches": [
          20,
          20
        ],
        "ceil_div": [
          22,
          22
        ],
        "chunk_start": [
          5,
          19
        ],
        "clamp": [
          21,
          21
        ],
        "inclusive_count": [
          21,
          21
        ],
        "inside": [
          21,
          22
        ],
        "page_offset": [
          17,
          19
        ],
        "retry": [
          16,
          21
        ],
        "window_end": [
          19,
          23
        ],
        "wrap": [
          16,
          24
        ]
      },
      "paired_tasks_vs_bf16": {
        "wins": 0,
        "losses": 34,
        "exact_mcnemar": 1.1641532182693481e-10,
        "template_label_swap_p": 0.03125,
        "mean": -0.1328125,
        "template95": [
          -0.2578125,
          -0.035856573705179286
        ],
        "templates": 12
      },
      "paired_sequences_vs_bf16_fixed_bank": {
        "wins": 0,
        "losses": 22,
        "exact_mcnemar": 4.76837158203125e-07
      },
      "paired_history_effect": {
        "wins": 19,
        "losses": 18,
        "exact_mcnemar": 1.0,
        "template_label_swap_p": 1.0,
        "mean": 0.00390625,
        "template95": [
          -0.11507936507936507,
          0.13108614232209737
        ],
        "templates": 12
      }
    }
  },
  "repair": {
    "summaries": {
      "attention_bf16": {
        "passed": 74,
        "total": 96,
        "mean": 0.7708333333333334,
        "template95": [
          0.5416666666666666,
          0.9583333333333334
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            6,
            8
          ],
          "wrap": [
            4,
            8
          ]
        }
      },
      "q4": {
        "passed": 88,
        "total": 96,
        "mean": 0.9166666666666666,
        "template95": [
          0.8229166666666666,
          1.0
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            5,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            6,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            5,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "bf16": {
        "passed": 89,
        "total": 96,
        "mean": 0.9270833333333334,
        "template95": [
          0.78125,
          1.0
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            1,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "q3": {
        "passed": 65,
        "total": 96,
        "mean": 0.6770833333333334,
        "template95": [
          0.4166666666666667,
          0.9166666666666666
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            2,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            7,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_bf16": {
        "passed": 95,
        "total": 96,
        "mean": 0.9895833333333334,
        "template95": [
          0.96875,
          1.0
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            7,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "mlp_budget_0": {
        "passed": 65,
        "total": 96,
        "mean": 0.6770833333333334,
        "template95": [
          0.4166666666666667,
          0.9166666666666666
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            7,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            2,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_budget_2": {
        "passed": 71,
        "total": 96,
        "mean": 0.7395833333333334,
        "template95": [
          0.5,
          0.9583333333333334
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            1,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            6,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_budget_1": {
        "passed": 80,
        "total": 96,
        "mean": 0.8333333333333334,
        "template95": [
          0.5833333333333334,
          1.0
        ],
        "templates": 12,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "batches": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      }
    },
    "comparisons": {
      "q3-minus-bf16": {
        "wins": 7,
        "losses": 31,
        "exact_mcnemar": 0.00011616700794547796,
        "template_label_swap_p": 0.15625,
        "mean": -0.25,
        "template95": [
          -0.5625,
          0.0625
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.0008131690556183457,
        "holm_adjusted_template_swap": 1
      },
      "q4-minus-bf16": {
        "wins": 7,
        "losses": 8,
        "exact_mcnemar": 1.0,
        "template_label_swap_p": 1.0,
        "mean": -0.010416666666666666,
        "template95": [
          -0.15625,
          0.1875
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 1,
        "holm_adjusted_template_swap": 1
      },
      "attention_bf16-minus-q3": {
        "wins": 11,
        "losses": 2,
        "exact_mcnemar": 0.0224609375,
        "template_label_swap_p": 0.375,
        "mean": 0.09375,
        "template95": [
          -0.03125,
          0.25
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.1123046875,
        "holm_adjusted_template_swap": 1
      },
      "mlp_bf16-minus-q3": {
        "wins": 31,
        "losses": 1,
        "exact_mcnemar": 1.5366822481155396e-08,
        "template_label_swap_p": 0.09375,
        "mean": 0.3125,
        "template95": [
          0.07291666666666667,
          0.5729166666666666
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 1.3830140233039856e-07,
        "holm_adjusted_template_swap": 0.84375
      },
      "mlp_bf16-minus-bf16": {
        "wins": 7,
        "losses": 1,
        "exact_mcnemar": 0.0703125,
        "template_label_swap_p": 1.0,
        "mean": 0.0625,
        "template95": [
          -0.03125,
          0.21875
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.28125,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_0-minus-attention_bf16": {
        "wins": 1,
        "losses": 10,
        "exact_mcnemar": 0.01171875,
        "template_label_swap_p": 0.25,
        "mean": -0.09375,
        "template95": [
          -0.20833333333333334,
          0.0
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.0703125,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_1-minus-attention_bf16": {
        "wins": 10,
        "losses": 4,
        "exact_mcnemar": 0.1795654296875,
        "template_label_swap_p": 0.75,
        "mean": 0.0625,
        "template95": [
          -0.10416666666666667,
          0.2708333333333333
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.5386962890625,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_2-minus-attention_bf16": {
        "wins": 8,
        "losses": 11,
        "exact_mcnemar": 0.6476058959960938,
        "template_label_swap_p": 0.875,
        "mean": -0.03125,
        "template95": [
          -0.23958333333333334,
          0.16666666666666666
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 1,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_1-minus-mlp_budget_0": {
        "wins": 15,
        "losses": 0,
        "exact_mcnemar": 6.103515625e-05,
        "template_label_swap_p": 0.25,
        "mean": 0.15625,
        "template95": [
          0.0,
          0.3645833333333333
        ],
        "templates": 12,
        "holm_adjusted_over_reported_comparisons": 0.00048828125,
        "holm_adjusted_template_swap": 1
      }
    }
  },
  "repair_without_batches": {
    "summaries": {
      "attention_bf16": {
        "passed": 66,
        "total": 88,
        "mean": 0.75,
        "template95": [
          0.5,
          0.9545454545454546
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            6,
            8
          ],
          "wrap": [
            4,
            8
          ]
        }
      },
      "q4": {
        "passed": 80,
        "total": 88,
        "mean": 0.9090909090909091,
        "template95": [
          0.8068181818181818,
          1.0
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            5,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            6,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            5,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "bf16": {
        "passed": 88,
        "total": 88,
        "mean": 1.0,
        "template95": [
          1.0,
          1.0
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "q3": {
        "passed": 57,
        "total": 88,
        "mean": 0.6477272727272727,
        "template95": [
          0.375,
          0.8977272727272727
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            2,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            7,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_bf16": {
        "passed": 87,
        "total": 88,
        "mean": 0.9886363636363636,
        "template95": [
          0.9659090909090909,
          1.0
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            8,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            7,
            8
          ],
          "wrap": [
            8,
            8
          ]
        }
      },
      "mlp_budget_0": {
        "passed": 57,
        "total": 88,
        "mean": 0.6477272727272727,
        "template95": [
          0.375,
          0.8977272727272727
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            7,
            8
          ],
          "page_offset": [
            0,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            2,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_budget_2": {
        "passed": 63,
        "total": 88,
        "mean": 0.7159090909090909,
        "template95": [
          0.45454545454545453,
          0.9545454545454546
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            1,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            6,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      },
      "mlp_budget_1": {
        "passed": 72,
        "total": 88,
        "mean": 0.8181818181818182,
        "template95": [
          0.5454545454545454,
          1.0
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            8,
            8
          ],
          "backoff": [
            8,
            8
          ],
          "ceil_div": [
            8,
            8
          ],
          "chunk_start": [
            0,
            8
          ],
          "clamp": [
            8,
            8
          ],
          "inclusive_count": [
            8,
            8
          ],
          "inside": [
            8,
            8
          ],
          "page_offset": [
            8,
            8
          ],
          "retry": [
            8,
            8
          ],
          "window_end": [
            8,
            8
          ],
          "wrap": [
            0,
            8
          ]
        }
      }
    },
    "comparisons": {
      "q3-minus-bf16": {
        "wins": 0,
        "losses": 31,
        "exact_mcnemar": 9.313225746154785e-10,
        "template_label_swap_p": 0.0625,
        "mean": -0.3522727272727273,
        "template95": [
          -0.625,
          -0.10227272727272728
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 8.381903171539307e-09,
        "holm_adjusted_template_swap": 0.5625
      },
      "q4-minus-bf16": {
        "wins": 0,
        "losses": 8,
        "exact_mcnemar": 0.0078125,
        "template_label_swap_p": 0.25,
        "mean": -0.09090909090909091,
        "template95": [
          -0.19318181818181818,
          0.0
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 0.046875,
        "holm_adjusted_template_swap": 1
      },
      "attention_bf16-minus-q3": {
        "wins": 11,
        "losses": 2,
        "exact_mcnemar": 0.0224609375,
        "template_label_swap_p": 0.375,
        "mean": 0.10227272727272728,
        "template95": [
          -0.03409090909090909,
          0.2727272727272727
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 0.08984375,
        "holm_adjusted_template_swap": 1
      },
      "mlp_bf16-minus-q3": {
        "wins": 31,
        "losses": 1,
        "exact_mcnemar": 1.5366822481155396e-08,
        "template_label_swap_p": 0.09375,
        "mean": 0.3409090909090909,
        "template95": [
          0.07954545454545454,
          0.6136363636363636
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 1.2293457984924316e-07,
        "holm_adjusted_template_swap": 0.75
      },
      "mlp_bf16-minus-bf16": {
        "wins": 0,
        "losses": 1,
        "exact_mcnemar": 1.0,
        "template_label_swap_p": 1.0,
        "mean": -0.011363636363636364,
        "template95": [
          -0.03409090909090909,
          0.0
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 1,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_0-minus-attention_bf16": {
        "wins": 1,
        "losses": 10,
        "exact_mcnemar": 0.01171875,
        "template_label_swap_p": 0.25,
        "mean": -0.10227272727272728,
        "template95": [
          -0.22727272727272727,
          0.0
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 0.05859375,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_1-minus-attention_bf16": {
        "wins": 10,
        "losses": 4,
        "exact_mcnemar": 0.1795654296875,
        "template_label_swap_p": 0.75,
        "mean": 0.06818181818181818,
        "template95": [
          -0.11363636363636363,
          0.29545454545454547
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 0.5386962890625,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_2-minus-attention_bf16": {
        "wins": 8,
        "losses": 11,
        "exact_mcnemar": 0.6476058959960938,
        "template_label_swap_p": 0.875,
        "mean": -0.03409090909090909,
        "template95": [
          -0.2727272727272727,
          0.18181818181818182
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 1,
        "holm_adjusted_template_swap": 1
      },
      "mlp_budget_1-minus-mlp_budget_0": {
        "wins": 15,
        "losses": 0,
        "exact_mcnemar": 6.103515625e-05,
        "template_label_swap_p": 0.25,
        "mean": 0.17045454545454544,
        "template95": [
          0.0,
          0.3977272727272727
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 0.00042724609375,
        "holm_adjusted_template_swap": 1
      }
    }
  },
  "sampling_without_batches": {
    "summaries": {
      "bf16": {
        "passed": 107,
        "total": 110,
        "mean": 0.9727272727272728,
        "template95": [
          0.9272727272727272,
          1.0
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            10,
            10
          ],
          "backoff": [
            10,
            10
          ],
          "ceil_div": [
            8,
            10
          ],
          "chunk_start": [
            10,
            10
          ],
          "clamp": [
            10,
            10
          ],
          "inclusive_count": [
            10,
            10
          ],
          "inside": [
            10,
            10
          ],
          "page_offset": [
            9,
            10
          ],
          "retry": [
            10,
            10
          ],
          "window_end": [
            10,
            10
          ],
          "wrap": [
            10,
            10
          ]
        }
      },
      "q3": {
        "passed": 75,
        "total": 110,
        "mean": 0.6818181818181818,
        "template95": [
          0.4090909090909091,
          0.9090909090909091
        ],
        "templates": 11,
        "per_template": {
          "align_up": [
            5,
            10
          ],
          "backoff": [
            10,
            10
          ],
          "ceil_div": [
            10,
            10
          ],
          "chunk_start": [
            0,
            10
          ],
          "clamp": [
            10,
            10
          ],
          "inclusive_count": [
            10,
            10
          ],
          "inside": [
            10,
            10
          ],
          "page_offset": [
            0,
            10
          ],
          "retry": [
            10,
            10
          ],
          "window_end": [
            10,
            10
          ],
          "wrap": [
            0,
            10
          ]
        }
      }
    },
    "comparisons": {
      "q3-minus-bf16": {
        "wins": 2,
        "losses": 34,
        "exact_mcnemar": 1.941225491464138e-08,
        "template_label_swap_p": 0.125,
        "mean": -0.2909090909090909,
        "template95": [
          -0.5636363636363636,
          -0.045454545454545456
        ],
        "templates": 11,
        "holm_adjusted_over_reported_comparisons": 1.941225491464138e-08,
        "holm_adjusted_template_swap": 0.125
      }
    }
  },
  "sweep": [
    {
      "layer": 0,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040794054977596,
      "quantized_token_probability": 0.9991877675056458,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 0,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0009110440732911229,
      "quantized_token_probability": 0.999081015586853,
      "margin_reference_over_quantized": -7.0
    },
    {
      "layer": 0,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.001169500290416181,
      "quantized_token_probability": 0.9988219141960144,
      "margin_reference_over_quantized": -6.75
    },
    {
      "layer": 1,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040794054977596,
      "quantized_token_probability": 0.9991877675056458,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 1,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0005527764442376792,
      "quantized_token_probability": 0.9994431734085083,
      "margin_reference_over_quantized": -7.5
    },
    {
      "layer": 1,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0009110440732911229,
      "quantized_token_probability": 0.999081015586853,
      "margin_reference_over_quantized": -7.0
    },
    {
      "layer": 2,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0009110440732911229,
      "quantized_token_probability": 0.999081015586853,
      "margin_reference_over_quantized": -7.0
    },
    {
      "layer": 2,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0009110440732911229,
      "quantized_token_probability": 0.999081015586853,
      "margin_reference_over_quantized": -7.0
    },
    {
      "layer": 2,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0010322220623493195,
      "quantized_token_probability": 0.9989591240882874,
      "margin_reference_over_quantized": -6.875
    },
    {
      "layer": 3,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.000804082490503788,
      "quantized_token_probability": 0.9991915822029114,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 3,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040794054977596,
      "quantized_token_probability": 0.9991877675056458,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 3,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0007096652989275753,
      "quantized_token_probability": 0.9992830753326416,
      "margin_reference_over_quantized": -7.25
    },
    {
      "layer": 4,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0004878551117144525,
      "quantized_token_probability": 0.9995080232620239,
      "margin_reference_over_quantized": -7.625
    },
    {
      "layer": 4,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0003799825208261609,
      "quantized_token_probability": 0.9996147751808167,
      "margin_reference_over_quantized": -7.875
    },
    {
      "layer": 4,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0007096652989275753,
      "quantized_token_probability": 0.9992830753326416,
      "margin_reference_over_quantized": -7.25
    },
    {
      "layer": 5,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0002959566772915423,
      "quantized_token_probability": 0.999702513217926,
      "margin_reference_over_quantized": -8.125
    },
    {
      "layer": 5,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0006263300310820341,
      "quantized_token_probability": 0.9993669390678406,
      "margin_reference_over_quantized": -7.375
    },
    {
      "layer": 5,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040794054977596,
      "quantized_token_probability": 0.9991877675056458,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 6,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0002959566772915423,
      "quantized_token_probability": 0.999702513217926,
      "margin_reference_over_quantized": -8.125
    },
    {
      "layer": 6,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0009110440732911229,
      "quantized_token_probability": 0.999081015586853,
      "margin_reference_over_quantized": -7.0
    },
    {
      "layer": 6,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0024725759867578745,
      "quantized_token_probability": 0.9975082874298096,
      "margin_reference_over_quantized": -6.0
    },
    {
      "layer": 7,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0002305062225786969,
      "quantized_token_probability": 0.9997673034667969,
      "margin_reference_over_quantized": -8.375
    },
    {
      "layer": 7,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.001169500290416181,
      "quantized_token_probability": 0.9988219141960144,
      "margin_reference_over_quantized": -6.75
    },
    {
      "layer": 7,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0005527764442376792,
      "quantized_token_probability": 0.9994431734085083,
      "margin_reference_over_quantized": -7.5
    },
    {
      "layer": 8,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00017952734197024256,
      "quantized_token_probability": 0.9998169541358948,
      "margin_reference_over_quantized": -8.625
    },
    {
      "layer": 8,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0007096652989275753,
      "quantized_token_probability": 0.9992830753326416,
      "margin_reference_over_quantized": -7.25
    },
    {
      "layer": 8,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0017006979323923588,
      "quantized_token_probability": 0.9982848763465881,
      "margin_reference_over_quantized": -6.375
    },
    {
      "layer": 9,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00013982190284878016,
      "quantized_token_probability": 0.9998587965965271,
      "margin_reference_over_quantized": -8.875
    },
    {
      "layer": 9,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0006263300310820341,
      "quantized_token_probability": 0.9993669390678406,
      "margin_reference_over_quantized": -7.375
    },
    {
      "layer": 9,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0011694958666339517,
      "quantized_token_probability": 0.9988180994987488,
      "margin_reference_over_quantized": -6.75
    },
    {
      "layer": 10,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00013982190284878016,
      "quantized_token_probability": 0.9998587965965271,
      "margin_reference_over_quantized": -8.875
    },
    {
      "layer": 10,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0005527764442376792,
      "quantized_token_probability": 0.9994431734085083,
      "margin_reference_over_quantized": -7.5
    },
    {
      "layer": 10,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0007096626213751733,
      "quantized_token_probability": 0.999279260635376,
      "margin_reference_over_quantized": -7.25
    },
    {
      "layer": 11,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 3.5356155422050506e-05,
      "quantized_token_probability": 0.9999618530273438,
      "margin_reference_over_quantized": -10.25
    },
    {
      "layer": 11,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0002959566772915423,
      "quantized_token_probability": 0.999702513217926,
      "margin_reference_over_quantized": -8.125
    },
    {
      "layer": 11,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0007096652989275753,
      "quantized_token_probability": 0.9992830753326416,
      "margin_reference_over_quantized": -7.25
    },
    {
      "layer": 12,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0001088967255782336,
      "quantized_token_probability": 0.9998893737792969,
      "margin_reference_over_quantized": -9.125
    },
    {
      "layer": 12,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00037998397601768374,
      "quantized_token_probability": 0.9996185898780823,
      "margin_reference_over_quantized": -7.875
    },
    {
      "layer": 12,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0017007108544930816,
      "quantized_token_probability": 0.9982924461364746,
      "margin_reference_over_quantized": -6.375
    },
    {
      "layer": 13,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00013982190284878016,
      "quantized_token_probability": 0.9998587965965271,
      "margin_reference_over_quantized": -8.875
    },
    {
      "layer": 13,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00033535005059093237,
      "quantized_token_probability": 0.9996643662452698,
      "margin_reference_over_quantized": -8.0
    },
    {
      "layer": 13,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.002800897927954793,
      "quantized_token_probability": 0.9971886873245239,
      "margin_reference_over_quantized": -5.875
    },
    {
      "layer": 14,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.003172671189531684,
      "quantized_token_probability": 0.9968236088752747,
      "margin_reference_over_quantized": -5.75
    },
    {
      "layer": 14,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.008577263914048672,
      "quantized_token_probability": 0.9913969039916992,
      "margin_reference_over_quantized": -4.75
    },
    {
      "layer": 14,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.04742114990949631,
      "quantized_token_probability": 0.9524792432785034,
      "margin_reference_over_quantized": -3.0
    },
    {
      "layer": 15,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0001795280259102583,
      "quantized_token_probability": 0.9998207092285156,
      "margin_reference_over_quantized": -8.625
    },
    {
      "layer": 15,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.05339233949780464,
      "quantized_token_probability": 0.946401834487915,
      "margin_reference_over_quantized": -2.875
    },
    {
      "layer": 15,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0002305062225786969,
      "quantized_token_probability": 0.9997673034667969,
      "margin_reference_over_quantized": -8.375
    },
    {
      "layer": 16,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0021827067248523235,
      "quantized_token_probability": 0.9978127479553223,
      "margin_reference_over_quantized": -6.125
    },
    {
      "layer": 16,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.002182698342949152,
      "quantized_token_probability": 0.9978089332580566,
      "margin_reference_over_quantized": -6.125
    },
    {
      "layer": 16,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 9.610212146071717e-05,
      "quantized_token_probability": 0.9999008178710938,
      "margin_reference_over_quantized": -9.25
    },
    {
      "layer": 17,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.008577395230531693,
      "quantized_token_probability": 0.9914120435714722,
      "margin_reference_over_quantized": -4.75
    },
    {
      "layer": 17,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.006692733149975538,
      "quantized_token_probability": 0.9932896494865417,
      "margin_reference_over_quantized": -5.0
    },
    {
      "layer": 17,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.1479906141757965,
      "quantized_token_probability": 0.8516271710395813,
      "margin_reference_over_quantized": -1.75
    },
    {
      "layer": 18,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.02033180743455887,
      "quantized_token_probability": 0.9796411991119385,
      "margin_reference_over_quantized": -3.875
    },
    {
      "layer": 18,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.01590513437986374,
      "quantized_token_probability": 0.9840158224105835,
      "margin_reference_over_quantized": -4.125
    },
    {
      "layer": 18,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.2688620984554291,
      "quantized_token_probability": 0.7308429479598999,
      "margin_reference_over_quantized": -1.0
    },
    {
      "layer": 19,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.008577460423111916,
      "quantized_token_probability": 0.9914196133613586,
      "margin_reference_over_quantized": -4.75
    },
    {
      "layer": 19,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 2.144491736544296e-05,
      "quantized_token_probability": 0.9999771118164062,
      "margin_reference_over_quantized": -10.75
    },
    {
      "layer": 19,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.26885801553726196,
      "quantized_token_probability": 0.7308318018913269,
      "margin_reference_over_quantized": -1.0
    },
    {
      "layer": 20,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.01406356692314148,
      "quantized_token_probability": 0.9859320521354675,
      "margin_reference_over_quantized": -4.25
    },
    {
      "layer": 20,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.014062815345823765,
      "quantized_token_probability": 0.9858794808387756,
      "margin_reference_over_quantized": -4.25
    },
    {
      "layer": 20,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0004878551117144525,
      "quantized_token_probability": 0.9995080232620239,
      "margin_reference_over_quantized": -7.625
    },
    {
      "layer": 21,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.5619815587997437,
      "quantized_token_probability": 0.4376716613769531,
      "margin_reference_over_quantized": 0.25
    },
    {
      "layer": 21,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.002800833899527788,
      "quantized_token_probability": 0.997165858745575,
      "margin_reference_over_quantized": -5.875
    },
    {
      "layer": 21,
      "component": "mlp",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.8802946209907532,
      "quantized_token_probability": 0.11913492530584335,
      "margin_reference_over_quantized": 2.0
    },
    {
      "layer": 22,
      "component": "residual",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.03732394054532051,
      "quantized_token_probability": 0.962597131729126,
      "margin_reference_over_quantized": -3.25
    },
    {
      "layer": 22,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 4.006361632491462e-05,
      "quantized_token_probability": 0.9999580383300781,
      "margin_reference_over_quantized": -10.125
    },
    {
      "layer": 22,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0015011755749583244,
      "quantized_token_probability": 0.9984943270683289,
      "margin_reference_over_quantized": -6.5
    },
    {
      "layer": 23,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.8625160455703735,
      "quantized_token_probability": 0.13227111101150513,
      "margin_reference_over_quantized": 1.875
    },
    {
      "layer": 23,
      "component": "attention",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.7744901180267334,
      "quantized_token_probability": 0.22189512848854065,
      "margin_reference_over_quantized": 1.25
    },
    {
      "layer": 23,
      "component": "mlp",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9919756650924683,
      "quantized_token_probability": 0.0066838799975812435,
      "margin_reference_over_quantized": 5.0
    },
    {
      "layer": 24,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9961564540863037,
      "quantized_token_probability": 0.00020268633670639247,
      "margin_reference_over_quantized": 8.5
    },
    {
      "layer": 24,
      "component": "attention",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.4980742037296295,
      "quantized_token_probability": 0.4980742037296295,
      "margin_reference_over_quantized": 0.0
    },
    {
      "layer": 24,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.02297690138220787,
      "quantized_token_probability": 0.977002739906311,
      "margin_reference_over_quantized": -3.75
    },
    {
      "layer": 25,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9980069398880005,
      "quantized_token_probability": 4.53094435215462e-05,
      "margin_reference_over_quantized": 10.0
    },
    {
      "layer": 25,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.010985689237713814,
      "quantized_token_probability": 0.9889002442359924,
      "margin_reference_over_quantized": -4.5
    },
    {
      "layer": 25,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.14800022542476654,
      "quantized_token_probability": 0.8516824245452881,
      "margin_reference_over_quantized": -1.75
    },
    {
      "layer": 26,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9992716312408447,
      "quantized_token_probability": 2.900206709455233e-06,
      "margin_reference_over_quantized": 12.75
    },
    {
      "layer": 26,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.010986528359353542,
      "quantized_token_probability": 0.9889756441116333,
      "margin_reference_over_quantized": -4.5
    },
    {
      "layer": 26,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.4065519869327545,
      "quantized_token_probability": 0.5915296673774719,
      "margin_reference_over_quantized": -0.375
    },
    {
      "layer": 27,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9996147751808167,
      "quantized_token_probability": 1.0672928283383953e-06,
      "margin_reference_over_quantized": 13.75
    },
    {
      "layer": 27,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0003353487700223923,
      "quantized_token_probability": 0.9996605515480042,
      "margin_reference_over_quantized": -8.0
    },
    {
      "layer": 27,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.042081620544195175,
      "quantized_token_probability": 0.9577732086181641,
      "margin_reference_over_quantized": -3.125
    },
    {
      "layer": 28,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9995652437210083,
      "quantized_token_probability": 1.067239850272017e-06,
      "margin_reference_over_quantized": 13.75
    },
    {
      "layer": 28,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.014063459821045399,
      "quantized_token_probability": 0.9859246015548706,
      "margin_reference_over_quantized": -4.25
    },
    {
      "layer": 28,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040794054977596,
      "quantized_token_probability": 0.9991877675056458,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 29,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9998398423194885,
      "quantized_token_probability": 2.1020970564222807e-07,
      "margin_reference_over_quantized": 15.375
    },
    {
      "layer": 29,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.001169500290416181,
      "quantized_token_probability": 0.9988219141960144,
      "margin_reference_over_quantized": -6.75
    },
    {
      "layer": 29,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.085059754550457,
      "quantized_token_probability": 0.914478600025177,
      "margin_reference_over_quantized": -2.375
    },
    {
      "layer": 30,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9998169541358948,
      "quantized_token_probability": 1.125145701053043e-07,
      "margin_reference_over_quantized": 16.0
    },
    {
      "layer": 30,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0010322182206436992,
      "quantized_token_probability": 0.9989553093910217,
      "margin_reference_over_quantized": -6.875
    },
    {
      "layer": 30,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0001088967255782336,
      "quantized_token_probability": 0.9998893737792969,
      "margin_reference_over_quantized": -9.125
    },
    {
      "layer": 31,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9998398423194885,
      "quantized_token_probability": 1.2749862321470573e-07,
      "margin_reference_over_quantized": 15.875
    },
    {
      "layer": 31,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.004609469790011644,
      "quantized_token_probability": 0.995368242263794,
      "margin_reference_over_quantized": -5.375
    },
    {
      "layer": 31,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.09524496644735336,
      "quantized_token_probability": 0.9036591053009033,
      "margin_reference_over_quantized": -2.25
    },
    {
      "layer": 32,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9995690584182739,
      "quantized_token_probability": 3.4648334690245974e-07,
      "margin_reference_over_quantized": 14.875
    },
    {
      "layer": 32,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.005219767335802317,
      "quantized_token_probability": 0.9947115778923035,
      "margin_reference_over_quantized": -5.25
    },
    {
      "layer": 32,
      "component": "mlp",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.8519131541252136,
      "quantized_token_probability": 0.148040309548378,
      "margin_reference_over_quantized": 1.75
    },
    {
      "layer": 33,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9997386932373047,
      "quantized_token_probability": 3.9268374507628323e-07,
      "margin_reference_over_quantized": 14.75
    },
    {
      "layer": 33,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00037998109473846853,
      "quantized_token_probability": 0.9996110200881958,
      "margin_reference_over_quantized": -7.875
    },
    {
      "layer": 33,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0008040855755098164,
      "quantized_token_probability": 0.999195396900177,
      "margin_reference_over_quantized": -7.125
    },
    {
      "layer": 34,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9998587965965271,
      "quantized_token_probability": 1.125192952144971e-07,
      "margin_reference_over_quantized": 16.0
    },
    {
      "layer": 34,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.0015011526411399245,
      "quantized_token_probability": 0.9984790682792664,
      "margin_reference_over_quantized": -6.5
    },
    {
      "layer": 34,
      "component": "mlp",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.05340181291103363,
      "quantized_token_probability": 0.9465697407722473,
      "margin_reference_over_quantized": -2.875
    },
    {
      "layer": 35,
      "component": "residual",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.9968160390853882,
      "quantized_token_probability": 1.440379406858483e-07,
      "margin_reference_over_quantized": 15.75
    },
    {
      "layer": 35,
      "component": "attention",
      "top_token": 488,
      "top_text": " +",
      "reference_token_probability": 0.00017952734197024256,
      "quantized_token_probability": 0.9998169541358948,
      "margin_reference_over_quantized": -8.625
    },
    {
      "layer": 35,
      "component": "mlp",
      "top_token": 8,
      "top_text": ")",
      "reference_token_probability": 0.998909592628479,
      "quantized_token_probability": 0.0009108876693062484,
      "margin_reference_over_quantized": 7.0
    }
  ],
  "benchmarks": [
    {
      "name": "MMLU-Pro (560 questions)",
      "bf16": 77.5,
      "cq": 71.79,
      "label": "MMLU-Pro"
    },
    {
      "name": "MMLU-ProX Lite, 10 languages (5,880)",
      "bf16": 76.04,
      "cq": 76.28,
      "label": "MMLU-ProX Lite"
    },
    {
      "name": "GPQA Diamond (198 \u00d7 8)",
      "bf16": 89.2,
      "cq": 85.54,
      "label": "GPQA Diamond"
    },
    {
      "name": "GSM8K, flexible extract (300)",
      "bf16": 94.0,
      "cq": 91.67,
      "label": "GSM8K"
    },
    {
      "name": "IFEval, prompt-level strict (300)",
      "bf16": 83.33,
      "cq": 82.67,
      "label": "IFEval"
    },
    {
      "name": "IFBench (300 \u00d7 8)",
      "bf16": 71.58,
      "cq": 65.31,
      "label": "IFBench"
    },
    {
      "name": "HumanEval (164)",
      "bf16": 95.12,
      "cq": 95.12,
      "label": "HumanEval"
    },
    {
      "name": "LiveCodeBench v6, thinking (1,055)",
      "bf16": 86.07,
      "cq": 81.04,
      "label": "LiveCodeBench v6"
    },
    {
      "name": "BigCodeBench, thinking, complete",
      "bf16": 65.26,
      "cq": 61.84,
      "label": "BigCodeBench \u00b7 thinking / complete"
    },
    {
      "name": "BigCodeBench, thinking, instruct",
      "bf16": 52.28,
      "cq": 51.05,
      "label": "BigCodeBench \u00b7 thinking / instruct"
    },
    {
      "name": "BigCodeBench, leaderboard, complete",
      "bf16": 61.58,
      "cq": 59.74,
      "label": "BigCodeBench \u00b7 leaderboard / complete"
    },
    {
      "name": "BigCodeBench, leaderboard, instruct",
      "bf16": 52.72,
      "cq": 50.18,
      "label": "BigCodeBench \u00b7 leaderboard / instruct"
    },
    {
      "name": "SciCode, subtask accuracy (65 \u00d7 8)",
      "bf16": 45.83,
      "cq": 41.23,
      "label": "SciCode"
    },
    {
      "name": "GraphWalks, prompts up to 262K tokens (650)",
      "bf16": 96.58,
      "cq": 88.03,
      "label": "GraphWalks"
    },
    {
      "name": "AA-LCR v1.1 (98 \u00d7 16)",
      "bf16": 67.35,
      "cq": 70.45,
      "label": "AA-LCR v1.1"
    },
    {
      "name": "V*, image (191 \u00d7 8)",
      "bf16": 86.32,
      "cq": 82.85,
      "label": "V*"
    },
    {
      "name": "MathVision, image (3,040)",
      "bf16": 81.28,
      "cq": 76.22,
      "label": "MathVision"
    },
    {
      "name": "SWE-bench Verified, mini-swe-agent, no network (500)",
      "bf16": 61.4,
      "cq": 63.6,
      "label": "SWE-bench Verified"
    }
  ]
}