{
  "quality": {
    "baseline": {
      "f1": {
        "fp16": 0.6890078214781351,
        "awq": 0.5781900284463714,
        "delta": -0.1108177930317638,
        "ci95": [
          -0.1489177946839921,
          -0.07611785304483927
        ],
        "significant": true
      },
      "contain": {
        "fp16": 0.975,
        "awq": 0.945,
        "delta": -0.03,
        "ci95": [
          -0.055,
          -0.01
        ],
        "significant": true
      },
      "comply": {
        "fp16": 0.825,
        "awq": 0.675,
        "delta": -0.15,
        "ci95": [
          -0.205,
          -0.1
        ],
        "significant": true
      },
      "mean_words": {
        "fp16": 9.025,
        "awq": 11.775
      }
    },
    "strict": {
      "f1": {
        "fp16": 0.8839629566340093,
        "awq": 0.8527801550444739,
        "delta": -0.031182801589535333,
        "ci95": [
          -0.055169174607642105,
          -0.007266396829864322
        ],
        "significant": true
      },
      "contain": {
        "fp16": 0.905,
        "awq": 0.905,
        "delta": 0.0,
        "ci95": [
          -0.03,
          0.03
        ],
        "significant": false
      },
      "comply": {
        "fp16": 0.95,
        "awq": 0.895,
        "delta": -0.055,
        "ci95": [
          -0.095,
          -0.02
        ],
        "significant": true
      },
      "mean_words": {
        "fp16": 2.76,
        "awq": 2.825
      }
    },
    "fewshot": {
      "f1": {
        "fp16": 0.8733949953486735,
        "awq": 0.8796027129956495,
        "delta": 0.006207717646975959,
        "ci95": [
          -0.011125326303075665,
          0.02473856578204404
        ],
        "significant": false
      },
      "contain": {
        "fp16": 0.915,
        "awq": 0.92,
        "delta": 0.005,
        "ci95": [
          -0.015,
          0.025
        ],
        "significant": false
      },
      "comply": {
        "fp16": 0.93,
        "awq": 0.94,
        "delta": 0.01,
        "ci95": [
          -0.01,
          0.03
        ],
        "significant": false
      },
      "mean_words": {
        "fp16": 4.115,
        "awq": 4.03
      }
    },
    "fewshot_cap": {
      "f1": {
        "fp16": 0.8744657587013726,
        "awq": 0.8801813495186258,
        "delta": 0.0057155908172532195,
        "ci95": [
          -0.01319746593276005,
          0.025788710578991906
        ],
        "significant": false
      },
      "contain": {
        "fp16": 0.89,
        "awq": 0.895,
        "delta": 0.005,
        "ci95": [
          -0.015,
          0.025
        ],
        "significant": false
      },
      "comply": {
        "fp16": 0.95,
        "awq": 0.965,
        "delta": 0.015,
        "ci95": [
          -0.005,
          0.04
        ],
        "significant": false
      },
      "mean_words": {
        "fp16": 3.38,
        "awq": 3.33
      }
    }
  },
  "throughput": {
    "1": {
      "fp16_tok_per_s": 76.40361571898039,
      "awq_tok_per_s": 161.27533287571305,
      "speedup": 2.110833778716687
    },
    "4": {
      "fp16_tok_per_s": 303.379780700721,
      "awq_tok_per_s": 598.7894682648688,
      "speedup": 1.9737289903824025
    },
    "16": {
      "fp16_tok_per_s": 1129.8218509924359,
      "awq_tok_per_s": 2083.033676869298,
      "speedup": 1.8436832984240483
    },
    "64": {
      "fp16_tok_per_s": 3498.6746228715065,
      "awq_tok_per_s": 4510.038791679322,
      "speedup": 1.289070656126847
    },
    "256": {
      "fp16_tok_per_s": 7982.753650520906,
      "awq_tok_per_s": 7251.141718519298,
      "speedup": 0.9083509320178171
    }
  },
  "H3_damage_is_format_not_content": false,
  "H4_prompting_rescues": {
    "rescued_by": [
      "fewshot",
      "fewshot_cap"
    ],
    "any": true
  },
  "H5_throughput": {
    "curve": [
      [
        1,
        2.110833778716687
      ],
      [
        4,
        1.9737289903824025
      ],
      [
        16,
        1.8436832984240483
      ],
      [
        64,
        1.289070656126847
      ],
      [
        256,
        0.9083509320178171
      ]
    ],
    "monotone_decreasing": true,
    "crossover_batch": 256
  }
}