{
  "task_slug": "flat-maxima-transferability-compute-matched",
  "question": "\u5728\u56fa\u5b9a\u68af\u5ea6\u8c03\u7528\u9884\u7b97(\u800c\u975e\u56fa\u5b9a\u8fed\u4ee3\u6570)\u4e0b,PGN \u5f0f\u5e73\u5766\u6781\u503c\u653b\u51fb\u76f8\u5bf9 MI-FGSM \u7684\u8fc1\u79fb\u6027\u589e\u76ca\u8fd8\u5269\u591a\u5c11?\u5e73\u5766\u5ea6\u672c\u8eab\u662f\u5426\u662f\u4e2d\u4ecb\u53d8\u91cf?",
  "falsifiable_prediction": "\u9884\u6d4b:\u76f8\u540c\u8fed\u4ee3\u6570\u4e0b\u5e73\u5766\u5316\u81c2\u5bf9 MI-FGSM \u7684\u5e73\u5747\u8fc1\u79fb\u6210\u529f\u7387\u4f18\u52bf >=5 \u4e2a\u767e\u5206\u70b9;\u6539\u4e3a\u76f8\u540c\u68af\u5ea6\u8c03\u7528\u9884\u7b97\u540e\u4f18\u52bf\u7f29\u5230 <=2 \u4e2a\u767e\u5206\u70b9\u6216\u53cd\u8f6c\u3002\u82e5\u5728\u76f8\u540c\u68af\u5ea6\u8c03\u7528\u9884\u7b97\u4e0b\u5e73\u5766\u5316\u81c2\u5728\u4e24\u4e2a\u9884\u7b97\u6863 x 3 \u4e2a\u79cd\u5b50\u7684\u5e73\u5747\u503c\u4ecd\u9886\u5148 >=5 \u4e2a\u767e\u5206\u70b9\u4e14\u6bcf\u4e2a\u79cd\u5b50\u90fd\u4e3a\u6b63,\u5219\u9884\u6d4b\u88ab\u63a8\u7ffb\u3002",
  "prediction_outcome": "inconclusive",
  "negative_result": false,
  "dataset": "CIFAR-10 test set (10000 images, torchvision-format via HF parquet mirror); per (grad_budget, seed) combo: 128 images sampled and filtered to be correctly classified by surrogate cifar10_resnet20 AND all 4 targets (cifar10_vgg11_bn, cifar10_mobilenetv2_x1_0, cifar10_shufflenetv2_x1_0, cifar10_repvgg_a0).",
  "env": {
    "python": "3.12.3",
    "torch": "2.13.0+cpu",
    "extra_packages": [
      "pyarrow==25.0.0"
    ]
  },
  "seeds": [
    0,
    1,
    2
  ],
  "arms": [
    {
      "name": "baseline-mi-equal-budget-gb36",
      "is_baseline": true,
      "what": "MI-FGSM (decay=1.0), T=grad_budget -- consumes exactly the same number of fwd+bwd calls as PGN (the compute-matched, fair comparison). grad_budget=36.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            1.0,
            1.0,
            1.0
          ],
          "mean": 1.0,
          "std": 0.0
        },
        "transfer_success_rate": {
          "per_seed": [
            0.65625,
            0.6640625,
            0.6640625
          ],
          "mean": 0.6614583333333334,
          "std": 0.003682847818679935
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            59.67692947387695,
            61.93803024291992,
            62.29187774658203
          ],
          "mean": 61.30227915445963,
          "std": 1.1583388111399435
        },
        "fwd_bwd_calls": {
          "per_seed": [
            36,
            36,
            36
          ],
          "mean": 36.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.2265625,
            0.2421875,
            0.2421875
          ],
          "mean": 0.23697916666666666,
          "std": 0.00736569563735987
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.875,
            0.84375,
            0.8359375
          ],
          "mean": 0.8515625,
          "std": 0.016876928902103804
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.7109375,
            0.734375,
            0.734375
          ],
          "mean": 0.7265625,
          "std": 0.011048543456039806
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.8125,
            0.8359375,
            0.84375
          ],
          "mean": 0.8307291666666666,
          "std": 0.01327869664998121
        }
      }
    },
    {
      "name": "baseline-mi-equal-budget-gb72",
      "is_baseline": true,
      "what": "MI-FGSM (decay=1.0), T=grad_budget -- consumes exactly the same number of fwd+bwd calls as PGN (the compute-matched, fair comparison). grad_budget=72.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            1.0,
            1.0,
            1.0
          ],
          "mean": 1.0,
          "std": 0.0
        },
        "transfer_success_rate": {
          "per_seed": [
            0.66015625,
            0.65625,
            0.6484375
          ],
          "mean": 0.6549479166666666,
          "std": 0.0048719497223619025
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            60.99873733520508,
            62.1788444519043,
            65.28673553466797
          ],
          "mean": 62.82143910725912,
          "std": 1.8085773521319974
        },
        "fwd_bwd_calls": {
          "per_seed": [
            72,
            72,
            72
          ],
          "mean": 72.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.2109375,
            0.2421875,
            0.21875
          ],
          "mean": 0.22395833333333334,
          "std": 0.01327869664998121
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.8671875,
            0.84375,
            0.8515625
          ],
          "mean": 0.8541666666666666,
          "std": 0.009743899444723805
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.7421875,
            0.71875,
            0.6953125
          ],
          "mean": 0.71875,
          "std": 0.019136638615493577
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.8203125,
            0.8203125,
            0.828125
          ],
          "mean": 0.8229166666666666,
          "std": 0.003682847818679935
        }
      }
    },
    {
      "name": "baseline-mi-equal-steps-gb36",
      "is_baseline": true,
      "what": "MI-FGSM (decay=1.0), T=grad_budget/6 -- same iteration count as PGN but only grad_budget/6 gradient calls (the unfair, literature-standard comparison). grad_budget=36.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            1.0,
            1.0,
            1.0
          ],
          "mean": 1.0,
          "std": 0.0
        },
        "transfer_success_rate": {
          "per_seed": [
            0.64453125,
            0.65625,
            0.671875
          ],
          "mean": 0.6575520833333334,
          "std": 0.01120094435812842
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            57.883506774902344,
            57.86724853515625,
            59.88411331176758
          ],
          "mean": 58.54495620727539,
          "std": 0.946950331493627
        },
        "fwd_bwd_calls": {
          "per_seed": [
            6,
            6,
            6
          ],
          "mean": 6.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.25,
            0.3046875,
            0.2890625
          ],
          "mean": 0.28125,
          "std": 0.0229993772560621
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.8515625,
            0.8203125,
            0.828125
          ],
          "mean": 0.8333333333333334,
          "std": 0.01327869664998121
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.7109375,
            0.6953125,
            0.7421875
          ],
          "mean": 0.7161458333333334,
          "std": 0.01948779888944761
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.765625,
            0.8046875,
            0.828125
          ],
          "mean": 0.7994791666666666,
          "std": 0.025779934730759544
        }
      }
    },
    {
      "name": "baseline-mi-equal-steps-gb72",
      "is_baseline": true,
      "what": "MI-FGSM (decay=1.0), T=grad_budget/6 -- same iteration count as PGN but only grad_budget/6 gradient calls (the unfair, literature-standard comparison). grad_budget=72.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            1.0,
            1.0,
            1.0
          ],
          "mean": 1.0,
          "std": 0.0
        },
        "transfer_success_rate": {
          "per_seed": [
            0.6640625,
            0.66015625,
            0.662109375
          ],
          "mean": 0.662109375,
          "std": 0.001594719884624465
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            55.61458206176758,
            57.224510192871094,
            58.798065185546875
          ],
          "mean": 57.21238581339518,
          "std": 1.2996798196171635
        },
        "fwd_bwd_calls": {
          "per_seed": [
            12,
            12,
            12
          ],
          "mean": 12.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.234375,
            0.2890625,
            0.265625
          ],
          "mean": 0.2630208333333333,
          "std": 0.02240188871625684
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.8828125,
            0.84375,
            0.828125
          ],
          "mean": 0.8515625,
          "std": 0.0229993772560621
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.7421875,
            0.7109375,
            0.7109375
          ],
          "mean": 0.7213541666666666,
          "std": 0.01473139127471974
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.796875,
            0.796875,
            0.84375
          ],
          "mean": 0.8125,
          "std": 0.02209708691207961
        }
      }
    },
    {
      "name": "pgn-flat-maxima-gb36",
      "is_baseline": false,
      "what": "PGN-style flat-maxima attack: per iteration, 3 neighborhood samples each contribute 2 gradient calls (sample point + sign-step prediction point), weighted 0.5/0.5 and averaged, momentum decay=1.0. 6 grad calls/iteration, T=grad_budget/6 iterations -> grad_budget total calls. grad_budget=36.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            1.0,
            0.96875,
            0.9765625
          ],
          "mean": 0.9817708333333334,
          "std": 0.01327869664998121
        },
        "transfer_success_rate": {
          "per_seed": [
            0.708984375,
            0.607421875,
            0.650390625
          ],
          "mean": 0.6555989583333334,
          "std": 0.041625956674858806
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            76.63130187988281,
            77.9410171508789,
            78.58625793457031
          ],
          "mean": 77.71952565511067,
          "std": 0.8133294354946482
        },
        "fwd_bwd_calls": {
          "per_seed": [
            36,
            36,
            36
          ],
          "mean": 36.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.3046875,
            0.234375,
            0.265625
          ],
          "mean": 0.2682291666666667,
          "std": 0.028763960982258495
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.8671875,
            0.78125,
            0.8203125
          ],
          "mean": 0.8229166666666666,
          "std": 0.03513212907091677
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.7734375,
            0.65625,
            0.71875
          ],
          "mean": 0.7161458333333334,
          "std": 0.047877021642839156
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.890625,
            0.7578125,
            0.796875
          ],
          "mean": 0.8151041666666666,
          "std": 0.05573160041414765
        }
      }
    },
    {
      "name": "pgn-flat-maxima-gb72",
      "is_baseline": false,
      "what": "PGN-style flat-maxima attack: per iteration, 3 neighborhood samples each contribute 2 gradient calls (sample point + sign-step prediction point), weighted 0.5/0.5 and averaged, momentum decay=1.0. 6 grad calls/iteration, T=grad_budget/6 iterations -> grad_budget total calls. grad_budget=72.",
      "metrics": {
        "whitebox_success_rate": {
          "per_seed": [
            0.9921875,
            0.9921875,
            0.9921875
          ],
          "mean": 0.9921875,
          "std": 0.0
        },
        "transfer_success_rate": {
          "per_seed": [
            0.763671875,
            0.703125,
            0.72265625
          ],
          "mean": 0.7298177083333334,
          "std": 0.02523153955088034
        },
        "neighborhood_grad_norm": {
          "per_seed": [
            69.86393737792969,
            71.73055267333984,
            71.49815368652344
          ],
          "mean": 71.03088124593098,
          "std": 0.8305904789436261
        },
        "fwd_bwd_calls": {
          "per_seed": [
            72,
            72,
            72
          ],
          "mean": 72.0,
          "std": 0.0
        },
        "transfer_success_rate__cifar10_vgg11_bn": {
          "per_seed": [
            0.375,
            0.296875,
            0.3046875
          ],
          "mean": 0.3255208333333333,
          "std": 0.03513212907091678
        },
        "transfer_success_rate__cifar10_mobilenetv2_x1_0": {
          "per_seed": [
            0.9375,
            0.90625,
            0.875
          ],
          "mean": 0.90625,
          "std": 0.02551551815399144
        },
        "transfer_success_rate__cifar10_shufflenetv2_x1_0": {
          "per_seed": [
            0.84375,
            0.734375,
            0.8125
          ],
          "mean": 0.796875,
          "std": 0.0459987545121242
        },
        "transfer_success_rate__cifar10_repvgg_a0": {
          "per_seed": [
            0.8984375,
            0.875,
            0.8984375
          ],
          "mean": 0.890625,
          "std": 0.011048543456039806
        }
      }
    }
  ],
  "headline": {
    "metric": "transfer_success_rate",
    "baseline_mean": 0.6549479166666666,
    "proposed_mean": 0.7298177083333334,
    "delta": 0.07486979166666674,
    "claim": "At grad_budget=72, MI-FGSM=65.5%, PGN-flat-maxima=73.0% transfer success; Part A premise failed: equal-steps advantage mean=3.29pp < 5pp, so the literature-bias premise the prediction is built on did not even replicate here."
  },
  "deviations": [],
  "runtime_sec": 234.0882225036621,
  "diagnostics": {
    "eps": 0.03137254901960784,
    "grad_budgets_swept": [
      36,
      72
    ],
    "mean_delta_pp_equal_steps": 3.2877604166666665,
    "mean_delta_pp_equal_budget": 3.4505208333333335,
    "per_run_delta_pp_equal_budget": [
      5.2734375,
      -5.6640625,
      -1.3671875,
      10.3515625,
      4.6875,
      7.421875
    ],
    "all_positive_equal_budget": false,
    "outcome_reason": "Part A premise failed: equal-steps advantage mean=3.29pp < 5pp, so the literature-bias premise the prediction is built on did not even replicate here.",
    "pgn_hyperparams": {
      "N": 3,
      "zeta": 3.0,
      "balance": 0.5,
      "chi": 0.03137254901960784,
      "decay": 1.0
    }
  }
}