{
  "kind": "comparison",
  "compare_name": "compare_gw6_iterative_attention_downstream_blind_coco",
  "run_name": "gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24",
  "dataset": "COCO val2017",
  "eval_samples": 200,
  "eval_offset": 2000,
  "attention_models": 22,
  "skipped_models": 1,
  "metrics": {
    "lpips": "decoded RepTok image vs COCO-val image; lower is better",
    "clip_score": "ViT-B-32/openai image-caption cosine; higher is better"
  },
  "delta_sign": "positive means better than the reference",
  "direct": {
    "lpips": 0.7020319291949272,
    "clip_score": 0.2832008783519268
  },
  "single_step_attention": {
    "lpips": 0.6994143104553223,
    "clip_score": 0.2866644821316004
  },
  "selected_qualitative_models": [
    "A04",
    "A02",
    "A14",
    "A09"
  ],
  "anonymous_rankings": [
    {
      "anon_id": "A01",
      "lpips": 0.7163630950450898,
      "clip_score": 0.24648099459707737,
      "delta_lpips_vs_direct": -0.014331165850162586,
      "delta_clip_vs_direct": -0.03671988375484944,
      "delta_lpips_vs_single_step": -0.01694878458976745,
      "delta_clip_vs_single_step": -0.04018348753452303
    },
    {
      "anon_id": "A02",
      "lpips": 0.6976378881931304,
      "clip_score": 0.2906032217293978,
      "delta_lpips_vs_direct": 0.004394041001796745,
      "delta_clip_vs_direct": 0.007402343377470977,
      "delta_lpips_vs_single_step": 0.001776422262191879,
      "delta_clip_vs_single_step": 0.003938739597797392
    },
    {
      "anon_id": "A03",
      "lpips": 0.6969562846422196,
      "clip_score": 0.29011776857078075,
      "delta_lpips_vs_direct": 0.005075644552707592,
      "delta_clip_vs_direct": 0.006916890218853944,
      "delta_lpips_vs_single_step": 0.0024580258131027266,
      "delta_clip_vs_single_step": 0.0034532864391803586
    },
    {
      "anon_id": "A04",
      "lpips": 0.6958863013982772,
      "clip_score": 0.2891111347079277,
      "delta_lpips_vs_direct": 0.006145627796649955,
      "delta_clip_vs_direct": 0.005910256356000898,
      "delta_lpips_vs_single_step": 0.0035280090570450895,
      "delta_clip_vs_single_step": 0.002446652576327313
    },
    {
      "anon_id": "A05",
      "lpips": 0.6982038164138794,
      "clip_score": 0.2891671937704086,
      "delta_lpips_vs_direct": 0.0038281127810477766,
      "delta_clip_vs_direct": 0.005966315418481805,
      "delta_lpips_vs_single_step": 0.001210494041442911,
      "delta_clip_vs_single_step": 0.0025027116388082193
    },
    {
      "anon_id": "A06",
      "lpips": 0.6976402926445008,
      "clip_score": 0.2899773107469082,
      "delta_lpips_vs_direct": 0.004391636550426403,
      "delta_clip_vs_direct": 0.00677643239498138,
      "delta_lpips_vs_single_step": 0.0017740178108215376,
      "delta_clip_vs_single_step": 0.0033128286153077946
    },
    {
      "anon_id": "A07",
      "lpips": 0.7023392334580422,
      "clip_score": 0.2850860933959484,
      "delta_lpips_vs_direct": -0.0003073042631149825,
      "delta_clip_vs_direct": 0.0018852150440216153,
      "delta_lpips_vs_single_step": -0.002924923002719848,
      "delta_clip_vs_single_step": -0.00157838873565197
    },
    {
      "anon_id": "A08",
      "lpips": 0.7068918228149415,
      "clip_score": 0.2754242030531168,
      "delta_lpips_vs_direct": -0.004859893620014266,
      "delta_clip_vs_direct": -0.007776675298809987,
      "delta_lpips_vs_single_step": -0.007477512359619132,
      "delta_clip_vs_single_step": -0.011240279078483573
    },
    {
      "anon_id": "A09",
      "lpips": 0.6966526180505752,
      "clip_score": 0.2903703351318836,
      "delta_lpips_vs_direct": 0.0053793111443519726,
      "delta_clip_vs_direct": 0.007169456779956784,
      "delta_lpips_vs_single_step": 0.002761692404747107,
      "delta_clip_vs_single_step": 0.003705853000283199
    },
    {
      "anon_id": "A10",
      "lpips": 0.6969745773077011,
      "clip_score": 0.28952035509049895,
      "delta_lpips_vs_direct": 0.005057351887226114,
      "delta_clip_vs_direct": 0.006319476738572138,
      "delta_lpips_vs_single_step": 0.002439733147621248,
      "delta_clip_vs_single_step": 0.002855872958898553
    },
    {
      "anon_id": "A11",
      "lpips": 0.7172600582242012,
      "clip_score": 0.24555890008807182,
      "delta_lpips_vs_direct": -0.015228129029274062,
      "delta_clip_vs_direct": -0.037641978263854986,
      "delta_lpips_vs_single_step": -0.017845747768878928,
      "delta_clip_vs_single_step": -0.04110558204352857
    },
    {
      "anon_id": "A12",
      "lpips": 0.7291279324889183,
      "clip_score": 0.20739201024174692,
      "delta_lpips_vs_direct": -0.02709600329399109,
      "delta_clip_vs_direct": -0.0758088681101799,
      "delta_lpips_vs_single_step": -0.029713622033595954,
      "delta_clip_vs_single_step": -0.07927247188985348
    },
    {
      "anon_id": "A13",
      "lpips": 0.697997884452343,
      "clip_score": 0.28744016997516153,
      "delta_lpips_vs_direct": 0.004034044742584175,
      "delta_clip_vs_direct": 0.00423929162323472,
      "delta_lpips_vs_single_step": 0.0014164260029793097,
      "delta_clip_vs_single_step": 0.0007756878435611347
    },
    {
      "anon_id": "A14",
      "lpips": 0.6963877332210541,
      "clip_score": 0.2903259391337633,
      "delta_lpips_vs_direct": 0.005644195973873112,
      "delta_clip_vs_direct": 0.007125060781836501,
      "delta_lpips_vs_single_step": 0.003026577234268246,
      "delta_clip_vs_single_step": 0.0036614570021629156
    },
    {
      "anon_id": "A15",
      "lpips": 0.7085346308350563,
      "clip_score": 0.26907712027430536,
      "delta_lpips_vs_direct": -0.006502701640129116,
      "delta_clip_vs_direct": -0.014123758077621451,
      "delta_lpips_vs_single_step": -0.009120320379733982,
      "delta_clip_vs_single_step": -0.017587361857295036
    },
    {
      "anon_id": "A16",
      "lpips": 0.7053588297963143,
      "clip_score": 0.2726739403605461,
      "delta_lpips_vs_direct": -0.0033269006013870728,
      "delta_clip_vs_direct": -0.010526937991380714,
      "delta_lpips_vs_single_step": -0.005944519340991938,
      "delta_clip_vs_single_step": -0.013990541771054299
    },
    {
      "anon_id": "A17",
      "lpips": 0.699482134282589,
      "clip_score": 0.28659861370921136,
      "delta_lpips_vs_direct": 0.002549794912338199,
      "delta_clip_vs_direct": 0.003397735357284548,
      "delta_lpips_vs_single_step": -6.782382726666647e-05,
      "delta_clip_vs_single_step": -6.586842238903712e-05
    },
    {
      "anon_id": "A18",
      "lpips": 0.7089871662855148,
      "clip_score": 0.2704461593925953,
      "delta_lpips_vs_direct": -0.0069552370905876515,
      "delta_clip_vs_direct": -0.012754718959331501,
      "delta_lpips_vs_single_step": -0.009572855830192517,
      "delta_clip_vs_single_step": -0.016218322739005087
    },
    {
      "anon_id": "A19",
      "lpips": 0.6975564736127854,
      "clip_score": 0.2856007231026888,
      "delta_lpips_vs_direct": 0.004475455582141796,
      "delta_clip_vs_direct": 0.0023998447507619924,
      "delta_lpips_vs_single_step": 0.0018578368425369307,
      "delta_clip_vs_single_step": -0.0010637590289115928
    },
    {
      "anon_id": "A20",
      "lpips": 0.7211284145712853,
      "clip_score": 0.22670649070292712,
      "delta_lpips_vs_direct": -0.019096485376358108,
      "delta_clip_vs_direct": -0.056494387648999694,
      "delta_lpips_vs_single_step": -0.021714104115962973,
      "delta_clip_vs_single_step": -0.05995799142867328
    },
    {
      "anon_id": "A21",
      "lpips": 0.6986803662776947,
      "clip_score": 0.28673091121017935,
      "delta_lpips_vs_direct": 0.0033515629172324868,
      "delta_clip_vs_direct": 0.003530032858252541,
      "delta_lpips_vs_single_step": 0.0007339441776276212,
      "delta_clip_vs_single_step": 6.642907857895564e-05
    },
    {
      "anon_id": "A22",
      "lpips": 0.7973870122432709,
      "clip_score": 0.1768489959463477,
      "delta_lpips_vs_direct": -0.09535508304834373,
      "delta_clip_vs_direct": -0.1063518824055791,
      "delta_lpips_vs_single_step": -0.09797270178794859,
      "delta_clip_vs_single_step": -0.10981548618525269
    }
  ],
  "artifacts": {
    "delta_bars": "assets/downstream_delta_bars.jpg",
    "ranked_bars": "assets/downstream_delta_ranked.jpg",
    "qualitative_grid": "assets/coco_qualitative_grid.jpg",
    "csv": "anonymous_downstream_metrics.csv"
  }
}