{
  "kind": "comparison",
  "compare_name": "compare_gw6_modsweep_bge_reptok_canonical_attention_coco_k",
  "source_metrics": "gw_setup/outputs/gw6_modsweep57_bge_reptok_canonical_attention/coco_eval200/metrics.csv",
  "eval_samples": 200,
  "rows": [
    {
      "index": 0,
      "k": 2,
      "modality_code": "rt_be",
      "direct_lpips": 0.6946425431966782,
      "attention_lpips": 0.6891761744022369,
      "delta_lpips_vs_direct": 0.00546636879444129,
      "direct_clip": 0.2923094003647566,
      "attention_clip": 0.2960548632591963,
      "delta_clip_vs_direct": 0.003745462894439666,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k2_rt_be_s03"
    },
    {
      "index": 1,
      "k": 3,
      "modality_code": "bg_rt_be",
      "direct_lpips": 0.6951199761033058,
      "attention_lpips": 0.6900810170173645,
      "delta_lpips_vs_direct": 0.005038959085941319,
      "direct_clip": 0.26810608357191085,
      "attention_clip": 0.26862156368792056,
      "delta_clip_vs_direct": 0.0005154801160097144,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k3_bg_rt_be_s03"
    },
    {
      "index": 2,
      "k": 3,
      "modality_code": "di_rt_be",
      "direct_lpips": 0.6964975968003273,
      "attention_lpips": 0.7237538564205169,
      "delta_lpips_vs_direct": -0.027256259620189627,
      "direct_clip": 0.2926125704497099,
      "attention_clip": 0.19778637070208788,
      "delta_clip_vs_direct": -0.09482619974762202,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k3_di_rt_be_s03"
    },
    {
      "index": 3,
      "k": 3,
      "modality_code": "rt_be_so",
      "direct_lpips": 0.6965517631173134,
      "attention_lpips": 0.6936281800270081,
      "delta_lpips_vs_direct": 0.0029235830903052706,
      "direct_clip": 0.28488163121044635,
      "attention_clip": 0.281304986551404,
      "delta_clip_vs_direct": -0.0035766446590423384,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k3_rt_be_so_s03"
    },
    {
      "index": 4,
      "k": 3,
      "modality_code": "rt_be_sy",
      "direct_lpips": 0.7015174832940102,
      "attention_lpips": 0.7007648327946663,
      "delta_lpips_vs_direct": 0.0007526504993439209,
      "direct_clip": 0.2797954845428467,
      "attention_clip": 0.27969396866858004,
      "delta_clip_vs_direct": -0.00010151587426665554,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k3_rt_be_sy_s03"
    },
    {
      "index": 5,
      "k": 4,
      "modality_code": "bg_di_rt_be",
      "direct_lpips": 0.698287358880043,
      "attention_lpips": 0.6950971798598766,
      "delta_lpips_vs_direct": 0.0031901790201663838,
      "direct_clip": 0.2724348816275597,
      "attention_clip": 0.2746531254798174,
      "delta_clip_vs_direct": 0.0022182438522577175,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_bg_di_rt_be_s03"
    },
    {
      "index": 6,
      "k": 4,
      "modality_code": "bg_rt_be_so",
      "direct_lpips": 0.702110243588686,
      "attention_lpips": 0.7011216703057289,
      "delta_lpips_vs_direct": 0.0009885732829570948,
      "direct_clip": 0.2722639490664005,
      "attention_clip": 0.2691884867846966,
      "delta_clip_vs_direct": -0.0030754622817039112,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_bg_rt_be_so_s03"
    },
    {
      "index": 7,
      "k": 4,
      "modality_code": "bg_rt_be_sy",
      "direct_lpips": 0.6983869457244873,
      "attention_lpips": 0.6969764158129692,
      "delta_lpips_vs_direct": 0.0014105299115181325,
      "direct_clip": 0.27104941844940184,
      "attention_clip": 0.2707375483214855,
      "delta_clip_vs_direct": -0.00031187012791633606,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_bg_rt_be_sy_s03"
    },
    {
      "index": 8,
      "k": 4,
      "modality_code": "di_rt_be_so",
      "direct_lpips": 0.7059093928337097,
      "attention_lpips": 0.6969085597991943,
      "delta_lpips_vs_direct": 0.009000833034515376,
      "direct_clip": 0.2837478034943342,
      "attention_clip": 0.2823435402661562,
      "delta_clip_vs_direct": -0.0014042632281779932,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_di_rt_be_so_s03"
    },
    {
      "index": 9,
      "k": 4,
      "modality_code": "di_rt_be_sy",
      "direct_lpips": 0.6978310823440552,
      "attention_lpips": 0.7059476271271705,
      "delta_lpips_vs_direct": -0.008116544783115298,
      "direct_clip": 0.2834233907610178,
      "attention_clip": 0.27975772745907307,
      "delta_clip_vs_direct": -0.0036656633019447327,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_di_rt_be_sy_s03"
    },
    {
      "index": 10,
      "k": 4,
      "modality_code": "rt_be_so_sy",
      "direct_lpips": 0.7015163719654083,
      "attention_lpips": 0.7026036587357521,
      "delta_lpips_vs_direct": -0.001087286770343865,
      "direct_clip": 0.282396869584918,
      "attention_clip": 0.2804735929518938,
      "delta_clip_vs_direct": -0.001923276633024218,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k4_rt_be_so_sy_s03"
    },
    {
      "index": 11,
      "k": 5,
      "modality_code": "bg_di_rt_be_so",
      "direct_lpips": 0.6987403684854507,
      "attention_lpips": 0.6916531336307525,
      "delta_lpips_vs_direct": 0.007087234854698199,
      "direct_clip": 0.2732295624911785,
      "attention_clip": 0.2712670163065195,
      "delta_clip_vs_direct": -0.0019625461846590198,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k5_bg_di_rt_be_so_s03"
    },
    {
      "index": 12,
      "k": 5,
      "modality_code": "bg_di_rt_be_sy",
      "direct_lpips": 0.7005267006158828,
      "attention_lpips": 0.6948844280838966,
      "delta_lpips_vs_direct": 0.005642272531986237,
      "direct_clip": 0.26684046499431135,
      "attention_clip": 0.2693017810583115,
      "delta_clip_vs_direct": 0.0024613160640001297,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k5_bg_di_rt_be_sy_s03"
    },
    {
      "index": 13,
      "k": 5,
      "modality_code": "bg_rt_be_so_sy",
      "direct_lpips": 0.6993399420380593,
      "attention_lpips": 0.6960565707087517,
      "delta_lpips_vs_direct": 0.0032833713293075606,
      "direct_clip": 0.27124196909368037,
      "attention_clip": 0.2720762015879154,
      "delta_clip_vs_direct": 0.0008342324942350321,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k5_bg_rt_be_so_sy_s04"
    },
    {
      "index": 14,
      "k": 5,
      "modality_code": "di_rt_be_so_sy",
      "direct_lpips": 0.6991266834735871,
      "attention_lpips": 0.6919016675651073,
      "delta_lpips_vs_direct": 0.007225015908479793,
      "direct_clip": 0.2834124954044819,
      "attention_clip": 0.2863132568448782,
      "delta_clip_vs_direct": 0.0029007614403963045,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k5_di_rt_be_so_sy_s03"
    },
    {
      "index": 15,
      "k": 6,
      "modality_code": "bg_di_rt_be_so_sy",
      "direct_lpips": 0.7013793513178825,
      "attention_lpips": 0.6988714531064033,
      "delta_lpips_vs_direct": 0.0025078982114792314,
      "direct_clip": 0.2668261819332838,
      "attention_clip": 0.2657234778255224,
      "delta_clip_vs_direct": -0.0011027041077613653,
      "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw256_k6_bg_di_rt_be_so_sy_s04"
    }
  ],
  "by_k": [
    {
      "k": 2,
      "n": 1,
      "direct_lpips_mean": 0.6946425431966782,
      "direct_lpips_sem": 0.0,
      "attention_lpips_mean": 0.6891761744022369,
      "attention_lpips_sem": 0.0,
      "delta_lpips_vs_direct_mean": 0.00546636879444129,
      "delta_lpips_vs_direct_sem": 0.0,
      "direct_clip_mean": 0.2923094003647566,
      "direct_clip_sem": 0.0,
      "attention_clip_mean": 0.2960548632591963,
      "attention_clip_sem": 0.0,
      "delta_clip_vs_direct_mean": 0.003745462894439666,
      "delta_clip_vs_direct_sem": 0.0
    },
    {
      "k": 3,
      "n": 4,
      "direct_lpips_mean": 0.6974217048287392,
      "direct_lpips_sem": 0.001404876346620254,
      "attention_lpips_mean": 0.702056971564889,
      "attention_lpips_sem": 0.007565780688695844,
      "delta_lpips_vs_direct_mean": -0.004635266736149779,
      "delta_lpips_vs_direct_sem": 0.007590925652558374,
      "direct_clip_mean": 0.28134894244372843,
      "direct_clip_sem": 0.005140815015866604,
      "attention_clip_mean": 0.2568517224024981,
      "attention_clip_sem": 0.019889225050668535,
      "delta_clip_vs_direct_mean": -0.024497220041230325,
      "delta_clip_vs_direct_sem": 0.023460288098227235
    },
    {
      "k": 4,
      "n": 6,
      "direct_lpips_mean": 0.7006735658893982,
      "direct_lpips_sem": 0.0012805022069457353,
      "attention_lpips_mean": 0.6997758519401153,
      "attention_lpips_sem": 0.0016916136930160922,
      "delta_lpips_vs_direct_mean": 0.0008977139492829708,
      "delta_lpips_vs_direct_sem": 0.002283157601450636,
      "direct_clip_mean": 0.27755271883060534,
      "direct_clip_sem": 0.0025348630373231633,
      "attention_clip_mean": 0.2761923368771871,
      "attention_clip_sem": 0.0022364873704268516,
      "delta_clip_vs_direct_mean": -0.0013603819534182455,
      "delta_clip_vs_direct_sem": 0.0008656507825562436
    },
    {
      "k": 5,
      "n": 4,
      "direct_lpips_mean": 0.699433423653245,
      "direct_lpips_sem": 0.0003849684329954075,
      "attention_lpips_mean": 0.6936239499971271,
      "attention_lpips_sem": 0.0010938015686686902,
      "delta_lpips_vs_direct_mean": 0.005809473656117947,
      "delta_lpips_vs_direct_sem": 0.0009149491210921078,
      "direct_clip_mean": 0.273681122995913,
      "direct_clip_sem": 0.0035077009267389684,
      "attention_clip_mean": 0.27473956394940613,
      "attention_clip_sem": 0.003901622912435551,
      "delta_clip_vs_direct_mean": 0.0010584409534931116,
      "delta_clip_vs_direct_sem": 0.0011007132564170524
    },
    {
      "k": 6,
      "n": 1,
      "direct_lpips_mean": 0.7013793513178825,
      "direct_lpips_sem": 0.0,
      "attention_lpips_mean": 0.6988714531064033,
      "attention_lpips_sem": 0.0,
      "delta_lpips_vs_direct_mean": 0.0025078982114792314,
      "delta_lpips_vs_direct_sem": 0.0,
      "direct_clip_mean": 0.2668261819332838,
      "direct_clip_sem": 0.0,
      "attention_clip_mean": 0.2657234778255224,
      "attention_clip_sem": 0.0,
      "delta_clip_vs_direct_mean": -0.0011027041077613653,
      "delta_clip_vs_direct_sem": 0.0
    }
  ],
  "artifacts": {
    "direct_by_k": "assets/direct_by_k.jpg",
    "grounded_by_k": "assets/grounded_by_k.jpg",
    "delta_by_k": "assets/delta_by_k.jpg",
    "delta_by_model": "assets/delta_by_model.jpg"
  }
}