{
  "kind": "comparison",
  "compare_name": "compare_bge_reptok_codebook_grounders_coco1000",
  "dataset": "COCO val2017",
  "caption_source": "COCO captions_val2017 first caption per image",
  "sample_seed": 20260630,
  "sample_count": 3000,
  "eval_offset": 2000,
  "eval_samples": 1000,
  "metrics_definition": {
    "lpips": "decoded/generated image vs paired COCO val image; lower is better",
    "clip_similarity": "ViT-B-32/openai image-caption cosine; higher is better",
    "fid": "generated images vs paired COCO val reference images; lower is better"
  },
  "metrics": [
    {
      "model": "gw_direct_bge",
      "n_samples": 1000,
      "lpips_mean": 0.7014688248932361,
      "lpips_std": 0.05542194062016755,
      "clip_similarity_mean": 0.28640163788199424,
      "clip_similarity_std": 0.03300513231734236,
      "fid": 77.60134887695312
    },
    {
      "model": "reptok_prior_796m",
      "n_samples": 1000,
      "lpips_mean": 0.7214722891151905,
      "lpips_std": 0.07157120731752703,
      "clip_similarity_mean": 0.3033647171854973,
      "clip_similarity_std": 0.036609110396594494,
      "fid": 65.88847351074219
    },
    {
      "model": "gw_short_t5_scalar",
      "n_samples": 1000,
      "lpips_mean": 0.6958555586636066,
      "lpips_std": 0.058191509654314716,
      "clip_similarity_mean": 0.2887599302381277,
      "clip_similarity_std": 0.03265254305688373,
      "fid": 67.09028625488281
    },
    {
      "model": "gw_codebook_clip",
      "n_samples": 1000,
      "lpips_mean": 0.7677462876439095,
      "lpips_std": 0.05791207369816877,
      "clip_similarity_mean": 0.16068525305390358,
      "clip_similarity_std": 0.03394100005924846,
      "fid": 294.34356689453125
    },
    {
      "model": "gw_codebook_lpips",
      "n_samples": 1000,
      "lpips_mean": 0.9091186586022377,
      "lpips_std": 0.05534217065439817,
      "clip_similarity_mean": 0.17375345944613219,
      "clip_similarity_std": 0.02829726001723797,
      "fid": 269.65020751953125
    },
    {
      "model": "gw_codebook_mse",
      "n_samples": 1000,
      "lpips_mean": 0.6979030154645443,
      "lpips_std": 0.05638174159819701,
      "clip_similarity_mean": 0.28725388584285977,
      "clip_similarity_std": 0.03138279828700053,
      "fid": 68.13221740722656
    },
    {
      "model": "gw_codebook_mse_percode",
      "n_samples": 1000,
      "lpips_mean": 0.707886289358139,
      "lpips_std": 0.05693148521772288,
      "clip_similarity_mean": 0.2798221201747656,
      "clip_similarity_std": 0.03209431940139787,
      "fid": 80.22473907470703
    }
  ],
  "generation": {
    "gw_checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/checkpoints/last.ckpt",
    "codebook_clip": {
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/bge_reptok_metric_codebook_clip_20r/attention_latest.pt",
      "generated": 0,
      "reused": 1000
    },
    "codebook_lpips": {
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/bge_reptok_metric_codebook_lpips_20r/attention_latest.pt",
      "generated": 0,
      "reused": 1000
    },
    "codebook_mse": {
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/bge_reptok_metric_codebook_mse_20r/attention_latest.pt",
      "generated": 0,
      "reused": 1000
    },
    "codebook_mse_percode": {
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/bge_reptok_metric_codebook_mse_percodekey_10e/attention_latest.pt",
      "generated": 1000,
      "reused": 0
    }
  },
  "existing_source_pages": {
    "prior_and_direct": "gw_setup/outputs/reptok_prior_vs_gw_direct_coco1000",
    "scalar_grounder": "gw_setup/outputs/gw_short_t5_grounded_coco1000"
  },
  "artifacts": {
    "metric_barplots": "assets/metric_barplots.jpg",
    "qualitative_grid": "assets/qualitative_grid.jpg",
    "metrics_csv": "metrics.csv",
    "per_sample_metrics_csv": "per_sample_metrics.csv",
    "manifest_public_jsonl": "manifest_public.jsonl"
  }
}