{
  "kind": "comparison",
  "compare_name": "compare_reptok_prior_vs_gw_direct_bge_coco1000",
  "dataset": "COCO val2017",
  "caption_source": "COCO captions_val2017 first caption per image",
  "sample_seed": 20260630,
  "sample_count": 3000,
  "eval_offset": 2000,
  "eval_samples": 1000,
  "gw_label": "GW-96hr ImageNet-only direct BGE->RepTok path",
  "reptok_prior_label": "ImageNet caption->RepTok-token prior trained by us",
  "gw_direct_policy": "BGE caption embedding to RepTok decoder output only; no grounding, attention, or fusion",
  "metrics_definition": {
    "lpips": "decoded/generated image vs paired COCO val image; lower is better",
    "clip_similarity": "ViT-B-32/openai image-caption cosine; higher is better",
    "fid": "generated 1000 images vs paired 1000 COCO val reference images; lower is better"
  },
  "generation": {
    "gw_direct_bge": {
      "gw_checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/checkpoints/last.ckpt",
      "generated": 0,
      "reused": 1000
    },
    "reptok_prior": {
      "prior_checkpoint": "gw_setup/outputs/reptok_t2i_prior_imagenet/runs/mixer_flow_paper_v1/checkpoints/last.pt",
      "generated": 0,
      "reused": 1000
    }
  },
  "metrics": [
    {
      "model": "gw_direct_bge",
      "n_samples": 1000,
      "lpips_mean": 0.7014688248932361,
      "lpips_std": 0.05542194062016755,
      "clip_similarity_mean": 0.28640163788199424,
      "clip_similarity_std": 0.03300513231734236,
      "fid": 77.60134887695312
    },
    {
      "model": "reptok_prior",
      "n_samples": 1000,
      "lpips_mean": 0.7214722891151905,
      "lpips_std": 0.07157120731752703,
      "clip_similarity_mean": 0.3033647171854973,
      "clip_similarity_std": 0.036609110396594494,
      "fid": 65.88847351074219
    }
  ],
  "artifacts": {
    "side_by_side_grid": "assets/side_by_side_grid.jpg",
    "metrics_csv": "metrics.csv",
    "per_sample_metrics_csv": "per_sample_metrics.csv",
    "manifest_public_jsonl": "manifest_public.jsonl",
    "metric_barplots_with_12h": "assets/metric_barplots_with_12h.svg",
    "parameter_metric_bubbles_with_12h": "assets/parameter_metric_bubbles_with_12h.svg",
    "side_by_side_grid_12h_activated": "assets/side_by_side_grid_12h_activated.jpg",
    "metrics_with_12h_csv": "metrics_with_12h.csv",
    "metrics_12h_activated_grounded_csv": "metrics_12h_activated_grounded.csv",
    "per_sample_metrics_12h_activated_grounded_csv": "per_sample_metrics_12h_activated_grounded.csv"
  },
  "grounded_short_t5_summary": {
    "kind": "comparison",
    "compare_name": "compare_gw6_short_t5_grounded_bge_coco1000",
    "dataset": "COCO val2017",
    "caption_source": "COCO captions_val2017 first caption per image",
    "sample_seed": 20260630,
    "sample_count": 3000,
    "eval_offset": 2000,
    "eval_samples": 1000,
    "gw_run_name": "gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24",
    "grounded_label": "GW short_T5_keep grounded BGE",
    "grounding_config": "short_T5_keep",
    "metrics_definition": {
      "lpips": "generated image vs paired COCO val image; lower is better",
      "clip_similarity": "ViT-B-32/openai image-caption cosine; higher is better",
      "fid": "generated images vs paired COCO val reference images; lower is better"
    },
    "metrics": [
      {
        "model": "gw_direct_bge",
        "n_samples": 1000,
        "lpips_mean": 0.7014688248932361,
        "lpips_std": 0.05542194062016755,
        "clip_similarity_mean": 0.28640163788199424,
        "clip_similarity_std": 0.03300513231734236,
        "fid": 77.60134887695312
      },
      {
        "model": "gw_short_t5_grounded",
        "n_samples": 1000,
        "lpips_mean": 0.6958555586636066,
        "lpips_std": 0.058191509654314716,
        "clip_similarity_mean": 0.2887599302381277,
        "clip_similarity_std": 0.03265254305688373,
        "fid": 67.09028625488281
      }
    ],
    "generation": {
      "gw_checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/checkpoints/last.ckpt",
      "attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/text_grounded_reptok_iterative_attention_tstate_mse_20260722_sigmoid_keep/short_T5_keep/attention.pt",
      "gw_direct_bge": {
        "generated": 1000,
        "reused": 0
      },
      "gw_short_t5_grounded": {
        "attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/text_grounded_reptok_iterative_attention_tstate_mse_20260722_sigmoid_keep/short_T5_keep/attention.pt",
        "generated": 1000,
        "reused": 0
      }
    },
    "artifacts": {
      "side_by_side_grid": "assets/side_by_side_grid.jpg",
      "metrics_csv": "metrics.csv",
      "per_sample_metrics_csv": "per_sample_metrics.csv",
      "manifest_public_jsonl": "manifest_public.jsonl"
    }
  },
  "bge_memory_prior_summary": {
    "kind": "comparison",
    "compare_name": "compare_reptok_bge_memory_prior_vs_gw_direct_bge_coco1000",
    "dataset": "COCO val2017",
    "caption_source": "COCO captions_val2017 first caption per image",
    "sample_seed": 20260630,
    "sample_count": 3000,
    "eval_offset": 2000,
    "eval_samples": 1000,
    "gw_label": "GW-96hr ImageNet-only direct BGE->RepTok path",
    "reptok_prior_label": "ImageNet caption->RepTok-token prior trained by us",
    "gw_direct_policy": "BGE caption embedding to RepTok decoder output only; no grounding, attention, or fusion",
    "metrics_definition": {
      "lpips": "decoded/generated image vs paired COCO val image; lower is better",
      "clip_similarity": "ViT-B-32/openai image-caption cosine; higher is better",
      "fid": "generated 1000 images vs paired 1000 COCO val reference images; lower is better"
    },
    "generation": {},
    "metrics": [
      {
        "model": "gw_direct_bge",
        "n_samples": 1000,
        "lpips_mean": 0.7014688248932361,
        "lpips_std": 0.05542194062016755,
        "clip_similarity_mean": 0.28640163788199424,
        "clip_similarity_std": 0.03300513231734236,
        "fid": 77.60134887695312
      },
      {
        "model": "reptok_prior",
        "n_samples": 1000,
        "lpips_mean": 0.7245100690126419,
        "lpips_std": 0.060922619101622055,
        "clip_similarity_mean": 0.29028694324195387,
        "clip_similarity_std": 0.036915632404643926,
        "fid": 70.26390838623047
      }
    ],
    "artifacts": {
      "side_by_side_grid": "assets/side_by_side_grid.jpg",
      "metrics_csv": "metrics.csv",
      "per_sample_metrics_csv": "per_sample_metrics.csv",
      "manifest_public_jsonl": "manifest_public.jsonl"
    }
  },
  "parameter_summary": {
    "gw_direct_bge": {
      "active_generator_params": 37750860,
      "checkpoint_tensor_params_including_loss_copy": 75501721,
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/checkpoints/last.ckpt"
    },
    "reptok_prior": {
      "prior_params": 796387072,
      "subgroups": {
        "image/time/text projections": 14754560,
        "latent transformer blocks": 780595200,
        "learned helper/null/position tokens": 50944,
        "output head": 986368,
        "total": 796387072
      },
      "checkpoint": "gw_setup/outputs/reptok_t2i_prior_imagenet/runs/mixer_flow_paper_v1/checkpoints/last.pt"
    },
    "notes": [
      "GW parameter count reports the active gw_mod generator copy.",
      "RepTok prior count excludes the frozen RepTok decoder and frozen text encoder."
    ],
    "reptok_prior_37m": {
      "prior_params": 36780672,
      "subgroups": {
        "image/time/text projections": 2182208,
        "latent transformer blocks": 34209280,
        "learned helper/null/position tokens": 43456,
        "output head": 345728,
        "total": 36780672
      },
      "checkpoint": "gw_setup/outputs/reptok_t2i_prior_imagenet/runs/latent_transformer_clip_token_37m_v1/checkpoints/last.pt"
    },
    "gw_short_t5_grounded": {
      "active_gw_generator_params": 37750860,
      "short_t5_attention_params": 526336,
      "combined_active_params": 38277196,
      "checkpoint": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/text_grounded_reptok_iterative_attention_tstate_mse_20260722_sigmoid_keep/short_T5_keep/attention.pt"
    },
    "reptok_prior_bge_memory": {
      "prior_params": 850131712,
      "subgroups": {
        "image/time/text projections": 14097920,
        "latent transformer blocks": 780595200,
        "learned helper/null/position tokens": 12288,
        "output head": 986368,
        "total": 850131712
      },
      "checkpoint": "gw_setup/outputs/reptok_t2i_prior_imagenet/runs/latent_transformer_bge_memory_v1/checkpoints/last.pt"
    }
  },
  "model_size_metrics": [
    {
      "model": "gw_direct_bge",
      "label": "GW direct BGE",
      "params": 37750860,
      "params_m": 37.75086,
      "params_label": "37.8M",
      "lpips": 0.7014688248932361,
      "clip_similarity": 0.28640163788199424,
      "fid": 77.60134887695312
    },
    {
      "model": "gw_short_t5_grounded",
      "label": "GW short_T5 grounded",
      "params": 38277196,
      "params_m": 38.277196,
      "params_label": "38.3M",
      "lpips": 0.6958555586636066,
      "clip_similarity": 0.2887599302381277,
      "fid": 67.09028625488281
    },
    {
      "model": "reptok_prior",
      "label": "RepTok prior 796M",
      "params": 796387072,
      "params_m": 796.387072,
      "params_label": "796.4M",
      "lpips": 0.7214722891151905,
      "clip_similarity": 0.3033647171854973,
      "fid": 65.88847351074219
    },
    {
      "model": "reptok_prior_37m",
      "label": "RepTok prior 37M",
      "params": 36780672,
      "params_m": 36.780672,
      "params_label": "36.8M",
      "lpips": 0.7331561633348465,
      "clip_similarity": 0.27740161304920913,
      "fid": 81.88143920898438
    },
    {
      "model": "reptok_prior_bge_memory",
      "label": "RepTok prior BGE memory",
      "params": 850131712,
      "params_m": 850.131712,
      "params_label": "850.1M",
      "lpips": 0.7245100690126419,
      "clip_similarity": 0.29028694324195387,
      "fid": 70.26390838623047
    }
  ],
  "gw12h_activated_grounded_summary": {
    "kind": "comparison",
    "compare_name": "compare_gw_short_t5_grounded_bge_coco1000_12h_activated_start_lr3e4_full_e100",
    "dataset": "COCO val2017",
    "caption_source": "COCO captions_val2017 first caption per image",
    "sample_seed": 20260630,
    "sample_count": 3000,
    "eval_offset": 2000,
    "eval_samples": 1000,
    "gw_run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03",
    "grounded_label": "GW short_T5_keep grounded BGE",
    "grounding_config": "short_T5_keep",
    "metrics_definition": {
      "lpips": "generated image vs paired COCO val image; lower is better",
      "clip_similarity": "ViT-B-32/openai image-caption cosine; higher is better",
      "fid": "generated images vs paired COCO val reference images; lower is better"
    },
    "metrics": [
      {
        "model": "gw_direct_bge",
        "n_samples": 1000,
        "lpips_mean": 0.6971765531003475,
        "lpips_std": 0.055730533628727055,
        "clip_similarity_mean": 0.28643184058368204,
        "clip_similarity_std": 0.03384197934216952,
        "fid": 75.60994720458984
      },
      {
        "model": "gw_short_t5_grounded",
        "n_samples": 1000,
        "lpips_mean": 0.6953469677269459,
        "lpips_std": 0.05903350333754925,
        "clip_similarity_mean": 0.2916463987007737,
        "clip_similarity_std": 0.03247407374524215,
        "fid": 68.92864227294922
      }
    ],
    "generation": {
      "gw_checkpoint": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/checkpoints/last.ckpt",
      "attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/text_grounded_reptok_iterative_attention_tstate_mse_20260817_activated_start_lr3e4_full_e100_pat20/short_T5_keep/attention.pt",
      "gw_direct_bge": {
        "generated": 1000,
        "reused": 0
      },
      "gw_short_t5_grounded": {
        "attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/text_grounded_reptok_iterative_attention_tstate_mse_20260817_activated_start_lr3e4_full_e100_pat20/short_T5_keep/attention.pt",
        "generated": 1000,
        "reused": 0
      }
    },
    "artifacts": {
      "side_by_side_grid": "assets/side_by_side_grid.jpg",
      "metrics_csv": "metrics.csv",
      "per_sample_metrics_csv": "per_sample_metrics.csv",
      "manifest_public_jsonl": "manifest_public.jsonl"
    }
  },
  "combined_metrics_with_12h": [
    {
      "model": "gw112h_direct_bge",
      "label": "GW 112h direct BGE",
      "n_samples": 1000,
      "lpips_mean": 0.7014688248932361,
      "lpips_std": 0.05542194062016755,
      "clip_similarity_mean": 0.28640163788199424,
      "clip_similarity_std": 0.03300513231734236,
      "fid": 77.60134887695312,
      "params_m": 37.8
    },
    {
      "model": "gw112h_short_t5_grounded",
      "label": "GW 112h short_T5 grounded",
      "n_samples": 1000,
      "lpips_mean": 0.6958555586636066,
      "lpips_std": 0.058191509654314716,
      "clip_similarity_mean": 0.2887599302381277,
      "clip_similarity_std": 0.03265254305688373,
      "fid": 67.09028625488281,
      "params_m": 38.3
    },
    {
      "model": "gw12h_direct_bge",
      "label": "GW 12h direct BGE",
      "n_samples": 1000,
      "lpips_mean": 0.6971765531003475,
      "lpips_std": 0.055730533628727055,
      "clip_similarity_mean": 0.28643184058368204,
      "clip_similarity_std": 0.03384197934216952,
      "fid": 75.60994720458984,
      "params_m": 51.0
    },
    {
      "model": "gw12h_activated_short_t5_grounded",
      "label": "GW 12h activated short_T5",
      "n_samples": 1000,
      "lpips_mean": 0.6953469677269459,
      "lpips_std": 0.05903350333754925,
      "clip_similarity_mean": 0.2916463987007737,
      "clip_similarity_std": 0.03247407374524215,
      "fid": 68.92864227294922,
      "params_m": 51.5
    },
    {
      "model": "reptok_prior_796m",
      "label": "RepTok prior 796M",
      "n_samples": 1000,
      "lpips_mean": 0.7214722891151905,
      "lpips_std": 0.07157120731752703,
      "clip_similarity_mean": 0.3033647171854973,
      "clip_similarity_std": 0.036609110396594494,
      "fid": 65.88847351074219,
      "params_m": 796.4
    },
    {
      "model": "reptok_prior_37m",
      "label": "RepTok prior 37M",
      "n_samples": 1000,
      "lpips_mean": 0.7331561633348465,
      "lpips_std": 0.06471643899597412,
      "clip_similarity_mean": 0.27740161304920913,
      "clip_similarity_std": 0.037047673184247454,
      "fid": 81.88143920898438,
      "params_m": 36.8
    },
    {
      "model": "reptok_prior_bge_memory",
      "label": "RepTok prior BGE memory",
      "n_samples": 1000,
      "lpips_mean": 0.7245100690126419,
      "lpips_std": 0.060922619101622055,
      "clip_similarity_mean": 0.29028694324195387,
      "clip_similarity_std": 0.036915632404643926,
      "fid": 70.26390838623047,
      "params_m": 850.1
    }
  ]
}