{ "kind": "comparison", "title": "RepTok-v-Sonar Stage-1 BLEU", "eval_n_samples": 1000, "sample_seed": 20260519, "target_lang": "eng_Latn", "sonar_max_seq_len": 256, "metrics": [ { "model": "SONAR oracle", "n": 1000, "bleu": 63.58353603294297, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "GW RepTok-to-SONAR", "n": 1000, "bleu": 11.64949459740809, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "GW RepTok-to-SONAR + short_T5_keep MSE", "n": 1000, "bleu": 11.691164326294691, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "GW RepTok-to-SONAR + short_T5_keep SONAR loss", "n": 1000, "bleu": 11.223213795574527, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "GW RepTok-to-SONAR + BLEU-probe grounder", "n": 1000, "bleu": 12.342476215734937, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "GW RepTok-to-SONAR + codebook short-grounder BLEU-probe", "n": 1000, "bleu": 13.83520373793914, "alignment_l2": "", "elementwise_mse": "", "cosine": "", "source": "matched 1000-sample controller eval" }, { "model": "RepTok-v-Sonar Stage-1 adaptation", "n": 1000, "bleu": 10.241169797365478, "alignment_l2": 0.0531977154314518, "elementwise_mse": 5.1950893976027146e-05, "cosine": 0.6947507262229919, "source": "new 1000-sample RepTok-v-Sonar eval" } ], "compute_estimates": [ { "model": "GW RepTok-to-SONAR", "wall_hours": 112.0, "gpu_hours": 112.0, "devices": 1, "method": "CHAIN_VERIFIED nominal_total_hours times devices", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 1.257984e+20, "peak_eflops": 125.7984 }, { "model": "GW RepTok-to-SONAR + short_T5_keep MSE", "wall_hours": 0.16945567226942834, "gpu_hours": 0.16945567226942834, "devices": 1, "method": "best checkpoint elapsed_minutes from controller summary.json; training was manually stopped early", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 1.9033261109302192e+17, "peak_eflops": 0.1903326110930219 }, { "model": "GW RepTok-to-SONAR + short_T5_keep SONAR loss", "wall_hours": 0.09836810485201163, "gpu_hours": 0.09836810485201163, "devices": 1, "method": "best checkpoint elapsed_minutes from controller summary.json; training was manually stopped early", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 1.1048705536977946e+17, "peak_eflops": 0.11048705536977946 }, { "model": "GW RepTok-to-SONAR + BLEU-probe grounder", "wall_hours": 1.7211136622755374, "gpu_hours": 1.7211136622755374, "devices": 1, "method": "elapsed_minutes at best fixed-XPERF BLEU round from BLEU-probe loop summary.json", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 1.9331548654678835e+18, "peak_eflops": 1.9331548654678836 }, { "model": "GW RepTok-to-SONAR + codebook short-grounder BLEU-probe", "wall_hours": 2.2617173090582297, "gpu_hours": 2.2617173090582297, "devices": 1, "method": "elapsed_minutes at best fixed-XPERF BLEU round from BLEU-probe loop summary.json", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 2.5403608815342034e+18, "peak_eflops": 2.5403608815342036 }, { "model": "RepTok-v-Sonar", "wall_hours": 32.62783031589455, "gpu_hours": 65.2556606317891, "devices": 2, "method": "sum of per-chunk elapsed_minutes from train_log.jsonl times world_size", "peak_bf16_tflops_per_gpu": 312.0, "peak_flops": 7.329515802162552e+19, "peak_eflops": 73.29515802162553 } ], "parameter_counts": [ { "model": "GW RepTok-to-SONAR", "trainable_params": 37750860, "trainable_params_m": 37.75086, "components": "active gw_mod.* generator parameters", "method": "counted state_dict tensors in checkpoints/last.ckpt with prefix gw_mod." }, { "model": "RepTok-v-Sonar", "trainable_params": 92096512, "trainable_params_m": 92.096512, "connector_params": 5512960, "reptok_params": 86583552, "components": "RepTok visual encoder plus v-Sonar connector after step 2,000", "method": "read from config.json total_params_after_unfreeze" }, { "model": "GW RepTok-to-SONAR + short_T5_keep MSE", "trainable_params": 526336, "trainable_params_m": 0.526336, "components": "short_T5_keep iterative attention controller only; frozen GW reused", "method": "read attention_params from controller summary.json" }, { "model": "GW RepTok-to-SONAR + short_T5_keep SONAR loss", "trainable_params": 526336, "trainable_params_m": 0.526336, "components": "short_T5_keep iterative attention controller only; frozen GW reused", "method": "read attention_params from controller summary.json" }, { "model": "GW RepTok-to-SONAR + BLEU-probe grounder", "trainable_params": 526336, "trainable_params_m": 0.526336, "components": "short_T5_keep iterative attention controller only; frozen GW reused", "method": "counted state_dict tensors in attention_best.pt" }, { "model": "GW RepTok-to-SONAR + codebook short-grounder BLEU-probe", "trainable_params": 400417, "trainable_params_m": 0.400417, "components": "dictionary short-grounder controller only; frozen GW reused", "method": "counted state_dict tensors in attention_best.pt" } ], "gw_run_identity": { "run_name": "gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24", "group_mode": "imagenet_only", "imagenet_modalities": [ "dinov2", "reptok", "bge", "sonar", "symbolic" ], "nominal_total_hours": 112, "source_hours": 16, "added_hours": 96, "devices": 1, "sonar_loss": { "mse_coef": 1.0, "cosine_coef": 16.0, "variance_coef": 2.5, "var_eps": 0.0001 } }, "grounding_metrics_json": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_iterative_grounding_bleu_matched_n1000/metrics.json", "trajectory_dir": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_iterative_grounding_bleu_matched_n1000/trajectory_plots", "trajectory_plots": [ "assets/trajectory_attention_mse.jpg", "assets/trajectory_attention_sonar_loss.jpg", "assets/trajectory_keep_gate.jpg" ], "codebook_attention_dir": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_iterative_grounding_bleu_matched_n1000/codebook_attention_plots", "codebook_attention_plots": [ "assets/codebook_attention_modality_weights.jpg", "assets/codebook_attention_gate.jpg", "assets/codebook_attention_components_dinov2.jpg", "assets/codebook_attention_components_reptok.jpg", "assets/codebook_attention_components_bge.jpg", "assets/codebook_attention_components_sonar.jpg", "assets/codebook_attention_components_symbolic.jpg" ], "codebook_concepts_dir": "gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_iterative_grounding_bleu_matched_n1000/codebook_concept_decodes", "codebook_concept_grid": "assets/codebook_concept_grid.jpg", "codebook_concept_count": 25, "notes": [ "RepTok-v-Sonar and SONAR oracle are computed by the new eval in eval-dir.", "GW RepTok-to-SONAR is read from gw-metrics-csv; the matched page build uses sonar_eval_reptok_matched_n1000.", "Controller rows are lightweight short_T5_keep add-ons trained after GW and evaluated on the same 1000 matched indices.", "Training FLOPs are peak-equivalent upper-bound estimates from GPU-hours, not measured achieved FLOPs.", "Parameter counts are active trainable parameters for GW and post-unfreeze trainable parameters for RepTok-v-Sonar." ] }