{ "kind": "reptok_sonar_bleu_grounding_12h", "title": "12h RepTok-to-SONAR BLEU Grounding", "gw_checkpoint": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/checkpoints/last.ckpt", "metrics": [ { "dataset": "ImageNet heldout", "variant": "Plain MSE", "variant_key": "mse", "training": "MSE trained only", "direct_bleu": 11.90354682489747, "grounded_bleu": 11.666574393112244, "delta_bleu": -0.23697243178522598, "sonar_oracle_bleu": 63.58353603294297, "n_samples": 1000, "sample_seed": 20260519 }, { "dataset": "COCO val2017", "variant": "Plain MSE", "variant_key": "mse", "training": "MSE trained only", "direct_bleu": 0.4917895416240927, "grounded_bleu": 0.5191006035213563, "delta_bleu": 0.027311061897263622, "sonar_oracle_bleu": "", "n_samples": 1000, "sample_seed": 20260630 }, { "dataset": "ImageNet heldout", "variant": "Scratch BLEU Probe", "variant_key": "scratch_bleu_probe", "training": "BLEU-probe trained from scratch", "direct_bleu": 11.90354682489747, "grounded_bleu": 12.023072557013721, "delta_bleu": 0.1195257321162515, "sonar_oracle_bleu": 63.58353603294297, "n_samples": 1000, "sample_seed": 20260519 }, { "dataset": "COCO val2017", "variant": "Scratch BLEU Probe", "variant_key": "scratch_bleu_probe", "training": "BLEU-probe trained from scratch", "direct_bleu": 0.4917895416240927, "grounded_bleu": 0.31902798725758863, "delta_bleu": -0.1727615543665041, "sonar_oracle_bleu": "", "n_samples": 1000, "sample_seed": 20260630 }, { "dataset": "ImageNet heldout", "variant": "MSE Init + BLEU Probe", "variant_key": "mse_init_bleu_probe", "training": "Diagnostic: BLEU-probe training initialized from MSE", "direct_bleu": 11.90354682489747, "grounded_bleu": 13.15015958147152, "delta_bleu": 1.2466127565740504, "sonar_oracle_bleu": 63.58353603294297, "n_samples": 1000, "sample_seed": 20260519 }, { "dataset": "COCO val2017", "variant": "MSE Init + BLEU Probe", "variant_key": "mse_init_bleu_probe", "training": "Diagnostic: BLEU-probe training initialized from MSE", "direct_bleu": 0.4917895416240927, "grounded_bleu": 0.4439163776441492, "delta_bleu": -0.04787316397994351, "sonar_oracle_bleu": "", "n_samples": 1000, "sample_seed": 20260630 } ], "variants": [ { "key": "mse", "label": "Plain MSE", "kind": "MSE trained only", "checkpoint": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_iterative_grounding_short_t5_mse_20260818_activated_start_lr3e4_full_e100_pat20/short_T5_keep/attention.pt", "summary": { "complete": false, "stopped_reason": "signal", "config": { "config_id": "short_T5_keep", "trajectory_steps": 5, "eval_steps": 5, "use_keep": true, "use_prev_hypotheses": false, "trajectory_policy": "attention", "detached_history": true, "initial_state_mode": "activated_encoder", "unrolled_loss": "last", "use_interpolation": false }, "run_name": "gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03", "checkpoint": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/checkpoints/last.ckpt", "input_modality": "reptok", "target_modality": "sonar", "objective_mode": "mse", "loss_coefficients": { "mse_coef": 1.0, "cosine_coef": 0.0, "variance_coef": 0.0 }, "var_eps": 0.0001, "modalities": [ "dinov2", "reptok", "bge", "sonar", "symbolic" ], "candidate_names": [ "dinov2", "reptok", "bge", "sonar", "symbolic" ], "attention_params": 2099200, "attention_hparams": { "grounder_kind": "attention", "head_size": 16, "num_heads": 16, "learning_rate": 0.0003, "weight_decay": 0.0, "workspace_dim": 1024, "use_keep": true, "use_prev_hypotheses": false, "modality_names": [ "dinov2", "reptok", "bge", "sonar", "symbolic" ], "codes_per_modality": 0 }, "epoch_samples": 1231167, "heldout_samples": 10000, "best_objective": 0.5098315399169921, "best_epoch": 21, "epochs_ran": 40, "best_metrics": { "epoch": 21, "train_objective": 0.5038964705236789, "train_mse": 0.5038964705236789, "train_cosine": 0.2912690160294001, "train_variance": 0.13342780553757017, "heldout_final_objective": 0.5098315399169921, "heldout_final_mse": 0.5098315399169921, "heldout_final_cosine": 0.2950563422203064, "heldout_final_variance": 0.13465513458251954, "heldout_best_step": 4, "best_objective": 0.5098315399169921, "best_epoch": 21, "bad_epochs": 0, "elapsed_minutes": 22.44158783357901, "weight_dinov2": 0.831916093826294, "weight_reptok": 0.12024985998868942, "weight_bge": 0.01196736004203558, "weight_sonar": 0.008278872817754745, "weight_symbolic": 0.02758777141571045, "weight_keep_gate": 0.988991748046875 }, "resume_state": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_iterative_grounding_short_t5_mse_20260818_activated_start_lr3e4_full_e100_pat20/short_T5_keep/resume_state.pt", "attention_checkpoint_materialized_from_resume_state": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_iterative_grounding_short_t5_mse_20260818_activated_start_lr3e4_full_e100_pat20/short_T5_keep/resume_state.pt", "attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_iterative_grounding_short_t5_mse_20260818_activated_start_lr3e4_full_e100_pat20/short_T5_keep/attention.pt" }, "history": [ { "epoch": "0", "train_objective": "0.5092266190046387", "train_mse": "0.5092266190046387", "train_cosine": "0.2938665754102765", "train_variance": "0.12764191190615118", "heldout_final_objective": "0.5122170421600342", "heldout_final_mse": "0.5122170421600342", "heldout_final_cosine": "0.29605700941085816", "heldout_final_variance": "0.12758533244132997", "heldout_best_step": "3", "best_objective": "0.5122170421600342", "best_epoch": "0", "bad_epochs": "0", "elapsed_minutes": "1.7493320951548716", "weight_dinov2": "0.9335590600967407", "weight_reptok": "0.02376142516732216", "weight_bge": "0.028627946972846985", "weight_sonar": "0.014051505364477634", "weight_symbolic": "3.2079890827096946e-11", "weight_keep_gate": "0.9075794799804687" }, { "epoch": "1", "train_objective": "0.505245937611728", "train_mse": "0.505245937611728", "train_cosine": "0.29195721397792307", "train_variance": "0.13276283680286768", "heldout_final_objective": "0.5108629593849182", "heldout_final_mse": "0.5108629593849182", "heldout_final_cosine": "0.2956215517997742", "heldout_final_variance": "0.1331391222000122", "heldout_best_step": "4", "best_objective": "0.5108629593849182", "best_epoch": "1", "bad_epochs": "0", "elapsed_minutes": "2.733060961899658", "weight_dinov2": "0.9239766597747803", "weight_reptok": "0.037058085203170776", "weight_bge": "0.024849390611052513", "weight_sonar": "0.014115804806351662", "weight_symbolic": "3.401487980170259e-09", "weight_keep_gate": "0.9550807495117187" }, { "epoch": "2", "train_objective": "0.5048012107947827", "train_mse": "0.5048012107947827", "train_cosine": "0.29172475619720434", "train_variance": "0.13324464734677924", "heldout_final_objective": "0.5105955723762512", "heldout_final_mse": "0.5105955723762512", "heldout_final_cosine": "0.2952192476272583", "heldout_final_variance": "0.1317433074951172", "heldout_best_step": "3", "best_objective": "0.5105955723762512", "best_epoch": "2", "bad_epochs": "0", "elapsed_minutes": "3.718125588229547", "weight_dinov2": "0.903292179107666", "weight_reptok": "0.06278401613235474", "weight_bge": "0.022077934816479683", "weight_sonar": "0.011812044307589531", "weight_symbolic": "3.382619979674928e-05", "weight_keep_gate": "0.9705372131347656" }, { "epoch": "3", "train_objective": "0.5044280438766446", "train_mse": "0.5044280438766446", "train_cosine": "0.2915106186722551", "train_variance": "0.13360354381277217", "heldout_final_objective": "0.5103712990760804", "heldout_final_mse": "0.5103712990760804", "heldout_final_cosine": "0.2951524037361145", "heldout_final_variance": "0.13323325448036194", "heldout_best_step": "5", "best_objective": "0.5103712990760804", "best_epoch": "3", "bad_epochs": "0", "elapsed_minutes": "4.703110063758989", "weight_dinov2": "0.8881158828735352", "weight_reptok": "0.06290632486343384", "weight_bge": "0.023634860292077065", "weight_sonar": "0.024630822241306305", "weight_symbolic": "0.0007120204390957952", "weight_keep_gate": "0.9891486511230468" }, { "epoch": "4", "train_objective": "0.5042551611530428", "train_mse": "0.5042551611530428", "train_cosine": "0.29144258512573373", "train_variance": "0.13364339922777133", "heldout_final_objective": "0.5103412722587586", "heldout_final_mse": "0.5103412722587586", "heldout_final_cosine": "0.2954042083740234", "heldout_final_variance": "0.1371758323907852", "heldout_best_step": "3", "best_objective": "0.5103412722587586", "best_epoch": "4", "bad_epochs": "0", "elapsed_minutes": "5.688459273862342", "weight_dinov2": "0.8519387245178223", "weight_reptok": "0.07100889831781387", "weight_bge": "0.018101651221513748", "weight_sonar": "0.0053708809427917", "weight_symbolic": "0.05357976630330086", "weight_keep_gate": "0.9869122741699219" }, { "epoch": "5", "train_objective": "0.5041749849864974", "train_mse": "0.5041749849864974", "train_cosine": "0.29139834456014896", "train_variance": "0.1335461180567835", "heldout_final_objective": "0.5103321364402771", "heldout_final_mse": "0.5103321364402771", "heldout_final_cosine": "0.29543390493392946", "heldout_final_variance": "0.1375710414171219", "heldout_best_step": "4", "best_objective": "0.5103412722587586", "best_epoch": "4", "bad_epochs": "1", "elapsed_minutes": "6.673680862877518", "weight_dinov2": "0.8455283045768738", "weight_reptok": "0.09950246661901474", "weight_bge": "0.017580538988113403", "weight_sonar": "0.008364000357687473", "weight_symbolic": "0.029024651274085045", "weight_keep_gate": "0.9831045959472656" }, { "epoch": "6", "train_objective": "0.5041179552567168", "train_mse": "0.5041179552567168", "train_cosine": "0.29137286243033583", "train_variance": "0.13355419583285855", "heldout_final_objective": "0.5100858342170715", "heldout_final_mse": "0.5100858342170715", "heldout_final_cosine": "0.2950035526275635", "heldout_final_variance": "0.13244176511764527", "heldout_best_step": "4", "best_objective": "0.5100858342170715", "best_epoch": "6", "bad_epochs": "0", "elapsed_minutes": "7.659665204848474", "weight_dinov2": "0.8431274890899658", "weight_reptok": "0.09768136590719223", "weight_bge": "0.015840182080864906", "weight_sonar": "0.01092004869133234", "weight_symbolic": "0.03243085369467735", "weight_keep_gate": "0.9892573181152344" }, { "epoch": "7", "train_objective": "0.5040529201966765", "train_mse": "0.5040529201966765", "train_cosine": "0.2913455493159006", "train_variance": "0.13345708536194648", "heldout_final_objective": "0.5100307795524597", "heldout_final_mse": "0.5100307795524597", "heldout_final_cosine": "0.29514936571121214", "heldout_final_variance": "0.1346011923789978", "heldout_best_step": "5", "best_objective": "0.5100307795524597", "best_epoch": "7", "bad_epochs": "0", "elapsed_minutes": "8.64559344512721", "weight_dinov2": "0.8286165595054626", "weight_reptok": "0.107342429459095", "weight_bge": "0.022846130654215813", "weight_sonar": "0.010444232262670994", "weight_symbolic": "0.03075055219233036", "weight_keep_gate": "0.9900732543945312" }, { "epoch": "8", "train_objective": "0.5040714180976389", "train_mse": "0.5040714180976389", "train_cosine": "0.29133475313737456", "train_variance": "0.13337549567566329", "heldout_final_objective": "0.5100508716583252", "heldout_final_mse": "0.5100508716583252", "heldout_final_cosine": "0.2951455955505371", "heldout_final_variance": "0.1342278022289276", "heldout_best_step": "4", "best_objective": "0.5100307795524597", "best_epoch": "7", "bad_epochs": "1", "elapsed_minutes": "9.630526844862228", "weight_dinov2": "0.8485094308853149", "weight_reptok": "0.11122021079063416", "weight_bge": "0.01984803006052971", "weight_sonar": "0.010380683466792107", "weight_symbolic": "0.010041622444987297", "weight_keep_gate": "0.992011376953125" }, { "epoch": "9", "train_objective": "0.5040165752354673", "train_mse": "0.5040165752354673", "train_cosine": "0.2913302536584988", "train_variance": "0.13359219750142856", "heldout_final_objective": "0.5099796702384949", "heldout_final_mse": "0.5099796702384949", "heldout_final_cosine": "0.29521587591171267", "heldout_final_variance": "0.13644328289031982", "heldout_best_step": "5", "best_objective": "0.5099796702384949", "best_epoch": "9", "bad_epochs": "0", "elapsed_minutes": "10.616053720377385", "weight_dinov2": "0.8353285789489746", "weight_reptok": "0.10101024806499481", "weight_bge": "0.02076185680925846", "weight_sonar": "0.009274449199438095", "weight_symbolic": "0.03362487629055977", "weight_keep_gate": "0.98947705078125" }, { "epoch": "10", "train_objective": "0.5040040900416365", "train_mse": "0.5040040900416365", "train_cosine": "0.29131855411430285", "train_variance": "0.13354025111720905", "heldout_final_objective": "0.5102707473754883", "heldout_final_mse": "0.5102707473754883", "heldout_final_cosine": "0.2949293592453003", "heldout_final_variance": "0.13072327787876128", "heldout_best_step": "3", "best_objective": "0.5099796702384949", "best_epoch": "9", "bad_epochs": "1", "elapsed_minutes": "11.60148017858155", "weight_dinov2": "0.8335095643997192", "weight_reptok": "0.10815797001123428", "weight_bge": "0.01466851495206356", "weight_sonar": "0.014963831752538681", "weight_symbolic": "0.028700176626443863", "weight_keep_gate": "0.9850114440917969" }, { "epoch": "11", "train_objective": "0.5039693787883818", "train_mse": "0.5039693787883818", "train_cosine": "0.29130465892444746", "train_variance": "0.13352897405363057", "heldout_final_objective": "0.5099730349540711", "heldout_final_mse": "0.5099730349540711", "heldout_final_cosine": "0.29525704612731934", "heldout_final_variance": "0.13791981782913207", "heldout_best_step": "5", "best_objective": "0.5099796702384949", "best_epoch": "9", "bad_epochs": "2", "elapsed_minutes": "12.587568345588322", "weight_dinov2": "0.82803875207901", "weight_reptok": "0.11702272295951843", "weight_bge": "0.0156850665807724", "weight_sonar": "0.0061616599559783936", "weight_symbolic": "0.03309169039130211", "weight_keep_gate": "0.98842294921875" }, { "epoch": "12", "train_objective": "0.503983949333182", "train_mse": "0.503983949333182", "train_cosine": "0.29129695805045114", "train_variance": "0.13344014493321033", "heldout_final_objective": "0.5101657153129577", "heldout_final_mse": "0.5101657153129577", "heldout_final_cosine": "0.29488272352218625", "heldout_final_variance": "0.13051139469146728", "heldout_best_step": "3", "best_objective": "0.5099796702384949", "best_epoch": "9", "bad_epochs": "3", "elapsed_minutes": "13.573580679266403", "weight_dinov2": "0.8359430432319641", "weight_reptok": "0.11074286699295044", "weight_bge": "0.026894664391875267", "weight_sonar": "0.012290559709072113", "weight_symbolic": "0.01412880513817072", "weight_keep_gate": "0.9913647888183593" }, { "epoch": "13", "train_objective": "0.5039674237134586", "train_mse": "0.5039674237134586", "train_cosine": "0.29129865291972595", "train_variance": "0.13342094597155418", "heldout_final_objective": "0.5099571459770202", "heldout_final_mse": "0.5099571459770202", "heldout_final_cosine": "0.29513560733795163", "heldout_final_variance": "0.13477914481163025", "heldout_best_step": "3", "best_objective": "0.5099571459770202", "best_epoch": "13", "bad_epochs": "0", "elapsed_minutes": "14.559157751640305", "weight_dinov2": "0.8188117742538452", "weight_reptok": "0.12739719450473785", "weight_bge": "0.01129450835287571", "weight_sonar": "0.01052841730415821", "weight_symbolic": "0.03196803107857704", "weight_keep_gate": "0.9853060852050781" }, { "epoch": "14", "train_objective": "0.5039235010944552", "train_mse": "0.5039235010944552", "train_cosine": "0.29128338205284693", "train_variance": "0.1335802820911948", "heldout_final_objective": "0.5099014642715454", "heldout_final_mse": "0.5099014642715454", "heldout_final_cosine": "0.29509057931900023", "heldout_final_variance": "0.13493735280036925", "heldout_best_step": "5", "best_objective": "0.5099014642715454", "best_epoch": "14", "bad_epochs": "0", "elapsed_minutes": "15.544530671881512", "weight_dinov2": "0.8166863322257996", "weight_reptok": "0.13518878817558289", "weight_bge": "0.012084096670150757", "weight_sonar": "0.012342176400125027", "weight_symbolic": "0.02369857020676136", "weight_keep_gate": "0.9876202941894531" }, { "epoch": "15", "train_objective": "0.5038957663515462", "train_mse": "0.5038957663515462", "train_cosine": "0.29127009291338274", "train_variance": "0.1335165955177682", "heldout_final_objective": "0.5098812775611877", "heldout_final_mse": "0.5098812775611877", "heldout_final_cosine": "0.2950976336479187", "heldout_final_variance": "0.13474820091724396", "heldout_best_step": "4", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "0", "elapsed_minutes": "16.53032191172242", "weight_dinov2": "0.8146432638168335", "weight_reptok": "0.13271233439445496", "weight_bge": "0.009936613030731678", "weight_sonar": "0.011006869375705719", "weight_symbolic": "0.03170085325837135", "weight_keep_gate": "0.9903739440917969" }, { "epoch": "16", "train_objective": "0.5038997221346865", "train_mse": "0.5038997221346865", "train_cosine": "0.2912727611350705", "train_variance": "0.13344352281698268", "heldout_final_objective": "0.5099541718482972", "heldout_final_mse": "0.5099541718482972", "heldout_final_cosine": "0.2952098505973816", "heldout_final_variance": "0.13658914935588837", "heldout_best_step": "5", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "1", "elapsed_minutes": "17.51598281081145", "weight_dinov2": "0.8306888341903687", "weight_reptok": "0.1127292811870575", "weight_bge": "0.012300309725105762", "weight_sonar": "0.006971886847168207", "weight_symbolic": "0.037309739738702774", "weight_keep_gate": "0.9912275207519531" }, { "epoch": "17", "train_objective": "0.5038739634067337", "train_mse": "0.5038739634067337", "train_cosine": "0.2912696121815337", "train_variance": "0.13356698493226551", "heldout_final_objective": "0.5099393829345703", "heldout_final_mse": "0.5099393829345703", "heldout_final_cosine": "0.2951978218078613", "heldout_final_variance": "0.13739569091796874", "heldout_best_step": "5", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "2", "elapsed_minutes": "18.50122200579693", "weight_dinov2": "0.8296763300895691", "weight_reptok": "0.11352282762527466", "weight_bge": "0.0184191707521677", "weight_sonar": "0.00700417859479785", "weight_symbolic": "0.03137752786278725", "weight_keep_gate": "0.9897397155761719" }, { "epoch": "18", "train_objective": "0.5040548411710871", "train_mse": "0.5040548411710871", "train_cosine": "0.2913364736683664", "train_variance": "0.1333726644564102", "heldout_final_objective": "0.5115122955322265", "heldout_final_mse": "0.5115122955322265", "heldout_final_cosine": "0.2951951798439026", "heldout_final_variance": "0.1283816968202591", "heldout_best_step": "2", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "3", "elapsed_minutes": "19.486215590219945", "weight_dinov2": "0.8328360319137573", "weight_reptok": "0.1105942502617836", "weight_bge": "0.04728357121348381", "weight_sonar": "0.009147166274487972", "weight_symbolic": "0.00013904168736189604", "weight_keep_gate": "0.9769519470214844" }, { "epoch": "19", "train_objective": "0.5039619487621589", "train_mse": "0.5039619487621589", "train_cosine": "0.2912944052087198", "train_variance": "0.13349519134551951", "heldout_final_objective": "0.5103337067604065", "heldout_final_mse": "0.5103337067604065", "heldout_final_cosine": "0.2948115900039673", "heldout_final_variance": "0.12960458023548127", "heldout_best_step": "3", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "4", "elapsed_minutes": "20.47157677526896", "weight_dinov2": "0.8508493900299072", "weight_reptok": "0.10358945280313492", "weight_bge": "0.021009203046560287", "weight_sonar": "0.022741196677088737", "weight_symbolic": "0.00181072938721627", "weight_keep_gate": "0.9902393371582031" }, { "epoch": "20", "train_objective": "0.5038565555024979", "train_mse": "0.5038565555024979", "train_cosine": "0.2912571975135262", "train_variance": "0.13347715236290333", "heldout_final_objective": "0.5100807611465454", "heldout_final_mse": "0.5100807611465454", "heldout_final_cosine": "0.29492210664749147", "heldout_final_variance": "0.1326355414390564", "heldout_best_step": "3", "best_objective": "0.5098812775611877", "best_epoch": "15", "bad_epochs": "5", "elapsed_minutes": "21.4565835771927", "weight_dinov2": "0.8491557240486145", "weight_reptok": "0.1112462654709816", "weight_bge": "0.026857495307922363", "weight_sonar": "0.012727688066661358", "weight_symbolic": "1.2783541933458764e-05", "weight_keep_gate": "0.995626953125" }, { "epoch": "21", "train_objective": "0.5038964705236789", "train_mse": "0.5038964705236789", "train_cosine": "0.2912690160294001", "train_variance": "0.13342780553757017", "heldout_final_objective": "0.5098315399169921", "heldout_final_mse": "0.5098315399169921", "heldout_final_cosine": "0.2950563422203064", "heldout_final_variance": "0.13465513458251954", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "0", "elapsed_minutes": "22.44158783357901", "weight_dinov2": "0.831916093826294", "weight_reptok": "0.12024985998868942", "weight_bge": "0.01196736004203558", "weight_sonar": "0.008278872817754745", "weight_symbolic": "0.02758777141571045", "weight_keep_gate": "0.988991748046875" }, { "epoch": "22", "train_objective": "0.5038973318133546", "train_mse": "0.5038973318133546", "train_cosine": "0.29127573762449044", "train_variance": "0.13335019837213816", "heldout_final_objective": "0.5100250489234924", "heldout_final_mse": "0.5100250489234924", "heldout_final_cosine": "0.2949574354171753", "heldout_final_variance": "0.13229503326416014", "heldout_best_step": "3", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "1", "elapsed_minutes": "23.426747951858367", "weight_dinov2": "0.8627172708511353", "weight_reptok": "0.10225729644298553", "weight_bge": "0.02312801592051983", "weight_sonar": "0.008930797688663006", "weight_symbolic": "0.0029665965121239424", "weight_keep_gate": "0.9906951843261719" }, { "epoch": "23", "train_objective": "0.5038516959880718", "train_mse": "0.5038516959880718", "train_cosine": "0.2912527189024996", "train_variance": "0.13342452586718545", "heldout_final_objective": "0.5098693059921264", "heldout_final_mse": "0.5098693059921264", "heldout_final_cosine": "0.29511414270401", "heldout_final_variance": "0.1352176142692566", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "2", "elapsed_minutes": "24.41169764357619", "weight_dinov2": "0.8307441473007202", "weight_reptok": "0.11366645246744156", "weight_bge": "0.012598435394465923", "weight_sonar": "0.010065587237477303", "weight_symbolic": "0.03292534127831459", "weight_keep_gate": "0.9899829711914062" }, { "epoch": "24", "train_objective": "0.5038534254895805", "train_mse": "0.5038534254895805", "train_cosine": "0.29124928076926804", "train_variance": "0.13363617405573008", "heldout_final_objective": "0.5098258728027344", "heldout_final_mse": "0.5098258728027344", "heldout_final_cosine": "0.2951540771484375", "heldout_final_variance": "0.136221205329895", "heldout_best_step": "5", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "3", "elapsed_minutes": "25.396400548766056", "weight_dinov2": "0.8178367018699646", "weight_reptok": "0.12740924954414368", "weight_bge": "0.01076376624405384", "weight_sonar": "0.007536548655480146", "weight_symbolic": "0.03645369037985802", "weight_keep_gate": "0.9900111145019531" }, { "epoch": "25", "train_objective": "0.5037936412351526", "train_mse": "0.5037936412351526", "train_cosine": "0.29122440684427864", "train_variance": "0.133488303361022", "heldout_final_objective": "0.5099784817695617", "heldout_final_mse": "0.5099784817695617", "heldout_final_cosine": "0.29477993774414063", "heldout_final_variance": "0.13136144938468933", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "4", "elapsed_minutes": "26.381828433461486", "weight_dinov2": "0.8088549971580505", "weight_reptok": "0.1307763010263443", "weight_bge": "0.012767483480274677", "weight_sonar": "0.008570559322834015", "weight_symbolic": "0.03903067111968994", "weight_keep_gate": "0.9902313110351563" }, { "epoch": "26", "train_objective": "0.5037751779665208", "train_mse": "0.5037751779665208", "train_cosine": "0.2912174582407119", "train_variance": "0.1335426196342366", "heldout_final_objective": "0.5098579598426819", "heldout_final_mse": "0.5098579598426819", "heldout_final_cosine": "0.2951386836051941", "heldout_final_variance": "0.135944757604599", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "5", "elapsed_minutes": "27.367210091774663", "weight_dinov2": "0.8155590295791626", "weight_reptok": "0.1391611397266388", "weight_bge": "0.0144877377897501", "weight_sonar": "0.007674544118344784", "weight_symbolic": "0.02311757393181324", "weight_keep_gate": "0.9899679016113281" }, { "epoch": "27", "train_objective": "0.5038114055049816", "train_mse": "0.5038114055049816", "train_cosine": "0.2912321292221274", "train_variance": "0.13347130311971878", "heldout_final_objective": "0.5101837644577026", "heldout_final_mse": "0.5101837644577026", "heldout_final_cosine": "0.29469953060150145", "heldout_final_variance": "0.12981276466846467", "heldout_best_step": "3", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "6", "elapsed_minutes": "28.35308942073025", "weight_dinov2": "0.8451594710350037", "weight_reptok": "0.12158706784248352", "weight_bge": "0.019837526604533195", "weight_sonar": "0.011482485570013523", "weight_symbolic": "0.0019334908574819565", "weight_keep_gate": "0.9914142517089843" }, { "epoch": "28", "train_objective": "0.5038082894354805", "train_mse": "0.5038082894354805", "train_cosine": "0.29123366203271867", "train_variance": "0.13341254598221258", "heldout_final_objective": "0.5099729490280152", "heldout_final_mse": "0.5099729490280152", "heldout_final_cosine": "0.29478017206192014", "heldout_final_variance": "0.13069596807956696", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "7", "elapsed_minutes": "29.338677312759682", "weight_dinov2": "0.8197586536407471", "weight_reptok": "0.13385693728923798", "weight_bge": "0.011284684762358665", "weight_sonar": "0.013846632093191147", "weight_symbolic": "0.021252961829304695", "weight_keep_gate": "0.990361669921875" }, { "epoch": "29", "train_objective": "0.5037664587412473", "train_mse": "0.5037664587412473", "train_cosine": "0.2912095691591821", "train_variance": "0.13352998332232663", "heldout_final_objective": "0.5100000958442688", "heldout_final_mse": "0.5100000958442688", "heldout_final_cosine": "0.2949688005447388", "heldout_final_variance": "0.13172356696128845", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "8", "elapsed_minutes": "30.324525091269365", "weight_dinov2": "0.7929439544677734", "weight_reptok": "0.14698392152786255", "weight_bge": "0.008176821283996105", "weight_sonar": "0.00795262772589922", "weight_symbolic": "0.043942663818597794", "weight_keep_gate": "0.990583642578125" }, { "epoch": "30", "train_objective": "0.5037971747743429", "train_mse": "0.5037971747743429", "train_cosine": "0.29122262128346693", "train_variance": "0.1332767270987714", "heldout_final_objective": "0.509856515789032", "heldout_final_mse": "0.509856515789032", "heldout_final_cosine": "0.29505635108947753", "heldout_final_variance": "0.13528141100406646", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "9", "elapsed_minutes": "31.309903383860366", "weight_dinov2": "0.8295477032661438", "weight_reptok": "0.12182359397411346", "weight_bge": "0.016134122386574745", "weight_sonar": "0.00857250951230526", "weight_symbolic": "0.023922059684991837", "weight_keep_gate": "0.9924492431640625" }, { "epoch": "31", "train_objective": "0.5037565551718415", "train_mse": "0.5037565551718415", "train_cosine": "0.29120677042616017", "train_variance": "0.13346196155877374", "heldout_final_objective": "0.5098546487808228", "heldout_final_mse": "0.5098546487808228", "heldout_final_cosine": "0.294995294380188", "heldout_final_variance": "0.13364262852668762", "heldout_best_step": "5", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "10", "elapsed_minutes": "32.29594267330443", "weight_dinov2": "0.8109658360481262", "weight_reptok": "0.1330646425485611", "weight_bge": "0.009505216963589191", "weight_sonar": "0.013130308128893375", "weight_symbolic": "0.033333949744701385", "weight_keep_gate": "0.9907854248046875" }, { "epoch": "32", "train_objective": "0.5037347423940808", "train_mse": "0.5037347423940808", "train_cosine": "0.2912004548871223", "train_variance": "0.13355280951462947", "heldout_final_objective": "0.5098395277023315", "heldout_final_mse": "0.5098395277023315", "heldout_final_cosine": "0.2949993257522583", "heldout_final_variance": "0.13409944272041321", "heldout_best_step": "5", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "11", "elapsed_minutes": "33.28113547530957", "weight_dinov2": "0.809954822063446", "weight_reptok": "0.14053404331207275", "weight_bge": "0.012891482561826706", "weight_sonar": "0.0104575389996171", "weight_symbolic": "0.026162156835198402", "weight_keep_gate": "0.9909522094726563" }, { "epoch": "33", "train_objective": "0.5037412785475764", "train_mse": "0.5037412785475764", "train_cosine": "0.29120178929616686", "train_variance": "0.13352700842913598", "heldout_final_objective": "0.509902000617981", "heldout_final_mse": "0.509902000617981", "heldout_final_cosine": "0.29520810565948485", "heldout_final_variance": "0.13660826058387757", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "12", "elapsed_minutes": "34.26657295785844", "weight_dinov2": "0.8180693984031677", "weight_reptok": "0.13925665616989136", "weight_bge": "0.018681298941373825", "weight_sonar": "0.017685478553175926", "weight_symbolic": "0.0063071707263588905", "weight_keep_gate": "0.9922325256347656" }, { "epoch": "34", "train_objective": "0.5037203862289021", "train_mse": "0.5037203862289021", "train_cosine": "0.29119065071018163", "train_variance": "0.13351610050526902", "heldout_final_objective": "0.5098272346496582", "heldout_final_mse": "0.5098272346496582", "heldout_final_cosine": "0.2951699064254761", "heldout_final_variance": "0.13690976815223693", "heldout_best_step": "5", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "13", "elapsed_minutes": "35.252151436467344", "weight_dinov2": "0.8011390566825867", "weight_reptok": "0.1461389660835266", "weight_bge": "0.010009434074163437", "weight_sonar": "0.007254353258758783", "weight_symbolic": "0.03545815870165825", "weight_keep_gate": "0.9919535278320313" }, { "epoch": "35", "train_objective": "0.5037040011479851", "train_mse": "0.5037040011479851", "train_cosine": "0.29118550965468404", "train_variance": "0.13347727174105786", "heldout_final_objective": "0.5099403876304627", "heldout_final_mse": "0.5099403876304627", "heldout_final_cosine": "0.2949080677986145", "heldout_final_variance": "0.13193899624347688", "heldout_best_step": "3", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "14", "elapsed_minutes": "36.24095758118977", "weight_dinov2": "0.8206513524055481", "weight_reptok": "0.14132335782051086", "weight_bge": "0.021445078775286674", "weight_sonar": "0.015079628676176071", "weight_symbolic": "0.0015005685854703188", "weight_keep_gate": "0.9927294250488281" }, { "epoch": "36", "train_objective": "0.5037204968681483", "train_mse": "0.5037204968681483", "train_cosine": "0.29119489072093463", "train_variance": "0.1334459706797371", "heldout_final_objective": "0.509854817199707", "heldout_final_mse": "0.509854817199707", "heldout_final_cosine": "0.2951285892486572", "heldout_final_variance": "0.13604874947071074", "heldout_best_step": "5", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "15", "elapsed_minutes": "37.226552463090044", "weight_dinov2": "0.8160204887390137", "weight_reptok": "0.13314048945903778", "weight_bge": "0.009411338716745377", "weight_sonar": "0.006944937631487846", "weight_symbolic": "0.03448271378874779", "weight_keep_gate": "0.9911638977050782" }, { "epoch": "37", "train_objective": "0.5036991348610951", "train_mse": "0.5036991348610951", "train_cosine": "0.2911817881954858", "train_variance": "0.13348591251323366", "heldout_final_objective": "0.5099702762603759", "heldout_final_mse": "0.5099702762603759", "heldout_final_cosine": "0.2949241950035095", "heldout_final_variance": "0.13217830653190613", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "16", "elapsed_minutes": "38.21208552548972", "weight_dinov2": "0.8291305303573608", "weight_reptok": "0.1326502412557602", "weight_bge": "0.016490722075104713", "weight_sonar": "0.015067302621901035", "weight_symbolic": "0.006661089137196541", "weight_keep_gate": "0.9923336364746094" }, { "epoch": "38", "train_objective": "0.5037326007723397", "train_mse": "0.5037326007723397", "train_cosine": "0.291195944492684", "train_variance": "0.13331944607414856", "heldout_final_objective": "0.5099192109107972", "heldout_final_mse": "0.5099192109107972", "heldout_final_cosine": "0.2947195915222168", "heldout_final_variance": "0.13046011672019958", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "17", "elapsed_minutes": "39.197215958979605", "weight_dinov2": "0.82135409116745", "weight_reptok": "0.13103646039962769", "weight_bge": "0.013518810272216797", "weight_sonar": "0.010269878432154655", "weight_symbolic": "0.023820754140615463", "weight_keep_gate": "0.9894626831054687" }, { "epoch": "39", "train_objective": "0.503710900376464", "train_mse": "0.503710900376464", "train_cosine": "0.29122267467032675", "train_variance": "0.13339059418995663", "heldout_final_objective": "0.5099506295204163", "heldout_final_mse": "0.5099506295204163", "heldout_final_cosine": "0.29499613618850706", "heldout_final_variance": "0.13324085071086883", "heldout_best_step": "4", "best_objective": "0.5098315399169921", "best_epoch": "21", "bad_epochs": "18", "elapsed_minutes": "39.71509624832931", "weight_dinov2": "0.849399209022522", "weight_reptok": "0.11610473692417145", "weight_bge": "0.023211831226944923", "weight_sonar": "0.009872954338788986", "weight_symbolic": "0.0014113221550360322", "weight_keep_gate": "0.9933649780273438" } ] }, { "key": "scratch_bleu_probe", "label": "Scratch BLEU Probe", "kind": "BLEU-probe trained from scratch", "checkpoint": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_fixedprobe_attention_validpath760k_scratch_activated_20260818/attention_latest.pt", "summary": { "grounder_kind": "attention", "fixed_probe": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_bleu_probe_fixed_validpath760k/probe_once/round_000/probes/probe_00.pt", "data_dir": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_bleu_probe_valid_path_pool_760k_dense", "initial_state_mode": "activated_encoder", "init_attention_checkpoint": null, "num_pool_rows": 760000, "xperf_samples": 200, "xperf_sentence_bleu_mean": 12.205832481384277, "xperf_corpus_bleu": 12.205833635546576, "history_last": { "epoch": 23, "grounder_lr": 0.003, "reward_mean": 12.08480926574707, "penalty_mean": 0.0, "probe_bleu_mean": 12.08480926574707, "probe_improvement_mean": -1.278890517692566, "attn_dinov2": 0.21898511291503905, "attn_reptok": 0.029912626190185546, "attn_bge": 0.6874918408203124, "attn_sonar": 0.0011022620797157287, "attn_symbolic": 0.06250815948486328, "gate_mean": 0.625 } }, "history": [ { "epoch": "0", "grounder_lr": "0.003", "reward_mean": "10.88462836669922", "penalty_mean": "0.0", "probe_bleu_mean": "10.88462836669922", "probe_improvement_mean": "-2.4978397770690917", "attn_dinov2": "0.1676986550807953", "attn_reptok": "0.01466345762848854", "attn_bge": "0.7558927014160156", "attn_sonar": "0.017442060111801258", "attn_symbolic": "0.04430312467336655", "gate_mean": "0.4843017156982422" }, { "epoch": "1", "grounder_lr": "0.003", "reward_mean": "11.56525651977539", "penalty_mean": "0.0", "probe_bleu_mean": "11.56525651977539", "probe_improvement_mean": "-1.7700744442749023", "attn_dinov2": "0.23243173278808593", "attn_reptok": "0.01901072765350342", "attn_bge": "0.7096234155273438", "attn_sonar": "5.800650455057621e-08", "attn_symbolic": "0.038934066863059995", "gate_mean": "0.35177590759277344" }, { "epoch": "2", "grounder_lr": "0.003", "reward_mean": "11.511931520080566", "penalty_mean": "0.0", "probe_bleu_mean": "11.511931520080566", "probe_improvement_mean": "-1.8680878684234619", "attn_dinov2": "0.21185600280761718", "attn_reptok": "0.03813214385986328", "attn_bge": "0.687479375", "attn_sonar": "1.25e-06", "attn_symbolic": "0.0625312289428711", "gate_mean": "0.4083217553710938" }, { "epoch": "3", "grounder_lr": "0.003", "reward_mean": "11.701885479431152", "penalty_mean": "0.0", "probe_bleu_mean": "11.701885479431152", "probe_improvement_mean": "-1.6047564865493775", "attn_dinov2": "0.20582481323242188", "attn_reptok": "0.04416276542663574", "attn_bge": "0.6874986694335937", "attn_sonar": "2.500000424268798e-06", "attn_symbolic": "0.06251125", "gate_mean": "0.49629014282226563" }, { "epoch": "4", "grounder_lr": "0.003", "reward_mean": "11.824784415283203", "penalty_mean": "0.0", "probe_bleu_mean": "11.824784415283203", "probe_improvement_mean": "-1.5109344047164917", "attn_dinov2": "0.20042396728515624", "attn_reptok": "0.0495532234954834", "attn_bge": "0.6874849951171875", "attn_sonar": "3.7503451707792827e-06", "attn_symbolic": "0.06253406372070312", "gate_mean": "0.555704287109375" }, { "epoch": "5", "grounder_lr": "0.003", "reward_mean": "11.939794059448243", "penalty_mean": "0.0", "probe_bleu_mean": "11.939794059448243", "probe_improvement_mean": "-1.4355954976654053", "attn_dinov2": "0.20392236389160157", "attn_reptok": "0.04606679756164551", "attn_bge": "0.687442138671875", "attn_sonar": "7.4663900583982464e-06", "attn_symbolic": "0.06256123039245605", "gate_mean": "0.60093576171875" }, { "epoch": "6", "grounder_lr": "0.003", "reward_mean": "12.039623599853515", "penalty_mean": "0.0", "probe_bleu_mean": "12.039623599853515", "probe_improvement_mean": "-1.340187070388794", "attn_dinov2": "0.20552587005615233", "attn_reptok": "0.04446780395507813", "attn_bge": "0.6871014135742187", "attn_sonar": "6.25e-06", "attn_symbolic": "0.06289866333007813", "gate_mean": "0.625" }, { "epoch": "7", "grounder_lr": "0.003", "reward_mean": "12.072891585693359", "penalty_mean": "0.0", "probe_bleu_mean": "12.072891585693359", "probe_improvement_mean": "-1.2998664595413207", "attn_dinov2": "0.1957023400878906", "attn_reptok": "0.04510912700653076", "attn_bge": "0.6875024291992188", "attn_sonar": "0.009177352745837416", "attn_symbolic": "0.06250875", "gate_mean": "0.6249999926757812" }, { "epoch": "8", "grounder_lr": "0.003", "reward_mean": "12.099214455871582", "penalty_mean": "0.0", "probe_bleu_mean": "12.099214455871582", "probe_improvement_mean": "-1.2669608753204347", "attn_dinov2": "0.20330639831542968", "attn_reptok": "0.027009047241210938", "attn_bge": "0.68749375", "attn_sonar": "0.01966955144882202", "attn_symbolic": "0.06252125", "gate_mean": "0.625" }, { "epoch": "9", "grounder_lr": "0.003", "reward_mean": "12.090485440063476", "penalty_mean": "0.0", "probe_bleu_mean": "12.090485440063476", "probe_improvement_mean": "-1.280975147666931", "attn_dinov2": "0.2021640591430664", "attn_reptok": "0.021767183303833006", "attn_bge": "0.6874988354492187", "attn_sonar": "0.026058672027587892", "attn_symbolic": "0.06251125", "gate_mean": "0.624999990234375" }, { "epoch": "10", "grounder_lr": "0.003", "reward_mean": "12.106442535705566", "penalty_mean": "0.0", "probe_bleu_mean": "12.106442535705566", "probe_improvement_mean": "-1.2122067555999756", "attn_dinov2": "0.1984944680786133", "attn_reptok": "0.044363313446044925", "attn_bge": "0.6874975", "attn_sonar": "0.007138467233181", "attn_symbolic": "0.06250625", "gate_mean": "0.6249999340820312" }, { "epoch": "11", "grounder_lr": "0.003", "reward_mean": "12.07603073272705", "penalty_mean": "0.0", "probe_bleu_mean": "12.07603073272705", "probe_improvement_mean": "-1.308482506904602", "attn_dinov2": "0.19575637176513672", "attn_reptok": "0.05226521553039551", "attn_bge": "0.6874938745117187", "attn_sonar": "0.001977037742137909", "attn_symbolic": "0.0625075", "gate_mean": "0.62499998046875" }, { "epoch": "12", "grounder_lr": "0.003", "reward_mean": "12.115396923828126", "penalty_mean": "0.0", "probe_bleu_mean": "12.115396923828126", "probe_improvement_mean": "-1.2156711840438843", "attn_dinov2": "0.20084577117919922", "attn_reptok": "0.046818733291625975", "attn_bge": "0.687495", "attn_sonar": "0.0023342448043823243", "attn_symbolic": "0.06250625", "gate_mean": "0.6249999877929687" }, { "epoch": "13", "grounder_lr": "0.003", "reward_mean": "12.112294946289062", "penalty_mean": "0.0", "probe_bleu_mean": "12.112294946289062", "probe_improvement_mean": "-1.2487746599960328", "attn_dinov2": "0.2023041305541992", "attn_reptok": "0.04376473701477051", "attn_bge": "0.687495", "attn_sonar": "0.003929882822036743", "attn_symbolic": "0.06250625", "gate_mean": "0.624999912109375" }, { "epoch": "14", "grounder_lr": "0.003", "reward_mean": "12.123591291503907", "penalty_mean": "0.0", "probe_bleu_mean": "12.123591291503907", "probe_improvement_mean": "-1.2532201410675048", "attn_dinov2": "0.21624920654296875", "attn_reptok": "0.03211429405212402", "attn_bge": "0.6874925", "attn_sonar": "0.0016364978528022767", "attn_symbolic": "0.0625075", "gate_mean": "0.6249999169921875" }, { "epoch": "15", "grounder_lr": "0.003", "reward_mean": "12.089634977416992", "penalty_mean": "0.0", "probe_bleu_mean": "12.089634977416992", "probe_improvement_mean": "-1.237399223098755", "attn_dinov2": "0.21532296020507813", "attn_reptok": "0.032059312210083006", "attn_bge": "0.687495", "attn_sonar": "0.002615226607322693", "attn_symbolic": "0.06250750061035157", "gate_mean": "0.625" }, { "epoch": "16", "grounder_lr": "0.003", "reward_mean": "12.14388571899414", "penalty_mean": "0.0", "probe_bleu_mean": "12.14388571899414", "probe_improvement_mean": "-1.2089018259429931", "attn_dinov2": "0.2180349346923828", "attn_reptok": "0.027005147666931153", "attn_bge": "0.6874987182617187", "attn_sonar": "0.004957448482513427", "attn_symbolic": "0.06250375", "gate_mean": "0.625" }, { "epoch": "17", "grounder_lr": "0.003", "reward_mean": "12.145653394470214", "penalty_mean": "0.0", "probe_bleu_mean": "12.145653394470214", "probe_improvement_mean": "-1.2380234526062013", "attn_dinov2": "0.22155733154296875", "attn_reptok": "0.02528397689819336", "attn_bge": "0.6874987475585937", "attn_sonar": "0.00315619348526001", "attn_symbolic": "0.06250375", "gate_mean": "0.625" }, { "epoch": "18", "grounder_lr": "0.003", "reward_mean": "12.067479884033203", "penalty_mean": "0.0", "probe_bleu_mean": "12.067479884033203", "probe_improvement_mean": "-1.2526547148895264", "attn_dinov2": "0.22328082458496093", "attn_reptok": "0.02500234573364258", "attn_bge": "0.687495", "attn_sonar": "0.0017143307161331177", "attn_symbolic": "0.0625075", "gate_mean": "0.625" }, { "epoch": "19", "grounder_lr": "0.003", "reward_mean": "12.111037466430664", "penalty_mean": "0.0", "probe_bleu_mean": "12.111037466430664", "probe_improvement_mean": "-1.2978308308792115", "attn_dinov2": "0.22049634948730468", "attn_reptok": "0.028337568588256836", "attn_bge": "0.68749625", "attn_sonar": "0.001164832580089569", "attn_symbolic": "0.062505", "gate_mean": "0.6250000024414063" }, { "epoch": "20", "grounder_lr": "0.003", "reward_mean": "12.111812652282715", "penalty_mean": "0.0", "probe_bleu_mean": "12.111812652282715", "probe_improvement_mean": "-1.226981814994812", "attn_dinov2": "0.221491982421875", "attn_reptok": "0.027471806869506836", "attn_bge": "0.68749625", "attn_sonar": "0.0010349592447280883", "attn_symbolic": "0.062505", "gate_mean": "0.6249999755859375" }, { "epoch": "21", "grounder_lr": "0.003", "reward_mean": "12.103694259643555", "penalty_mean": "0.0", "probe_bleu_mean": "12.103694259643555", "probe_improvement_mean": "-1.288998687095642", "attn_dinov2": "0.2185437286376953", "attn_reptok": "0.029456267776489256", "attn_bge": "0.68749375", "attn_sonar": "0.001997503669261932", "attn_symbolic": "0.06250875", "gate_mean": "0.6249999963378906" }, { "epoch": "22", "grounder_lr": "0.003", "reward_mean": "12.093752270202637", "penalty_mean": "0.0", "probe_bleu_mean": "12.093752270202637", "probe_improvement_mean": "-1.2709952585601807", "attn_dinov2": "0.21981218170166017", "attn_reptok": "0.028240793380737306", "attn_bge": "0.6874925", "attn_sonar": "0.0019445244884490967", "attn_symbolic": "0.06251", "gate_mean": "0.6249999584960938" }, { "epoch": "23", "grounder_lr": "0.003", "reward_mean": "12.08480926574707", "penalty_mean": "0.0", "probe_bleu_mean": "12.08480926574707", "probe_improvement_mean": "-1.278890517692566", "attn_dinov2": "0.21898511291503905", "attn_reptok": "0.029912626190185546", "attn_bge": "0.6874918408203124", "attn_sonar": "0.0011022620797157287", "attn_symbolic": "0.06250815948486328", "gate_mean": "0.625" } ] }, { "key": "mse_init_bleu_probe", "label": "MSE Init + BLEU Probe", "kind": "Diagnostic: BLEU-probe training initialized from MSE", "checkpoint": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_fixedprobe_attention_validpath760k_from_mse_activated_20260818/attention_latest.pt", "summary": { "grounder_kind": "attention", "fixed_probe": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_bleu_probe_fixed_validpath760k/probe_once/round_000/probes/probe_00.pt", "data_dir": "$PROJECT_ROOT/gw_setup/outputs/gw6_runs/gw6_imagenet_16h_plus96h_small_1gpu_sonar_m1_c16_v2p5_s24/reptok_sonar_bleu_probe_valid_path_pool_760k_dense", "initial_state_mode": "activated_encoder", "init_attention_checkpoint": "gw_setup/outputs/gw6_runs/gw6_modsweep57_12h_1gpu_fixedlr1e4_steps24900_gw1024_h4w1024_k5_di_rt_be_so_sy_s03/reptok_sonar_iterative_grounding_short_t5_mse_20260818_activated_start_lr3e4_full_e100_pat20/short_T5_keep/attention.pt", "num_pool_rows": 760000, "xperf_samples": 200, "xperf_sentence_bleu_mean": 12.847923278808594, "xperf_corpus_bleu": 12.847922656914447, "history_last": { "epoch": 7, "grounder_lr": 0.003, "reward_mean": 12.61102781982422, "penalty_mean": 0.0, "probe_bleu_mean": 12.61102781982422, "probe_improvement_mean": -0.7617302628707886, "attn_dinov2": 0.61583419921875, "attn_reptok": 0.12840067687988282, "attn_bge": 0.25576511657714845, "attn_sonar": 0.0, "attn_symbolic": 0.0, "gate_mean": 0.6516761157226563 } }, "history": [ { "epoch": "0", "grounder_lr": "0.003", "reward_mean": "12.704764717407226", "penalty_mean": "0.0", "probe_bleu_mean": "12.704764717407226", "probe_improvement_mean": "-0.6777033239555359", "attn_dinov2": "0.5744522436523437", "attn_reptok": "0.08094179786682129", "attn_bge": "0.3274293632507324", "attn_sonar": "0.0006732486291011882", "attn_symbolic": "0.016503350825610982", "gate_mean": "0.5498553479003906" }, { "epoch": "1", "grounder_lr": "0.003", "reward_mean": "12.311731846618652", "penalty_mean": "0.0", "probe_bleu_mean": "12.311731846618652", "probe_improvement_mean": "-1.0235992692184448", "attn_dinov2": "0.5591644219970703", "attn_reptok": "0.2185685317993164", "attn_bge": "0.21159610168457033", "attn_sonar": "2.7141628352724846e-05", "attn_symbolic": "0.010643798242023913", "gate_mean": "0.4642501843261719" }, { "epoch": "2", "grounder_lr": "0.003", "reward_mean": "12.203419510803222", "penalty_mean": "0.0", "probe_bleu_mean": "12.203419510803222", "probe_improvement_mean": "-1.1765999809074401", "attn_dinov2": "0.42290928527832033", "attn_reptok": "0.285150417175293", "attn_bge": "0.2616749865722656", "attn_sonar": "0.010380135772228242", "attn_symbolic": "0.019885178637504577", "gate_mean": "0.6069102038574219" }, { "epoch": "3", "grounder_lr": "0.003", "reward_mean": "12.599114219360352", "penalty_mean": "0.0", "probe_bleu_mean": "12.599114219360352", "probe_improvement_mean": "-0.7075277526664734", "attn_dinov2": "0.5918104919433593", "attn_reptok": "0.15517103713989258", "attn_bge": "0.2522218157958984", "attn_sonar": "0.0007966581399738788", "attn_symbolic": "2.8795483792559693e-17", "gate_mean": "0.6486981860351563" }, { "epoch": "4", "grounder_lr": "0.003", "reward_mean": "12.53028297241211", "penalty_mean": "0.0", "probe_bleu_mean": "12.53028297241211", "probe_improvement_mean": "-0.8054357763671876", "attn_dinov2": "0.6427160656738281", "attn_reptok": "0.0932621060180664", "attn_bge": "0.26402182891845705", "attn_sonar": "1.55144159349486e-22", "attn_symbolic": "0.0", "gate_mean": "0.6608368310546875" }, { "epoch": "5", "grounder_lr": "0.003", "reward_mean": "12.588148276672364", "penalty_mean": "0.0", "probe_bleu_mean": "12.588148276672364", "probe_improvement_mean": "-0.7872411705780029", "attn_dinov2": "0.5721079907226563", "attn_reptok": "0.16907184020996094", "attn_bge": "0.25881517547607424", "attn_sonar": "0.0", "attn_symbolic": "5e-06", "gate_mean": "0.6531617614746094" }, { "epoch": "6", "grounder_lr": "0.003", "reward_mean": "12.577016141662599", "penalty_mean": "0.0", "probe_bleu_mean": "12.577016141662599", "probe_improvement_mean": "-0.8027946344947815", "attn_dinov2": "0.624036357421875", "attn_reptok": "0.125899454498291", "attn_bge": "0.25006293029785154", "attn_sonar": "8.043358684517444e-09", "attn_symbolic": "1.25e-06", "gate_mean": "0.648493564453125" }, { "epoch": "7", "grounder_lr": "0.003", "reward_mean": "12.61102781982422", "penalty_mean": "0.0", "probe_bleu_mean": "12.61102781982422", "probe_improvement_mean": "-0.7617302628707886", "attn_dinov2": "0.61583419921875", "attn_reptok": "0.12840067687988282", "attn_bge": "0.25576511657714845", "attn_sonar": "0.0", "attn_symbolic": "0.0", "gate_mean": "0.6516761157226563" } ] } ], "loop_example": { "id": 2708, "reference": "A small bird with a grey body, black head, white cheek patches, and a prominent curved crest is perched on a broken, brown, textured branch in the center-right of the frame. The background consists of a dense, out-of-focus mass of green leaves and branches, creating a natural, leafy environment with soft, diffused lighting.", "direct": "A small brown-headed bird with a light brown spine, a black-brown beak, a white shell, and a dark gray tail, is perched on a narrow, light-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green", "mse": "A young brown bird with a black head, white crest, and black tail is perched on a thick, light-colored branching trunk in the center behind the frame. The background is a light-focused mixture of green and white buds, featuring dazzling green foliage, suggesting a natural, light-filled environment.", "scratch_bleu_probe": "A small brown-headed bird with a black crown, white shell, and a black tail and white tail perches on a thin, gray-bodied branch in the center of the frame, with a slightly back-centred branch on the top. The background is a loose, light-focused mixture of green and brown foliage, featuring light green and branched flowers, suggesting a natural, light-green enclosure with a brightly colored outdoor plant.", "mse_init_bleu_probe": "A small brown bird with a black white head, a brown crest, a white beak, and a dark brown tail is perched on a thin, gray-coloured branch of foliage on a center-right edge of the frame. The background is a soft-focused blend of light-green and brown outlines, featuring a soft, outward-looking green plant, suggesting a gentle, dark environment with a naturally occurring green outline." }, "imagenet_examples": [ { "id": 1297, "reference": "A rooster with vibrant orange and red feathers on its neck and head, a prominent red comb, and a large red wattle is positioned in the foreground, occupying the left and central portion of the frame. The background consists of out-of-focus green foliage and brown earth, suggesting a natural, outdoor setting with soft, diffused lighting.", "direct": "A red rooster with a prominent swollen coat and a vivid, reddish-brown beak, black and red oat leaf, and a prominent vertical and black hair coat, occupies the foreground and center of the tail, with a dark white background. The background appears to be in a thick green background. A background of dark wood and foliage is white, with a conspicuous green-green foliage outside, suggesting a natural, light green background.", "mse": "A red rooster with a prominent comma and yellow fur, a red and black waistcoat, and a prominent red and green tail, occupies the center of the post, with a dark green background.", "scratch_bleu_probe": "A large rooster with a bright red coat and umbilical tail, a bright red and oval feather, and a prominent black and red tail occupy the foreground, with a slightly downward-facing center. The background is in a pale green-green background, featuring dark, airy bushes on the front, with a white and glossy woodland visible on the rear, with dark roots on the outside.", "mse_init_bleu_probe": "A bright rooster with a prominent red comb and hairy wick, a bright red and oval comb, and a prominent dark and yellow coat, occupies the foreground of the frame, with a dark-green background. The background consists of a faint green and light-green foliage, with a conspicuously dark ground outside, a white and woody stem, with a natural black spot visible to the rear." }, { "id": 2056, "reference": "A male house finch, characterized by its bright red head and breast, mottled brown and gray wings, and a conical beak, is perched on a thin, light-brown branch in the center of the frame. The background consists of out-of-focus green leaves and branches, with some small, unopened buds visible, suggesting a natural, leafy environment under soft, diffused lighting.", "direct": "A white-headed tree frog with a brown head and chest, reddish brown head and beak, pale chest and brown tail, and a narrow, dark-brown branch on the center of the head, is nestled on a stiff, reddish-brown branch on the center of the frame. The background consists of a faint out-of-frame base and green foliage, with a light brown and green branches pointing out, featuring a patterned outdoor environment, with a light green and dusty background.", "mse": "A white-headed Phoenix star and its head, brown and red beak, and narrow-leaved head and neck, are nested on a reddish-brown branch in the center of the frame. The background consists of light-green outer branches and foliage, suggesting a light-green tree and foliage, with a natural outdoor setting.", "scratch_bleu_probe": "A white-headed tree frog with a reddish and brown head, brown chest and tail, and a dark-brown and white-throated tail, is perched on a center-left branch in the center of the frame. The background consists of a light-greenish-greenish-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green.", "mse_init_bleu_probe": "A white-headed bird with a red chest and brown head, brown forehead and tail, and a reddish-brown beak and lower wing, is nestled on a center-left branch in the center of the frame. A dark-brown background is centered on a loose-leaved branch of green and brown leaves and buds, featuring a faint green and brown background, depending on a clear, dense, outdoor setting with green light." }, { "id": 2708, "reference": "A small bird with a grey body, black head, white cheek patches, and a prominent curved crest is perched on a broken, brown, textured branch in the center-right of the frame. The background consists of a dense, out-of-focus mass of green leaves and branches, creating a natural, leafy environment with soft, diffused lighting.", "direct": "A small brown-headed bird with a light brown spine, a black-brown beak, a white shell, and a dark gray tail, is perched on a narrow, light-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green-green", "mse": "A young brown bird with a black head, white crest, and black tail is perched on a thick, light-colored branching trunk in the center behind the frame. The background is a light-focused mixture of green and white buds, featuring dazzling green foliage, suggesting a natural, light-filled environment.", "scratch_bleu_probe": "A small brown-headed bird with a black crown, white shell, and a black tail and white tail perches on a thin, gray-bodied branch in the center of the frame, with a slightly back-centred branch on the top. The background is a loose, light-focused mixture of green and brown foliage, featuring light green and branched flowers, suggesting a natural, light-green enclosure with a brightly colored outdoor plant.", "mse_init_bleu_probe": "A small brown bird with a black white head, a brown crest, a white beak, and a dark brown tail is perched on a thin, gray-coloured branch of foliage on a center-right edge of the frame. The background is a soft-focused blend of light-green and brown outlines, featuring a soft, outward-looking green plant, suggesting a gentle, dark environment with a naturally occurring green outline." }, { "id": 6030, "reference": "A box turtle with a dark, glossy shell patterned with bright yellow, irregular, lightning-bolt-like markings is positioned in the center of the frame, resting on a large, gray, textured rock. The turtle's head and front legs are visible, and it is surrounded by green leaves in the upper right and lower right corners, with natural daylight illuminating the scene.", "direct": "A turtle shell with a dark box-shaped patch and yellow spotted shell with dark orange patterns is positioned in the center of the frame, the shell resting on a dark and green leaf-like texture. The shell rests on a green leaf-like texture, with dark green leaves and bark.", "mse": "A turtle box with a dark, patterned yellow shell with dark and yellow markings is positioned in the center of the frame, with the shell resting on light green leaves. The shell is surrounded by a dark, green leafy texture, with some green and shiny leaves on the back, displaying a clear natural glow in the background.", "scratch_bleu_probe": "A dark turtle with a light brown shell and patterned dots of light grey is situated in the center of the frame, resting on a soft, turquoise-green leaf background. The background consists of a light green, dusty bark, with light green leaves and shells, with a light green pollen, and white spots visible in the background.", "mse_init_bleu_probe": "A turtle shell with a dark, yellow-shaped patch and lighter-coloured spots are situated in the center of the frame, resting on a soft, green leaf surface. The shell is surrounded by a dark, green leafy texture, with light brown and green leaves, and a white, yellowish texture, visible in the center." }, { "id": 6662, "reference": "A bright green lizard, likely an American chameleon, is in the foreground, positioned on a weathered, textured tree stump; its body is elongated and its limbs are splayed as it appears to be climbing or looking upward. The background consists of lush green foliage and small white flowers, with the scene illuminated by natural daylight, creating a vibrant, outdoor setting.", "direct": "A light green lighter-skinned lizard with a glossy green shell and a slender, long-stemmed leg is positioned in the center of the background, resting on a pale green background, with its head resting on a thin leaf. The background is surrounded by a lush green foliage, with dark green foliage and large leaves, with a pale green foliage behind.", "mse": "A bright green lizard with a smooth-skinned body and a short tail is positioned in the center of the frame, leaning to the left in a large, foliated body. The background consists of dark green foliage, with extensive foliage and white flowers, with a dim green background, illuminated by natural, glowing light.", "scratch_bleu_probe": "A bright green lighter-skinned creature with a smooth body and a slender tail is positioned at the front of the body, resting in the background in a light grey leaf. The background consists of lush green foliage forests and small flowers, with glowing green leaves behind, with some natural dusting, in the light of the sun.", "mse_init_bleu_probe": "A bright green glossy lizard with a smooth skin and a slender leg and a flattened frame rests in the center of the background, resting on a light green background. The background consists of a dark green foliage with large foliage leaves and a green shell, with some light green wood behind, visible in a clear, natural light." }, { "id": 6881, "reference": "A vibrant agama lizard, displaying a gradient of bright purple on its head and back transitioning to deep blue on its limbs and tail, lies horizontally across the lower portion of the frame on a rough, light gray rock surface. The background consists of a darker, textured rock face above the lizard, with a distinct shadow cast to the left, indicating a light source from the right.", "direct": "A luminous lizard lizard with a vivid reddish-brown body and a glossy, blue tail and dark-skinned body and a broad base is positioned in the center of the frame, resting on a reddish-brown rock, resting on a shallow, textured surface. The background is composed of a dark brown rock, with a translucent dusty surface with light, dark rocks on the surface.", "mse": "A luminous lizard with a vivid blue body and a dark, yellow tail and center position is positioned on the rear of the frame, resting in a dark, clay-like ground. The background consists of a shiny, dark brown surface with some rocks on the bottom, reflecting light and dark on the top, featuring a clear sunny background.", "scratch_bleu_probe": "A vibrant roseate lizard with a bright green face and a textured brown body and dark tail is positioned vertically on the top of the frame, resting in a light brown ground, resting on a rocky surface. The background consists of a light, dusty brown rock, with light brown surfaces on the outside.", "mse_init_bleu_probe": "A vibrant bright-red lizard with a glossy blue torso and a light-colored body and dark outer tail is positioned in the center of the frame, resting on a smooth, rocky surface. The body rests on a light-brown rock, with a dark, translucent ground skin over the top. A dark, light-brown shadow appears on the outside, with a bright red background." }, { "id": 8699, "reference": "A dark, slender snake with a smooth, glossy black body and a distinct yellow ring around its neck is positioned horizontally across the center of the frame, resting on a bed of small, light-colored pebbles. The snake is nestled between large, gray, rough-textured rocks that form the background, with patches of white and gray lichen visible on the rock surfaces under diffuse, natural lighting.", "direct": "A dark, slender grey stone snake with a loose, smooth, grey stone elbow is arranged across a smooth surface across the center of the frame, rolling horizontally on a shallow, rocky surface. The background is composed of a smooth, grainy stone rock, with dark rocky surfaces of different shapes, granite and gray rock, with a light, shaded grey dust surrounding the surface.", "mse": "A dark, light grey snake with a flat, silvery head is positioned around the frame in a smooth, rocky shape. The rock consists of rocky, rough stone surfaces on one side, with dark grey and granite pebbles on the other.", "scratch_bleu_probe": "A dark dark, dark grey stone with a smooth, smooth-shaped dragon ball rests on a flat surface in the middle of the rock, curving to a smooth-shaped bottom. The background of the rock consists of a dark gray, rocky surface with smooth granite rocks and gravels, with a light yellow rock on the sides, and a transparent grey light appearing on the outside of the stone.", "mse_init_bleu_probe": "A dark, light gray, slender serpent rock with a smooth, light gray body is arranged horizontally along a narrow edge of the frame in a smooth, rocky surface. The background consists of a dark, grey stone rock with smooth, granite rocks on one side and dark grey, silvery stones on the other, with a light gray shadow on the outside." }, { "id": 9684, "reference": "A vibrant green vine snake is positioned in the foreground, its slender, elongated body resting on a dark, textured piece of wood that occupies the lower-left portion of the frame. The background is a soft-focus blend of dark green and black, suggesting a natural, leafy environment, with the snake's head and upper body extending towards the right side of the image.", "direct": "A vivid green glowing green snake with a smooth, elongated, green shell with a slender, smooth-shaped body is arranged horizontally on the underside and a dark, horizontal head is resting on the top edge of the tree, with a smooth, dark brown body in the upper edge of the frame. The background is dimly scattered, with a dark, light-brown texture.", "mse": "A vibrant green snake with a slender, light green body is arranged in a slanted and horizontal shape, with the upper part of the head in a dark brown and the upper part of the head in a light brown.The background is dimly pointed, suggesting a dark and opaque texture.", "scratch_bleu_probe": "A vivid green green snake with a slender, glossy plant tail and a smooth, scalloped body is positioned horizontally on the top with a dark, branching head resting in the top-right corner of the frame. The background is a light-colored, dark-brown texture, suggesting a dark, organic green texture, with a clear, shaded green area in the background.", "mse_init_bleu_probe": "A vibrant green green snake with a smooth, glossy green snake and a slender, smooth-shaped shell is positioned in a horizontal and vertical position, with the back of the head in a dark, brownish texture. The background is a light-colored, dark-brown texture, with a light, dark green texture, suggesting a dark, hollow, green texture." } ], "coco_examples": [ { "id": 318908, "reference": "A white dog sitting on red table next to metal object.", "direct": "A small, light fluffy, fluffy little dog with a pale white fur and dark black eyes and a dark moustache is positioned in the center of the frame, looking directly at the center of the camera, with a dark painted face. The dog is seated in a dark white carpet with a bright, tan background, with a white painted frame on the left side and a dark, metallic lid on the right.", "mse": "A small, fluffy white dog with light black hair and dark eyes is positioned in the center of the frame, looking directly at the camera. The dog is dressed in a dark, fluorescent suit with a white wall on the front, a dark orange background and a bright white bag on the back.", "scratch_bleu_probe": "A small, soft, fluffy white puppy with small white eyes and dark faces is placed in the foreground, looking down at the center of the frame, with a slightly darker room. The background is covered in pink fabric, featuring a brightly colored fluffy wooden trunk with white and marbled borders, a white fluffy canvas visible to the right, and a dark, fluffy hooded object visible to the left in the center.", "mse_init_bleu_probe": "A small, light fluffy white puppy, black furry dog with light eyes and a dark nose and dark eyes is placed in the center of the frame, looking up to the center of the frame, looking directly at the camera. The background is covered with a light brown cloth, with a dark, fluffy wooden posture and a white canvas, visible on the left side, a light-colored frame, and a dark, dark baguette appearing on the right behind the puppy." }, { "id": 484893, "reference": "A couple of kids petting sheep inside of a corral", "direct": "A young white-haired girl with a light brown fur is in the foreground, seated on the left-hand side of the frame, with a light brown coat on the back. In the background is a white-haired cat, lying on a wooden pole. In the background, a small white-haired pig is resting in front of the frame, with a wooden blanket.", "mse": "A young white-haired cat with a light brown sleeve is in the foreground, lying on the left side of the frame, with the head resting in a light brown sleeve. In the background, a dog with a small wooden fleece is seen, flanked on the right by a white-colored fleece, with a large wooden cage in the background.", "scratch_bleu_probe": "In the foreground, a young child with light brown hair, plain fur coat, is lying on the left-hand side of the frame, with a small white-colored piggy in the center of the frame. In the background, a child with light brown cotton pants, rests in a white wooden cage with a light brown coat on the back.", "mse_init_bleu_probe": "A light-skinned boy with a white fur coat and a light-colored fleece is sitting in the foreground, on the left side of the frame, focusing on a large wooden bed. In the background, a man with a light-colored fleece is lying in the background, with a small wooden cage surrounded by a white cord." }, { "id": 458045, "reference": "Little boys playing on a baseball field in baseball uniforms.", "direct": "A baseball player in a blue-collar shirt jacket in a white helmet and black field kit is in the foreground, holding a grass-coloured baseball bat in front of him. A player in the background is standing in the middle of the field, with a light blue-collar jacket on the back.", "mse": "A baseball player in a white shirt and a black helmet is in the foreground, holding the field in front of him, with a gray-backed box behind him. In the background, a player with a black belt is standing in front of the field, with a chain-link fence.", "scratch_bleu_probe": "In the background, a white man in a baseball cap and a gray-white shirt is standing in the middle of the field, with a blue-collared fence behind the player's head. In the background, a chain-linked field is in the background, with a white-collared box and a gray field visible behind the player's right hand.", "mse_init_bleu_probe": "In a baseball playoff game, a white player in a blue shirt and a black helmet is in the foreground with a baseball bat in his left field, gazing into the center of the field with a gray chain behind him. In the background, a player is standing in the middle of a chain-link fence, with a black field jacket and chain behind him." }, { "id": 276055, "reference": "A woman holding up a large carrot in a backyard.", "direct": "A woman in a dark brown-haired top dress, wearing a blue lampshade, is standing in the foreground with a light-colored beard, holding a green-colored potato in her hand, with a white and tan cap in front. The background is framed by a bright yellow wooden door, with a green canopy on the left, a green flower garden with a white canopy on the right, and a bright green tree in the background.", "mse": "A woman in a dark brown hair and a light purple dress is in the foreground, holding a green t-shirt in her left hand, holding a brightly colored potato in her right hand. A green window with a wooden floor is placed in front of her, with a green lantern at the bottom, a green tree and a white canopy in the center, with bright red trees on the side.", "scratch_bleu_probe": "A woman in a light brown hair and a light green t-shirt stands in the foreground, holding a green cup in front of her, holding the center of the flower in a light yellow. The background features a lush green tree with a light green canopy, a tree with green lights on the side, a white oak tree underneath, and a bright green flower box on the right, with some green leaves visible in the background.", "mse_init_bleu_probe": "A woman in a light brown hair and a light pink waistcoat is in the foreground, holding a light blue canvas to the left, holding a large tree in her hand. The background is covered with a bright yellow cloth, with a green wooden tent in front, a green window with leaves, and a white cedar tree underneath, with a bright green fireplace and some green flooring visible in the far right." }, { "id": 502336, "reference": "A group of people walking down a street next to a tall building.", "direct": "In the left-hand corner, a large light-colored and brown poster stands on a sidewalk, with a white vertical sign in the foreground, surrounded by a black-and-white pedestal. The front of the building features a white background, with a black-and-yellow sign in the center. In the left-hand corner, there is a statue displaying a \"black-and-white\" traffic sign.", "mse": "A white, dark-colored pedestrian poster stands in the center with a white striped sign, viewed to the left in a large frame, with the word \"Street\" embedded in the center. A black and white sign displaying a shopping street, with the words \"A white sign\" and \"A white sign\" in front of it.", "scratch_bleu_probe": "A large black-and-white street sign is in the center of the picture, standing on a vertical sidewalk, with the small sign in front of the wall, occupying a white and brown parking lot in the foreground. In the background is a yellow street sign, with a large black-and-white building on the doorstep, a street sign with a white flag in front, and two buildings with a white sign on the bottom, and a street sign with white tiles on the bottom.", "mse_init_bleu_probe": "A white, dark-colored poster in the front of the vehicle is positioned in a central yellow and black signpost, with the left-hand side overlooking a large brick structure. In the background is a \"black\" street sign, with a large building in the background showing a black-and-white signpost, a street with a white signboard, a \"French street\" sign, and a white signboard with many buildings on it." }, { "id": 125778, "reference": "A living room with dark colored furniture and a fire place.", "direct": "A dark blue, upholstered cushioned bed chair is located in the foreground on the left-hand side of the frame, with a light-colored sofa in the background. In the background is a white-washed tablecloth, with a colored lampshade on a left-hand side. A white-washed tablecloth, with a white lampshade on a left-hand side, and a green tablecloth, with a white lampshade on a left-hand side, are featured.", "mse": "A dark blue, upholstered couch chair with a rectangular seat is placed in the foreground at the bottom of the frame, with a dark leathered dining table in the center. In the background, a light-coloured bamboo table with a white drape dominates the room, with a green tablecloth illuminated to the left, a white tapestry with a fireplace in the center, and a white tablecloth illuminated to the right.", "scratch_bleu_probe": "A dark, thick, leather-colored sofa bed is seated in the foreground on the left side of the frame. In the background, a light-colored carpeted table with dark draperies overlaps the table, with a light-colored lamplight illuminating on each side, and red chairs are hidden from view.", "mse_init_bleu_probe": "A dark brown bed with a textured cushioned couch in a white frame is placed at the rear of the seat in the center of the background. In the background, a dark wood-coloured sofa occupies a table with a bright white canvas, with a lampshade on the left, a green tablecloth illuminated by a tablecloth, and a bright sunlight on the left." }, { "id": 563604, "reference": "A sign outside of a closed road redirecting traffic", "direct": "A large, red, rectangular black centerfold with an illuminated yellow background stands in the foreground, highlighted in black on the background, with a large illuminated table in left and right. The background is depicted in yellow background, with a number of snowboards in front.", "mse": "A large, red rectangular scoreboard, with a white background, stands in the foreground of the figure, featuring a yellow and white background. In the background, a large, red rectangular scoreboard, with a white background, is positioned in the middle of the figure. A large, red rectangular scoreboard, with a white background, stands in the foreground, featuring a yellow and white background. A large, red rectangular scoreboard, with a white background, is positioned in the middle of the figure.", "scratch_bleu_probe": "A large, red, black-colored scoreboard with a central indicator stands in the background in yellow, highlighting the center of the scene in black and white, with a numbered floor behind it. In the background is a green children's playground, with a black-and-white wall in front of it, trees in the background, and a white flag surrounding it.", "mse_init_bleu_probe": "A large, red, rectangular black center scoreboard with an illuminated black background stands at the top of the figure, centrally located in the left-hand side of the screen. A large red, rectangular scoreboard with a white background is displayed in the background, with a yellow flag behind it." }, { "id": 370677, "reference": "Three workers stand next to each other with their baked goods behind them.", "direct": "In the front of the two men, two men stand in a green skirt, dressed in shorts, one man is standing on the left-hand side in a pale jacket; the other man is wearing a white shirt with shorts in front; the man in the left-hand side is wearing a pink jacket and a light-colored suitcase, while the front man is wearing a light-colored beige jacket and bags; the background is covered with a light-colored white cardboard with shoelaces, and two men are wearing a light-colored leaflet on the front.", "mse": "In the front are two men, dressed in a blue sweater, the one on the left is holding a shirt; the one on the right is wearing a pink jacket, while the one on the left is wearing a light-colored suitcase; the one on the right is covered with brightly colored banknotes and labels, and the one on the left is wearing a green jacket.", "scratch_bleu_probe": "In the foreground, a man in a white shirt and brown trousers stands in the foreground, wearing a light-colored shirt in the left, with a man behind in a green shirt and gloves; the background is a light-colored wooden box with many shades of purple, with a storefront covered with green boxes and flowers.", "mse_init_bleu_probe": "In the front of the picture, two men stand in a brown shirt and white shorts, one standing with a pink bag in the foreground, the other with a yellow shirt in the center; the back is covered with a light-colored men's suitcase, with a light-colored barbecue container in the foreground." } ] }