{"benchmarks":{"adversarial_nli":{"dated_n":0,"featured":false,"has_cost":false,"id":"adversarial_nli","latest_eval":null,"metrics_n":0,"n":15,"name":"Adversarial Nli","score_col":"Score","source_status":"ok","source_url":"https://arxiv.org/pdf/1910.14599","trace_n":0,"uncertainty_n":0},"aider_polyglot":{"cost_basis":"reported","cost_col":"Cost","cost_label":"Reported cost (USD)","dated_n":61,"featured":true,"has_cost":true,"id":"aider_polyglot","latest_eval":"2025-10-03","metrics_n":61,"n":72,"name":"Aider Polyglot","score_col":"Percent correct","source_status":"ok","source_url":"https://aider.chat/docs/leaderboards/","trace_n":0,"uncertainty_n":0},"ale_bench":{"cost_basis":"reported","cost_col":"Cost","cost_label":"Reported cost (USD)","dated_n":0,"featured":false,"has_cost":true,"id":"ale_bench","latest_eval":null,"metrics_n":132,"n":132,"name":"ALE-Bench","score_col":"Performance","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"algotune":{"dated_n":0,"featured":false,"has_cost":false,"id":"algotune","latest_eval":null,"metrics_n":0,"n":18,"name":"AlgoTune","score_col":"Score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"apex_agents":{"dated_n":0,"featured":false,"has_cost":false,"id":"apex_agents","latest_eval":null,"metrics_n":0,"n":60,"name":"APEX Agents","score_col":"Pass@1 score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":60},"arc_agi":{"cost_basis":"task","cost_col":"Cost per task","cost_label":"$ per task","dated_n":0,"featured":false,"has_cost":true,"id":"arc_agi","latest_eval":null,"metrics_n":0,"n":309,"name":"Arc Agi","score_col":"Score","source_status":"ok","source_url":"https://arcprize.org/leaderboard","trace_n":0,"uncertainty_n":0},"arc_agi_2":{"cost_basis":"task","cost_col":"Cost per task","cost_label":"$ per task","dated_n":0,"featured":true,"has_cost":true,"id":"arc_agi_2","latest_eval":null,"metrics_n":0,"n":290,"name":"ARC-AGI-2","score_col":"Score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"arc_ai2":{"dated_n":0,"featured":false,"has_cost":false,"id":"arc_ai2","latest_eval":null,"metrics_n":0,"n":134,"name":"Arc Ai2","score_col":"Challenge score","source_status":"ok","source_url":"http://arxiv.org/abs/2403.04652","trace_n":0,"uncertainty_n":0},"balrog":{"dated_n":0,"featured":false,"has_cost":false,"id":"balrog","latest_eval":null,"metrics_n":0,"n":41,"name":"BALROG","score_col":"Average progress","source_status":"ok","source_url":"https://balrogai.com/","trace_n":14,"uncertainty_n":41},"bbh":{"dated_n":0,"featured":false,"has_cost":false,"id":"bbh","latest_eval":null,"metrics_n":0,"n":88,"name":"BIG-Bench Hard","score_col":"Average","source_status":"ok","source_url":"https://arxiv.org/pdf/2309.16609","trace_n":0,"uncertainty_n":0},"blueprint_bench_2":{"dated_n":0,"featured":false,"has_cost":false,"id":"blueprint_bench_2","latest_eval":null,"metrics_n":0,"n":35,"name":"Blueprint Bench 2","score_col":"Score","source_status":"ok","source_url":"https://andonlabs.com/evals/blueprint-bench-2","trace_n":0,"uncertainty_n":35},"bool_q":{"dated_n":0,"featured":false,"has_cost":false,"id":"bool_q","latest_eval":null,"metrics_n":0,"n":136,"name":"Bool Q","score_col":"Score","source_status":"ok","source_url":"https://arxiv.org/abs/2309.16609","trace_n":0,"uncertainty_n":0},"btf3":{"dated_n":0,"featured":false,"has_cost":false,"id":"btf3","latest_eval":null,"metrics_n":0,"n":15,"name":"BTF3","score_col":"Pooled score","source_status":"ok","source_url":"https://evals.futuresearch.ai/#btf3","trace_n":0,"uncertainty_n":15},"cad_eval":{"dated_n":0,"featured":false,"has_cost":false,"id":"cad_eval","latest_eval":null,"metrics_n":0,"n":15,"name":"Cad Eval","score_col":"Overall pass (%)","source_status":"ok","source_url":"https://willpatrick.xyz/cadevalresults_20250422_095709/","trace_n":0,"uncertainty_n":0},"chess_puzzles":{"dated_n":162,"featured":false,"has_cost":false,"id":"chess_puzzles","latest_eval":"2026-11-06","metrics_n":0,"n":162,"name":"Chess Puzzles","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":162,"uncertainty_n":162},"cl_bench":{"dated_n":0,"featured":false,"has_cost":false,"id":"cl_bench","latest_eval":null,"metrics_n":0,"n":23,"name":"CL-Bench","score_col":"Overall","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":23},"cl_bench_life":{"dated_n":0,"featured":false,"has_cost":false,"id":"cl_bench_life","latest_eval":null,"metrics_n":0,"n":17,"name":"CL-Bench Life","score_col":"Overall","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":17},"common_sense_qa_2":{"dated_n":0,"featured":false,"has_cost":false,"id":"common_sense_qa_2","latest_eval":null,"metrics_n":0,"n":6,"name":"Common Sense Qa 2","score_col":"Score","source_status":"ok","source_url":"https://arxiv.org/pdf/2201.05320","trace_n":0,"uncertainty_n":0},"critpt":{"cost_basis":"reported","cost_col":"Cost","cost_label":"Reported cost (USD)","dated_n":0,"featured":false,"has_cost":true,"id":"critpt","latest_eval":null,"metrics_n":0,"n":260,"name":"CritPt","score_col":"Accuracy","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"cursorbench":{"cost_basis":"task","cost_col":"Cost per task","cost_label":"$ per task","dated_n":0,"featured":false,"has_cost":true,"id":"cursorbench","latest_eval":null,"metrics_n":82,"n":82,"name":"Cursorbench","score_col":"Score","source_status":"ok","source_url":"https://cursor.com/cursorbench","trace_n":0,"uncertainty_n":0},"cybench":{"dated_n":0,"featured":true,"has_cost":false,"id":"cybench","latest_eval":null,"metrics_n":0,"n":22,"name":"Cybench","score_col":"Unguided % Solved","source_status":"ok","source_url":"https://www-cdn.anthropic.com/0dd865075ad3132672ee0ab40b05a53f14cf5288.pdf","trace_n":0,"uncertainty_n":0},"deepresearchbench":{"dated_n":0,"featured":false,"has_cost":false,"id":"deepresearchbench","latest_eval":null,"metrics_n":0,"n":41,"name":"Deepresearchbench","score_col":"Average score","source_status":"ok","source_url":"https://evals.futuresearch.ai/#drb","trace_n":0,"uncertainty_n":0},"deepswe":{"cost_basis":"task","cost_col":"Mean cost (USD)","cost_label":"Mean $ per task","dated_n":0,"featured":true,"has_cost":true,"id":"deepswe","latest_eval":null,"metrics_n":69,"n":69,"name":"DeepSWE","score_col":"Pass@1","source_error":"generated_at is not current: 2026-09-22T06:27:15.860279+00:00","source_status":"degraded","source_url":"https://deepswe.datacurve.ai/","trace_n":0,"uncertainty_n":69,"via":"Epoch AI (fallback)"},"enigma_eval":{"dated_n":0,"featured":false,"has_cost":false,"id":"enigma_eval","latest_eval":null,"metrics_n":0,"n":46,"name":"Enigma Eval","score_col":"Accuracy","source_status":"ok","source_url":"https://labs.scale.com/leaderboard/enigma_eval","trace_n":0,"uncertainty_n":46},"exploitbench":{"dated_n":0,"featured":false,"has_cost":false,"id":"exploitbench","latest_eval":null,"metrics_n":20,"n":20,"name":"Exploitbench","score_col":"Mean capability","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"fictionlivebench":{"dated_n":0,"featured":false,"has_cost":false,"id":"fictionlivebench","latest_eval":null,"metrics_n":0,"n":42,"name":"Fictionlivebench","score_col":"120k token score","source_status":"ok","source_url":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","trace_n":0,"uncertainty_n":0},"forecastbench":{"dated_n":0,"featured":false,"has_cost":false,"id":"forecastbench","latest_eval":null,"metrics_n":0,"n":92,"name":"Forecastbench","score_col":"Overall score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":92},"frontiercode":{"dated_n":0,"featured":false,"has_cost":false,"id":"frontiercode","latest_eval":null,"metrics_n":0,"n":46,"name":"Frontiercode","score_col":"Main score","source_status":"ok","source_url":"https://cognition.com/frontiercode","trace_n":0,"uncertainty_n":0},"frontiermath":{"dated_n":100,"featured":true,"has_cost":false,"id":"frontiermath","latest_eval":"2026-06-08","metrics_n":0,"n":101,"name":"FrontierMath","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":79,"uncertainty_n":101},"frontiermath_tier_4":{"dated_n":71,"featured":false,"has_cost":false,"id":"frontiermath_tier_4","latest_eval":"2026-06-08","metrics_n":0,"n":72,"name":"FrontierMath Tier 4","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":46,"uncertainty_n":72},"gbaeval":{"dated_n":0,"featured":false,"has_cost":false,"id":"gbaeval","latest_eval":null,"metrics_n":23,"n":23,"name":"Gbaeval","score_col":"Overall score","source_status":"ok","source_url":"https://gbaeval.com/model/claude-opus-5","trace_n":0,"uncertainty_n":0},"gdp_pdf":{"dated_n":0,"featured":false,"has_cost":false,"id":"gdp_pdf","latest_eval":null,"metrics_n":0,"n":56,"name":"Gdp Pdf","score_col":"GDP.pdf score","source_status":"ok","source_url":"https://surgehq.ai/leaderboards/gdp-pdf","trace_n":0,"uncertainty_n":0},"gdpval":{"dated_n":0,"featured":true,"has_cost":false,"id":"gdpval","latest_eval":null,"metrics_n":0,"n":11,"name":"GDPval","score_col":"Win Rate (%)","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"geobench":{"dated_n":0,"featured":false,"has_cost":false,"id":"geobench","latest_eval":null,"metrics_n":0,"n":32,"name":"Geobench","score_col":"ACW Avg Score","source_status":"ok","source_url":"https://geobench.org/","trace_n":0,"uncertainty_n":0},"gpqa_diamond":{"dated_n":333,"featured":true,"has_cost":false,"id":"gpqa_diamond","latest_eval":"2026-11-06","metrics_n":0,"n":334,"name":"GPQA Diamond","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":331,"uncertainty_n":334},"gsm8k":{"dated_n":0,"featured":false,"has_cost":false,"id":"gsm8k","latest_eval":null,"metrics_n":0,"n":170,"name":"GSM8K","score_col":"EM","source_status":"ok","source_url":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","trace_n":0,"uncertainty_n":0},"gso":{"dated_n":36,"featured":false,"has_cost":false,"id":"gso","latest_eval":"2026-09-27","metrics_n":0,"n":42,"name":"GSO","score_col":"Score OPT@1","source_status":"ok","source_url":"https://gso-bench.github.io/index.html","trace_n":0,"uncertainty_n":0},"hella_swag":{"dated_n":0,"featured":false,"has_cost":false,"id":"hella_swag","latest_eval":null,"metrics_n":0,"n":113,"name":"HellaSwag","score_col":"Overall accuracy","source_status":"ok","source_url":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","trace_n":0,"uncertainty_n":0},"hle":{"dated_n":0,"featured":true,"has_cost":false,"id":"hle","latest_eval":null,"metrics_n":54,"n":56,"name":"Humanity's Last Exam","score_col":"Accuracy","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":56},"lambada":{"dated_n":0,"featured":false,"has_cost":false,"id":"lambada","latest_eval":null,"metrics_n":0,"n":53,"name":"Lambada","score_col":"Score","source_status":"ok","source_url":"https://arxiv.org/abs/2309.16609","trace_n":0,"uncertainty_n":0},"lech_mazur_writing":{"dated_n":0,"featured":false,"has_cost":false,"id":"lech_mazur_writing","latest_eval":null,"metrics_n":0,"n":49,"name":"Lech Mazur Writing","score_col":"Mean score","source_status":"ok","source_url":"https://github.com/lechmazur/Writing","trace_n":0,"uncertainty_n":0},"live_bench":{"dated_n":0,"featured":false,"has_cost":false,"id":"live_bench","latest_eval":null,"metrics_n":0,"n":54,"name":"LiveBench","score_col":"Global average","source_status":"ok","source_url":"https://livebench.ai/#/","trace_n":0,"uncertainty_n":0,"versions":["LiveBench-2024-11-25"]},"math_level_5":{"dated_n":108,"featured":false,"has_cost":false,"id":"math_level_5","latest_eval":"2025-10-30","metrics_n":0,"n":108,"name":"MATH Level 5","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":108,"uncertainty_n":108},"metr_time_horizons":{"dated_n":0,"featured":true,"has_cost":false,"id":"metr_time_horizons","latest_eval":null,"metrics_n":0,"n":49,"name":"METR Time Horizons","score_col":"Time horizon","source_status":"ok","source_url":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","trace_n":0,"uncertainty_n":49,"versions":["METR-Horizon-v1.0","METR-Horizon-v1.1"]},"mindcube":{"dated_n":0,"featured":false,"has_cost":false,"id":"mindcube","latest_eval":null,"metrics_n":0,"n":5,"name":"Mindcube","score_col":"Overall score","source_status":"ok","source_url":"https://mind-cube.github.io/","trace_n":0,"uncertainty_n":0},"mmlu":{"dated_n":0,"featured":false,"has_cost":false,"id":"mmlu","latest_eval":null,"metrics_n":0,"n":217,"name":"MMLU","score_col":"EM","source_status":"ok","source_url":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","trace_n":0,"uncertainty_n":0},"mystery_game_puzzles":{"dated_n":144,"featured":false,"has_cost":false,"id":"mystery_game_puzzles","latest_eval":"2026-11-06","metrics_n":0,"n":144,"name":"Mystery Game Puzzles","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":92,"uncertainty_n":144},"open_book_qa":{"dated_n":0,"featured":false,"has_cost":false,"id":"open_book_qa","latest_eval":null,"metrics_n":0,"n":70,"name":"Open Book Qa","score_col":"Accuracy","source_status":"ok","source_url":"http://arxiv.org/abs/2309.05463","trace_n":0,"uncertainty_n":0},"os_world":{"dated_n":0,"featured":false,"has_cost":false,"id":"os_world","latest_eval":null,"metrics_n":0,"n":20,"name":"OSWorld","score_col":"Score","source_status":"ok","source_url":"https://os-world.github.io/","trace_n":0,"uncertainty_n":0},"osworld_2":{"cost_basis":"estimated","cost_col":"Estimated cost (USD)","cost_label":"Estimated cost (USD)","dated_n":0,"featured":true,"has_cost":true,"id":"osworld_2","latest_eval":null,"metrics_n":0,"n":16,"name":"OSWorld 2","score_col":"Binary accuracy","source_status":"ok","source_url":"https://osworld-v2.xlang.ai/","trace_n":0,"uncertainty_n":0},"otis_mock_aime_2024_2025":{"dated_n":313,"featured":true,"has_cost":false,"id":"otis_mock_aime_2024_2025","latest_eval":"2026-11-06","metrics_n":0,"n":314,"name":"OTIS Mock AIME 24-25","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":312,"uncertainty_n":314},"piqa":{"dated_n":0,"featured":false,"has_cost":false,"id":"piqa","latest_eval":null,"metrics_n":0,"n":113,"name":"Piqa","score_col":"Score","source_status":"ok","source_url":"http://arxiv.org/abs/2403.08295","trace_n":0,"uncertainty_n":0},"posttrainbench":{"dated_n":0,"featured":false,"has_cost":false,"id":"posttrainbench","latest_eval":null,"metrics_n":0,"n":12,"name":"Posttrainbench","score_col":"Average (%)","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"proofbench":{"cost_basis":"test","cost_col":"Cost per test (USD)","cost_label":"$ per test","dated_n":0,"featured":false,"has_cost":true,"id":"proofbench","latest_eval":null,"metrics_n":84,"n":84,"name":"Proofbench","score_col":"Accuracy","source_status":"ok","source_url":"https://www.vals.ai/benchmarks/proof_bench","trace_n":0,"uncertainty_n":57},"rli":{"dated_n":0,"featured":false,"has_cost":false,"id":"rli","latest_eval":null,"metrics_n":0,"n":18,"name":"RLI","score_col":"Score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"scicode":{"dated_n":0,"featured":true,"has_cost":false,"id":"scicode","latest_eval":null,"metrics_n":0,"n":235,"name":"SciCode","score_col":"Score","source_status":"ok","source_url":"https://artificialanalysis.ai/evaluations/scicode","trace_n":0,"uncertainty_n":0},"science_qa":{"dated_n":0,"featured":false,"has_cost":false,"id":"science_qa","latest_eval":null,"metrics_n":0,"n":26,"name":"Science Qa","score_col":"Score","source_status":"ok","source_url":"https://scienceqa.github.io/leaderboard.html","trace_n":0,"uncertainty_n":0},"simplebench":{"dated_n":0,"featured":true,"has_cost":false,"id":"simplebench","latest_eval":null,"metrics_n":0,"n":102,"name":"SimpleBench","score_col":"Score (AVG@5)","source_status":"ok","source_url":"https://lmcouncil.ai/benchmarks","trace_n":0,"uncertainty_n":0},"simpleqa_verified":{"dated_n":101,"featured":false,"has_cost":false,"id":"simpleqa_verified","latest_eval":"2026-11-06","metrics_n":0,"n":101,"name":"SimpleQA Verified","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":89,"uncertainty_n":101},"spatialviz_bench":{"dated_n":0,"featured":false,"has_cost":false,"id":"spatialviz_bench","latest_eval":null,"metrics_n":0,"n":8,"name":"Spatialviz Bench","score_col":"Overall score","source_status":"ok","source_url":"https://arxiv.org/abs/2507.07610","trace_n":0,"uncertainty_n":0},"superglue":{"dated_n":0,"featured":false,"has_cost":false,"id":"superglue","latest_eval":null,"metrics_n":0,"n":10,"name":"Superglue","score_col":"Score","source_status":"ok","source_url":"http://arxiv.org/abs/2005.14165","trace_n":0,"uncertainty_n":0},"surface_evolver_bench":{"cost_basis":"total","cost_col":"Total cost (USD)","cost_label":"Total run cost (USD)","dated_n":0,"featured":false,"has_cost":true,"id":"surface_evolver_bench","latest_eval":null,"metrics_n":29,"n":29,"name":"Surface Evolver Bench","score_col":"Mean score","source_status":"ok","source_url":"https://yhenon.github.io/surface-evolver-llm-eval/","trace_n":0,"uncertainty_n":0},"swe_bench_verified":{"dated_n":35,"featured":true,"has_cost":false,"id":"swe_bench_verified","latest_eval":"2026-06-25","metrics_n":0,"n":35,"name":"SWE-bench Verified","score_col":"mean_score","source_status":"ok","source_url":null,"trace_n":34,"uncertainty_n":35},"terminalbench":{"dated_n":202,"featured":true,"has_cost":false,"id":"terminalbench","latest_eval":"2026-05-14","metrics_n":0,"n":202,"name":"Terminal-Bench","score_col":"Accuracy mean","source_status":"ok","source_url":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","trace_n":0,"uncertainty_n":193},"the_agent_company":{"cost_basis":"average","cost_col":"Average costs","cost_label":"Average reported cost (USD)","dated_n":0,"featured":false,"has_cost":true,"id":"the_agent_company","latest_eval":null,"metrics_n":15,"n":16,"name":"The Agent Company","score_col":"% Score","source_status":"ok","source_url":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-sonnet-20241022","trace_n":0,"uncertainty_n":0},"trivia_qa":{"dated_n":0,"featured":false,"has_cost":false,"id":"trivia_qa","latest_eval":null,"metrics_n":0,"n":115,"name":"Trivia Qa","score_col":"EM","source_status":"ok","source_url":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","trace_n":0,"uncertainty_n":0},"vending_bench_2":{"dated_n":0,"featured":false,"has_cost":false,"id":"vending_bench_2","latest_eval":null,"metrics_n":0,"n":71,"name":"Vending-Bench 2","score_col":"Score","source_status":"ok","source_url":null,"trace_n":0,"uncertainty_n":0},"video_mme":{"dated_n":0,"featured":false,"has_cost":false,"id":"video_mme","latest_eval":null,"metrics_n":0,"n":50,"name":"Video Mme","score_col":"Overall (no subtitles)","source_status":"ok","source_url":"https://video-mme.github.io/home_page.html#leaderboard","trace_n":0,"uncertainty_n":0},"vpct":{"dated_n":0,"featured":false,"has_cost":false,"id":"vpct","latest_eval":null,"metrics_n":0,"n":38,"name":"VPCT","score_col":"Correct","source_status":"ok","source_url":"https://cbrower.dev/vpct","trace_n":0,"uncertainty_n":0},"webdev_arena":{"dated_n":0,"featured":false,"has_cost":false,"id":"webdev_arena","latest_eval":null,"metrics_n":135,"n":135,"name":"WebDev Arena","score_col":"Arena Score","source_status":"ok","source_url":"https://web.lmarena.ai/leaderboard","trace_n":0,"uncertainty_n":135},"weirdml":{"cost_basis":"run","cost_col":"Cost per run","cost_label":"$ per run","dated_n":0,"featured":false,"has_cost":true,"id":"weirdml","latest_eval":null,"metrics_n":0,"n":173,"name":"WeirdML","score_col":"Accuracy","source_status":"ok","source_url":"https://htihle.github.io/weirdml.html","trace_n":0,"uncertainty_n":158},"wino_grande":{"dated_n":0,"featured":false,"has_cost":false,"id":"wino_grande","latest_eval":null,"metrics_n":0,"n":137,"name":"WinoGrande","score_col":"Accuracy","source_status":"ok","source_url":"https://arxiv.org/pdf/2305.10403","trace_n":0,"uncertainty_n":0}},"models":{"ada":{"country":"United States of America","display":"ada (350M)","family":"ada","flop":641000000000000000000,"flop_notes":"Table D.1\nhttps://arxiv.org/abs/2005.14165","org":"OpenAI"},"amazon-nova-lite-v10":{"country":"United States of America","display":"Nova 2 Lite V1","family":"amazon-nova-lite-v10","org":"Amazon","release_date":"2024-12-03"},"amazon-nova-micro-v10":{"country":"United States of America","display":"Nova Micro","family":"amazon-nova-micro-v10","org":"Amazon","release_date":"2024-12-03"},"amazon-nova-pro-v10":{"country":"United States of America","display":"Nova Pro","family":"amazon-nova-pro-v10","flop":6.000010000000001e+24,"flop_notes":"\"probably just below 1e25 stemming from the Llama 70B serving speed.  If Llama 70B is trained proportionally to 405B, then it's at ~ 6.6e24. Nova Pro is served at 100tk/s, while Llama 70B is served at 70tk/s on average, and 100tk/s by together.ai at FP8. So Nova Pro would be \u003e1e25 if they roughly 2x the amount of training compared to Llama 70B which [seems unlikely]\"","org":"Amazon","release_date":"2024-12-03"},"aria":{"display":"Aria","family":"aria","release_date":"2024-09-30"},"babbage":{"country":"United States of America","display":"babbage (1.3B)","family":"babbage","flop":2.38e+21,"flop_notes":"Table D.1\nhttps://arxiv.org/abs/2005.14165","org":"OpenAI","release_date":"2020-06-22"},"baichuan-13b-base":{"country":"China","display":"Baichuan 1-13B-Base","family":"baichuan-13b-base","flop":9.36e+22,"flop_notes":"13b parameters * 1.2t tokens * 6 FLOP / parameter / token = 9.36e22 FLOP","org":"Baichuan","release_date":"2023-07-11"},"baichuan-2-13b-base":{"country":"China","display":"Baichuan2 13B","family":"baichuan-2-13b-base","flop":2.03e+23,"flop_notes":"They describe the dataset as having 2.6T tokens, but the checkpoint graph makes it clear that's also the number of tokens the model was trained on.\n\n13b * 2.6t * 6 = 2.03e23","org":"Baichuan","release_date":"2023-09-06"},"baichuan-2-7b-base":{"country":"China","display":"Baichuan2 7B","family":"baichuan-2-7b-base","flop":1.092e+23,"flop_notes":"7b * 2.6t * 6 = 1.092e23\nAlso mentions 1,024 NVIDIA A800 GPUs at 180 TFLOPS per GPU","org":"Baichuan","release_date":"2023-09-20"},"baichuan-7b":{"country":"China","display":"Baichuan 1-7B","family":"baichuan-7b","flop":5.04e+22,"flop_notes":"7b parameters * 1.2t tokens * 6 FLOP / parameter / token = 5.04e22 FLOP","org":"Baichuan","release_date":"2023-06-01"},"baichuan2-13b-chat":{"display":"Baichuan2-Chat 13B","family":"baichuan2-13b-chat","release_date":"2023-09-06"},"beam-1-0":{"country":"United States of America","display":"Beam","family":"beam-1-0","flop_notes":"Pre-training \n\nby 6ND: 6 * 23e9 active parameters * 23.8e12 tokens = 3.28e24 FLOP.\n\n\"Beam was pretrained end-to-end in under four weeks on a cluster of 6,144 NVIDIA GB300 NVL72 GPUs.\"\n\nRL\n \"To scale reinforcement learning, we deployed 10.5K NVIDIA GB300 GPUs for four weeks generating more than 100 million rollouts with a maximum context length of 256K tokens.\"\n\nAssuming an average context length of 128k tokens,","org":"Reflection","release_date":"2026-10-05"},"blip2-opt-2-7b":{"country":"United States of America","display":"BLIP-2","family":"blip2-opt-2-7b","flop":1.20000000001e+21,"flop_notes":"https://www.wolframalpha.com/input?i=312+teraFLOPS+*+16+*+200+hours+*+0.33","org":"Salesforce Research","release_date":"2023-02-06"},"bloom":{"country":"United States of America,France","display":"BLOOM (176B)","family":"bloom","flop":3.65664e+23,"flop_notes":"https://bigscience.huggingface.co/blog/bloom Blog post says 117 days.\n\n384 A100 GPUs * 314 TFLOPS throughput per GPU * 117 days * 0.3 (utilization assumption) = 3.65664e23\nhttps://www.wolframalpha.com/input?i=384+*+314+TFLOPS+*+117+days+*+0.3","open_weights":true,"org":"Hugging Face,BigScience","params":176247271424,"release_date":"2022-07-06"},"bytevideollm-14b":{"display":"ByteVideoLLM","family":"bytevideollm-14b","release_date":"2024-10-13"},"c4ai-command-a-03-2025":{"country":"Canada","display":"Command A","family":"c4ai-command-a-03-2025","flop_notes":"Unlikely to be \u003e1e25 FLOP. Trained on \"trillions of tokens\", so C=6ND would suggest at most 6*111e9*10e12 = 7e24 FLOP if it used 10T tokens.","org":"Cohere","release_date":"2025-03-13"},"c4ai-command-r-08-2024":{"display":"Command R","family":"c4ai-command-r-08-2024","release_date":"2024-08-30"},"c4ai-command-r-plus-08-2024":{"country":"Canada","display":"Command R Plus","family":"c4ai-command-r-plus-08-2024","org":"Cohere,Cohere Labs (formerly Cohere for AI)","release_date":"2024-08-30"},"cerebras-gpt-13b":{"country":"United States of America","display":"Cerebras-GPT-13B","family":"cerebras-gpt-13b","flop":2.3e+22,"flop_notes":"2.3e22, per table 2","org":"Cerebras Systems","release_date":"2023-03-20"},"chat-uni-vi-7b-v1-5-100k-sg-wv":{"display":"ShareGeminiXMU","family":"chat-uni-vi-7b-v1-5-100k-sg-wv","release_date":"2024-06-20"},"chat-univi-7b-v1-5":{"display":"Chat-UniVi-v1.5","family":"chat-univi-7b-v1-5","release_date":"2024-04-23"},"chatglm2-6b":{"display":"ChatGLM2 6B","family":"chatglm2-6b","release_date":"2023-06-24"},"chatgpt-4o-01-29-2025":{"country":"United States of America","display":"Chatgpt 4o 01.29","family":"chatgpt-4o-01-29-2025","flop_notes":"Training compute estimated to be 3.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2025-01-29"},"chatgpt-4o-03-27-2025":{"country":"United States of America","display":"Chatgpt 4o 03.27","family":"chatgpt-4o-03-27-2025","flop_notes":"Training compute estimated to be 3.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2025-03-27"},"chinchilla-70b":{"country":"United Kingdom","display":"Chinchilla 70B","family":"chinchilla-70b","flop":5.76e+23,"flop_notes":"\"Both Chinchilla and Gopher have been trained for the same number of FLOPs but differ in the size of the model and the number of training tokens.\"\n\nWe see the number of flops in table 3","org":"DeepMind","release_date":"2022-03-29"},"chutesllama-4-maverick-17b-128e-instruct":{"country":"United States of America","display":"Llama-4 Maverick","family":"chutesllama-4-maverick-17b-128e-instruct","flop":2.244000000001e+24,"flop_notes":"22T training tokens per model card:\n\nhttps://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md   \n\nMaverick was trained using co-distillation from Llama 4 Behemoth. It isn't 100% clear that all 22T tokens used distillation, but we assume this for the time being.\n\nEstimating training compute from parameters and tokens:\nCompute = 6 FLOP per token per parameter * 17B active parameters * 22T tokens = 2.244e24 FLOP\n(Implying mean throughput was 262 TFLOPS/GPU, or 13.2% MFU in FP8)\n\n\nThe model card also states that Llama 4 Maverick used 2.38M H100-hours.\nThe blog post gives a figure of 390 TFLOPS/GPU, but this may have been the utilization rate for Behemoth, or all of the models together. Using this utilization, we have:\nCompute = 390 TFLOP/s * 2.38 million hours = 3.342e24 FLOP\n(This value is higher than the compute implied by parameters and tokens, and suggests utilization may have been lower for Maverick than for Behemoth.)","open_weights":true,"org":"Meta","params":400000000000,"release_date":"2025-04-06"},"chutesllama-4-scout-17b-16e-instruct":{"country":"United States of America","display":"Chutesllama 4 Scout 17B 16e Instruct","family":"chutesllama-4-scout-17b-16e-instruct","flop":4.08e+24,"flop_notes":"40T training tokens per model card:\n\nhttps://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md  \n\nEstimating training compute from parameters and tokens:\n6 FLOP per token per parameter * 17B active parameters * 40T tokens = 4.08e24 FLOP\n(Implying mean throughput was 227 TFLOPS/GPU, or 11.5% MFU in FP8)\n\n\nThe model card also states that Llama 4 Scout used 5.0M H100-hours.\nThe blog post gives a figure of 390 TFLOPS/GPU, but this may have been the utilization rate for Behemoth, or all of the models together. Using this utilization, we have:\nCompute = 390 TFLOP/s * 5 million hours = 7.02e24 FLOP\n(This value is higher than the compute implied by parameters and tokens, and suggests utilization may have been lower for Scout than for Behemoth.)","org":"Meta","release_date":"2025-04-06"},"chutesqwen3-235b-a22b-thinking-2507":{"country":"China","display":"Chutesqwen3 235B A22b","family":"chutesqwen3-235b-a22b-thinking-2507","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","org":"Alibaba","release_date":"2025-07-25"},"chutesqwen3-next-80b-a3b-instruct":{"display":"Chutesqwen3 Next 80B A3b Instruct","family":"chutesqwen3-next-80b-a3b-instruct","release_date":"2025-09-11"},"claude-1-3":{"country":"United States of America","display":"Claude 1.3","family":"claude-1-3","org":"Anthropic","release_date":"2023-04-18"},"claude-2-0":{"country":"United States of America","display":"Claude 2","family":"claude-2-0","flop":3.866e+24,"flop_notes":"https://colab.research.google.com/drive/1MdPuhS4Emaf23VXYZ-ooExDW-5GXZkw0#scrollTo=Ds0Q5X8aMnOY","open_weights":false,"org":"Anthropic","release_date":"2023-07-11"},"claude-2-1":{"country":"United States of America","display":"claude-2.1†","family":"claude-2-1","open_weights":false,"org":"Anthropic","release_date":"2023-11-21"},"claude-3-5-haiku-2024-10-22":{"country":"United States of America","display":"Claude 3.5 Haiku","family":"claude-3-5-haiku","org":"Anthropic","release_date":"2024-10-22"},"claude-3-5-sonnet-2024-06-20":{"country":"United States of America","display":"claude-3-5-sonnet-20240620†","family":"claude-3-5-sonnet","flop":2.700000000000001e+25,"flop_notes":"Blog post by Dario Amodei includes some info on 3.5 Sonnet compute: https://darioamodei.com/on-deepseek-and-export-controls\n\"Claude 3.5 Sonnet is a mid-sized model that cost a few $10M's to train (I won't give an exact number). Also, 3.5 Sonnet was not trained in any way that involved a larger or more expensive model (contrary to some rumors).\"\n\nUsing assumptions about GPU pricing, this lets us estimate compute. https://docs.google.com/spreadsheets/d/1-p-ab6t6dkUM6T7GwnFp85ePTMpZMW7LFY7fW2t8POs/","open_weights":false,"org":"Anthropic","release_date":"2024-06-20"},"claude-3-5-sonnet-2024-10-22":{"country":"United States of America","display":"Claude 3.5 Sonnet","family":"claude-3-5-sonnet","flop":2.700000000000001e+25,"open_weights":false,"org":"Anthropic","release_date":"2024-10-22"},"claude-3-7-sonnet-2025-02-19":{"country":"United States of America","display":"Claude 3.7","family":"claude-3-7-sonnet","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-12k":{"country":"United States of America","display":"Claude 3.7 Sonnet 20250219 12K","family":"claude-3-7-sonnet-20250219-12k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-15k":{"country":"United States of America","display":"Claude 3.7 Sonnet 20250219 15K","family":"claude-3-7-sonnet-20250219-15k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-16k":{"country":"United States of America","display":"Claude 3.7 (16K)","family":"claude-3-7-sonnet-20250219-16k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-1k":{"country":"United States of America","display":"Claude 3.7 (1K)","family":"claude-3-7-sonnet-20250219-1k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-2k":{"country":"United States of America","display":"Claude 3.7 Sonnet 20250219 2K","family":"claude-3-7-sonnet-20250219-2k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-32k":{"country":"United States of America","display":"Claude 3.7 Sonnet 20250219 32K","family":"claude-3-7-sonnet-20250219-32k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-64k":{"country":"United States of America","display":"Claude 3.7 Sonnet 20250219 64K","family":"claude-3-7-sonnet-20250219-64k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-7-sonnet-20250219-8k":{"country":"United States of America","display":"Claude 3.7 (8K)","family":"claude-3-7-sonnet-20250219-8k","flop":3.35e+25,"flop_notes":"https://docs.google.com/spreadsheets/d/10bhwdVrfHI8tysVIz62ZxtvQ30L-HojYvmU18_b-WIM/edit?gid=0#gid=0","org":"Anthropic","release_date":"2025-02-24"},"claude-3-haiku-2024-03-07":{"country":"United States of America","display":"Claude 3 Haiku","family":"claude-3-haiku","org":"Anthropic","release_date":"2024-03-07"},"claude-3-opus-2024-02-29":{"country":"United States of America","display":"Claude 3 Opus","family":"claude-3-opus","flop_notes":"Training compute estimated to be 1.64e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"Anthropic","release_date":"2024-02-29"},"claude-3-sonnet-2024-02-29":{"country":"United States of America","display":"Claude 3 Sonnet (20240229)","family":"claude-3-sonnet","org":"Anthropic","release_date":"2024-02-29"},"claude-fable-5":{"aa":{"agentic_index":50.7,"coding_index":76.5,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1781007515,"display":"Claude Fable 5","family":"claude-fable-5","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-fable-5","org":"Anthropic","output_modalities":["text"],"price_cache":1,"price_in":10,"price_match":"exact","price_out":50,"release_date":"2026-06-09","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-fable-5-1":{"aa":{"agentic_index":57.2,"coding_index":80.7,"intelligence_index":50},"context":1000000,"country":"United States of America","created":1788285838,"display":"Claude Fable 5.1","family":"claude-fable-5-1","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-fable-5.1","org":"Anthropic","output_modalities":["text"],"price_cache":0.25,"price_in":10,"price_match":"exact","price_out":50,"release_date":"2026-09-01","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-haiku-4-5-2025-10-01":{"aa":{"agentic_index":8,"coding_index":43.9,"intelligence_index":7.4},"context":200000,"country":"United States of America","created":1760547638,"display":"Claude Haiku 4.5","family":"claude-haiku-4-5","input_modalities":["text","image","file"],"max_output":64000,"open_weights":false,"or_id":"anthropic/claude-haiku-4.5","org":"Anthropic","output_modalities":["text"],"price_cache":0.1,"price_in":1,"price_match":"alias","price_out":5,"release_date":"2025-10-15","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"claude-haiku-4-5-20251001-16k":{"country":"United States of America","display":"Claude Haiku 4.5 (Thinking 16K)","family":"claude-haiku-4-5-20251001-16k","org":"Anthropic","release_date":"2025-10-15"},"claude-haiku-4-5-20251001-1k":{"country":"United States of America","display":"Claude Haiku 4.5 (Thinking 1K)","family":"claude-haiku-4-5-20251001-1k","org":"Anthropic","release_date":"2025-10-15"},"claude-haiku-4-5-20251001-32k":{"country":"United States of America","display":"Claude 4.5 Haiku","family":"claude-haiku-4-5-20251001-32k","org":"Anthropic","release_date":"2025-10-15"},"claude-haiku-4-5-20251001-59k":{"country":"United States of America","display":"Claude Haiku 4.5.20251001 59K","family":"claude-haiku-4-5-20251001-59k","org":"Anthropic","release_date":"2025-10-15"},"claude-haiku-4-5-20251001-8k":{"country":"United States of America","display":"Claude Haiku 4.5 (Thinking 8K)","family":"claude-haiku-4-5-20251001-8k","org":"Anthropic","release_date":"2025-10-15"},"claude-haiku-5-5":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":36.5},"context":1000000,"country":"United States of America","created":1791397883,"display":"Claude Haiku 5.5","family":"claude-haiku-5-5","input_modalities":["text","image","file"],"max_output":128000,"or_id":"anthropic/claude-haiku-5.5","org":"Anthropic","output_modalities":["text"],"price_cache":0.01,"price_in":0.1,"price_match":"exact","price_out":0.5,"release_date":"2026-10-07","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-instant-1-1":{"country":"United States of America","display":"Claude Instant 1.1","family":"claude-instant-1-1","org":"Anthropic"},"claude-instant-1-2":{"country":"United States of America","display":"Claude Instant 1.2","family":"claude-instant-1-2","org":"Anthropic","release_date":"2023-08-09"},"claude-mythos-preview-early":{"country":"United States of America","display":"Claude Mythos Preview (AutoNudge)","family":"claude-mythos-preview-early","flop_notes":"Not disclosed.","org":"Anthropic","release_date":"2026-04-07"},"claude-opus-4-1-2025-08-05":{"context":200000,"country":"United States of America","created":1754411591,"display":"Claude Opus 4.1","family":"claude-opus-4-1","input_modalities":["image","text","file"],"knowledge_cutoff":"2025-01-31","max_output":32000,"open_weights":false,"or_id":"anthropic/claude-opus-4.1","org":"Anthropic","output_modalities":["text"],"price_cache":1.5,"price_in":15,"price_match":"alias","price_out":75,"release_date":"2025-08-05","supported_parameters":["include_reasoning","max_tokens","reasoning","stop","temperature","tool_choice","tools","top_k","top_p"]},"claude-opus-4-1-20250805-16k":{"country":"United States of America","display":"Claude 4.1 Opus","family":"claude-opus-4-1-20250805-16k","org":"Anthropic","release_date":"2025-08-05"},"claude-opus-4-1-20250805-24k":{"country":"United States of America","display":"Claude Opus 4.1.20250805 24K","family":"claude-opus-4-1-20250805-24k","org":"Anthropic","release_date":"2025-08-05"},"claude-opus-4-1-20250805-27k":{"country":"United States of America","display":"Claude Opus 4.1.20250805 27K","family":"claude-opus-4-1-20250805-27k","org":"Anthropic","release_date":"2025-08-05"},"claude-opus-4-2025-05-14":{"country":"United States of America","display":"Claude Opus 4 (high)","family":"claude-opus-4","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-12k":{"country":"United States of America","display":"Claude Opus 4.20250514 12K","family":"claude-opus-4-20250514-12k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-16k":{"country":"United States of America","display":"Claude Opus 4 (Thinking 16K)","family":"claude-opus-4-20250514-16k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-1k":{"country":"United States of America","display":"Claude Opus 4 (Thinking 1K)","family":"claude-opus-4-20250514-1k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-27k":{"country":"United States of America","display":"Claude Opus 4.20250514 27K","family":"claude-opus-4-20250514-27k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-2k":{"country":"United States of America","display":"Claude Opus 4.20250514 2K","family":"claude-opus-4-20250514-2k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-32k":{"country":"United States of America","display":"Claude Opus 4.20250514 32K","family":"claude-opus-4-20250514-32k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-20250514-8k":{"country":"United States of America","display":"Claude Opus 4 (Thinking 8K)","family":"claude-opus-4-20250514-8k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-opus-4-5-2025-11-01":{"country":"United States of America","display":"Claude Opus 4.5 (high)","family":"claude-opus-4-5","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-128k":{"country":"United States of America","display":"Claude Opus 4.5.20251101 128K","family":"claude-opus-4-5-20251101-128k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-16k":{"country":"United States of America","display":"Claude 4.5 Opus","family":"claude-opus-4-5-20251101-16k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-32k":{"country":"United States of America","display":"Opus 4.5 (Thinking, 32K)","family":"claude-opus-4-5-20251101-32k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-48k":{"country":"United States of America","display":"Claude Opus 4.5.20251101 48K","family":"claude-opus-4-5-20251101-48k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-59k":{"country":"United States of America","display":"Claude Opus 4.5.20251101 59K","family":"claude-opus-4-5-20251101-59k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-64k":{"country":"United States of America","display":"Opus 4.5 (Thinking, 64K)","family":"claude-opus-4-5-20251101-64k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-5-20251101-8k":{"country":"United States of America","display":"Opus 4.5 (Thinking, 8K)","family":"claude-opus-4-5-20251101-8k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-11-24"},"claude-opus-4-6":{"context":1000000,"country":"United States of America","created":1770219050,"display":"Claude Opus 4.6","family":"claude-opus-4-6","input_modalities":["text","image","file"],"max_output":128000,"or_id":"anthropic/claude-opus-4.6","org":"Anthropic","output_modalities":["text"],"price_cache":0.5,"price_in":5,"price_match":"exact","price_out":25,"release_date":"2026-02-05","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p","verbosity"]},"claude-opus-4-6-120k":{"country":"United States of America","display":"Claude Opus 4.6 (120K, High)","family":"claude-opus-4-6-120k","org":"Anthropic","release_date":"2026-02-05"},"claude-opus-4-6-32k":{"country":"United States of America","display":"Claude Opus 4.6 32K","family":"claude-opus-4-6-32k","org":"Anthropic","release_date":"2026-02-05"},"claude-opus-4-6-64k":{"country":"United States of America","display":"Claude Opus 4.6 64K","family":"claude-opus-4-6-64k","org":"Anthropic","release_date":"2026-02-05"},"claude-opus-4-7":{"aa":{"agentic_index":38.6,"coding_index":73.6,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1776351100,"display":"Claude Opus 4.7","family":"claude-opus-4-7","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-opus-4.7","org":"Anthropic","output_modalities":["text"],"price_cache":0.5,"price_in":5,"price_match":"exact","price_out":25,"release_date":"2026-04-16","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-opus-4-8":{"aa":{"agentic_index":41.9,"coding_index":74.3,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1779905091,"display":"Claude Opus 4.8","family":"claude-opus-4-8","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-opus-4.8","org":"Anthropic","output_modalities":["text"],"price_cache":0.5,"price_in":5,"price_match":"exact","price_out":25,"release_date":"2026-05-28","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-opus-4-8-24k":{"country":"United States of America","display":"claude-opus-4-8-adaptive-thinking-high-24000†","family":"claude-opus-4-8-24k","org":"Anthropic","release_date":"2026-05-28"},"claude-opus-5":{"aa":{"agentic_index":56.5,"coding_index":78,"intelligence_index":44.2},"context":1000000,"country":"United States of America","created":1784912544,"display":"Claude Opus 5","family":"claude-opus-5","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-opus-5","org":"Anthropic","output_modalities":["text"],"price_cache":0.5,"price_in":5,"price_match":"exact","price_out":25,"release_date":"2026-07-24","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-opus-5-5":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":59.1},"context":1000000,"country":"United States of America","created":1790094732,"display":"Claude Opus 5.5","family":"claude-opus-5-5","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-opus-5.5","org":"Anthropic","output_modalities":["text"],"price_cache":0.2,"price_in":4,"price_match":"exact","price_out":20,"release_date":"2026-09-22","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-sonnet-4-2025-05-14":{"country":"United States of America","display":"Claude Sonnet 4","family":"claude-sonnet-4","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","open_weights":false,"org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-12k":{"country":"United States of America","display":"Claude Sonnet 4.20250514 12K","family":"claude-sonnet-4-20250514-12k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-16k":{"country":"United States of America","display":"Claude Sonnet 4 (Thinking 16K)","family":"claude-sonnet-4-20250514-16k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-1k":{"country":"United States of America","display":"Claude Sonnet 4 (Thinking 1K)","family":"claude-sonnet-4-20250514-1k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-2k":{"country":"United States of America","display":"Claude Sonnet 4.20250514 2K","family":"claude-sonnet-4-20250514-2k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-32k":{"country":"United States of America","display":"Claude 4 Sonnet","family":"claude-sonnet-4-20250514-32k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-59k":{"country":"United States of America","display":"Claude Sonnet 4.20250514 59K","family":"claude-sonnet-4-20250514-59k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-20250514-8k":{"country":"United States of America","display":"Claude Sonnet 4 (Thinking 8K)","family":"claude-sonnet-4-20250514-8k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Anthropic","release_date":"2025-05-22"},"claude-sonnet-4-5-2025-09-29":{"aa":{"agentic_index":15.8,"coding_index":52.1,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1759161676,"display":"Claude Sonnet 4.5","family":"claude-sonnet-4-5","input_modalities":["text","image","file"],"knowledge_cutoff":"2025-01-31","max_output":64000,"open_weights":false,"or_id":"anthropic/claude-sonnet-4.5","org":"Anthropic","output_modalities":["text"],"price_cache":0.3,"price_in":3,"price_match":"alias","price_out":15,"release_date":"2025-09-29","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"claude-sonnet-4-5-20250929-12k":{"country":"United States of America","display":"Claude Sonnet 4.5.20250929 12K","family":"claude-sonnet-4-5-20250929-12k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-16k":{"country":"United States of America","display":"Claude Sonnet 4.5 (Thinking 16K)","family":"claude-sonnet-4-5-20250929-16k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-1k":{"country":"United States of America","display":"Claude Sonnet 4.5 (Thinking 1K)","family":"claude-sonnet-4-5-20250929-1k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-2k":{"country":"United States of America","display":"Claude Sonnet 4.5.20250929 2K","family":"claude-sonnet-4-5-20250929-2k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-32k":{"country":"United States of America","display":"Claude 4.5 Sonnet","family":"claude-sonnet-4-5-20250929-32k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-48k":{"country":"United States of America","display":"Claude Sonnet 4.5.20250929 48K","family":"claude-sonnet-4-5-20250929-48k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-59k":{"country":"United States of America","display":"Claude Sonnet 4.5.20250929 59K","family":"claude-sonnet-4-5-20250929-59k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-5-20250929-8k":{"country":"United States of America","display":"Claude Sonnet 4.5 (Thinking 8K)","family":"claude-sonnet-4-5-20250929-8k","org":"Anthropic","release_date":"2025-09-29"},"claude-sonnet-4-6":{"aa":{"agentic_index":31.8,"coding_index":63,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1771342990,"display":"Claude Sonnet 4.6","family":"claude-sonnet-4-6","input_modalities":["text","image","file"],"max_output":128000,"or_id":"anthropic/claude-sonnet-4.6","org":"Anthropic","output_modalities":["text"],"price_cache":0.3,"price_in":3,"price_match":"exact","price_out":15,"release_date":"2026-02-17","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p","verbosity"]},"claude-sonnet-4-6-16k":{"country":"United States of America","display":"claude-sonnet-4-6-adaptive-thinking-16000†","family":"claude-sonnet-4-6-16k","org":"Anthropic","release_date":"2026-02-17"},"claude-sonnet-4-6-32k":{"country":"United States of America","display":"Claude Sonnet 4.6 32K","family":"claude-sonnet-4-6-32k","org":"Anthropic","release_date":"2026-02-17"},"claude-sonnet-5":{"aa":{"agentic_index":43.6,"coding_index":71.5,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1782843083,"display":"Claude Sonnet 5","family":"claude-sonnet-5","input_modalities":["text","image","file"],"max_output":128000,"or_id":"anthropic/claude-sonnet-5","org":"Anthropic","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-06-30","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"claude-sonnet-5-16k":{"country":"United States of America","display":"claude-sonnet-5-adaptive-thinking-16000†","family":"claude-sonnet-5-16k","org":"Anthropic","release_date":"2026-06-30"},"claude-sonnet-5-5":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":54.5},"context":1000000,"country":"United States of America","created":1790618686,"display":"Claude Sonnet 5.5","family":"claude-sonnet-5-5","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"anthropic/claude-sonnet-5.5","org":"Anthropic","output_modalities":["text"],"price_cache":0.1,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-09-28","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","stop","structured_outputs","tool_choice","tools","verbosity"]},"code-davinci-002":{"country":"United States of America","display":"Code Davinci 002","family":"code-davinci-002","flop":2.578e+24,"flop_notes":"https://colab.research.google.com/drive/1QSxa8YCWjEBQU7mrXLhw6TP1VX5oqgdW#scrollTo=Gt6Z6oZ26clI","open_weights":false,"org":"OpenAI","release_date":"2022-03-15"},"codeqwen1-5-7b":{"display":"CodeQwen1.5-7B","family":"codeqwen1-5-7b","release_date":"2024-04-15"},"codestral-2501":{"country":"France","display":"Codestral","family":"codestral-2501","org":"Mistral","release_date":"2025-01-13"},"codestral-2508":{"context":256000,"country":"France","created":1754079630,"display":"Codestral 2508","family":"codestral-2508","input_modalities":["text","file"],"knowledge_cutoff":"2025-03-31","max_output":204800,"or_id":"mistralai/codestral-2508","org":"Mistral","output_modalities":["text"],"price_cache":0.03,"price_in":0.3,"price_match":"exact","price_out":0.9,"release_date":"2025-07-30","supported_parameters":["frequency_penalty","max_tokens","prediction","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"codex-mini-2025-05-16":{"display":"Codex Mini (Latest)","family":"codex-mini","release_date":"2025-05-16"},"cogito-671b-v2-1":{"display":"Cogito v2.1 (Reasoning)","family":"cogito-671b-v2-1","org":"Deep Cogito","release_date":"2025-11-19"},"command-a-plus-05-2026":{"aa":{"agentic_index":1,"coding_index":27.8,"intelligence_index":6.2},"context":192000,"country":"Canada","created":1790102896,"display":"Command A+","family":"command-a-plus-05-2026","input_modalities":["text","image"],"max_output":64000,"or_id":"cohere/command-a-plus","org":"Cohere,Cohere Labs (formerly Cohere for AI)","output_modalities":["text"],"price_cache":0.15,"price_in":0.3,"price_match":"exact","price_out":1.5,"release_date":"2026-05-20","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"composer-2-5":{"country":"United States of America","display":"Composer 2.5","family":"composer-2-5","flop":3.87e+25,"flop_notes":"Said by Cursor that 85% of the total compute was performed by their team (from figure following \"Composer 2.5 is built on the same open-source checkpoint as Composer 2, Moonshot's Kimi K2.5.\" here https://cursor.com/blog/composer-2-5) - 5.8e24 / 0.15 = 3.8666666667E25 FLOP","open_weights":false,"org":"Cursor","params":1040000000000,"release_date":"2026-05-18"},"computer-use-preview-2025-03-11":{"display":"Computer Use Preview","family":"computer-use","release_date":"2025-03-11"},"curie":{"country":"United States of America","display":"curie (6.7B)","family":"curie","flop":1.2e+22,"flop_notes":"Table D.1\nhttps://arxiv.org/abs/2005.14165","org":"OpenAI"},"davinci":{"country":"United States of America","display":"davinci (175B)","family":"davinci","flop":3.14e+23,"flop_notes":"Table D.1\nhttps://arxiv.org/abs/2005.14165","open_weights":false,"org":"OpenAI","params":174600000000,"release_date":"2020-05-28"},"davinci-002":{"country":"United States of America","display":"Davinci 002","family":"davinci-002","flop":3.14e+23,"flop_notes":"Table D.1\nhttps://arxiv.org/abs/2005.14165","open_weights":false,"org":"OpenAI","params":174600000000,"release_date":"2020-05-28"},"dbrx-instruct":{"country":"United States of America","display":"Dbrx Instruct","family":"dbrx-instruct","flop":2.6e+24,"flop_notes":"Mixture of Experts (MoE)\n\n36 billion active params * 12 trillion tokens * 6 ~= 2.6e24\nhttps://www.wolframalpha.com/input?i=6+FLOP+*+36+billion+*+12+trillion\n\nalso, it was trained on 3072 NVIDIA H100s, but with an unclear timeframe (end-end process was three months, including evals and red-teaming).","org":"Databricks","release_date":"2024-03-27"},"deepinfraqwen3-next-80b-a3b-instruct":{"display":"Deepinfraqwen3 Next 80B A3b Instruct","family":"deepinfraqwen3-next-80b-a3b-instruct","release_date":"2025-09-11"},"deepseek-chat":{"context":163840,"country":"China","created":1735241320,"display":"DeepSeek V3","family":"deepseek-chat","flop":4.2e+24,"flop_notes":"Pre-trained base was DeepSeek-V3.1-Terminus\n3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","input_modalities":["text"],"knowledge_cutoff":"2024-07-31","max_output":16384,"open_weights":true,"or_id":"deepseek/deepseek-chat","org":"DeepSeek","output_modalities":["text"],"params":671000000000,"price_in":0.32,"price_match":"exact","price_out":0.89,"release_date":"2025-12-01","supported_parameters":["frequency_penalty","logit_bias","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"deepseek-coder-1-3b-base":{"country":"China","display":"DS-Coder-1.3B-Base","family":"deepseek-coder-1-3b-base","flop":1.56e+22,"flop_notes":"2T tokens * 1.3B parameters * 6 FLOP / parameter / token = 15.6 * 10^21 = 1.56 * 10^22 FLOP","org":"DeepSeek","release_date":"2023-11-02"},"deepseek-coder-33b-base":{"country":"China","display":"DS-Coder-33B-Base","family":"deepseek-coder-33b-base","flop":3.96e+23,"flop_notes":"\"Step 1: Initially pre-trained with a dataset consisting of 87% code, 10% code-related language (Github Markdown and StackExchange), and 3% non-code-related Chinese language. Models are pre-trained using 1.8T tokens and a 4K window size in this step.\nStep 2: Further Pre-training using an extended 16K window size on an additional 200B tokens, resulting in foundational models (DeepSeek-Coder-Base).\nStep 3: Instruction Fine-tuning on 2B tokens of instruction data, resulting in instruction-tuned models (DeepSeek-Coder-Instruct).\"\n\nThis means it was trained on 2T tokens. 2T * 33B * 6 = 3.96e23","org":"DeepSeek","release_date":"2023-11-02"},"deepseek-coder-6-7b-base":{"country":"China","display":"DS-Coder-6.7B-Base","family":"deepseek-coder-6-7b-base","flop":8.04e+22,"flop_notes":"2T tokens * 6.7B parameters * 6 FLOP / parameter / token = 8.04*10^22 FLOP","org":"DeepSeek","release_date":"2023-11-02"},"deepseek-coder-v2-base":{"country":"China","display":"DS-Coder-V2-Base","family":"deepseek-coder-v2-base","flop":1.2852e+24,"flop_notes":"Trained on a total of 10.2T tokens\n6NC: 6 * 10.2T * 21B active parameters = 1.285e24","org":"DeepSeek","release_date":"2024-06-17"},"deepseek-coder-v2-instruct":{"country":"China","display":"DS-Coder-V2-Instruct","family":"deepseek-coder-v2-instruct","flop":1.2852e+24,"flop_notes":"Trained on a total of 10.2T tokens\n6NC: 6 * 10.2T * 21B active parameters = 1.285e24","org":"DeepSeek","release_date":"2024-06-17"},"deepseek-coder-v2-lite-base":{"display":"DS-Coder-V2-Lite-Base","family":"deepseek-coder-v2-lite-base","release_date":"2024-06-13"},"deepseek-coder-v2-lite-instruct":{"display":"DS-Coder-V2-Lite-Instruct","family":"deepseek-coder-v2-lite-instruct","release_date":"2024-06-13"},"deepseek-llm-67b-chat":{"country":"China","display":"DeepSeek Llm 67B Chat","family":"deepseek-llm-67b-chat","flop":8.04e+23,"flop_notes":"67B parameters * 2T tokens * 6 FLOP / token / parameter = 8.04e23 FLOP","org":"DeepSeek","release_date":"2023-11-29"},"deepseek-r1":{"country":"China","display":"Deepseek R1","family":"deepseek-r1","flop":3.3e+24,"flop_notes":"Base model is DeepSeek-V3/R1-Zero (3.3e24). Post-training RL FLOP is likely a small fraction (around 2e22 FLOP, or \u003c1%) of base FLOP. https://joshbuild.substack.com/p/deepseek-r1-zeros-rl-training-used \n\nthis supersedes Ege Erdil's earlier estimate that R1's RL cost was around 20% of pre-training, using updated RL hyperparameters disclosed by DeepSeek. R1 used cold-start SFT data from R1-Zero (i.e. it was distilled from R1-Zero), plus additional RL training","open_weights":true,"org":"DeepSeek","params":671000000000,"release_date":"2025-01-20"},"deepseek-r1-0528":{"context":163840,"country":"China","created":1748455170,"display":"R1 0528","family":"deepseek-r1-0528","flop":3.3e+24,"flop_notes":"Incremental update on DeepSeek-R1. Presumed to be similar in FLOP cost, if the additional training was post-training/RL instead of continued pre-trianing","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":32768,"or_id":"deepseek/deepseek-r1-0528","org":"DeepSeek","output_modalities":["text"],"price_cache":0.35,"price_in":0.5,"price_match":"exact","price_out":2.15,"release_date":"2025-05-28","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"deepseek-r1-0528-qwen3-8b":{"display":"DeepSeek R1.0528 Qwen3 8B","family":"deepseek-r1-0528-qwen3-8b","release_date":"2025-05-28"},"deepseek-r1-distill-llama-70b":{"country":"China","display":"DeepSeek R1 Distill Llama 70B","family":"deepseek-r1-distill-llama-70b","flop_notes":"base model compute: 6.8649768e+24 FLOP\nfine tune compute: 2.016e+22 FLOP","org":"DeepSeek","release_date":"2025-01-20"},"deepseek-r1-distill-qwen-1-5b":{"country":"China","display":"DeepSeek R1 Distill Qwen 1 5B","family":"deepseek-r1-distill-qwen-1-5b","org":"DeepSeek","release_date":"2025-01-20"},"deepseek-r1-distill-qwen-14b":{"country":"China","display":"DeepSeek R1 Distill Qwen 14B","family":"deepseek-r1-distill-qwen-14b","org":"DeepSeek","release_date":"2025-01-20"},"deepseek-r1-distill-qwen-32b":{"country":"China","display":"DeepSeek R1 Distill Qwen 32B","family":"deepseek-r1-distill-qwen-32b","flop_notes":"Qwen2.5-32B: 3.51e+24 FLOP\nFine-tune compute: 9.22e21 FLOP","org":"DeepSeek","release_date":"2025-01-20"},"deepseek-reasoner":{"country":"China","display":"DeepSeek Reasoner","family":"deepseek-reasoner","flop":4.2e+24,"flop_notes":"Pre-trained base was DeepSeek-V3.1-Terminus\n3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","org":"DeepSeek","release_date":"2025-12-01"},"deepseek-v2":{"country":"China","display":"DeepSeek-V2 Base","family":"deepseek-v2","flop":1.02e+24,"flop_notes":"21b active params * 8.1 trillion * 6 = 1.02e24","org":"DeepSeek","release_date":"2024-05-07"},"deepseek-v2-5":{"country":"China","display":"DeepSeek V2.5","family":"deepseek-v2-5","flop":1.7892e+24,"flop_notes":"V2.5 is a merge of V2-coder and V2-chat\nV2-coder is trained for 6T additional tokens from an intermediate checkpoint of V2, which had been trained for 4.2T tokens. Total: 10.2T\nV2-chat is fine-tuned from V2, saw 8.2T tokens in pre-training\nUnique steps: 8.2T + 6T = 14.2T\nFLOPs: 6 * 21B * 14.2T = 1.7892e24","open_weights":true,"org":"DeepSeek","params":236000000000,"release_date":"2024-09-06"},"deepseek-v3":{"country":"China","display":"DeepSeek-V3 Base","family":"deepseek-v3","flop":3.3e+24,"flop_notes":"6 × 37B × 14.8T = 3.29e24 pretraining FLOPs, 1e22 posttrainining FLOPs - see doc below\n\nhttps://docs.google.com/document/d/1C4zBvHh4UNnzaorwQSdG61zQJsBV9X3i3A70NeelxHY/edit?usp=sharing","open_weights":true,"org":"DeepSeek","params":671000000000,"release_date":"2024-12-26"},"deepseek-v3-0324":{"country":"China","display":"DeepSeek V3 0324","family":"deepseek-v3-0324","flop":3.3e+24,"flop_notes":"6 × 37B × 14.8T = 3.29e24 pretraining FLOPs, 1e22 posttrainining FLOPs - see doc below\n\nhttps://docs.google.com/document/d/1C4zBvHh4UNnzaorwQSdG61zQJsBV9X3i3A70NeelxHY/edit?usp=sharing","org":"DeepSeek","release_date":"2025-03-24"},"deepseek-v3-1":{"country":"China","display":"deepseek-v3.1†","family":"deepseek-v3-1","flop":3.594058e+24,"flop_notes":"This was based on V3 which was trained on 14.8T tokens for a total of 14.8T + 840B = 15.64T pretraining tokens. Adding the FLOPs from continued pretraining onto our estimate for V3, which is 3.4e24 FLOPs, we get 3.4e24 + 1.9e23 = 3.59e24","org":"DeepSeek","release_date":"2025-08-21"},"deepseek-v3-1-terminus":{"aa":{"agentic_index":6.8,"coding_index":43.5,"intelligence_index":null},"context":163840,"country":"China","created":1758548275,"display":"DeepSeek V3.1 Terminus","family":"deepseek-v3-1-terminus","flop":3.594058e+24,"flop_notes":"This was based on V3 which was trained on 14.8T tokens for a total of 14.8T + 840B = 15.64T pretraining tokens. Adding the FLOPs from continued pretraining onto our estimate for V3, which is 3.4e24 FLOPs, we get 3.4e24 + 1.9e23 = 3.59e24","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":147456,"or_id":"deepseek/deepseek-v3.1-terminus","org":"DeepSeek","output_modalities":["text"],"price_in":0.27,"price_match":"exact","price_out":1,"release_date":"2025-09-22","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","reasoning","response_format","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"deepseek-v3-2-exp":{"context":163840,"country":"China","created":1759150481,"display":"DeepSeek V3.2 Exp","family":"deepseek-v3-2","flop":4.18e+24,"flop_notes":"3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","input_modalities":["text"],"knowledge_cutoff":"2025-07-31","max_output":147456,"or_id":"deepseek/deepseek-v3.2-exp","org":"DeepSeek","output_modalities":["text"],"price_in":0.27,"price_match":"exact","price_out":0.41,"release_date":"2025-09-29","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","reasoning","response_format","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"deepseek-v3-2-speciale":{"country":"China","display":"DeepSeek V3.2 Speciale","family":"deepseek-v3-2-speciale","flop":4.2e+24,"flop_notes":"Pre-trained base was DeepSeek-V3.1-Terminus\n3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","org":"DeepSeek","release_date":"2025-12-01"},"deepseek-v3-base":{"country":"China","display":"DeepSeek-V3 Base","family":"deepseek-v3-base","flop":3.3e+24,"flop_notes":"6 × 37B × 14.8T = 3.29e24 pretraining FLOPs, 1e22 posttrainining FLOPs - see doc below\n\nhttps://docs.google.com/document/d/1C4zBvHh4UNnzaorwQSdG61zQJsBV9X3i3A70NeelxHY/edit?usp=sharing","org":"DeepSeek","release_date":"2024-12-26"},"deepseek-v4-1-flash":{"context":1048576,"country":"China","created":1789021285,"display":"DeepSeek V4.1 Flash","family":"deepseek-v4-1-flash","flop":4.32e+24,"flop_notes":"(Pre-training) - 6 * 16e9 active parameters * 45e12 tokens = 4.32e+24 FLOP","input_modalities":["text","image"],"max_output":943718,"open_weights":true,"or_id":"deepseek/deepseek-v4.1-flash","org":"DeepSeek","output_modalities":["text"],"params":552000000000,"price_cache":0.018,"price_in":0.051,"price_match":"exact","price_out":0.567,"release_date":"2026-09-09","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"deepseek-v4-1-flash-1017":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":28.6},"context":1048576,"country":"China","created":1792225624,"display":"DeepSeek V4.1 Flash 1017","family":"deepseek-v4-1-flash-1017","flop_notes":"A","input_modalities":["text","image"],"max_output":943718,"or_id":"deepseek/deepseek-v4.1-flash-1017","org":"DeepSeek","output_modalities":["text"],"price_cache":0.005,"price_in":0.006,"price_match":"exact","price_out":0.5,"release_date":"2026-10-17","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"deepseek-v4-flash":{"aa":{"agentic_index":22.2,"coding_index":56.2,"intelligence_index":null},"context":1048576,"country":"China","created":1777000666,"display":"DeepSeek V4 Flash 0423","family":"deepseek-v4-flash","flop":2.496e+24,"flop_notes":"6 * 13e9 active parameters * 32e12 tokens = 2.496E24 for pre-training","input_modalities":["text"],"max_output":943717,"open_weights":true,"or_id":"deepseek/deepseek-v4-flash","org":"DeepSeek","output_modalities":["text"],"params":284000000000,"price_cache":0.0182,"price_in":0.091,"price_match":"exact","price_out":0.182,"release_date":"2026-04-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"deepseek-v4-flash-0731":{"aa":{"agentic_index":41,"coding_index":69.1,"intelligence_index":null},"context":1048576,"country":"China","created":1785478908,"display":"DeepSeek V4 Flash 0731","family":"deepseek-v4-flash-0731","flop":2.496e+24,"flop_notes":"From https://api-docs.deepseek.com/updates/: \"DeepSeek-V4-Flash-0731 keeps the same model architecture and size as DeepSeek-V4-Flash-Preview, and was only re-post-trained.\"\n\n6 * 13e9 active parameters * 32e12 tokens = 2.496E24 for pre-training","input_modalities":["text"],"max_output":943718,"or_id":"deepseek/deepseek-v4-flash-0731","org":"DeepSeek","output_modalities":["text"],"price_cache":0.035,"price_in":0.0434,"price_match":"exact","price_out":0.3,"release_date":"2026-07-31","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"deepseek-v4-pro":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":12.6},"context":1048576,"country":"China","created":1777000679,"display":"DeepSeek V4 Pro 0423","family":"deepseek-v4-pro","flop":9.702e+24,"flop_notes":"6 * 49e9 active parameters * 33e12 tokens = 9.702E24 for pre-training","input_modalities":["text"],"max_output":384000,"open_weights":true,"or_id":"deepseek/deepseek-v4-pro","org":"DeepSeek","output_modalities":["text"],"params":1600000000000,"price_cache":0.079605,"price_in":0.9553,"price_match":"exact","price_out":1.9105,"release_date":"2026-04-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"deepseek-v4-pro-0813":{"aa":{"agentic_index":41.3,"coding_index":68.8,"intelligence_index":23.1},"context":1048576,"country":"China","created":1786549364,"display":"DeepSeek V4 Pro 0813","family":"deepseek-v4-pro-0813","flop":9.702e+24,"flop_notes":"From https://api-docs.deepseek.com/news/news260813/: built on the DeepSeek-V4-Pro (Preview) model structure with a DSpark speculative-decoding module; agent-focused post-training upgrades. Pre-training compute carried over from DeepSeek-V4-Pro (6 * 49e9 active parameters * 33e12 tokens = 9.702E24).","input_modalities":["text"],"max_output":943718,"or_id":"deepseek/deepseek-v4-pro-0813","org":"DeepSeek","output_modalities":["text"],"price_cache":0.097,"price_in":0.111,"price_match":"exact","price_out":4.44,"release_date":"2026-08-13","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"deepseekdeepseek-v3-2":{"country":"China","display":"DeepSeek-V3.2","family":"deepseekdeepseek-v3-2","flop":4.2e+24,"flop_notes":"Pre-trained base was DeepSeek-V3.1-Terminus\n3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","org":"DeepSeek","release_date":"2025-12-01"},"devstral-small-2512":{"country":"France","display":"Devstral Small 2","family":"devstral-small-2512","org":"Mistral","release_date":"2025-12-09"},"dolly-v2-12b":{"country":"United States of America","display":"Dolly-v2-12B","family":"dolly-v2-12b","org":"Databricks","release_date":"2023-04-11"},"doubao-seed-2-0-lite-260428":{"country":"China","display":"Doubao Seed 2.0 Lite 260428","family":"doubao-seed-2-0-lite-260428","org":"ByteDance","release_date":"2026-04-28"},"doubao-seed-2-0-pro-260215":{"country":"China","display":"Doubao Seed 2.0 Pro 260215","family":"doubao-seed-2-0-pro-260215","org":"ByteDance","release_date":"2026-02-14"},"doubao-seed-2-1-pro-260628":{"country":"China","display":"Doubao Seed 2.1 Pro 260628","family":"doubao-seed-2-1-pro-260628","org":"ByteDance","release_date":"2026-06-23"},"doubao-seed-2-1-turbo-260628":{"country":"China","display":"Doubao Seed 2.1 Turbo 260628","family":"doubao-seed-2-1-turbo-260628","org":"ByteDance","release_date":"2026-06-23"},"doubao-seed-evolving":{"country":"China","display":"Doubao Seed Evolving","family":"doubao-seed-evolving","org":"ByteDance","release_date":"2026-11-02"},"dracarys2-72b-instruct":{"display":"Dracarys2 72B Instruct","family":"dracarys2-72b-instruct","release_date":"2024-09-30"},"dracarys2-llama-3-1-70b-instruct":{"display":"Dracarys2 Llama 3.1 70B Instruct","family":"dracarys2-llama-3-1-70b-instruct","release_date":"2024-08-14"},"eurus-2-7b-prime":{"country":"China,United States of America","display":"Eurus 2 7B Prime","family":"eurus-2-7b-prime","flop":401915904000000000000,"flop_notes":"Hardware-based estimate from the RL training: 592 steps (from Table 1) * 680s/step (Table 2, PRIME row) * 8 (they used A800) * 312,000,000,000,000 FLOPS * 0.4 assumed utilization = 4.01915904E20 FLOPs","open_weights":true,"org":"Tsinghua University,University of Illinois Urbana-Champaign (UIUC),Shanghai AI Lab,Peking University,Shanghai Jiao Tong University,CUHK Shenzhen Research Institute","params":7000000000,"release_date":"2024-12-31"},"falcon-11b":{"country":"United Arab Emirates","display":"Falcon2-11B stage 4","family":"falcon-11b","flop":3.6e+23,"flop_notes":"trained on 5.5T tokens\n\n6 * 11B * 5.5T = 3.6e23","org":"Technology Innovation Institute","release_date":"2024-05-09"},"falcon-11b-vlm":{"display":"Falcon2-11B VLM","family":"falcon-11b-vlm","release_date":"2024-05-21"},"falcon-180b":{"country":"United Arab Emirates","display":"Falcon 180B","family":"falcon-180b","flop":3.76e+24,"flop_notes":"43,500 petaflop-days per Table 1 of the paper\n\n43500 * 1e15 * 24 * 3600 = 3.76e24\n\n\nC = 6ND = 6 FLOP/token/parameter * 3.5 trillion tokens * 180 billion parameters = 3.78*10^24 FLOP","open_weights":true,"org":"Technology Innovation Institute","params":180000000000,"release_date":"2023-09-06"},"falcon-40b":{"country":"United Arab Emirates","display":"Falcon 40B","family":"falcon-40b","flop":2.4e+23,"flop_notes":"C = 6ND = 6 * 40B * 1000B = 2.4e+23 FLOP (assuming one epoch)\n\nTable 1 from https://arxiv.org/pdf/2311.16867 Falcon paper\n\n2,800 petaflop-days * 1e15 * 24 * 3600 = 2.4192e+23 FLOPs","open_weights":true,"org":"Technology Innovation Institute","params":40000000000,"release_date":"2023-03-15"},"falcon-40b-instruct":{"country":"United Arab Emirates","display":"Falcon-Instruct (40B)","family":"falcon-40b-instruct","flop":2.4e+23,"flop_notes":"C = 6ND = 6 * 40B * 1000B = 2.4e+23 FLOP (assuming one epoch)\n\nTable 1 from https://arxiv.org/pdf/2311.16867 Falcon paper\n\n2,800 petaflop-days * 1e15 * 24 * 3600 = 2.4192e+23 FLOPs","org":"Technology Innovation Institute","release_date":"2023-05-25"},"falcon-7b":{"country":"United Arab Emirates","display":"Falcon 7B","family":"falcon-7b","flop":6.3e+22,"flop_notes":"6ND = 6 * 7B * 1.5T = 6.3e22\n\"Falcon-7B is a 7B parameters causal decoder-only model built by TII and trained on 1,500B tokens of RefinedWeb enhanced with curated corpora. It is made available under the Apache 2.0 license.\"\n\nTable 1 from https://arxiv.org/pdf/2311.16867 Falcon paper\n\n730 petaflop-days * 1e15 * 24 * 3600 = 6.3072e+22 FLOPs","org":"Technology Innovation Institute","release_date":"2023-04-24"},"fireworksdeepseek-v3p2":{"country":"China","display":"fireworks/deepseek-v3p2","family":"fireworksdeepseek-v3p2","flop":4.2e+24,"flop_notes":"Pre-trained base was DeepSeek-V3.1-Terminus\n3.594058e+24 FLOP [base model] + 2.095014e+23 FLOP = 3.8035594e+24 FLOP for pre-training plus continued pre-training\n\nThen, \"this framework allocates a\npost-training computational budget exceeding 10% of the pre-training cost\", so 4.18e24","org":"DeepSeek","release_date":"2025-12-01"},"fireworkskimi-k2-instruct-0905":{"country":"China","display":"Fireworkskimi K2 Instruct","family":"fireworkskimi-k2-instruct-0905","flop":2.976e+24,"flop_notes":"6 FLOP / parameter / token * 32 * 10^9 activated parameters * 15.5 * 10^12 tokens = 2.976e+24 FLOP","org":"Moonshot AI","release_date":"2025-09-05"},"fireworkskimi-k2p5":{"country":"China","display":"Fireworkskimi K2p5","family":"fireworkskimi-k2p5","flop":5.8e+24,"flop_notes":"Per section 3.2, \"The joint pre-training stage continues from a near-end Kimi K2 checkpoint over additional 15T vision-text tokens at 4K sequence length\"\n\nKimi K2 was pretrained on 15T tokens. A near-end checkpoint of K2 was trained into K2.5 over another 15T tokens (or 15.2 to 15.5T tokens). This makes for a total of ~30T.\n\nThere are an additional 1T tokens in \"Stage 1\" used to train the ViT encoder, which is likely much smaller than the full LLM, so we omit those tokens to calculate training compute.\n\n30e12 tokens * 32B parameters * 6 = 5.76e24 FLOP\n\nPer Kimi talk (timestamp 26:00), the 15 trillion tokens of K2.5 were additive, for 30T training tokens overall: https://x.com/Kimi_Moonshot/status/2035011542115393973?s=20","org":"Moonshot AI","release_date":"2026-01-27"},"gdm-ai-co-mathematician":{"country":"United States of America","display":"Gdm AI Co Mathematician","family":"gdm-ai-co-mathematician","org":"Google DeepMind","release_date":"2026-05-08"},"gemini-1-0-pro-001":{"country":"United States of America","display":"Gemini 1.0 Pro (001)","family":"gemini-1-0-pro-001","flop_notes":"Training compute estimated to be 1.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing \n\nOur reasoning and calculations for Gemini 1 Ultra are detailed in this Colab notebook.\nhttps://colab.research.google.com/drive/1sfG91UfiYpEYnj_xB5YRy07T5dv-9O_c","org":"Google DeepMind","release_date":"2023-12-13"},"gemini-1-0-pro-vision":{"country":"United States of America","display":"Gemini 1.0 Pro V","family":"gemini-1-0-pro-vision","flop_notes":"To the extent that we believe Gemini 1 Pro was below 1e25 FLOP, Gemini 1 Pro Vision is unlikely to exceed it.","org":"Google DeepMind","release_date":"2024-01-04"},"gemini-1-5-flash-001":{"country":"United States of America","display":"Gemini-1.5 Flash","family":"gemini-1-5-flash-001","flop_notes":"\"Gemini 1.5 Flash is a dense Transformer based model that is online distilled [...] from Gemini 1.5 Pro.\"","org":"Google DeepMind","release_date":"2024-05-23"},"gemini-1-5-flash-002":{"country":"United States of America","display":"Gemini 1.5 Flash (002)","family":"gemini-1-5-flash-002","flop_notes":"\"Gemini 1.5 Flash is a dense Transformer based model that is online distilled [...] from Gemini 1.5 Pro.\"","org":"Google DeepMind","release_date":"2024-09-24"},"gemini-1-5-flash-0514":{"country":"United States of America","display":"Gemini 1.5 Flash (0514 preview)","family":"gemini-1-5-flash-0514","flop_notes":"\"Gemini 1.5 Flash is a dense Transformer based model that is online distilled [...] from Gemini 1.5 Pro.\"","org":"Google DeepMind","release_date":"2024-05-14"},"gemini-1-5-flash-8b-001":{"country":"United States of America","display":"Gemini 1.5 Flash 8B 001","family":"gemini-1-5-flash-8b-001","org":"Google DeepMind","release_date":"2024-10-03"},"gemini-1-5-pro-001":{"country":"United States of America","display":"Gemini 1.5 Pro (May 2024)","family":"gemini-1-5-pro-001","flop_notes":"Training compute imputed to be 1.58e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"Google DeepMind","release_date":"2024-05-14"},"gemini-1-5-pro-001-feb24":{"country":"United States of America","display":"Gemini 1.5 Pro (Feb 2024)","family":"gemini-1-5-pro-001-feb24","flop_notes":"Training compute imputed to be 1.58e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"Google DeepMind","release_date":"2024-02-15"},"gemini-1-5-pro-002":{"country":"United States of America","display":"Gemini 1.5 Pro","family":"gemini-1-5-pro-002","flop_notes":"Training compute imputed to be 1.58e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"Google DeepMind","release_date":"2024-09-24"},"gemini-2-0-flash-001":{"country":"United States of America","display":"Gemini 2.0 Flash","family":"gemini-2-0-flash-001","flop_notes":"\"We used Trillium TPUs to train the new Gemini 2.0, Google’s most capable AI model yet\" according to https://cloud.google.com/blog/products/compute/trillium-tpu-is-ga","org":"Google DeepMind","release_date":"2025-02-05"},"gemini-2-0-flash-02-05":{"country":"United States of America","display":"Gemini 2.0 Flash (February 2025)","family":"gemini-2-0-flash-02-05","flop_notes":"\"We used Trillium TPUs to train the new Gemini 2.0, Google’s most capable AI model yet\" according to https://cloud.google.com/blog/products/compute/trillium-tpu-is-ga","org":"Google DeepMind","release_date":"2025-02-05"},"gemini-2-0-flash-exp":{"country":"United States of America","display":"Gemini 2.0 Flash (Experimental)","family":"gemini-2-0-flash","flop_notes":"\"We used Trillium TPUs to train the new Gemini 2.0, Google’s most capable AI model yet\" according to https://cloud.google.com/blog/products/compute/trillium-tpu-is-ga","org":"Google DeepMind","release_date":"2024-12-11"},"gemini-2-0-flash-lite":{"country":"United States of America","display":"Gemini 2.0 Flash Lite","family":"gemini-2-0-flash-lite","org":"Google DeepMind","release_date":"2025-02-05"},"gemini-2-0-flash-lite-001":{"country":"United States of America","display":"gemini-2.0-flash-lite-001†","family":"gemini-2-0-flash-lite-001","org":"Google DeepMind","release_date":"2025-02-25"},"gemini-2-0-flash-lite-preview-02-05":{"country":"United States of America","display":"Gemini 2.0 Flash Lite Preview 02.05","family":"gemini-2-0-flash-lite-preview-02-05","org":"Google DeepMind","release_date":"2025-02-05"},"gemini-2-0-flash-thinking-exp-01-21":{"country":"United States of America","display":"Gemini 2.0 Flash Thinking (January 2025)","family":"gemini-2-0-flash-thinking-exp-01-21","org":"Google DeepMind","release_date":"2025-01-21"},"gemini-2-0-pro-exp-02-05":{"country":"United States of America","display":"Gemini 2.0 Pro Experimental (Feb 2025)","family":"gemini-2-0-pro-exp-02-05","flop_notes":"Flagship model from a leading developer in early 2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-02-05"},"gemini-2-5-deep-think-2025-08-01-webapp":{"country":"United States of America","display":"Gemini 2.5 Deep Think 2025.08.01 Webapp","family":"gemini-2-5-deep-think-2025-08-01-webapp","org":"Google DeepMind","release_date":"2025-08-01"},"gemini-2-5-flash":{"context":1048576,"country":"United States of America","created":1750172488,"display":"Gemini 2.5 Flash","family":"gemini-2-5-flash","input_modalities":["file","image","text","audio","video"],"knowledge_cutoff":"2025-01-31","max_output":65535,"or_id":"google/gemini-2.5-flash","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.03,"price_in":0.3,"price_match":"exact","price_out":2.5,"release_date":"2025-06-17","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-2-5-flash-lite-preview-06-17-16k":{"country":"United States of America","display":"Gemini 2.5 Flash Lite Preview 06.17 16K","family":"gemini-2-5-flash-lite-preview-06-17-16k","org":"Google DeepMind","release_date":"2025-06-17"},"gemini-2-5-flash-lite-preview-06-17-thinking":{"country":"United States of America","display":"Gemini 2.5 Flash Lite","family":"gemini-2-5-flash-lite-preview-06-17-thinking","org":"Google DeepMind","release_date":"2025-06-17"},"gemini-2-5-flash-lite-preview-09-2025":{"country":"United States of America","display":"Gemini 2.5 Flash Lite Preview 09","family":"gemini-2-5-flash-lite-preview-09-2025","org":"Google DeepMind","release_date":"2025-09-25"},"gemini-2-5-flash-preview-04-17":{"country":"United States of America","display":"gemini-2.5-flash-preview-04-17†","family":"gemini-2-5-flash-preview-04-17","org":"Google DeepMind","release_date":"2025-04-17"},"gemini-2-5-flash-preview-04-17-16k-thinking":{"country":"United States of America","display":"Gemini 2.5 Flash Preview 04.17 16K","family":"gemini-2-5-flash-preview-04-17-16k-thinking","org":"Google DeepMind","release_date":"2025-04-17"},"gemini-2-5-flash-preview-04-17-24k-thinking":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview) (Thinking 24K)","family":"gemini-2-5-flash-preview-04-17-24k-thinking","org":"Google DeepMind","release_date":"2025-04-17"},"gemini-2-5-flash-preview-05-20":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview)","family":"gemini-2-5-flash-preview-05-20","org":"Google DeepMind","release_date":"2025-05-20"},"gemini-2-5-flash-preview-05-20-16k":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview) (Thinking 16K)","family":"gemini-2-5-flash-preview-05-20-16k","org":"Google DeepMind","release_date":"2025-05-20"},"gemini-2-5-flash-preview-05-20-1k":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview) (Thinking 1K)","family":"gemini-2-5-flash-preview-05-20-1k","org":"Google DeepMind","release_date":"2025-05-20"},"gemini-2-5-flash-preview-05-20-23k":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview) (Thinking 24K)","family":"gemini-2-5-flash-preview-05-20-23k","org":"Google DeepMind","release_date":"2025-05-20"},"gemini-2-5-flash-preview-05-20-8k":{"country":"United States of America","display":"Gemini 2.5 Flash (Preview) (Thinking 8K)","family":"gemini-2-5-flash-preview-05-20-8k","org":"Google DeepMind","release_date":"2025-05-20"},"gemini-2-5-flash-preview-09-2025":{"country":"United States of America","display":"MUSE + Gemini 2.5 Flash","family":"gemini-2-5-flash-preview-09-2025","org":"Google DeepMind","release_date":"2025-09-25"},"gemini-2-5-flash-preview-09-2025-16k":{"country":"United States of America","display":"Gemini 2.5 Flash Preview 09.2025 16K","family":"gemini-2-5-flash-preview-09-2025-16k","org":"Google DeepMind","release_date":"2025-09-25"},"gemini-2-5-pro":{"aa":{"agentic_index":1.6,"coding_index":33.3,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1750169544,"display":"Gemini 2.5 Pro","family":"gemini-2-5-pro","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","input_modalities":["text","image","file","audio","video"],"knowledge_cutoff":"2025-01-31","max_output":65536,"or_id":"google/gemini-2.5-pro","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.125,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-06-17","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-2-5-pro-16k":{"country":"United States of America","display":"Gemini 2.5 Pro (Thinking 16K)","family":"gemini-2-5-pro-16k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-06-17"},"gemini-2-5-pro-32k":{"country":"United States of America","display":"Gemini 2.5 Pro","family":"gemini-2-5-pro-32k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-06-17"},"gemini-2-5-pro-8k":{"country":"United States of America","display":"Gemini 2.5 Pro (Thinking 8K)","family":"gemini-2-5-pro-8k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-06-17"},"gemini-2-5-pro-exp-03-25":{"country":"United States of America","display":"Gemini 2.5 Pro Experimental (March 2025)","family":"gemini-2-5-pro-exp-03-25","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-03-25"},"gemini-2-5-pro-preview-03-25":{"country":"United States of America","display":"Gemini 2.5 Pro (Preview)","family":"gemini-2-5-pro-preview-03-25","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-03-31"},"gemini-2-5-pro-preview-05-06":{"country":"United States of America","display":"Gemini 2.5 Pro Preview (May 06 2025)","family":"gemini-2-5-pro-preview-05-06","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-05-06"},"gemini-2-5-pro-preview-06-05":{"context":1048576,"country":"United States of America","created":1749137257,"display":"Gemini 2.5 Pro Preview 06-05","family":"gemini-2-5-pro-preview-06-05","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","input_modalities":["file","image","text","audio"],"knowledge_cutoff":"2025-01-31","max_output":65536,"or_id":"google/gemini-2.5-pro-preview","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.125,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-06-05","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-2-5-pro-preview-06-05-1k":{"country":"United States of America","display":"Gemini 2.5 Pro (Thinking 1K)","family":"gemini-2-5-pro-preview-06-05-1k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-06-05"},"gemini-2-5-pro-preview-06-05-32k":{"country":"United States of America","display":"Gemini 2.5 Pro Preview 06.05 32K","family":"gemini-2-5-pro-preview-06-05-32k","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","org":"Google DeepMind","release_date":"2025-06-05"},"gemini-3-1-flash-lite":{"context":1048576,"country":"United States of America","created":1778168828,"display":"Gemini 3.1 Flash Lite","family":"gemini-3-1-flash-lite","input_modalities":["text","image","video","file","audio"],"max_output":65536,"or_id":"google/gemini-3.1-flash-lite","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.025,"price_in":0.25,"price_match":"exact","price_out":1.5,"release_date":"2026-03-03","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-1-pro-preview":{"aa":{"agentic_index":8.2,"coding_index":68.8,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1771509627,"display":"Gemini 3.1 Pro Preview","family":"gemini-3-1-pro","input_modalities":["audio","file","image","text","video"],"max_output":65536,"or_id":"google/gemini-3.1-pro-preview","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":12,"release_date":"2026-02-19","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-1-pro-preview-customtools":{"context":1048576,"country":"United States of America","created":1772045923,"display":"Gemini 3.1 Pro Preview Custom Tools","family":"gemini-3-1-pro-preview-customtools","input_modalities":["text","audio","image","video","file"],"max_output":65536,"or_id":"google/gemini-3.1-pro-preview-customtools","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":12,"release_date":"2026-02-19","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-5-flash":{"aa":{"agentic_index":26,"coding_index":70.1,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1779193800,"display":"Gemini 3.5 Flash","family":"gemini-3-5-flash","input_modalities":["text","image","video","file","audio"],"knowledge_cutoff":"2025-01-01","max_output":65536,"open_weights":false,"or_id":"google/gemini-3.5-flash","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.15,"price_in":1.5,"price_match":"exact","price_out":9,"release_date":"2026-05-19","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-5-flash-lite":{"aa":{"agentic_index":14.3,"coding_index":49.3,"intelligence_index":11.7},"context":1048576,"country":"United States of America","created":1784646726,"display":"Gemini 3.5 Flash Lite","family":"gemini-3-5-flash-lite","input_modalities":["text","image","video","file","audio"],"max_output":65536,"or_id":"google/gemini-3.5-flash-lite","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.03,"price_in":0.3,"price_match":"exact","price_out":2.5,"release_date":"2026-07-21","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-6-flash":{"aa":{"agentic_index":29,"coding_index":69.2,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1784646733,"display":"Gemini 3.6 Flash","family":"gemini-3-6-flash","input_modalities":["text","image","video","file","audio"],"max_output":65536,"or_id":"google/gemini-3.6-flash","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.075,"price_in":0.75,"price_match":"exact","price_out":3.75,"release_date":"2026-07-21","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-7-flash":{"aa":{"agentic_index":35.2,"coding_index":76.1,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1786640581,"display":"Gemini 3.7 Flash","family":"gemini-3-7-flash","input_modalities":["text","image","video","file","audio"],"max_output":65536,"or_id":"google/gemini-3.7-flash","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.075,"price_in":0.75,"price_match":"exact","price_out":3.75,"release_date":"2026-08-13","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-8-flash":{"aa":{"agentic_index":40.2,"coding_index":76.3,"intelligence_index":30.8},"context":1048576,"country":"United States of America","created":1788362056,"display":"Gemini 3.8 Flash","family":"gemini-3-8-flash","input_modalities":["text","image","video","file","audio"],"max_output":65536,"open_weights":false,"or_id":"google/gemini-3.8-flash","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.0825,"price_in":0.825,"price_match":"exact","price_out":4.125,"release_date":"2026-09-02","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-deep-think-preview":{"country":"United States of America","display":"Gemini 3 Deep Think (2/26)","family":"gemini-3-deep-think","org":"Google DeepMind","release_date":"2026-02-12"},"gemini-3-flash-preview":{"context":1048576,"country":"United States of America","created":1765987078,"display":"Gemini 3 Flash Preview","family":"gemini-3-flash","input_modalities":["text","image","file","audio","video"],"max_output":65536,"or_id":"google/gemini-3-flash-preview","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.05,"price_in":0.5,"price_match":"exact","price_out":3,"release_date":"2025-12-17","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-3-pro-preview":{"country":"United States of America","display":"Gemini 3 Pro Preview","family":"gemini-3-pro","flop_notes":"Almost certainly \u003e1e25","org":"Google DeepMind","release_date":"2025-11-18"},"gemini-4-argon":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":50.4},"context":1048576,"country":"United States of America","created":1791907477,"display":"Gemini 4 Argon","family":"gemini-4-argon","input_modalities":["audio","file","image","text","video"],"max_output":943718,"or_id":"google/gemini-4-argon","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.1,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-09-30","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"gemini-4-carbon":{"country":"United States of America","display":"Gemini 4 Carbon","family":"gemini-4-carbon","open_weights":false,"org":"Google DeepMind","release_date":"2026-10-28"},"gemini-exp-1206":{"country":"United States of America","display":"Gemini Exp","family":"gemini-exp-1206","flop_notes":"\"We used Trillium TPUs to train the new Gemini 2.0, Google’s most capable AI model yet\" according to https://cloud.google.com/blog/products/compute/trillium-tpu-is-ga","org":"Google DeepMind","release_date":"2024-12-06"},"gemini-robotics-er-1-5-preview":{"display":"Gemini Robotics Er 1.5 Preview","family":"gemini-robotics-er-1-5","release_date":"2025-09-26"},"gemini-robotics-er-1-6-preview":{"country":"United States of America","display":"Gemini Robotics-ER 1.6","family":"gemini-robotics-er-1-6","org":"Google DeepMind","release_date":"2026-04-14"},"gemma-2-27b-it":{"context":8192,"country":"United States of America","created":1720828800,"display":"Gemma 2 27B","family":"gemma-2-27b-it","flop":2.106e+24,"flop_notes":"\"For the 27B model, we train on an 8x24x32 configuration of\nTPUv5p, totaling 6144 chips\"\n\ntrained on 13T tokens\n\n6ND = 6*27000000000*13000000000000=2.106e+24","input_modalities":["text"],"knowledge_cutoff":"2024-06-30","max_output":2048,"or_id":"google/gemma-2-27b-it","org":"Google DeepMind","output_modalities":["text"],"price_in":0.65,"price_match":"exact","price_out":0.65,"release_date":"2024-06-24","supported_parameters":["frequency_penalty","max_tokens","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","top_p"]},"gemma-2-9b":{"country":"United States of America","display":"Gemma-2 9B","family":"gemma-2-9b","flop":4.32e+23,"flop_notes":"\"For the 9B model, we train on an 8x16x32 configuration of TPUv4, totaling 4096 chips\"\n\n6ND = 6 FLOP / token / parameter * 9000000000 parameters * 8000000000000 tokens = 4.32e+23 FLOP","org":"Google DeepMind"},"gemma-2-9b-it":{"country":"United States of America","display":"Gemma 2 Instruct (9B)","family":"gemma-2-9b-it","flop":4.32e+23,"flop_notes":"\"For the 9B model, we train on an 8x16x32 configuration of TPUv4, totaling 4096 chips\"\n\n6ND = 6 FLOP / token / parameter * 9000000000 parameters * 8000000000000 tokens = 4.32e+23 FLOP","org":"Google DeepMind","release_date":"2024-06-24"},"gemma-2b":{"country":"United States of America","display":"Gemma 2B","family":"gemma-2b","flop":4.5115822e+22,"flop_notes":"6ND = 6*2506434560.00 parameters * 3*10^12 training tokens = 4.5115822e+22\n\n(assuming 1 epoch)","org":"Google DeepMind","release_date":"2024-02-21"},"gemma-3-12b-it":{"aa":{"agentic_index":0.1,"coding_index":5.8,"intelligence_index":null},"context":131072,"country":"United States of America","created":1741902625,"display":"Gemma 3 12B","family":"gemma-3-12b-it","flop":8.64e+23,"flop_notes":"6ND = 6 * 12B parameters * 12T training tokens = 8.64 × 10^23 FLOP","input_modalities":["text","image"],"knowledge_cutoff":"2024-08-31","max_output":16384,"or_id":"google/gemma-3-12b-it","org":"Google DeepMind","output_modalities":["text"],"price_in":0.05,"price_match":"exact","price_out":0.15,"release_date":"2025-03-12","supported_parameters":["frequency_penalty","logit_bias","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"gemma-3-1b-it":{"country":"United States of America","display":"Gemma 3 1B IT","family":"gemma-3-1b-it","flop":1.2e+22,"flop_notes":"6ND = 6 * 1B parameters * 2T training tokens = 1.2 × 10^22 FLOP","org":"Google DeepMind","release_date":"2025-03-12"},"gemma-3-270m-it":{"country":"United States of America","display":"Gemma 3 270M","family":"gemma-3-270m-it","flop":9.72e+21,"flop_notes":"6 FLOP / parameter / token * 270 * 10^9 parameters * 6 * 10^12 tokens = 9.72e+21 FLOP","org":"Google DeepMind","release_date":"2025-08-14"},"gemma-3-27b-it":{"aa":{"agentic_index":0.1,"coding_index":10.1,"intelligence_index":null},"context":131072,"country":"United States of America","created":1741756359,"display":"Gemma 3 27B","family":"gemma-3-27b-it","flop":2.268e+24,"flop_notes":"6ND =  6 * 27B parameters * 14T training tokens = 2.268 × 10^24 FLOP","input_modalities":["text","image"],"knowledge_cutoff":"2024-08-31","max_output":117964,"or_id":"google/gemma-3-27b-it","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.04,"price_in":0.08,"price_match":"exact","price_out":0.45,"release_date":"2025-03-12","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"gemma-3-4b-it":{"aa":{"agentic_index":null,"coding_index":2.7,"intelligence_index":null},"context":131072,"country":"United States of America","created":1741905510,"display":"Gemma 3 4B","family":"gemma-3-4b-it","flop":9.6e+22,"flop_notes":"6ND = 6 * 4B parameters * 4T training tokens = 9.6 × 10^22 FLOP","input_modalities":["text","image"],"knowledge_cutoff":"2024-08-31","max_output":16384,"or_id":"google/gemma-3-4b-it","org":"Google DeepMind","output_modalities":["text"],"price_in":0.05,"price_match":"exact","price_out":0.1,"release_date":"2025-03-12","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","top_k","top_logprobs","top_p"]},"gemma-4-26b-a4b":{"country":"United States of America","display":"Gemma 4 26B A4b IT","family":"gemma-4-26b-a4b","org":"Google DeepMind","release_date":"2026-04-02"},"gemma-4-26b-a4b-it":{"aa":{"agentic_index":2.9,"coding_index":39.3,"intelligence_index":null},"context":262144,"country":"United States of America","created":1775227989,"display":"Gemma 4 26B A4B ","family":"gemma-4-26b-a4b-it","input_modalities":["image","text","video"],"max_output":235929,"or_id":"google/gemma-4-26b-a4b-it","org":"Google DeepMind","output_modalities":["text"],"price_cache":0.05,"price_in":0.09,"price_match":"exact","price_out":0.3,"release_date":"2026-04-02","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"gemma-4-31b-it":{"aa":{"agentic_index":4.2,"coding_index":43.4,"intelligence_index":4.4},"context":262144,"country":"United States of America","created":1775148486,"display":"Gemma 4 31B","family":"gemma-4-31b-it","input_modalities":["image","text","video"],"max_output":16384,"or_id":"google/gemma-4-31b-it","org":"Google DeepMind","output_modalities":["text"],"price_in":0.2,"price_match":"exact","price_out":0.4,"release_date":"2026-04-02","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"gemma-7b":{"country":"United States of America","display":"Gemma 7b","family":"gemma-7b","flop":3.07e+23,"flop_notes":"6ND aproximation 6*8.54B*6T = 3.07e23\n\"Gemma 2B and 7B are trained on 2T and 6T tokens respectively of primarily-English data from web documents, mathematics, and code.\"\n\nAs confirmation: \"We estimate the carbon emissions from pretraining the Gemma models to be ∼ 131 𝑡𝐶𝑂2𝑒𝑞. \"\n\nU.S. avg CO2 per kWh is ~0.87lbs \n131 tCO2 * 2000 lb/t * (1 kWh/0.87lb) = 3.01e5 kWh\n\nPer SemiAnalysis TPU v5e uses ~ 5x less power than H100, so ~140 W TDP\n3.01e5 kWh * 1000 W/kW * 1 TPUv5e/140 W  = 2.15e6 TPUv5e-hours\n\nIn bf16 precision, TPUv5e has peak performance of 197 TF/s, so:\n2.15e6 * 3600 * 197e12 * 0.3 = 4.57e23","org":"Google DeepMind","release_date":"2024-02-21"},"glam-moe":{"country":"United States of America","display":"GLaM","family":"glam-moe","flop":3.6363112434e+23,"flop_notes":"The network activates 96.6 billion parameters per token and trained for 600B tokens.\n\n6 * 600B * 96.6B = 3.478e23\n\nDigitizing figure 4 (d) indicates 139.67 TPU-years of training. \n2.75e14 * 139.67 * 365.25 * 24 * 3600 * 0.3 = 3.636e23\n\nSince these are close, we will use the 6NC estimate and derive hardware utilization from the training time information.\n\nLater they say they measured 326W power usage per chip, which could maybe be used to estimate utilization.","open_weights":false,"org":"Google DeepMind","params":1200000000000,"release_date":"2021-12-13"},"glm-4-5":{"context":131072,"country":"China","created":1753471347,"display":"GLM 4.5","expiration_date":"2026-12-31","family":"glm-4-5","flop":4.42e+24,"flop_notes":"(6 FLOP/token/parameter) * (23T tokens) * (32B active parameters) = 4.42e24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2024-12-31","max_output":98304,"open_weights":true,"or_id":"z-ai/glm-4.5","org":"Z.ai","output_modalities":["text"],"params":355000000000,"price_cache":0.11,"price_in":0.6,"price_match":"exact","price_out":2.2,"release_date":"2025-08-03","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","temperature","tool_choice","tools","top_k","top_p"]},"glm-4-5-air":{"context":131072,"country":"China","created":1753471258,"display":"GLM 4.5 Air","family":"glm-4-5-air","flop":1.656e+24,"flop_notes":"6 FLOP / parameter / token * 12 * 10^9 active parameters * 23 * 10^12 tokens = 1.656e+24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2024-12-31","max_output":98304,"or_id":"z-ai/glm-4.5-air","org":"Z.ai","output_modalities":["text"],"price_cache":0.03,"price_in":0.2,"price_match":"exact","price_out":1.1,"release_date":"2025-07-20","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","repetition_penalty","seed","stop","temperature","tool_choice","tools","top_k","top_p"]},"glm-4-6":{"aa":{"agentic_index":null,"coding_index":45.8,"intelligence_index":null},"context":204800,"country":"China","created":1759235576,"display":"GLM 4.6","family":"glm-4-6","flop":4.42e+24,"flop_notes":"6 FLOP/parameter/token * 32000000000 active parameters [very likely assumption - everything else is reported to be same as at GLM 4.5] * 23000000000000 tokens = 4.42e24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":131072,"open_weights":true,"or_id":"z-ai/glm-4.6","org":"Z.ai","output_modalities":["text"],"params":357000000000,"price_cache":0.1,"price_in":0.5,"price_match":"exact","price_out":2,"release_date":"2025-09-30","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"glm-4-7":{"aa":{"agentic_index":null,"coding_index":45.3,"intelligence_index":null},"context":204800,"country":"China","created":1766378014,"display":"GLM 4.7","family":"glm-4-7","flop":4.42e+24,"flop_notes":"Training tokens not disclosed.\n\nPresuming this is a post-training update from GLM-4.5 (23T tokens) with minor additional compute, training compute is at least:\n\n6 FLOP/parameter/token * 32B active parameters * 23T tokens = 4.42e24 FLOP","input_modalities":["text"],"max_output":131072,"open_weights":true,"or_id":"z-ai/glm-4.7","org":"Z.ai","output_modalities":["text"],"params":358000000000,"price_cache":0.11,"price_in":0.6,"price_match":"exact","price_out":2.2,"release_date":"2025-12-22","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_p"]},"glm-4-7-flash":{"context":200000,"created":1768833913,"display":"GLM 4.7 Flash","family":"glm-4-7-flash","input_modalities":["text"],"max_output":117964,"or_id":"z-ai/glm-4.7-flash","output_modalities":["text"],"price_in":0.0605,"price_match":"exact","price_out":0.4,"supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"glm-5":{"context":204800,"country":"China","created":1770829182,"display":"GLM 5","family":"glm-5","flop":6.84e+24,"flop_notes":"6 * 40B parameters * 28.5e12 tokens = 6.84e24 FLOP estimated","input_modalities":["text"],"max_output":131072,"open_weights":true,"or_id":"z-ai/glm-5","org":"Z.ai","output_modalities":["text"],"params":744000000000,"price_cache":0.2,"price_in":1,"price_match":"exact","price_out":3.2,"release_date":"2026-02-11","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"glm-5-1":{"aa":{"agentic_index":23.9,"coding_index":55.8,"intelligence_index":null},"context":204800,"country":"China","created":1775578025,"display":"GLM 5.1","family":"glm-5-1","input_modalities":["text"],"max_output":131072,"open_weights":true,"or_id":"z-ai/glm-5.1","org":"Z.ai","output_modalities":["text"],"params":754000000000,"price_cache":0.26,"price_in":1.4,"price_match":"exact","price_out":4.4,"release_date":"2026-04-07","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"glm-5-2":{"aa":{"agentic_index":38.4,"coding_index":68.8,"intelligence_index":null},"context":1048576,"country":"China","created":1781631930,"display":"GLM 5.2","family":"glm-5-2","input_modalities":["text"],"max_output":943718,"or_id":"z-ai/glm-5.2","org":"Z.ai","output_modalities":["text"],"price_cache":0.005,"price_in":0.006,"price_match":"exact","price_out":6.8,"release_date":"2026-06-16","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"glm-5-3":{"aa":{"agentic_index":53.1,"coding_index":74.8,"intelligence_index":31.7},"context":1048576,"country":"China","created":1787086655,"display":"GLM 5.3","family":"glm-5-3","input_modalities":["text"],"max_output":943718,"open_weights":true,"or_id":"z-ai/glm-5.3","org":"Z.ai","output_modalities":["text"],"params":744000000000,"price_cache":0.0689,"price_in":0.0699,"price_match":"exact","price_out":1.4,"release_date":"2026-08-14","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"glm-5-3-flash":{"aa":{"agentic_index":50.9,"coding_index":71.5,"intelligence_index":27.8},"context":1048576,"country":"China","created":1787752741,"display":"GLM 5.3 Flash","family":"glm-5-3-flash","flop":3.24e+24,"flop_notes":"6 * 18e9 active parameters * 30e12 tokens = 3.24e24 FLOP. \n\nhttps://docs.z.ai/guides/vlm/glm-5.3-flash says \"Combined with our latest 30T-token multimodal pre-training corpus, these changes let GLM-5.3-Flash produce more intelligence with less compute.\", which is not definitive in saying that all 30T tokens were used exactly once; likely confidence.","input_modalities":["text","image","video"],"max_output":131072,"open_weights":true,"or_id":"z-ai/glm-5.3-flash","org":"Z.ai","output_modalities":["text"],"params":320000000000,"price_cache":0.03,"price_in":0.0335,"price_match":"exact","price_out":0.083,"release_date":"2026-08-20","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"gopher-280b":{"country":"United Kingdom","display":"Gopher 280B","family":"gopher-280b","flop":6.31e+23,"flop_notes":"Table A26\n6.31E+08 Train PFLOPs","open_weights":false,"org":"DeepMind","params":280000000000,"release_date":"2021-12-08"},"gpt-3-5-turbo-0125":{"country":"United States of America","display":"gpt-3.5-turbo-0125†","family":"gpt-3-5-turbo-0125","org":"OpenAI","release_date":"2024-01-25"},"gpt-3-5-turbo-0613":{"context":4095,"country":"United States of America","created":1706140800,"display":"GPT-3.5 Turbo (older v0613)","family":"gpt-3-5-turbo-0613","input_modalities":["text"],"knowledge_cutoff":"2021-09-30","max_output":3685,"or_id":"openai/gpt-3.5-turbo-0613","org":"OpenAI","output_modalities":["text"],"price_in":1,"price_match":"exact","price_out":2,"release_date":"2023-06-13","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_completion_tokens","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"gpt-3-5-turbo-1106":{"country":"United States of America","display":"GPT-3.5 version 1106","family":"gpt-3-5-turbo-1106","org":"OpenAI","release_date":"2023-11-06"},"gpt-3-5-turbo-instruct":{"country":"United States of America","display":"GPT 3.5 Turbo Instruct","family":"gpt-3-5-turbo-instruct","open_weights":false,"org":"OpenAI","params":20000000000,"release_date":"2023-09-18"},"gpt-4-0125-preview":{"country":"United States of America","display":"GPT 4.0125 Preview","family":"gpt-4-0125","flop_notes":"Training compute estimated to be 2.2e25 FLOP using benchmark imputation. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2024-01-25"},"gpt-4-0314":{"country":"United States of America","display":"GPT-4","family":"gpt-4-0314","flop":2.1e+25,"flop_notes":"90% CI: 8.2E+24 to 4.4E+25\n\nNOTE: this is a rough estimate based on public information, much less information than most other systems in the database.\n\nCalculation and confidence intervals here: https://colab.research.google.com/drive/1O99z9b1I5O66bT78r9ScslE_nOj5irN9?usp=sharing","open_weights":false,"org":"OpenAI","params":1800000000000,"release_date":"2023-03-14"},"gpt-4-0613":{"country":"United States of America","display":"GPT-4","family":"gpt-4-0613","flop":2.1e+25,"flop_notes":"90% CI: 8.2E+24 to 4.4E+25\n\nNOTE: this is a rough estimate based on public information, much less information than most other systems in the database.\n\nCalculation and confidence intervals here: https://colab.research.google.com/drive/1O99z9b1I5O66bT78r9ScslE_nOj5irN9?usp=sharing","open_weights":false,"org":"OpenAI","params":1800000000000,"release_date":"2023-06-13"},"gpt-4-1-2025-04-14":{"context":1047576,"country":"United States of America","created":1744651385,"display":"GPT-4.1","family":"gpt-4-1","flop_notes":"Flagship model from a leading developer in mid-2025; very likely it used \u003e1e25 FLOP.","input_modalities":["image","text","file"],"knowledge_cutoff":"2024-06-30","max_output":32768,"or_id":"openai/gpt-4.1","org":"OpenAI","output_modalities":["text"],"price_cache":0.5,"price_in":2,"price_match":"exact","price_out":8,"release_date":"2025-04-14","supported_parameters":["max_completion_tokens","max_tokens","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_p"]},"gpt-4-1-mini-2025-04-14":{"aa":{"agentic_index":null,"coding_index":20.2,"intelligence_index":null},"context":1047576,"country":"United States of America","created":1744651381,"display":"GPT-4.1 Mini","family":"gpt-4-1-mini","input_modalities":["image","text","file"],"knowledge_cutoff":"2024-06-30","max_output":32768,"or_id":"openai/gpt-4.1-mini","org":"OpenAI","output_modalities":["text"],"price_cache":0.1,"price_in":0.4,"price_match":"exact","price_out":1.6,"release_date":"2025-04-14","supported_parameters":["max_completion_tokens","max_tokens","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_p"]},"gpt-4-1-nano-2025-04-14":{"aa":{"agentic_index":null,"coding_index":11.1,"intelligence_index":null},"context":1047576,"country":"United States of America","created":1744651369,"display":"GPT-4.1 Nano","family":"gpt-4-1-nano","input_modalities":["image","text","file"],"knowledge_cutoff":"2024-06-30","max_output":942818,"or_id":"openai/gpt-4.1-nano","org":"OpenAI","output_modalities":["text"],"price_cache":0.03,"price_in":0.1,"price_match":"exact","price_out":0.4,"release_date":"2025-04-14","supported_parameters":["max_completion_tokens","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_p"]},"gpt-4-1106-preview":{"country":"United States of America","display":"GPT 4.1106 Preview","family":"gpt-4-1106","flop_notes":"Training compute estimated to be 2.2e25 FLOP using benchmark imputation. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2023-11-06"},"gpt-4-1106-vision-preview":{"country":"United States of America","display":"GPT-4V","family":"gpt-4-1106-vision","open_weights":false,"org":"OpenAI","release_date":"2023-11-06"},"gpt-4-32k-0314":{"country":"United States of America","display":"GPT-4","family":"gpt-4-32k-0314","flop":2.1e+25,"flop_notes":"90% CI: 8.2E+24 to 4.4E+25\n\nNOTE: this is a rough estimate based on public information, much less information than most other systems in the database.\n\nCalculation and confidence intervals here: https://colab.research.google.com/drive/1O99z9b1I5O66bT78r9ScslE_nOj5irN9?usp=sharing","open_weights":false,"org":"OpenAI","params":1800000000000,"release_date":"2023-03-14"},"gpt-4-5-preview-2025-02-27":{"country":"United States of America","display":"GPT-4.5","family":"gpt-4-5","flop":3.8e+26,"flop_notes":"Analysis of GPT-4.5's training cluster yields a median estimate of 187M H100-hours of training. The utilization assumptions we used for the Grok 3 estimate (maybe worth revisiting) were 20 to 40% under the H100 FP8 spec of 2000 teraflop/s. This leads to an estimate of 2.7e26 to 5.4e26 FLOP, or a geomean of 3.8e26\n\nAlternatively, using a plausible range of 20 to 50% utilization, given the possibility of FP8 training, yields a median estimate of ~2e25 FLOP. See notebook below for details.\n\nhttps://colab.research.google.com/drive/1QBmVPm64Ti0xucN0EsZTgSz_I7Mj9hAZ#scrollTo=NYH1ABJuLJlw \n\nThis is consistent with OpenAI's statement that GPT-4.5 was a “new order of magnitude in compute” compared to previous models (e.g. GPT-4, which was ~2e25), suggesting around 2e26 FLOP. But they could have meant this somewhat loosely. \n\nIn the \"Pretraining GPT-4.5\" interview, they state they used multi-cluster training: https://youtu.be/6nJZopACRuQ?si=FFJC-gEmGPZjvoPM\u0026t=617","open_weights":false,"org":"OpenAI","release_date":"2025-02-27"},"gpt-4-turbo":{"country":"United States of America","display":"GPT-4 Turbo (1106 preview)","family":"gpt-4-turbo","flop_notes":"Training compute estimated to be 2.2e25 FLOP using benchmark imputation. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2023-11-06"},"gpt-4-turbo-2024-04-09":{"country":"United States of America","display":"gpt-4-turbo-2024-04-09†","family":"gpt-4-turbo","flop_notes":"Training compute estimated to be 2.2e25 FLOP using benchmark imputation. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","open_weights":false,"org":"OpenAI","release_date":"2024-04-09"},"gpt-4o-2024-05-13":{"aa":{"agentic_index":null,"coding_index":24.2,"intelligence_index":null},"context":128000,"country":"United States of America","created":1715558400,"display":"GPT-4o (2024-05-13)","family":"gpt-4o","flop_notes":"Training compute estimated to be 3.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","input_modalities":["text","image","file"],"knowledge_cutoff":"2023-10-31","max_output":4096,"open_weights":false,"or_id":"openai/gpt-4o-2024-05-13","org":"OpenAI","output_modalities":["text"],"price_in":5,"price_match":"exact","price_out":15,"release_date":"2024-05-13","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_completion_tokens","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p","web_search_options"]},"gpt-4o-2024-08-06":{"context":128000,"country":"United States of America","created":1722902400,"display":"GPT-4o (2024-08-06)","family":"gpt-4o","flop_notes":"Training compute estimated to be 3.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","input_modalities":["text","image","file"],"knowledge_cutoff":"2023-10-31","max_output":16384,"open_weights":false,"or_id":"openai/gpt-4o-2024-08-06","org":"OpenAI","output_modalities":["text"],"price_cache":1.25,"price_in":2.5,"price_match":"exact","price_out":10,"release_date":"2024-08-06","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_completion_tokens","max_tokens","prediction","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p","web_search_options"]},"gpt-4o-2024-11-20":{"context":128000,"country":"United States of America","created":1732127594,"display":"GPT-4o (2024-11-20)","family":"gpt-4o","flop_notes":"Training compute estimated to be 3.8e25 FLOP from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing","input_modalities":["text","image","file"],"knowledge_cutoff":"2023-10-31","max_output":16384,"open_weights":false,"or_id":"openai/gpt-4o-2024-11-20","org":"OpenAI","output_modalities":["text"],"price_cache":1.25,"price_in":2.5,"price_match":"exact","price_out":10,"release_date":"2024-11-20","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","prediction","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p","web_search_options"]},"gpt-4o-mini-2024-07-18":{"context":128000,"country":"United States of America","created":1721260800,"display":"GPT-4o-mini (2024-07-18)","family":"gpt-4o-mini","flop_notes":"Training compute estimated to be 7.36001e+24 from benchmark scores. https://colab.research.google.com/drive/1r3pUMhB7Kh0Gls9eG-v_XefWrye9fVQR?usp=sharing\n\n90% CI [3.23e+24, 2.05e+25]","input_modalities":["text","image","file"],"knowledge_cutoff":"2023-10-31","max_output":16384,"or_id":"openai/gpt-4o-mini-2024-07-18","org":"OpenAI","output_modalities":["text"],"price_cache":0.075,"price_in":0.15,"price_match":"exact","price_out":0.6,"release_date":"2024-07-18","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","prediction","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p","web_search_options"]},"gpt-5-1-2025-11-13":{"aa":{"agentic_index":null,"coding_index":49.4,"intelligence_index":null},"context":400000,"country":"United States of America","created":1763060305,"display":"GPT-5.1","family":"gpt-5-1","flop_notes":"Almost certainly \u003e 1e25","input_modalities":["image","text","file"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.1","org":"OpenAI","output_modalities":["text"],"price_cache":0.125,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-11-13","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-1-codex":{"context":400000,"country":"United States of America","created":1763060298,"display":"GPT-5.1-Codex","family":"gpt-5-1-codex","input_modalities":["text","image"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.1-codex","org":"OpenAI","output_modalities":["text"],"price_cache":0.13,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-11-12","supported_parameters":["include_reasoning","max_completion_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-1-codex-max":{"context":400000,"country":"United States of America","created":1764878934,"display":"GPT-5.1-Codex-Max","family":"gpt-5-1-codex-max","input_modalities":["text","image"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.1-codex-max","org":"OpenAI","output_modalities":["text"],"price_cache":0.125,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-11-19","supported_parameters":["include_reasoning","max_completion_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-1-codex-mini":{"context":400000,"country":"United States of America","created":1763057820,"display":"GPT-5.1-Codex-Mini","family":"gpt-5-1-codex-mini","input_modalities":["image","text"],"max_output":128000,"or_id":"openai/gpt-5.1-codex-mini","org":"OpenAI","output_modalities":["text"],"price_cache":0.03,"price_in":0.25,"price_match":"exact","price_out":2,"release_date":"2025-11-12","supported_parameters":["include_reasoning","max_completion_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-2-2025-12-11":{"context":400000,"country":"United States of America","created":1765389775,"display":"GPT-5.2","family":"gpt-5-2","input_modalities":["file","image","text"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.2","org":"OpenAI","output_modalities":["text"],"price_cache":0.175,"price_in":1.75,"price_match":"exact","price_out":14,"release_date":"2025-12-11","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-2-codex":{"context":400000,"country":"United States of America","created":1768409315,"display":"GPT-5.2-Codex","family":"gpt-5-2-codex","input_modalities":["text","image"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.2-codex","org":"OpenAI","output_modalities":["text"],"price_cache":0.175,"price_in":1.75,"price_match":"exact","price_out":14,"release_date":"2025-12-18","supported_parameters":["include_reasoning","max_completion_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-2-pro-2025-12-11":{"context":400000,"country":"United States of America","created":1765389780,"display":"GPT-5.2 Pro","family":"gpt-5-2-pro","input_modalities":["image","text","file"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.2-pro","org":"OpenAI","output_modalities":["text"],"price_in":21,"price_match":"exact","price_out":168,"release_date":"2025-12-11","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-2-pro-2025-12-11-webapp":{"country":"United States of America","display":"GPT 5.2 Pro 2025.12.11 Webapp","family":"gpt-5-2-pro-2025-12-11-webapp","org":"OpenAI","release_date":"2025-12-11"},"gpt-5-2025-08-07":{"aa":{"agentic_index":null,"coding_index":37.8,"intelligence_index":null},"context":400000,"country":"United States of America","created":1754587413,"display":"GPT-5","family":"gpt-5","flop":6.6e+25,"flop_notes":"Fairly speculative, possibly around 6e25 [CI: 2e25 to 2e26] FLOP. See document below for details\n\nhttps://docs.google.com/document/d/1V2jIk365LnhH4WDoCw5dYJjZr1Htw8IHaK1noMf5Y48/edit?tab=t.z871imftkus","input_modalities":["text","image","file"],"knowledge_cutoff":"2024-09-30","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5","org":"OpenAI","output_modalities":["text"],"price_cache":0.125,"price_in":1.25,"price_match":"exact","price_out":10,"release_date":"2025-08-07","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-3-codex":{"context":400000,"country":"United States of America","created":1771959164,"display":"GPT-5.3-Codex","family":"gpt-5-3-codex","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.3-codex","org":"OpenAI","output_modalities":["text"],"price_cache":0.175,"price_in":1.75,"price_match":"exact","price_out":14,"release_date":"2026-02-05","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-4-2026-03-05":{"aa":{"agentic_index":null,"coding_index":71.1,"intelligence_index":null},"context":1050000,"country":"United States of America","created":1772734352,"display":"GPT-5.4","family":"gpt-5-4","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.4","org":"OpenAI","output_modalities":["text"],"price_cache":0.25,"price_in":2.5,"price_match":"exact","price_out":15,"release_date":"2026-03-05","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-4-mini-2026-03-17":{"aa":{"agentic_index":17.9,"coding_index":56.1,"intelligence_index":null},"context":400000,"country":"United States of America","created":1773748178,"display":"GPT-5.4 Mini","family":"gpt-5-4-mini","input_modalities":["file","image","text"],"knowledge_cutoff":"2025-08-31","max_output":128000,"or_id":"openai/gpt-5.4-mini","org":"OpenAI","output_modalities":["text"],"price_cache":0.075,"price_in":0.75,"price_match":"exact","price_out":4.5,"release_date":"2026-03-17","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-4-nano-2026-03-17":{"aa":{"agentic_index":16,"coding_index":56.1,"intelligence_index":null},"context":400000,"country":"United States of America","created":1773748187,"display":"GPT-5.4 Nano","family":"gpt-5-4-nano","input_modalities":["file","image","text"],"knowledge_cutoff":"2025-08-31","max_output":128000,"or_id":"openai/gpt-5.4-nano","org":"OpenAI","output_modalities":["text"],"price_cache":0.02,"price_in":0.2,"price_match":"exact","price_out":1.25,"release_date":"2026-03-17","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-4-pro-2026-03-05":{"context":1050000,"country":"United States of America","created":1772734366,"display":"GPT-5.4 Pro","family":"gpt-5-4-pro","input_modalities":["text","image","file"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.4-pro","org":"OpenAI","output_modalities":["text"],"price_in":30,"price_match":"exact","price_out":180,"release_date":"2026-03-05","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-4-pro-2026-03-05-web-app":{"country":"United States of America","display":"GPT 5.4 Pro 2026.03.05 Web App","family":"gpt-5-4-pro-2026-03-05-web-app","org":"OpenAI","release_date":"2026-03-05"},"gpt-5-5":{"aa":{"agentic_index":36.4,"coding_index":74.9,"intelligence_index":null},"context":1050000,"country":"United States of America","created":1777051893,"display":"GPT-5.5","family":"gpt-5-5","input_modalities":["file","image","text"],"knowledge_cutoff":"2025-12-01","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.5","org":"OpenAI","output_modalities":["text"],"price_cache":0.5,"price_in":5,"price_match":"exact","price_out":30,"release_date":"2026-04-23","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-5-instant":{"country":"United States of America","display":"GPT-5.5 Instant (June 2026)","family":"gpt-5-5-instant","open_weights":false,"org":"OpenAI","release_date":"2026-05-05"},"gpt-5-5-pre-release":{"country":"United States of America","display":"GPT 5.5 Pre Release","family":"gpt-5-5-pre-release","org":"OpenAI","release_date":"2026-04-23"},"gpt-5-5-pro":{"context":1050000,"country":"United States of America","created":1777051896,"display":"GPT-5.5 Pro","family":"gpt-5-5-pro","input_modalities":["file","image","text"],"knowledge_cutoff":"2025-12-01","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5.5-pro","org":"OpenAI","output_modalities":["text"],"price_in":30,"price_match":"exact","price_out":180,"release_date":"2026-04-23","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-5-pro-pre-release":{"country":"United States of America","display":"GPT 5.5 Pro Pre Release","family":"gpt-5-5-pro-pre-release","org":"OpenAI","release_date":"2026-04-23"},"gpt-5-6-luna":{"aa":{"agentic_index":42.1,"coding_index":71.4,"intelligence_index":23.8},"context":1050000,"country":"United States of America","created":1783590864,"display":"GPT-5.6 Luna","family":"gpt-5-6-luna","input_modalities":["file","image","text"],"knowledge_cutoff":"2026-02-16","max_output":128000,"or_id":"openai/gpt-5.6-luna","org":"OpenAI","output_modalities":["text"],"price_cache":0.02,"price_in":0.2,"price_match":"exact","price_out":1.2,"release_date":"2026-07-09","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-6-sol":{"aa":{"agentic_index":47.4,"coding_index":78.3,"intelligence_index":null},"context":1050000,"country":"United States of America","created":1783590850,"display":"GPT-5.6 Sol","family":"gpt-5-6-sol","input_modalities":["file","image","text"],"knowledge_cutoff":"2026-02-16","max_output":128000,"or_id":"openai/gpt-5.6-sol","org":"OpenAI","output_modalities":["text"],"price_cache":0.4,"price_in":4,"price_match":"exact","price_out":20,"release_date":"2026-07-09","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-6-sol-prounknown":{"country":"United States of America","display":"GPT 5.6 Sol Prounknown","family":"gpt-5-6-sol-prounknown","org":"OpenAI","release_date":"2026-07-09"},"gpt-5-6-sol-proxhigh":{"country":"United States of America","display":"GPT 5.6 Sol Proxhigh","family":"gpt-5-6-sol-proxhigh","org":"OpenAI","release_date":"2026-07-09"},"gpt-5-6-terra":{"aa":{"agentic_index":43.2,"coding_index":76.7,"intelligence_index":32.3},"context":1050000,"country":"United States of America","created":1783590857,"display":"GPT-5.6 Terra","family":"gpt-5-6-terra","input_modalities":["file","image","text"],"knowledge_cutoff":"2026-02-16","max_output":128000,"or_id":"openai/gpt-5.6-terra","org":"OpenAI","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":12,"release_date":"2026-07-09","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-chat":{"country":"United States of America","display":"GPT 5 Chat","family":"gpt-5-chat","flop":6.6e+25,"flop_notes":"Fairly speculative, possibly around 6e25 [CI: 2e25 to 2e26] FLOP. See document below for details\n\nhttps://docs.google.com/document/d/1V2jIk365LnhH4WDoCw5dYJjZr1Htw8IHaK1noMf5Y48/edit?tab=t.z871imftkus","org":"OpenAI"},"gpt-5-codex":{"country":"United States of America","display":"GPT-5-Codex","family":"gpt-5-codex","flop_notes":"OpenAI says this is a version of GPT-5. The meaning of this is not completely clear, but likely means a shared base model and a similar/overlapping post-training history. But it is not clear whether 5-codex was produced via additional fine-tuning of a published GPT-5 checkpoint.","org":"OpenAI","release_date":"2025-09-15"},"gpt-5-mini-2025-08-07":{"aa":{"agentic_index":6.8,"coding_index":15.6,"intelligence_index":null},"context":400000,"country":"United States of America","created":1754587407,"display":"GPT-5 Mini","family":"gpt-5-mini","input_modalities":["text","image","file"],"knowledge_cutoff":"2024-05-31","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5-mini","org":"OpenAI","output_modalities":["text"],"price_cache":0.025,"price_in":0.25,"price_match":"exact","price_out":2,"release_date":"2025-08-07","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-nano-2025-08-07":{"context":400000,"country":"United States of America","created":1754587402,"display":"GPT-5 Nano","family":"gpt-5-nano","input_modalities":["text","image","file"],"knowledge_cutoff":"2024-05-31","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5-nano","org":"OpenAI","output_modalities":["text"],"price_cache":0.005,"price_in":0.05,"price_match":"exact","price_out":0.4,"release_date":"2025-08-07","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-5-pro-2025-10-06":{"context":400000,"country":"United States of America","created":1759776663,"display":"GPT-5 Pro","family":"gpt-5-pro","input_modalities":["image","text","file"],"knowledge_cutoff":"2024-09-30","max_output":128000,"open_weights":false,"or_id":"openai/gpt-5-pro","org":"OpenAI","output_modalities":["text"],"price_in":15,"price_match":"exact","price_out":120,"release_date":"2025-10-07","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-6-1-sol":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":54.6},"context":1050000,"country":"United States of America","created":1790702882,"display":"GPT-6.1 Sol","family":"gpt-6-1-sol","input_modalities":["file","image","text"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-6.1-sol","org":"OpenAI","output_modalities":["text"],"price_cache":0.1,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-09-29","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-6-1-terra":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":51.3},"context":1050000,"country":"United States of America","created":1793038409,"display":"GPT-6.1 Terra","family":"gpt-6-1-terra","input_modalities":["file","image","text"],"max_output":128000,"or_id":"openai/gpt-6.1-terra","org":"OpenAI","output_modalities":["text"],"price_cache":0.05,"price_in":1,"price_match":"exact","price_out":5,"release_date":"2026-10-26","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-6-astra":{"aa":{"agentic_index":51,"coding_index":76.9,"intelligence_index":56.9},"context":1050000,"country":"United States of America","created":1788552838,"display":"GPT-6 Astra","family":"gpt-6-astra","flop":9.4e+26,"flop_notes":"per OpenAI and affiliates (e.g. Nvidia), 6 Astra was trained on at least 100,000 GB200s in Abilene, Texas (so 100,000 in \"Hardware quantity\" is a lower bound/underestimate).\n\nThis suggests around 9e26 FLOP (corresponding to ~100k GB200s over 90 days at 25% FP8 MFU). See more detailed estimate in this notebook, yielding a CI of ~[5e26, 2e27] FLOP.\nhttps://colab.research.google.com/drive/1DRlugIQs-_zoV66AJT1gQg6I1-NX7NqN","input_modalities":["file","image","text"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-6-astra","org":"OpenAI","output_modalities":["text"],"price_cache":1,"price_in":10,"price_match":"exact","price_out":50,"release_date":"2026-09-03","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-6-luna":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":27.9},"context":1050000,"country":"United States of America","created":1790100786,"display":"GPT-6 Luna","family":"gpt-6-luna","input_modalities":["file","image","text"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-6-luna","org":"OpenAI","output_modalities":["text"],"price_cache":0.01,"price_in":0.1,"price_match":"exact","price_out":0.5,"release_date":"2026-09-22","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-6-sol":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":44.2},"context":1050000,"country":"United States of America","created":1790100775,"display":"GPT-6 Sol","family":"gpt-6-sol","input_modalities":["file","image","text"],"max_output":128000,"open_weights":false,"or_id":"openai/gpt-6-sol","org":"OpenAI","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-09-22","supported_parameters":["include_reasoning","max_completion_tokens","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","tool_choice","tools","verbosity"]},"gpt-j-6b":{"country":"Germany,United States of America","display":"GPT-J-6B","family":"gpt-j-6b","flop":1.5e+22,"flop_notes":"source: zero shot evaluation table in GitHub","org":"EleutherAI,LAION","release_date":"2021-08-05"},"gpt-neo-2-7b":{"country":"United States of America","display":"GPT-Neo-2.7B","family":"gpt-neo-2-7b","flop":7.9e+21,"flop_notes":"source: https://www.aitracker.org/\n\n6 FLOP / token / parameter * 2.7 * 10^9 parameters * 420000000000 tokens [see dataset size notes] = 6.804e+21 FLOP","org":"EleutherAI","release_date":"2021-03-30"},"gpt-neox-20b":{"country":"United States of America","display":"GPT-neox-20B","family":"gpt-neox-20b","flop":9.31627008e+22,"flop_notes":"Trained for 3 months on 96 A100s (according to correspondence with author). Let's say 0.4 utilization rate.","open_weights":true,"org":"EleutherAI","params":20000000000,"release_date":"2022-04-07"},"gpt-oss-120b":{"aa":{"agentic_index":3.7,"coding_index":30.4,"intelligence_index":null},"context":131072,"country":"United States of America","created":1754414231,"display":"gpt-oss-120b","family":"gpt-oss-120b","flop":4.94e+24,"flop_notes":"\"The training run for gpt-oss-120b required 2.1 million H100-hours to complete\"\n(2.1e6 hours)*(1,979 H100 FLOP/s)*(30% utilization)*(60*60) = 4.49e24\nThey also do post training similar to o3, which we assume adds at least 10% as much compute, so we multiply this estimate by 1.1 to get 4.94e24","input_modalities":["text"],"knowledge_cutoff":"2024-06-30","max_output":65536,"open_weights":true,"or_id":"openai/gpt-oss-120b","org":"OpenAI","output_modalities":["text"],"params":116830000000,"price_cache":0.075,"price_in":0.15,"price_match":"exact","price_out":0.6,"release_date":"2025-08-05","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"gpt-oss-20b":{"aa":{"agentic_index":1.2,"coding_index":20.7,"intelligence_index":null},"context":131072,"country":"United States of America","created":1754414229,"display":"gpt-oss-20b","family":"gpt-oss-20b","flop":5.49e+23,"flop_notes":"\"The training run for gpt-oss-120b required 2.1\nmillion H100-hours to complete, with gpt-oss-20b needing almost 10x fewer\"\nassuming \"almost 10x fewer\" means ~9x fewer: 4.94e24/9 = 5.49e23","input_modalities":["text"],"knowledge_cutoff":"2024-06-30","max_output":117964,"open_weights":true,"or_id":"openai/gpt-oss-20b","org":"OpenAI","output_modalities":["text"],"params":20910000000,"price_cache":0.03,"price_in":0.03,"price_match":"exact","price_out":0.13,"release_date":"2025-08-05","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"gpt2-xl":{"country":"United States of America","display":"GPT2-XL-1.5B","family":"gpt2-xl","flop":1.920000000001e+21,"flop_notes":"Estimating based on compute = 6 FLOP/token/param * epochs * parameters * tokens.\n\n40GB dataset is approximately 8B words, or 1/0.75 * 8B = 10.66B tokens.\n\nThe number of epochs is not reported, but another paper [1] claims in table 1 that it is 20 or 100 epochs, and another paper [2] claims 12 epochs based on communication with the GPT-2 authors (page 4).\n\n12 epochs is the modal, most credible value. Mean of probability mass is probably around 20 epochs, so calculating from that value:\n\n6 * (40 * 200 million * 1/0.75 * 20) * 1.5 billion parameters = 1.92e21\nhttps://www.wolframalpha.com/input?i=6+FLOP+*+20+*+%2840+billion+%2F+5+*+%284%2F3%29%29+*+1.5+billion\n\n[1] https://arxiv.org/abs/1906.06669 One Epoch Is All You Need\n[2] https://www.usenix.org/system/files/sec21-carlini-extracting.pdf Extracting Data From Large Language Models\n\nIt also appears the model was trained on TPU v3 chips:\nhttps://huggingface.co/openai-community/gpt2","org":"OpenAI","release_date":"2019-11-05"},"granite-4-0-1b":{"display":"Granite 4.0 1B","family":"granite-4-0-1b","release_date":"2025-10-28"},"granite-4-0-350m":{"display":"Granite 4.0 350M","family":"granite-4-0-350m","release_date":"2025-10-28"},"granite-4-0-micro":{"display":"Granite 4.0 Micro","family":"granite-4-0-micro","release_date":"2025-10-02"},"granite-4-1-30b":{"country":"United States of America","display":"Granite 4.1 30B","family":"granite-4-1-30b","flop":2.7e+24,"flop_notes":"6 * 30e9 parameters * 15e12 tokens = 2.7e24 FLOP","org":"IBM","release_date":"2026-04-29"},"grok-2-1212":{"country":"United States of America","display":"Grok 2","family":"grok-2-1212","flop":2.96e+25,"flop_notes":"Estimate based on xAI statements comparing Grok-2 compute to GPT-4 and Grok-3. Full estimate here: https://docs.google.com/document/d/1C_dABuZrAqYE_ui4_GZ4bRLtq3TBjIGoBSktaPElhEU/edit?usp=sharing","open_weights":false,"org":"xAI","release_date":"2024-12-12"},"grok-3":{"country":"United States of America","display":"Grok 3","family":"grok-3","flop":3.5e+26,"flop_notes":"Estimate based on a cluster of 80,000 H100s per the xai website and an estimated training time of approximately three months.\n\nFull estimate here: https://docs.google.com/document/d/1MIUFviULJ3YI_XjyzL8cwG0cBRANKNxVEB4DrUcFiNs/edit?usp=sharing","open_weights":false,"org":"xAI","params":3000000000000,"release_date":"2025-04-09"},"grok-3-beta":{"country":"United States of America","display":"Grok 3 Beta","family":"grok-3-beta","flop":3.5e+26,"flop_notes":"Estimate based on a cluster of 80,000 H100s per the xai website and an estimated training time of approximately three months.\n\nFull estimate here: https://docs.google.com/document/d/1MIUFviULJ3YI_XjyzL8cwG0cBRANKNxVEB4DrUcFiNs/edit?usp=sharing","open_weights":false,"org":"xAI","params":3000000000000,"release_date":"2025-04-09"},"grok-3-mini":{"country":"United States of America","display":"Grok 3 Mini","family":"grok-3-mini","org":"xAI","release_date":"2025-06-24"},"grok-3-mini-beta":{"country":"United States of America","display":"Grok 3 Mini Beta","family":"grok-3-mini-beta","org":"xAI","release_date":"2025-04-09"},"grok-4-0709":{"country":"United States of America","display":"grok-4-0709†","family":"grok-4-0709","flop":5.0000000000001e+26,"flop_notes":"We think that RL relative to pre-compute is between our estimate for o3 (10% of pre-training) and the 100% implied by this slide in the launch ( https://archive.is/f0vJU ). Assuming the same pre-training as Grok 3 (also implied by that slide, and much more consistent) and that Grok 3 used a tenth as much RL, we get:\n\n2 * (grok3/1.1) in the high case (rl is 10% of grok 3, so grok3/1.1 is grok3 precompute, and in this case twice that is grok 4)\n1.1 * (grok3/1.01) in the low case\nThe geometric mean is (rounded to one sig fig): 5e26","open_weights":false,"org":"xAI","params":3000000000000,"release_date":"2025-07-09"},"grok-4-1":{"country":"United States of America","display":"Grok 4.1","family":"grok-4-1","open_weights":false,"org":"xAI","release_date":"2025-11-17"},"grok-4-1-fast-non-reasoning":{"country":"United States of America","display":"Grok 4.1 Fast Non Reasoning","family":"grok-4-1-fast-non-reasoning","org":"xAI","release_date":"2025-11-19"},"grok-4-1-fast-reasoning":{"country":"United States of America","display":"Grok 4.1 Fast","family":"grok-4-1-fast-reasoning","open_weights":false,"org":"xAI","release_date":"2025-11-19"},"grok-4-20":{"context":2000000,"country":"United States of America","created":1774979019,"display":"Grok 4.20","family":"grok-4-20","input_modalities":["text","image","file"],"knowledge_cutoff":"2025-09-01","max_output":1800000,"open_weights":false,"or_id":"x-ai/grok-4.20","org":"xAI","output_modalities":["text"],"params":500000000000,"price_cache":0.2,"price_in":1.25,"price_match":"exact","price_out":2.5,"release_date":"2026-02-17","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"grok-4-20-0309-reasoning":{"country":"United States of America","display":"grok-4.20-0309-reasoning†","family":"grok-4-20-0309-reasoning","org":"xAI","release_date":"2026-02-17"},"grok-4-3":{"aa":{"agentic_index":15.5,"coding_index":42.2,"intelligence_index":null},"context":1000000,"country":"United States of America","created":1777591821,"display":"Grok 4.3","family":"grok-4-3","input_modalities":["text","image","file"],"max_output":900000,"or_id":"x-ai/grok-4.3","org":"xAI","output_modalities":["text"],"price_cache":0.2,"price_in":1.25,"price_match":"exact","price_out":2.5,"release_date":"2026-04-17","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"grok-4-5":{"aa":{"agentic_index":41.2,"coding_index":72.4,"intelligence_index":null},"context":500000,"country":"United States of America","created":1783523154,"display":"Grok 4.5","family":"grok-4-5","input_modalities":["text","image","file"],"max_output":450000,"or_id":"x-ai/grok-4.5","org":"xAI","output_modalities":["text"],"price_cache":0.3,"price_in":2,"price_match":"exact","price_out":6,"release_date":"2026-07-08","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","reasoning_effort","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"grok-4-6":{"aa":{"agentic_index":52.5,"coding_index":75.9,"intelligence_index":31},"context":500000,"country":"United States of America","created":1786548957,"display":"Grok 4.6","family":"grok-4-6","input_modalities":["text","image","file"],"max_output":450000,"or_id":"x-ai/grok-4.6","org":"xAI","output_modalities":["text"],"price_cache":0.5,"price_in":2,"price_match":"exact","price_out":6,"release_date":"2026-08-12","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"grok-4-7":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":35.5},"context":500000,"country":"United States of America","created":1790007541,"display":"Grok 4.7","family":"grok-4-7","input_modalities":["text","image","file"],"max_output":450000,"open_weights":false,"or_id":"x-ai/grok-4.7","org":"xAI","output_modalities":["text"],"params":2100000000,"price_cache":0.5,"price_in":2,"price_match":"exact","price_out":6,"release_date":"2026-09-21","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"grok-4-fast":{"country":"United States of America","display":"Grok 4 (Fast Reasoning)","family":"grok-4-fast","org":"xAI","release_date":"2025-09-19"},"grok-4-heavy-web-app":{"country":"United States of America","display":"Grok 4 Heavy Web App","family":"grok-4-heavy-web-app","org":"xAI","release_date":"2025-07-10"},"grok-build-0-1":{"aa":{"agentic_index":null,"coding_index":51.5,"intelligence_index":null},"context":256000,"country":"United States of America","created":1779298123,"display":"Grok Build 0.1","family":"grok-build-0-1","input_modalities":["text","image","file"],"max_output":230400,"or_id":"x-ai/grok-build-0.1","org":"xAI","output_modalities":["text"],"price_cache":0.2,"price_in":1,"price_match":"exact","price_out":2,"release_date":"2026-05-29","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"grok-code-fast-1":{"display":"Grok Code Fast 1","family":"grok-code-fast-1","release_date":"2025-08-28"},"hermes-2-theta-llama-3-70b":{"country":"United States of America","display":"Hermes 2 Theta Llama 3 70B","family":"hermes-2-theta-llama-3-70b","org":"Nous Research,Arcee AI","release_date":"2024-06-20"},"hy4-preview":{"context":1048576,"country":"China","created":1787897375,"display":"Hy4 preview","family":"hy4","input_modalities":["text"],"max_output":64000,"or_id":"tencent/hy4-preview","org":"Tencent","output_modalities":["text"],"price_cache":0.042,"price_in":0.834,"price_match":"exact","price_out":2.501,"release_date":"2026-08-28","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"inflection-1":{"country":"United States of America","display":"Inflection-1","family":"inflection-1","flop":1.0001e+24,"flop_notes":"\u003c= 2.5e24\n\nThey define two \"compute classes\", one for models with more compute than PaLM 540B, i.e. GPT-4 and PaLM 2, and one for models with as much compute or less, i.e. GPT-3.5, Chinchilla, LLaMA, and Inflection-1.\n\nPaLM 540B required 2.5e24 FLOP to train (confirmed by Google)","org":"Inflection AI","release_date":"2023-06-22"},"inkling":{"aa":{"agentic_index":22.5,"coding_index":52.1,"intelligence_index":12.5},"context":524288,"country":"United States of America","created":1784325956,"display":"Inkling","family":"inkling","flop":1.1e+25,"flop_notes":"6ND = 6 * 41e9 active parameters * 45e12 tokens = 1.1e25 FLOP","input_modalities":["text","image","audio"],"max_output":471859,"open_weights":true,"or_id":"thinkingmachines/inkling","org":"Thinking Machines","output_modalities":["text"],"params":975000000000,"price_cache":0.17,"price_in":1,"price_match":"exact","price_out":4.05,"release_date":"2026-07-15","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","stop","temperature","tool_choice","tools","top_k","top_p"]},"inkling-small":{"aa":{"agentic_index":23.5,"coding_index":52.9,"intelligence_index":12.9},"context":524288,"country":"United States of America","created":1785443117,"display":"Inkling Small","family":"inkling-small","input_modalities":["text","image","audio"],"max_output":262144,"or_id":"thinkingmachines/inkling-small","org":"Thinking Machines","output_modalities":["text"],"price_cache":0.1,"price_in":0.45,"price_match":"exact","price_out":1.2,"release_date":"2026-07-15","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","seed","stop","temperature","tool_choice","tools","top_k","top_p"]},"instructblip-vicuna-13b":{"display":"InstructBLIP-13B (Vicuna)","family":"instructblip-vicuna-13b","release_date":"2023-12-25"},"instructblip-vicuna-7b":{"display":"InstructBLIP-7B (Vicuna)","family":"instructblip-vicuna-7b","release_date":"2023-05-22"},"intellect-1-instruct":{"country":"United States of America","display":"INTELLECT-1-Instruct","family":"intellect-1-instruct","flop":6.000001e+22,"flop_notes":"10B parameters trained on 1T tokens: 6 * 10B * 1T = 6e22","org":"Prime Intellect,Hugging Face,Arcee AI","release_date":"2024-11-29"},"internlm-20b":{"display":"InternLM 20B","family":"internlm-20b","release_date":"2023-09-18"},"internlm-7b":{"display":"InternLM 7B","family":"internlm-7b","release_date":"2023-07-05"},"internlm-chat-20b":{"display":"InternLM-Chat 20B","family":"internlm-chat-20b","release_date":"2023-09-17"},"internvl-chat-v1-5":{"display":"InternVL-Chat-V1.5","family":"internvl-chat-v1-5","release_date":"2024-04-18"},"internvl-chat-vit-6b-vicuna-13b":{"display":"InternVL-13B","family":"internvl-chat-vit-6b-vicuna-13b","release_date":"2024-08-16"},"internvl-chat-vit-6b-vicuna-7b":{"display":"InternVL-7B","family":"internvl-chat-vit-6b-vicuna-7b","release_date":"2023-12-25"},"internvl2-40b":{"country":"China","display":"InternVL2","family":"internvl2-40b","org":"Shanghai AI Lab","release_date":"2024-07-08"},"internvl2-5-78b":{"country":"Hong Kong,China","display":"InternVL2.5","family":"internvl2-5-78b","org":"Shanghai AI Lab,SenseTime,Tsinghua University,Nanjing University,Fudan University,Chinese University of Hong Kong (CUHK),Shanghai Jiao Tong University","release_date":"2024-12-06"},"kangaroo":{"display":"Kangaroo","family":"kangaroo","release_date":"2024-11-13"},"kimi-k2-0711-preview":{"country":"China","display":"Kimi K2.0711 Preview","family":"kimi-k2-0711","flop":2.976e+24,"flop_notes":"6 FLOP / parameter / token * 32 * 10^9 activated parameters * 15.5 * 10^12 tokens = 2.976e+24 FLOP","org":"Moonshot AI","release_date":"2025-07-12"},"kimi-k2-5":{"aa":{"agentic_index":null,"coding_index":46.8,"intelligence_index":null},"context":262144,"country":"China","created":1769487076,"display":"Kimi K2.5","expiration_date":"2026-11-18","family":"kimi-k2-5","flop":5.8e+24,"flop_notes":"Per section 3.2, \"The joint pre-training stage continues from a near-end Kimi K2 checkpoint over additional 15T vision-text tokens at 4K sequence length\"\n\nKimi K2 was pretrained on 15T tokens. A near-end checkpoint of K2 was trained into K2.5 over another 15T tokens (or 15.2 to 15.5T tokens). This makes for a total of ~30T.\n\nThere are an additional 1T tokens in \"Stage 1\" used to train the ViT encoder, which is likely much smaller than the full LLM, so we omit those tokens to calculate training compute.\n\n30e12 tokens * 32B parameters * 6 = 5.76e24 FLOP\n\nPer Kimi talk (timestamp 26:00), the 15 trillion tokens of K2.5 were additive, for 30T training tokens overall: https://x.com/Kimi_Moonshot/status/2035011542115393973?s=20","input_modalities":["text","image"],"max_output":65536,"open_weights":true,"or_id":"moonshotai/kimi-k2.5","org":"Moonshot AI","output_modalities":["text"],"params":1040000000000,"price_cache":0.209,"price_in":0.532,"price_match":"exact","price_out":3.325,"release_date":"2026-01-27","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"kimi-k2-6":{"aa":{"agentic_index":20.5,"coding_index":61.8,"intelligence_index":null},"context":262144,"country":"China","created":1776699402,"display":"Kimi K2.6","family":"kimi-k2-6","input_modalities":["text","image"],"max_output":235929,"open_weights":true,"or_id":"moonshotai/kimi-k2.6","org":"Moonshot AI","output_modalities":["text"],"params":1040000000000,"price_cache":0.15,"price_in":0.65,"price_match":"exact","price_out":3.41,"release_date":"2026-04-20","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"kimi-k2-7-code":{"aa":{"agentic_index":21,"coding_index":60.8,"intelligence_index":null},"context":262144,"country":"China","created":1781266361,"display":"Kimi K2.7 Code","family":"kimi-k2-7-code","input_modalities":["text","image"],"max_output":235929,"or_id":"moonshotai/kimi-k2.7-code","org":"Moonshot AI","output_modalities":["text"],"price_cache":0.16,"price_in":0.6,"price_match":"exact","price_out":3.5,"release_date":"2026-06-12","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"kimi-k2-instruct":{"country":"China","display":"kimi-k2-instruct†","family":"kimi-k2-instruct","flop":2.976e+24,"flop_notes":"6 FLOP / parameter / token * 32 * 10^9 activated parameters * 15.5 * 10^12 tokens = 2.976e+24 FLOP","org":"Moonshot AI","release_date":"2025-07-12"},"kimi-k2-instruct-0905":{"country":"China","display":"kimi-k2-instruct-0905†","family":"kimi-k2-instruct-0905","flop":2.976e+24,"flop_notes":"6 FLOP / parameter / token * 32 * 10^9 activated parameters * 15.5 * 10^12 tokens = 2.976e+24 FLOP","org":"Moonshot AI","release_date":"2025-09-05"},"kimi-k2-thinking":{"aa":{"agentic_index":null,"coding_index":21,"intelligence_index":null},"context":262144,"country":"China","created":1762440622,"display":"Kimi K2 Thinking","family":"kimi-k2-thinking","flop":4.2e+24,"flop_notes":"Assuming the additional post-training contributed between 1% and 100% of Kimi K2's training compute (which we confidently estimate at 2.976e+24), we get a range of 3.0e24 to 6.0e24, and a geometric mean of 4.2e24.","input_modalities":["text"],"max_output":98304,"open_weights":true,"or_id":"moonshotai/kimi-k2-thinking","org":"Moonshot AI","output_modalities":["text"],"params":1000000000000,"price_cache":0.15,"price_in":0.6,"price_match":"exact","price_out":2.5,"release_date":"2025-11-06","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"kimi-k2-thinking-turbo":{"country":"China","display":"Kimi K2 Thinking Turbo","family":"kimi-k2-thinking-turbo","flop":4.2e+24,"flop_notes":"Assuming the additional post-training contributed between 1% and 100% of Kimi K2's training compute (which we confidently estimate at 2.976e+24), we get a range of 3.0e24 to 6.0e24, and a geometric mean of 4.2e24.","org":"Moonshot AI","release_date":"2025-11-06"},"kimi-k3":{"aa":{"agentic_index":29.3,"coding_index":72,"intelligence_index":17.4},"context":1048576,"country":"China","created":1784215858,"display":"Kimi K3","family":"kimi-k3","flop":2.0001e+25,"flop_notes":"Based on peer models such as Kimi K2.6 and DeepSeek V4 Pro, plausibly was trained on roughly 30T tokens (likely between 15T and 60T). Using the heuristic that C = 6ND, 30T pretraining tokens would imply ~1.9e25 FLOP. This is a somewhat uncertain estimate (more uncertain than estimates based on disclosed active params and token counts, but comparable to estimates based on disclosed GPU-hours).\n\nK3 was likely pretrained on over 1e25 FLOP. Speculatively, K3's relatively large size compared to other Chinese models may have reduced the training token budget vs smaller models.","input_modalities":["text","image","video"],"max_output":943718,"open_weights":true,"or_id":"moonshotai/kimi-k3","org":"Moonshot AI","output_modalities":["text"],"params":2800000000000,"price_cache":0.182,"price_in":0.192,"price_match":"exact","price_out":16,"release_date":"2026-07-16","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"kimi-k3-1-pro":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":46.5},"context":1048576,"country":"China","created":1792778188,"display":"Kimi K3.1 Pro","family":"kimi-k3-1-pro","flop":2.0001e+25,"flop_notes":"Kimi K3.1 Pro is post-trained from Kimi K3, so its training compute is at least Kimi K3's. Moonshot: \"Kimi K3.1 Pro is a post-trained version of Kimi K3, with substantially larger post-training compute\" (https://www.kimi.ai/blog/kimi-k3-1-pro). The value here is Kimi K3's pretraining estimate (6ND with ~30T tokens assumed, range 15–60T); the additional post-training compute is unknown and not included, so this is likely an underestimate. The model card reports no token counts.","input_modalities":["text","image","video"],"max_output":943718,"or_id":"moonshotai/kimi-k3.1-pro","org":"Moonshot AI","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":10,"release_date":"2026-10-23","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"kimi-k3-1-pro-together":{"country":"China","display":"Kimi K3.1 Pro Together","family":"kimi-k3-1-pro-together","flop":2.0001e+25,"flop_notes":"Kimi K3.1 Pro is post-trained from Kimi K3, so its training compute is at least Kimi K3's. Moonshot: \"Kimi K3.1 Pro is a post-trained version of Kimi K3, with substantially larger post-training compute\" (https://www.kimi.ai/blog/kimi-k3-1-pro). The value here is Kimi K3's pretraining estimate (6ND with ~30T tokens assumed, range 15–60T); the additional post-training compute is unknown and not included, so this is likely an underestimate. The model card reports no token counts.","org":"Moonshot AI","release_date":"2026-10-23"},"laguna-m-1":{"country":"United States of America","display":"poolside/laguna-m.1","family":"laguna-m-1","org":"Poolside","release_date":"2026-04-28"},"laguna-xs-2":{"country":"United States of America","display":"poolside/laguna-xs.2","family":"laguna-xs-2","flop_notes":"6 * e12 tokens * 33.4e9 parameters = \n\nTokens trained on:\nPre-training: 30T\nMid-training: 60B","org":"Poolside","release_date":"2026-04-28"},"learnlm-1-5-pro-experimental":{"display":"Learnlm 1.5 Pro Experimental","family":"learnlm-1-5-pro-experimental"},"linvt":{"display":"LinVT","family":"linvt","release_date":"2024-12-06"},"livecc-7b-instruct":{"display":"LiveCC","family":"livecc-7b-instruct","release_date":"2024-04-12"},"llama-13b":{"country":"United States of America","display":"LLaMA 13B","family":"llama-13b","flop":7.8e+22,"flop_notes":"1T tokens * 13B parameters * 6 FLOP/token/parameter = 7.8e22\n\nfrom paper, Llama-13B took 135,168 GPU hours using A100s\n\n312 trillion * 135,168 * 3600 = 1.518e23 FLOPs at full utilization\nThis implies that the actual utilization was:\nMFU = 7.8e22/1.518e23 = 0.514","org":"Meta","release_date":"2023-02-24"},"llama-2-13b":{"country":"United States of America","display":"LLaMA-2 13B","family":"llama-2-13b","flop":1.6e+23,"flop_notes":"13 billion parameters * 2 trillion tokens * 6 FLOP / token / parameter = 1.6e23 FLOP","org":"Meta","release_date":"2023-07-18"},"llama-2-13b-chat":{"country":"United States of America","display":"LLaMA2-Chat 13B","family":"llama-2-13b-chat","flop":1.6e+23,"flop_notes":"13 billion parameters * 2 trillion tokens * 6 FLOP / token / parameter = 1.6e23 FLOP","org":"Meta","release_date":"2023-07-18"},"llama-2-34b":{"country":"United States of America","display":"LLaMA-2 34B","family":"llama-2-34b","flop":4.08e+23,"flop_notes":"All models sizes trained on 2.0T tokens, per table 1\n2T * 34b * 6 = 4.08e23\n\nAlso trained on 1038336 A100-hours, which is 3.5e23 at 30% utilization. So the utilization was probably around 35%.","org":"Meta","release_date":"2023-07-18"},"llama-2-70b-chat":{"country":"United States of America","display":"LLaMA2-Chat 70B","family":"llama-2-70b-chat","flop":8.1e+23,"flop_notes":"\"Pretraining utilized a cumulative 3.3M GPU hours of computation on hardware of type A100-80GB\" of which 1720320 GPU hours were used to train the 70B model.\n\n311.84 BF16 TFLOP/s * 1720320 hours * 0.40 utilization = 7.725e+23 FLOP.\n\nAlternatively: the model was trained for 1 epoch on 2 trillion tokens and has 70B parameters. C = 6ND = 6*70B*2T = 8.4e+23 FLOP.","org":"Meta","release_date":"2023-07-18"},"llama-2-70b-chat-hf":{"country":"United States of America","display":"llama-2-70b-chat-hf†","family":"llama-2-70b-chat-hf","flop":8.1e+23,"flop_notes":"\"Pretraining utilized a cumulative 3.3M GPU hours of computation on hardware of type A100-80GB\" of which 1720320 GPU hours were used to train the 70B model.\n\n311.84 BF16 TFLOP/s * 1720320 hours * 0.40 utilization = 7.725e+23 FLOP.\n\nAlternatively: the model was trained for 1 epoch on 2 trillion tokens and has 70B parameters. C = 6ND = 6*70B*2T = 8.4e+23 FLOP.","org":"Meta","release_date":"2023-07-18"},"llama-2-70b-hf":{"country":"United States of America","display":"LLaMA-2 70B","family":"llama-2-70b-hf","flop":8.1e+23,"flop_notes":"\"Pretraining utilized a cumulative 3.3M GPU hours of computation on hardware of type A100-80GB\" of which 1720320 GPU hours were used to train the 70B model.\n\n311.84 BF16 TFLOP/s * 1720320 hours * 0.40 utilization = 7.725e+23 FLOP.\n\nAlternatively: the model was trained for 1 epoch on 2 trillion tokens and has 70B parameters. C = 6ND = 6*70B*2T = 8.4e+23 FLOP.","open_weights":true,"org":"Meta","params":70000000000,"release_date":"2023-07-18"},"llama-2-7b":{"country":"United States of America","display":"LLaMA-2 7B","family":"llama-2-7b","flop":8.4e+22,"flop_notes":"Trained on 2 trillion tokens per Table 1. \n\nC = 6ND = 6 FLOP / token / parameter * 7B parameters * 2T tokens = 8.4e+22 FLOP.\n\nAlso, 7B model was trained on 184320 GPU-hours\n\n312 trillion * 184320 GPU-hours * 3600 sec/hour * 0.3 [utilization] = 6.21e22 FLOP","open_weights":true,"org":"Meta","params":7000000000,"release_date":"2023-07-18"},"llama-3-1-405b":{"country":"United States of America","display":"LLaMA-3.1 405B Base","family":"llama-3-1-405b","flop":3.8e+25,"flop_notes":"Stated in paper.\n\nAlso, 6 * 405B * 15.6T training tokens = 3.8e25","open_weights":true,"org":"Meta","params":405000000000,"release_date":"2024-07-23"},"llama-3-1-405b-instruct":{"country":"United States of America","display":"Llama 3.1 Instruct 405B","family":"llama-3-1-405b-instruct","flop":3.8e+25,"flop_notes":"Stated in paper.\n\nAlso, 6 * 405B * 15.6T training tokens = 3.8e25","open_weights":true,"org":"Meta","params":405000000000,"release_date":"2024-07-23"},"llama-3-1-70b-instruct":{"context":131072,"country":"United States of America","created":1721692800,"display":"Llama 3.1 70B Instruct","family":"llama-3-1-70b-instruct","flop":7.929e+24,"flop_notes":"Huggingface page says 3.1-70B used 7.0M H100 hours and trained over 15T tokens. https://huggingface.co/meta-llama/Llama-3.1-70B\nThe paper also says that 3.1-405B got MFU of between 38-43%; presumably 70B was around the same or a bit higher. I'll assume utilization of 40%\n\n6ND:\n6 * 15T * 70B = 6.3e24 FLOPs\n\nHardware:\n7M * 9.9e14 * 3600 * 0.4 = 9.98e24 FLOPs\n\nGeometric mean: sqrt(6.3e24 * 9.98e24) = 7.929e24\n\nNote that Llama 3-70B also said it used 15T tokens, but only 6.4M H100 hours. This suggests 3.1 might have used a bit more than 15T tokens.\n\nTraining compute upper bound: 7M H100-hours * 989 TFLOPS * 50% utilization = 1.25e25 FLOP","input_modalities":["text"],"knowledge_cutoff":"2023-12-31","max_output":8192,"or_id":"meta-llama/llama-3.1-70b-instruct","org":"Meta","output_modalities":["text"],"price_in":0.72,"price_match":"exact","price_out":0.72,"release_date":"2024-07-23","supported_parameters":["max_tokens","stop","temperature","top_k","top_p"]},"llama-3-1-8b-instruct":{"aa":{"agentic_index":null,"coding_index":5.4,"intelligence_index":null},"context":131072,"country":"United States of America","created":1721692800,"display":"Llama 3.1 8B Instruct","family":"llama-3-1-8b-instruct","flop":1.224e+24,"flop_notes":"Huggingface page says 3.1-8B used 1.46M H100 hours and trained over 15T tokens. https://huggingface.co/meta-llama/Llama-3.1-70B\nThe paper also says that 3.1-405B got MFU of between 38-43%; presumably 8B was around the same or a bit higher. I'll assume utilization of 40%\n\n6ND:\n6 * 15T * 8B = 7.2e23 FLOPs\n\nHardware:\n1.46M * 9.9e14 * 3600 * 0.4 = 2.08e24 FLOPs\n\nGeometric mean: sqrt(7.2e23 * 2.08e24) = 1.224e24\n\nNote that Llama 3-8B also said it used 15T tokens, but only 1.3M H100 hours. This suggests 3.1 might have used a bit more than 15T tokens.","input_modalities":["text"],"knowledge_cutoff":"2023-12-31","max_output":16384,"or_id":"meta-llama/llama-3.1-8b-instruct","org":"Meta","output_modalities":["text"],"price_in":0.02,"price_match":"exact","price_out":0.04,"release_date":"2024-07-23","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"llama-3-1-tulu-3-70b-dpo":{"country":"United States of America","display":"Llama 3.1 Tulu 3 70B Dpo","family":"llama-3-1-tulu-3-70b-dpo","org":"Allen Institute for AI,University of Washington","release_date":"2024-11-21"},"llama-3-2-11b-vision-instruct":{"country":"United States of America","display":"Llama 3.2 Instruct 11B (Vision)","family":"llama-3-2-11b-vision-instruct","flop":5.79e+23,"flop_notes":"Tensor type is BF16 (https://huggingface.co/meta-llama/Llama-3.2-11B-Vision-Instruct).\n\n“Training utilized a cumulative of 2.02M GPU hours of computation on H100-80GB (TDP of 700W) type hardware, per the table below. Training time is the total GPU time required for training each model and power consumption is the peak power capacity per GPU device used, adjusted for power usage efficiency… Training time: Stage 1 pretraining: 147K H100 hours Stage 2 annealing: 98K H100 hours SFT: 896 H100 hours RLHF: 224 H100 hours.” (https://github.com/meta-llama/llama-models/blob/main/models/llama3_2/MODEL_CARD_VISION.md#hardware-and-software).\n\nThe Nvidia H100 80GB is the H100 SXM. BFLOAT16 Tensor Core peak FLOPS with sparsity is 1,979 teraFLOPS (https://resources.nvidia.com/en-us-tensor-core/nvidia-tensor-core-gpu-datasheet?ncid=no-ncid).\n\nAssuming 33% utilization rate,\nTraining compute\n~= 0.33 * ( 147000 + 98000 + 896 + 224 ) hours * 3600 s / hour * 1979e12 FLOPS / GPU\n~= 5.79e23 FLOPS","org":"Meta","release_date":"2024-09-24"},"llama-3-2-1b-instruct":{"context":60000,"country":"United States of America","created":1727222400,"display":"Llama 3.2 1B Instruct","expiration_date":"2026-11-20","family":"llama-3-2-1b-instruct","flop":6.642e+22,"flop_notes":"6ND = 6*1230000000.00*9000000000000 = 6.642e+22\n\n370000 hours * 3600 s * 133800000000000 FLOPS/s* 0.3 = 5.346648e+22","input_modalities":["text"],"knowledge_cutoff":"2023-12-31","max_output":54000,"or_id":"meta-llama/llama-3.2-1b-instruct","org":"Meta","output_modalities":["text"],"price_in":0.027,"price_match":"exact","price_out":0.201,"release_date":"2024-09-24","supported_parameters":["frequency_penalty","logit_bias","max_tokens","min_p","presence_penalty","repetition_penalty","seed","stop","temperature","top_k","top_p"]},"llama-3-2-3b-instruct":{"context":131072,"country":"United States of America","created":1727222400,"display":"Llama 3.2 3B Instruct","family":"llama-3-2-3b-instruct","flop":1.7334e+23,"flop_notes":"6ND = 6*3210000000.00*9000000000000 = 1.7334e+23\n\n460000 hours * 3600 s * 133800000000000 FLOPS/s* 0.3 = 6.647184e+22","input_modalities":["text"],"knowledge_cutoff":"2023-12-31","max_output":117964,"or_id":"meta-llama/llama-3.2-3b-instruct","org":"Meta","output_modalities":["text"],"price_in":0.05,"price_match":"exact","price_out":0.33,"release_date":"2024-09-24","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","min_p","presence_penalty","repetition_penalty","seed","stop","structured_outputs","temperature","top_k","top_logprobs","top_p"]},"llama-3-2-90b-vision-instruct":{"country":"United States of America","display":"Llama 3.2 90B Vision Instruct","family":"llama-3-2-90b-vision-instruct","org":"Meta","release_date":"2024-09-24"},"llama-3-3-70b-instruct":{"aa":{"agentic_index":null,"coding_index":11.9,"intelligence_index":null},"context":131072,"country":"United States of America","created":1733506137,"display":"Llama 3.3 70B Instruct","family":"llama-3-3-70b-instruct","flop":6.8649768e+24,"flop_notes":"6ND = 6 FLOP / parameter / token * 70*10^9 parameters * 15*10^12 tokens = 6.3e+24 FLOP\n\n7000000 GPU-hours * 3600 sec / hour * 989500000000000 FLOP / second * 0.3 [assumed utilization]= 7.48062e+24 FLOP\n\nsqrt(7.48062e+24*6.3e+24) = 6.8649768e+24","input_modalities":["text"],"knowledge_cutoff":"2023-12-31","max_output":32768,"or_id":"meta-llama/llama-3.3-70b-instruct","org":"Meta","output_modalities":["text"],"price_cache":0.295,"price_in":0.59,"price_match":"exact","price_out":0.79,"release_date":"2024-12-06","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"llama-33b":{"country":"United States of America","display":"LLaMA 33B","family":"llama-33b","flop":2.7300000000001e+23,"flop_notes":"1.4T tokens * 32.5B params * 6 FLOP/token/param = 2.73e+23 FLOP","org":"Meta","release_date":"2023-02-24"},"llama-4-maverick-17b-128e-instruct":{"aa":{"agentic_index":0.6,"coding_index":16.3,"intelligence_index":null},"context":524288,"country":"United States of America","created":1743881822,"display":"Llama 4 Maverick","family":"llama-4-maverick-17b-128e-instruct","flop":2.244000000001e+24,"flop_notes":"22T training tokens per model card:\n\nhttps://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md   \n\nMaverick was trained using co-distillation from Llama 4 Behemoth. It isn't 100% clear that all 22T tokens used distillation, but we assume this for the time being.\n\nEstimating training compute from parameters and tokens:\nCompute = 6 FLOP per token per parameter * 17B active parameters * 22T tokens = 2.244e24 FLOP\n(Implying mean throughput was 262 TFLOPS/GPU, or 13.2% MFU in FP8)\n\n\nThe model card also states that Llama 4 Maverick used 2.38M H100-hours.\nThe blog post gives a figure of 390 TFLOPS/GPU, but this may have been the utilization rate for Behemoth, or all of the models together. Using this utilization, we have:\nCompute = 390 TFLOP/s * 2.38 million hours = 3.342e24 FLOP\n(This value is higher than the compute implied by parameters and tokens, and suggests utilization may have been lower for Maverick than for Behemoth.)","input_modalities":["text","image"],"knowledge_cutoff":"2024-08-31","max_output":32768,"open_weights":true,"or_id":"meta-llama/llama-4-maverick","org":"Meta","output_modalities":["text"],"params":400000000000,"price_cache":0.17,"price_in":0.35,"price_match":"exact","price_out":1,"release_date":"2025-04-05","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"llama-4-maverick-17b-128e-instruct-fp8":{"country":"United States of America","display":"Llama 4 Maverick","family":"llama-4-maverick-17b-128e-instruct-fp8","flop":2.244000000001e+24,"flop_notes":"22T training tokens per model card:\n\nhttps://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md   \n\nMaverick was trained using co-distillation from Llama 4 Behemoth. It isn't 100% clear that all 22T tokens used distillation, but we assume this for the time being.\n\nEstimating training compute from parameters and tokens:\nCompute = 6 FLOP per token per parameter * 17B active parameters * 22T tokens = 2.244e24 FLOP\n(Implying mean throughput was 262 TFLOPS/GPU, or 13.2% MFU in FP8)\n\n\nThe model card also states that Llama 4 Maverick used 2.38M H100-hours.\nThe blog post gives a figure of 390 TFLOPS/GPU, but this may have been the utilization rate for Behemoth, or all of the models together. Using this utilization, we have:\nCompute = 390 TFLOP/s * 2.38 million hours = 3.342e24 FLOP\n(This value is higher than the compute implied by parameters and tokens, and suggests utilization may have been lower for Maverick than for Behemoth.)","open_weights":true,"org":"Meta","params":400000000000,"release_date":"2025-04-05"},"llama-4-scout-17b-16e-instruct":{"aa":{"agentic_index":0.5,"coding_index":8.2,"intelligence_index":null},"context":1310720,"country":"United States of America","created":1743881519,"display":"Llama 4 Scout","family":"llama-4-scout-17b-16e-instruct","flop":4.08e+24,"flop_notes":"40T training tokens per model card:\n\nhttps://github.com/meta-llama/llama-models/blob/main/models/llama4/MODEL_CARD.md  \n\nEstimating training compute from parameters and tokens:\n6 FLOP per token per parameter * 17B active parameters * 40T tokens = 4.08e24 FLOP\n(Implying mean throughput was 227 TFLOPS/GPU, or 11.5% MFU in FP8)\n\n\nThe model card also states that Llama 4 Scout used 5.0M H100-hours.\nThe blog post gives a figure of 390 TFLOPS/GPU, but this may have been the utilization rate for Behemoth, or all of the models together. Using this utilization, we have:\nCompute = 390 TFLOP/s * 5 million hours = 7.02e24 FLOP\n(This value is higher than the compute implied by parameters and tokens, and suggests utilization may have been lower for Scout than for Behemoth.)","input_modalities":["text","image"],"knowledge_cutoff":"2024-08-31","max_output":16384,"open_weights":true,"or_id":"meta-llama/llama-4-scout","org":"Meta","output_modalities":["text"],"params":109000000000,"price_in":0.1,"price_match":"exact","price_out":0.3,"release_date":"2025-04-05","supported_parameters":["frequency_penalty","logit_bias","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"llama-65b":{"country":"United States of America","display":"LLaMA 65B","family":"llama-65b","flop":5.5e+23,"flop_notes":"1.4e12 tokens * 6.52e10 parameters * 6 FLOP/token/parameter = 5.5e23 FLOP","open_weights":true,"org":"Meta","params":65200000000,"release_date":"2023-02-24"},"llama-7b":{"country":"United States of America","display":"LLaMA 7B","family":"llama-7b","flop":4.00000001e+22,"flop_notes":"1T tokens * 6.7B parameters * 6 FLOP/token/parameter = 4e22 FLOP","org":"Meta","release_date":"2023-02-24"},"llama3-llava-next-8b":{"display":"LLaVA-Next Llama3-8b","family":"llama3-llava-next-8b","release_date":"2024-04-20"},"llava-onevision-72b":{"display":"LLaVA-OneVision","family":"llava-onevision-72b","release_date":"2024-08-06"},"llava-v1-5-7b":{"display":"LLaVA-1.5-7B","family":"llava-v1-5-7b","release_date":"2023-10-05"},"llava-v1-6-mistral-7b":{"display":"LLaVA1.6-Mistral-7B","family":"llava-v1-6-mistral-7b","release_date":"2024-01-31"},"llava-v1-6-vicuna-13b":{"display":"LLaVA1.6-Vicuna-13B","family":"llava-v1-6-vicuna-13b","release_date":"2024-01-31"},"llava-v1-6-vicuna-7b":{"display":"LLaVA-1.6 Vicuna-7b","family":"llava-v1-6-vicuna-7b","release_date":"2024-01-31"},"llava-video-72b-qwen2":{"display":"LLaVA-Video","family":"llava-video-72b-qwen2","release_date":"2024-09-02"},"llava-video-7b-qwen2":{"display":"LLaVA-Video-Qwen-7B","family":"llava-video-7b-qwen2","release_date":"2024-09-02"},"llava-video-7b-qwen2-tpo":{"display":"TPO","family":"llava-video-7b-qwen2-tpo","release_date":"2025-01-19"},"long-llava-qwen2-7b":{"display":"Long-LLaVA","family":"long-llava-qwen2-7b","release_date":"2024-08-30"},"longva-7b":{"display":"LongVA-7B","family":"longva-7b","release_date":"2024-06-13"},"magistral-medium-2506":{"country":"France","display":"Magistral","family":"magistral-medium-2506","org":"Mistral","release_date":"2025-06-10"},"magistral-medium-2509":{"country":"France","display":"Magistral Medium 1.2","family":"magistral-medium-2509","org":"Mistral","release_date":"2025-09-18"},"magistral-small-2506":{"country":"France","display":"Magistral Small","family":"magistral-small-2506","org":"Mistral","release_date":"2025-06-10"},"magistral-small-2509":{"country":"France","display":"Magistral Small 1.2","family":"magistral-small-2509","org":"Mistral","release_date":"2025-09-18"},"megatron-turing-nlg-530b":{"country":"United States of America","display":"MT-NLG (ours)","family":"megatron-turing-nlg-530b","flop":8.586e+23,"flop_notes":"https://www.lesswrong.com/posts/bGuMrzhJdENCo8BxX/nvidia-and-microsoft-releases-530b-parameter-transformer?commentId=HSJSNspKp94tFcSCx\n\nsource: https://lair.lighton.ai/akronomicon/\n9938 PF-days * 3600 * 24 * 10^15  = 8.586432e+23\n\n6ND estimate: 6 * 530B * 270B = 8.586000e+23","open_weights":false,"org":"Microsoft,Nvidia","params":530000000000,"release_date":"2021-10-11"},"mercury-2":{"country":"United States of America","display":"Mercury 2","family":"mercury-2","org":"Inception Labs"},"mercury-2-5":{"country":"United States of America","display":"Mercury 2.5","family":"mercury-2-5","open_weights":false,"org":"Inception Labs","release_date":"2026-09-08"},"meta-llama-3-70b":{"country":"United States of America","display":"llama-3-70b-chat-hf†","family":"meta-llama-3-70b","flop":7.861e+24,"flop_notes":"Arithmetic calculation:\n6 * 15T tokens * 70B parameters = 6.3e24\n\nGPU calculation:\nhttps://huggingface.co/meta-llama/Meta-Llama-3-70B indicates training took 6.4M GPU-hours\nWe also know their larger scale training runs for 405B were getting between 0.38-0.41 MFU. Presumably the 70B model gets at least 0.43 utilization (405B has to be split across two nodes, while 70B should fit on one).\n990 TFLOPS per GPU * 6.4 million GPU hours * 3600s * 0.43 = 9.808e24\n\nGeometric mean: sqrt(6.3e24 * 9.808e24) = 7.861e24","org":"Meta","release_date":"2024-04-18"},"meta-llama-3-70b-instruct":{"country":"United States of America","display":"Llama 3 (70B)","family":"meta-llama-3-70b-instruct","flop":7.861e+24,"flop_notes":"Arithmetic calculation:\n6 * 15T tokens * 70B parameters = 6.3e24\n\nGPU calculation:\nhttps://huggingface.co/meta-llama/Meta-Llama-3-70B indicates training took 6.4M GPU-hours\nWe also know their larger scale training runs for 405B were getting between 0.38-0.41 MFU. Presumably the 70B model gets at least 0.43 utilization (405B has to be split across two nodes, while 70B should fit on one).\n990 TFLOPS per GPU * 6.4 million GPU hours * 3600s * 0.43 = 9.808e24\n\nGeometric mean: sqrt(6.3e24 * 9.808e24) = 7.861e24","open_weights":true,"org":"Meta","params":70000000000,"release_date":"2024-04-18"},"meta-llama-3-8b":{"country":"United States of America","display":"LLaMA 3 8B","family":"meta-llama-3-8b","flop":7.2e+23,"flop_notes":"Counting operations\n15000000000000 tokens*8000000000.00 parameters*6 FLOP / token / parameter = 7.2×10^23 FLOP\n\nGPU calculation\n400 TFLOPS per GPU * 1.3M GPU hours * 3600s=1.872×10^24 \n(it is not confident that 400 TFLOPs applies to the Llama 3-8B training run)","org":"Meta","release_date":"2024-04-18"},"meta-llama-3-8b-instruct":{"country":"United States of America","display":"Llama-3-In 8b","family":"meta-llama-3-8b-instruct","flop":7.2e+23,"flop_notes":"Counting operations\n15000000000000 tokens*8000000000.00 parameters*6 FLOP / token / parameter = 7.2×10^23 FLOP\n\nGPU calculation\n400 TFLOPS per GPU * 1.3M GPU hours * 3600s=1.872×10^24 \n(it is not confident that 400 TFLOPs applies to the Llama 3-8B training run)","org":"Meta","release_date":"2024-04-18"},"mimo-v2-5":{"aa":{"agentic_index":15.8,"coding_index":56.8,"intelligence_index":null},"context":1000000,"country":"China","created":1776874269,"display":"MiMo-V2.5","family":"mimo-v2-5","flop":4.32e+24,"flop_notes":"MiMo-V2.5 is a 310B-parameter Sparse MoE model (15B active) trained on 48T tokens\n\n6ND:\n6 * 15B * 48T = 4.32e24","input_modalities":["text","audio","image","video"],"max_output":65536,"or_id":"xiaomi/mimo-v2.5","org":"Xiaomi Corp","output_modalities":["text"],"price_cache":0.08,"price_in":0.4,"price_match":"exact","price_out":2,"supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"mimo-v2-5-pro":{"aa":{"agentic_index":21.3,"coding_index":60.2,"intelligence_index":null},"context":1050000,"country":"China","created":1776874273,"display":"MiMo-V2.5-Pro","family":"mimo-v2-5-pro","flop":6.804e+24,"flop_notes":"6 FLOPS/token/active parameter * 27e12 tokens * 42e9 active parameters = 6.804E24 FLOP","input_modalities":["text"],"max_output":235929,"open_weights":true,"or_id":"xiaomi/mimo-v2.5-pro","org":"Xiaomi Corp","output_modalities":["text"],"params":1020000000000,"price_cache":0.096,"price_in":0.48,"price_match":"exact","price_out":1.8,"release_date":"2026-04-23","supported_parameters":["include_reasoning","logprobs","max_tokens","reasoning","response_format","seed","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"mimo-v2-6-flash":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":26.7},"context":1050000,"country":"China","created":1790021264,"display":"MiMo-V2.6-Flash","family":"mimo-v2-6-flash","input_modalities":["text","image","video","audio"],"max_output":131072,"or_id":"xiaomi/mimo-v2.6-flash","org":"Xiaomi Corp","output_modalities":["text"],"price_cache":0.0028,"price_in":0.14,"price_match":"exact","price_out":0.28,"release_date":"2026-09-21","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","parallel_tool_calls","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"mimo-v2-6-pro":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":29.9},"context":1050000,"country":"China","created":1790021259,"display":"MiMo-V2.6-Pro","family":"mimo-v2-6-pro","input_modalities":["text","image","video","audio"],"max_output":131072,"or_id":"xiaomi/mimo-v2.6-pro","org":"Xiaomi Corp","output_modalities":["text"],"price_cache":0.0036,"price_in":0.435,"price_match":"exact","price_out":0.87,"release_date":"2026-09-21","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"mimo-v2-flash":{"country":"China","display":"Mimo V2 Flash","family":"mimo-v2-flash","flop":2.43e+24,"flop_notes":"6 * 15e9 active parameters * 27e12 tokens = 2.43e+24","org":"Xiaomi Corp"},"mimo-v2-pro":{"country":"China","display":"Mimo V2 Pro","family":"mimo-v2-pro","open_weights":false,"org":"Xiaomi Corp","params":1020000000000,"release_date":"2026-03-18"},"minicpm-o-2-6":{"display":"MiniCPM-o 2.6","family":"minicpm-o-2-6","release_date":"2025-01-12"},"minicpm-v-2-6":{"display":"MiniCPM-V 2.6","family":"minicpm-v-2-6","release_date":"2024-08-03"},"minimax-m1-80k":{"country":"China","display":"MiniMax M1 80K","family":"minimax-m1-80k","flop":4.3240062e+24,"flop_notes":"1.9828799999999997e+24 FLOP [base model compute] + 2.3411262e+24 FLOP [see finetune compute notes] = 4.3240062e+24 FLOP","org":"MiniMax","release_date":"2025-06-13"},"minimax-m2":{"country":"China","display":"MiniMax M2","family":"minimax-m2","open_weights":true,"org":"MiniMax","params":229000000000,"release_date":"2025-10-27"},"minimax-m2-1":{"country":"China","display":"MiniMax M2.1","family":"minimax-m2-1","open_weights":true,"org":"MiniMax","params":229000000000,"release_date":"2025-12-23"},"minimax-m2-5":{"context":204800,"country":"China","created":1770908502,"display":"MiniMax M2.5","family":"minimax-m2-5","input_modalities":["text"],"max_output":58982,"or_id":"minimax/minimax-m2.5","org":"MiniMax","output_modalities":["text"],"price_cache":0.06,"price_in":0.3,"price_match":"exact","price_out":1.2,"release_date":"2026-02-12","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"minimax-m2-7":{"aa":{"agentic_index":15.3,"coding_index":52.6,"intelligence_index":null},"context":204800,"country":"China","created":1773836697,"display":"MiniMax M2.7","family":"minimax-m2-7","input_modalities":["text"],"max_output":176947,"or_id":"minimax/minimax-m2.7","org":"MiniMax","output_modalities":["text"],"price_cache":0.042,"price_in":0.21,"price_match":"exact","price_out":0.84,"release_date":"2026-03-18","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","temperature","tool_choice","tools","top_k","top_p"]},"minimax-m3":{"aa":{"agentic_index":29.5,"coding_index":58.6,"intelligence_index":18.9},"context":1048576,"country":"China","created":1780245374,"display":"MiniMax M3","family":"minimax-m3","input_modalities":["text","image","video"],"max_output":512000,"open_weights":true,"or_id":"minimax/minimax-m3","org":"MiniMax","output_modalities":["text"],"params":428000000000,"price_cache":0.06,"price_in":0.3,"price_match":"exact","price_out":1.2,"release_date":"2026-06-01","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"ministral-3b-2410":{"country":"France","display":"Ministral 3B","family":"ministral-3b-2410","org":"Mistral","release_date":"2024-10-16"},"ministral-8b-2410":{"country":"France","display":"Ministral 8B","family":"ministral-8b-2410","org":"Mistral","release_date":"2024-10-16"},"mistral-7b-instruct-v0-1":{"country":"France","display":"Mistral 7B","family":"mistral-7b-instruct-v0-1","org":"Mistral","release_date":"2023-09-27"},"mistral-7b-instruct-v0-2":{"country":"France","display":"Mistral 7b","family":"mistral-7b-instruct-v0-2","org":"Mistral","release_date":"2023-12-11"},"mistral-7b-instruct-v0-3":{"country":"France","display":"Mistral Instruct v0.3 (7B)","family":"mistral-7b-instruct-v0-3","org":"Mistral","release_date":"2024-05-27"},"mistral-7b-v0-1":{"country":"France","display":"Mistral 7b","family":"mistral-7b-v0-1","org":"Mistral","release_date":"2023-09-27"},"mistral-large-2402":{"country":"France","display":"Mistral Large (2402)","family":"mistral-large-2402","flop_notes":"https://www.wsj.com/tech/ai/the-9-month-old-ai-startup-challenging-silicon-valleys-giants-ee2e4c48\n\nMistral spent \u003c20 million euro (meaning approximately 20 million?) to train Mistral Large.\n\n\nAssuming bf16 or fp16, H100 SXM performance is 989 TFLOPS\nAt 1.9 euro per H100-hour and 30% utilization, spending 20M euro produces 1.12*10^25 FLOP.\nhttps://www.wolframalpha.com/input?i=20+million+%2F+%281.9%2Fhour%29+*+989+TFLOPS+*+0.30","org":"Mistral","release_date":"2024-02-26"},"mistral-large-2407":{"context":131072,"country":"France","created":1731978415,"display":"Mistral Large 2407","family":"mistral-large-2407","flop_notes":"[Editor's note: this record previously contained a training compute estimate of ~1e25 FLOP. This is better described as an upper bound instead. See note below for details]\n\nDetails are sparse, but we can hazard a guess based on evidence about the training cluster they may have used, the scale up in compute they likely would have used relative to Mistral Large 1, and from the model's MMLU score. Extended reasoning given here: https://docs.google.com/document/d/1I2ZWBLFMpRZYcdMMUfKAGZFJrOJpduNDS9ZeVFIHnd8/edit?usp=sharing","input_modalities":["text","file"],"knowledge_cutoff":"2024-03-31","max_output":104857,"or_id":"mistralai/mistral-large-2407","org":"Mistral","output_modalities":["text"],"price_cache":0.2,"price_in":2,"price_match":"exact","price_out":6,"release_date":"2024-07-24","supported_parameters":["frequency_penalty","max_tokens","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"mistral-large-2411":{"country":"France","display":"mistral-large-2411†","family":"mistral-large-2411","flop_notes":"[Editor's note: this record previously contained a training compute estimate of ~1e25 FLOP. This is better described as an upper bound instead. See note below for details]\n\nDetails are sparse, but we can hazard a guess based on evidence about the training cluster they may have used, the scale up in compute they likely would have used relative to Mistral Large 1, and from the model's MMLU score. Extended reasoning given here: https://docs.google.com/document/d/1I2ZWBLFMpRZYcdMMUfKAGZFJrOJpduNDS9ZeVFIHnd8/edit?usp=sharing","org":"Mistral","release_date":"2024-11-18"},"mistral-large-2512":{"aa":{"agentic_index":1,"coding_index":20.1,"intelligence_index":null},"context":262144,"country":"France","created":1764624472,"display":"Mistral Large 3 2512","family":"mistral-large-2512","input_modalities":["text","image","file"],"max_output":209715,"or_id":"mistralai/mistral-large-2512","org":"Mistral","output_modalities":["text"],"price_cache":0.05,"price_in":0.5,"price_match":"exact","price_out":1.5,"release_date":"2025-12-02","supported_parameters":["frequency_penalty","max_tokens","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"mistral-large-4-0":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":26.1},"context":1048576,"country":"France","created":1791294122,"display":"Mistral Large 4","family":"mistral-large-4-0","flop":1.24e+25,"flop_notes":"From tech report: \"Mistral Large 4 was pretrained on 3,840 NVIDIA GB200 GPUs in a Mistral data center near Paris.\nOur internal training stack is built on PyTorch and CUDA. Pretraining took 45 days, or about 4.15 million GPU-hours, with a model FLOPs utilization (MFU) of 36% and a goodput of 94.1%. Webmeasure MFU against peak hardware BF16 FLOP/s, include matrix-multiplication and attention\nFLOPs.\"\n\nGB200 is 2500 TFLOP/s BF16 peak, so 36% MFU and 94.1% goodput is 32.9% overall utilization. \n3840 * 45 * 24*3600 * 0.329 * 2500 trillion = 1.23e25 FLOP pretraining.\n\nThis suggests roughly 40T training tokens, using 6ND: 1.23e25 = 6* 52 billion * D.\n\nFor post-training, RL involved 162.5 billion output tokens during rollouts for RL, so RL FLOP was \u003c1% of pre-training. SFT and mid-training tokens were not disclosed.\n\nMFU is reported against bf16 spec, but report mentions FP8 in the pretraining without clarifying the primary format.","input_modalities":["text","image"],"max_output":262144,"open_weights":true,"or_id":"mistralai/mistral-large-4-0","org":"Mistral","output_modalities":["text"],"params":1050000000000,"price_cache":0.07,"price_in":0.7,"price_match":"exact","price_out":2.1,"release_date":"2026-10-06","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_logprobs","top_p"]},"mistral-large-4-0-oss":{"country":"France","display":"Mistral Large 4.0 OSS","family":"mistral-large-4-0-oss","flop":1.24e+25,"flop_notes":"From tech report: \"Mistral Large 4 was pretrained on 3,840 NVIDIA GB200 GPUs in a Mistral data center near Paris.\nOur internal training stack is built on PyTorch and CUDA. Pretraining took 45 days, or about 4.15 million GPU-hours, with a model FLOPs utilization (MFU) of 36% and a goodput of 94.1%. Webmeasure MFU against peak hardware BF16 FLOP/s, include matrix-multiplication and attention\nFLOPs.\"\n\nGB200 is 2500 TFLOP/s BF16 peak, so 36% MFU and 94.1% goodput is 32.9% overall utilization. \n3840 * 45 * 24*3600 * 0.329 * 2500 trillion = 1.23e25 FLOP pretraining.\n\nThis suggests roughly 40T training tokens, using 6ND: 1.23e25 = 6* 52 billion * D.\n\nFor post-training, RL involved 162.5 billion output tokens during rollouts for RL, so RL FLOP was \u003c1% of pre-training. SFT and mid-training tokens were not disclosed.\n\nMFU is reported against bf16 spec, but report mentions FP8 in the pretraining without clarifying the primary format.","org":"Mistral","release_date":"2026-10-29"},"mistral-medium-2505":{"country":"France","display":"Mistral Medium 3","family":"mistral-medium-2505","flop_notes":"Benchmarks match with models like GPT-4o, Mistral's previous largest runs were ~1e25 FLOP scale, so plausibly they might have trained Medium with this much compute.","org":"Mistral","release_date":"2025-05-07"},"mistral-medium-2508":{"country":"France","display":"Mistral Medium 3.1","family":"mistral-medium-2508","org":"Mistral","release_date":"2025-08-12"},"mistral-medium-2604":{"country":"France","display":"Mistral Medium 3.5","family":"mistral-medium-2604","flop_notes":"Commentary from Mistral researcher - https://x.com/mertunsal2020/status/2049551864556143094?s=20 - says \"nowhere close to those flops\" in response to someone suggesting 2.3e25FLOPS was used","org":"Mistral","release_date":"2026-04-28"},"mistral-nemo-base-2407":{"country":"France","display":"Mistral-Nemo 12B","family":"mistral-nemo-base-2407","org":"Mistral","release_date":"2024-07-18"},"mistral-nemo-instruct-2407":{"country":"France","display":"Mistral Nemo Instruct","family":"mistral-nemo-instruct-2407","org":"Mistral","release_date":"2024-07-18"},"mistral-small-2402":{"display":"Mistral Small (2402)","family":"mistral-small-2402","release_date":"2024-02-26"},"mistral-small-2501":{"country":"France","display":"Mistral Small","family":"mistral-small-2501","flop":1.152e+24,"flop_notes":"6ND = 6*8T tokens * 24B parameters = 1.152e+24 FLOP","org":"Mistral","release_date":"2025-01-25"},"mistral-small-2503":{"country":"France","display":"Mistral Small 3.1","family":"mistral-small-2503","flop_notes":"At least 1.152e+24 FLOP (base model Mistral Small 3 training compute)","org":"Mistral","release_date":"2025-03-17"},"mistral-small-2506":{"country":"France","display":"Mistral Small 3.2","family":"mistral-small-2506","flop_notes":"Likely to use \u003e1e23 FLOP because Mistral Small 3 used about 1e24 FLOP. However, if this was continued pretraining (or new post-training), then not clear whether it would have used \u003e1e23 FLOP.","org":"Mistral","release_date":"2025-06-20"},"mistral-small-2603":{"aa":{"agentic_index":0.8,"coding_index":26.6,"intelligence_index":4.3},"context":262144,"country":"France","created":1773695685,"display":"Mistral Small 4","family":"mistral-small-2603","input_modalities":["text","image"],"max_output":209715,"or_id":"mistralai/mistral-small-2603","org":"Mistral","output_modalities":["text"],"price_cache":0.015,"price_in":0.15,"price_match":"exact","price_out":0.6,"release_date":"2026-03-16","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"mistral-small-3-1-2503":{"country":"France","display":"Mistral Small 3.1","family":"mistral-small-3-1-2503","flop_notes":"At least 1.152e+24 FLOP (base model Mistral Small 3 training compute)","org":"Mistral","release_date":"2025-03-17"},"mistral-small-3-2-2506":{"country":"France","display":"Mistral Small 3.2","family":"mistral-small-3-2-2506","flop_notes":"Likely to use \u003e1e23 FLOP because Mistral Small 3 used about 1e24 FLOP. However, if this was continued pretraining (or new post-training), then not clear whether it would have used \u003e1e23 FLOP.","org":"Mistral","release_date":"2025-06-20"},"mistral-small-3-2501":{"country":"France","display":"Mistral Small 3","family":"mistral-small-3-2501","flop":1.152e+24,"flop_notes":"6ND = 6*8T tokens * 24B parameters = 1.152e+24 FLOP","org":"Mistral","release_date":"2025-01-30"},"mixtral-8x22b-instruct-v0-1":{"country":"France","display":"mixtral-8x22b-instruct-v0.1†","family":"mixtral-8x22b-instruct-v0-1","flop":2.34e+24,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 39 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 2.34e+24 FLOP","org":"Mistral","release_date":"2024-04-17"},"mixtral-8x22b-v0-1":{"country":"France","display":"Mixtral (8x22B)","family":"mixtral-8x22b-v0-1","flop":2.34e+24,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 39 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 2.34e+24 FLOP","org":"Mistral","release_date":"2024-04-17"},"mixtral-8x7b-instruct-v0-1":{"country":"France","display":"mixtral-8x7b-instruct-v0.1†","family":"mixtral-8x7b-instruct-v0-1","flop":7.74e+23,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 12.9 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 7.74e+23 FLOP","org":"Mistral","release_date":"2023-12-11"},"mixtral-8x7b-v0-1":{"country":"France","display":"Mixtral 8x7b","family":"mixtral-8x7b-v0-1","flop":7.74e+23,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 12.9 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 7.74e+23 FLOP","open_weights":true,"org":"Mistral","params":46700000000,"release_date":"2023-12-11"},"ml-elephant":{"display":"Zoo Ml Text To Cad","family":"ml-elephant"},"mm1-3b-chat":{"display":"MM1-3B-Chat 3.6b","family":"mm1-3b-chat","release_date":"2024-03-14"},"mm1-7b-chat":{"display":"MM1-7B-Chat 7.6b","family":"mm1-7b-chat","release_date":"2024-03-14"},"moonshotaikimi-k2-0905":{"country":"China","display":"Kimi K2","family":"moonshotaikimi-k2-0905","flop":2.976e+24,"flop_notes":"6 FLOP / parameter / token * 32 * 10^9 activated parameters * 15.5 * 10^12 tokens = 2.976e+24 FLOP","open_weights":true,"org":"Moonshot AI","params":1000000000000,"release_date":"2025-09-05"},"moonshotaikimi-k2-thinking":{"country":"China","display":"Moonshotaikimi K2","family":"moonshotaikimi-k2-thinking","flop":4.2e+24,"flop_notes":"Assuming the additional post-training contributed between 1% and 100% of Kimi K2's training compute (which we confidently estimate at 2.976e+24), we get a range of 3.0e24 to 6.0e24, and a geometric mean of 4.2e24.","org":"Moonshot AI","release_date":"2025-11-06"},"mplug-owl3-7b-241101":{"display":"mPLUG-Owl3-7B-241101","family":"mplug-owl3-7b-241101","release_date":"2024-11-26"},"mpt-30b":{"country":"United States of America","display":"MPT 30B","family":"mpt-30b","flop":1.8900000000001e+23,"flop_notes":"According to their blog post,\n\"MPT-30B FLOPs ~= 6 * 30e9 [params] * 1.05e12 [tokens] = 1.89e23 FLOPs\"","org":"MosaicML","release_date":"2023-06-22"},"mpt-30b-instruct":{"country":"United States of America","display":"MPT-Instruct (30B)","family":"mpt-30b-instruct","flop":1.8900000000001e+23,"flop_notes":"According to their blog post,\n\"MPT-30B FLOPs ~= 6 * 30e9 [params] * 1.05e12 [tokens] = 1.89e23 FLOPs\"","org":"MosaicML","release_date":"2023-06-22"},"mpt-7b":{"country":"United States of America","display":"MPT 7B","family":"mpt-7b","flop":4.2000000000000004e+22,"flop_notes":"\"As shown in Table 3, nearly all of the training budget was spent on the base MPT-7B model, which took ~9.5 days to train on 440xA100-40GB GPUs, and cost ~$200k.\"","org":"MosaicML","release_date":"2023-05-05"},"muse-glimmer":{"country":"United States of America","display":"Muse Glimmer","family":"muse-glimmer","org":"Meta","release_date":"2026-08-10"},"muse-spark":{"country":"United States of America","display":"Muse Spark","family":"muse-spark","open_weights":false,"org":"Meta","release_date":"2026-04-08"},"muse-spark-1-1":{"aa":{"agentic_index":25.8,"coding_index":71.3,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1784215741,"display":"Muse Spark 1.1","family":"muse-spark-1-1","input_modalities":["text","image","video","file"],"max_output":943718,"or_id":"meta/muse-spark-1.1","org":"Meta","output_modalities":["text"],"price_cache":0.15,"price_in":1.25,"price_match":"exact","price_out":4.25,"release_date":"2026-07-09","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","repetition_penalty","response_format","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"muse-spark-1-2":{"aa":{"agentic_index":43.2,"coding_index":72.2,"intelligence_index":null},"context":1048576,"country":"United States of America","created":1785959287,"display":"Muse Spark 1.2","family":"muse-spark-1-2","input_modalities":["text","image","video","file"],"max_output":943718,"open_weights":false,"or_id":"meta/muse-spark-1.2","org":"Meta","output_modalities":["text"],"price_cache":0.15,"price_in":1.25,"price_match":"exact","price_out":4.25,"release_date":"2026-08-05","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","repetition_penalty","response_format","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"muse-spark-1-3":{"aa":{"agentic_index":55.5,"coding_index":75.8,"intelligence_index":36.6},"context":1048576,"country":"United States of America","created":1788378359,"display":"Muse Spark 1.3","family":"muse-spark-1-3","input_modalities":["text","image","video","file"],"max_output":943718,"open_weights":false,"or_id":"meta/muse-spark-1.3","org":"Meta","output_modalities":["text"],"price_cache":0.15,"price_in":1.25,"price_match":"exact","price_out":4.25,"release_date":"2026-09-02","supported_parameters":["include_reasoning","max_tokens","reasoning","reasoning_effort","repetition_penalty","response_format","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"muse-spark-open-816b-a42b":{"country":"United States of America","display":"Muse Spark Open","family":"muse-spark-open-816b-a42b","org":"Meta","release_date":"2026-10-15"},"muse-wonder-1-0":{"country":"United States of America","display":"Muse Wonder 1.0","family":"muse-wonder-1-0","org":"Meta","release_date":"2026-10-12"},"nemotron-3-super":{"country":"United States of America","display":"Nemotron 3 Super","family":"nemotron-3-super","open_weights":true,"org":"Nvidia","params":120000000000,"release_date":"2026-03-11"},"nemotron-3-ultra":{"country":"United States of America","display":"Nemotron 3 Ultra","family":"nemotron-3-ultra","flop":6.6e+24,"flop_notes":"55 billion active parameters, ~20T tokens","open_weights":true,"org":"Nvidia","params":550000000000,"release_date":"2026-06-04"},"nemotron-4-15b":{"country":"United States of America","display":"Nemotron-4 15B","family":"nemotron-4-15b","flop":7.5005116e+23,"flop_notes":"6ND = 6 FLOP/token/parameter * 15*10^9 parameters * 8*10^12 tokens = 7.2e+23 FLOP\n\n\"Nemotron-4 was trained using 384 DGX H100 nodes; each node contains 8 H100 80GB SXM5 GPUs based on the NVIDIA Hopper architecture (NVIDIA, 2022). Each H100 GPU has a peak throughput of 989 teraFLOP/s when doing 16-bit floating point (bfloat16) arithmetic without sparsity.\n\nTable 2 reports more detailed training schedule:\n\n989*10^12 FLOP/sec * 3600 sec/hour * 24 hours * (768 gpus * 0.343 [reported utilization] * 0.8 days +  1536 gpus * 0.333 [reported utilization] * 0.4 days + 2304 gpus * 0.305 [reported utilization] * 11.9 days) = 7.5005116e+23 FLOP","org":"Nvidia","release_date":"2024-02-26"},"nova-2-0-pro-preview":{"country":"United States of America","display":"Nova 2.0 Pro Preview (Non-reasoning)","family":"nova-2-0-pro","org":"Amazon","release_date":"2025-12-02"},"nvidia-nemotron-nano-9b-v2":{"display":"NVIDIA Nemotron Nano 9B V2 (Reasoning)","family":"nvidia-nemotron-nano-9b-v2","release_date":"2025-08-18"},"nvila-8b":{"country":"China,United States of America","display":"NVILA","family":"nvila-8b","flop":2.2794518e+21,"flop_notes":"\"We train all models using 128 NVIDIA H100 GPUs with a global batch size of 2048 across all stages.\"\n\nfrom Figure 1\nNVILA 8B takes 4.5x times less gpu hours than LLAVA OneVision which is reported to take 400 GPU days (\"For example, training a state-of-the-art 7B VLM [5] can take up to 400 GPU days\")\n\n400/4.5 ~ 89 GPU days ~ 2133 GPU hours\n\n989500000000000 FLOP / sec / GPU * 2133 GPU*hours * 3600 sec / hour * 0.3 [assumed utilization] = 2.2794518e+21 FLOP\n\nConfidence: Likely (precision is FP8 not FP16 and utilization could be different from 0.3)","org":"Nvidia,Massachusetts Institute of Technology (MIT),University of California (UC) Berkeley,University of California San Diego,University of Washington,Tsinghua University","release_date":"2024-12-10"},"o1-2024-12-17":{"country":"United States of America","display":"O1","family":"o1","open_weights":false,"org":"OpenAI","release_date":"2024-12-17"},"o1-mini-2024-09-12":{"country":"United States of America","display":"O1 Mini","family":"o1-mini","flop_notes":"We can’t make a precise estimate, but seems unlikely to exceed 10^25 FLOP. We think active parameter count is 10-30B. This would require \u003e55T tokens to reach 10^25 FLOP at the large size, i.e. well beyond 10x overtraining relative to Chinchilla.","open_weights":false,"org":"OpenAI","release_date":"2024-09-12"},"o1-preview-2024-09-12":{"country":"United States of America","display":"O1 Preview","family":"o1","open_weights":false,"org":"OpenAI","release_date":"2024-09-12"},"o1-pro-2025-03-19":{"country":"United States of America","display":"o1 Pro (March 2025)","family":"o1-pro","org":"OpenAI","release_date":"2025-03-19"},"o3-2025-04-16":{"context":200000,"country":"United States of America","created":1744823457,"display":"o3","family":"o3","input_modalities":["image","text","file"],"knowledge_cutoff":"2024-06-30","max_output":100000,"open_weights":false,"or_id":"openai/o3","org":"OpenAI","output_modalities":["text"],"price_cache":0.5,"price_in":2,"price_match":"exact","price_out":8,"release_date":"2025-04-16","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","seed","structured_outputs","tool_choice","tools"]},"o3-mini-2025-01-31":{"country":"United States of America","display":"O3 Mini","family":"o3-mini","flop_notes":"We can’t make a precise estimate, but seems unlikely to exceed 10^25 FLOP. We think active parameter count is 10-30B. This would require \u003e55T tokens to reach 10^25 FLOP at the large size, i.e. well beyond 10x overtraining relative to Chinchilla.","open_weights":false,"org":"OpenAI","release_date":"2025-01-31"},"o3-pro-2025-06-10":{"context":200000,"country":"United States of America","created":1749598352,"display":"o3 Pro","family":"o3-pro","input_modalities":["text","file","image"],"knowledge_cutoff":"2024-06-30","max_output":100000,"or_id":"openai/o3-pro","org":"OpenAI","output_modalities":["text"],"price_in":20,"price_match":"exact","price_out":80,"release_date":"2025-06-10","supported_parameters":["include_reasoning","max_tokens","reasoning","response_format","seed","structured_outputs","tool_choice","tools"]},"o4-mini-2025-04-16":{"country":"United States of America","display":"O4 Mini","family":"o4-mini","flop_notes":"We can’t make a precise estimate, but seems unlikely to exceed 10^25 FLOP. We think active parameter count is 10-30B. This would require \u003e55T tokens to reach 10^25 FLOP at the large size, i.e. well beyond 10x overtraining relative to Chinchilla.","org":"OpenAI","release_date":"2025-04-16"},"olmo-2-1124-13b-instruct":{"country":"United States of America","display":"OLMo 2.1124 13B Instruct","family":"olmo-2-1124-13b-instruct","flop":4.6e+23,"flop_notes":"4.6*10^23 FLOPs (Table 6 - developers calculated using 6ND formula)","org":"Allen Institute for AI,University of Washington,New York University (NYU)","release_date":"2024-12-31"},"open-llama-7b":{"display":"OpenLLaMA-7B","family":"open-llama-7b","release_date":"2023-06-07"},"open-mistral-7b":{"country":"France","display":"Open Mistral 7B","family":"open-mistral-7b","org":"Mistral","release_date":"2023-09-27"},"open-mistral-nemo-2407":{"country":"France","display":"Open Mistral Nemo","family":"open-mistral-nemo-2407","org":"Mistral","release_date":"2024-07-18"},"open-mixtral-8x22b":{"country":"France","display":"Open Mixtral 8x22b","family":"open-mixtral-8x22b","flop":2.34e+24,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 39 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 2.34e+24 FLOP","org":"Mistral","release_date":"2024-04-17"},"open-mixtral-8x7b":{"country":"France","display":"Open Mixtral 8x7b","family":"open-mixtral-8x7b","flop":7.74e+23,"flop_notes":"Assuming the model was trained on ~1-10 trillions of tokens (same OOM as the models from the comparison in Figure 1. Llama 2 was trained on 2T tokens) + Mistral Small 3 was trained on 8T of tokens, we can estimate training compute with \"speculative\" confidence:\n\n6 FLOP / token / parameter * 12.9 * 10^9 active parameters * 10*10^12 tokens [speculatively] = 7.74e+23 FLOP","org":"Mistral","release_date":"2023-12-11"},"openaigpt-oss-120b":{"country":"United States of America","display":"Openaigpt OSS 120B","family":"openaigpt-oss-120b","flop":4.94e+24,"flop_notes":"\"The training run for gpt-oss-120b required 2.1 million H100-hours to complete\"\n(2.1e6 hours)*(1,979 H100 FLOP/s)*(30% utilization)*(60*60) = 4.49e24\nThey also do post training similar to o3, which we assume adds at least 10% as much compute, so we multiply this estimate by 1.1 to get 4.94e24","org":"OpenAI","release_date":"2025-08-05"},"openhands-lm-32b-v0-1":{"display":"Openhands Lm 32B V0.1","family":"openhands-lm-32b-v0-1","release_date":"2024-03-26"},"opt-1-3b":{"country":"United States of America","display":"OPT-1.3B","family":"opt-1-3b","org":"Meta","release_date":"2022-05-11"},"opt-13b":{"display":"OPT-13B","family":"opt-13b","release_date":"2022-05-11"},"opt-175b":{"country":"United States of America","display":"OPT (175B)","family":"opt-175b","flop":4.3e+23,"flop_notes":"https://github.com/facebookresearch/metaseq/blob/main/projects/OPT/chronicles/final_update.md\n\n\"As of yesterday, at 12:46pm PST on January 6, our 175B model finally completed its training run on 300B tokens. This required ~4.30E+23 FLOPs of compute\"","open_weights":true,"org":"Meta","params":175000000000,"release_date":"2022-05-02"},"opt-66b":{"country":"United States of America","display":"OPT (66B)","family":"opt-66b","flop":1.100000000001e+23,"flop_notes":"OPT-66B was trained for 140k steps, using a batch size of 2M tokens (see the OPT baselines logbook and Table 1 in Zhang et al. (2022), respectively), so training took 140e3 ∗ 2e6 ∗ 66e9 ∗ 6 = 1.1e23 FLOP","org":"Meta","release_date":"2022-05-03"},"oryx-1-5-32b":{"display":"Oryx-1.5","family":"oryx-1-5-32b","release_date":"2024-10-22"},"palm-2-l":{"display":"PaLM 2-L","family":"palm-2-l","release_date":"2023-05-17"},"palm-2-m":{"display":"PaLM 2-M","family":"palm-2-m","release_date":"2023-05-17"},"palm-2-s":{"display":"PaLM 2-S","family":"palm-2-s","release_date":"2023-05-17"},"palm-540b":{"country":"United States of America","display":"PaLM","family":"palm-540b","flop":2.5272e+24,"flop_notes":"See Table 20.\n\n6144 TPUv4 for 1200 hours + 3072 TPUv4 for 336 hours.\nEquivalent to 6144 TPUv4 for 1368 hours.\n\n46.2% model FLOPs utilization\n\n\"The 540B-parameter PaLM model sustained a remarkable 57.8% of the peak hardware floating point performance over 50 days while training on TPU v4 supercomputers.\" https://cloud.google.com/blog/topics/systems/tpu-v4-enables-performance-energy-and-co2e-efficiency-gains","open_weights":false,"org":"Google DeepMind","params":540350000000,"release_date":"2022-04-04"},"palm-62b":{"display":"PaLM 62B","family":"palm-62b","release_date":"2022-04-04"},"parasail-qwen3-235b-a22b-instruct-2507":{"country":"China","display":"Parasail Qwen3 235B A22b Instruct","family":"parasail-qwen3-235b-a22b-instruct-2507","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","org":"Alibaba","release_date":"2025-09-01"},"phi-1-5":{"country":"United States of America","display":"phi-1.5 (1.3B)","family":"phi-1-5","flop":1.17e+21,"flop_notes":"150B training tokens\n\n150B*1.3B*6 = 1.17e21\n\nalso, took 1.5k GPU-hours with A100s, per Table 1\n\n1500 * 312 trillion * 3600 * 0.3 (utilization assumption) = 5.05e20\n\nso utilization was likely ~60%. may be high due to the relatively small cluster.","org":"Microsoft","release_date":"2023-09-11"},"phi-2":{"country":"United States of America","display":"Phi-2 2.7b","family":"phi-2","flop":2.27e+22,"flop_notes":"2.7B params, trained on 1.4T tokens\n\n2.7 billion * 1.4 trillion * 6 = 2.27e22\n\n96*14 A100-days\n\n14 * 96 * 312 trillion * 24 * 3600 * 0.3 = 1.09e22","org":"Microsoft","release_date":"2023-12-12"},"phi-3-5-mini-instruct":{"country":"United States of America","display":"Phi-3.5-mini","family":"phi-3-5-mini-instruct","flop":3.7101154e+22,"flop_notes":"6ND = 6*3800000000.00 parameters *3400000000000 tokens  = 7.752e+22\n\n512 GPUs *133800000000000 FLOP/s *240 hours *3600 sec/hour *0.3 [assumed utilization] = 1.7756652e+22\n\ngeometric mean sqrt (7.752e+22*1.7756652e+22) = 3.7101154e+22","org":"Microsoft","release_date":"2024-08-16"},"phi-3-5-moe-instruct":{"country":"United States of America","display":"Phi-3.5-MoE","family":"phi-3-5-moe-instruct","flop":3.0202896e+23,"flop_notes":"512 GPUs * 989500000000000 FLOP / sec * 552 hours * 3600 sec / hour * 0.3 [assumed utilization] = 3.0202896e+23 FLOP\n\n6 FLOP / token / parameter * 4900000000000 tokens * 6.6*10^9 active parameters = 1.9404e+23 FLOP (slightly less confidence than hardware estimation since the 6ND formula is less accurate for MoE)","org":"Microsoft","release_date":"2024-08-17"},"phi-3-5-vision-instruct":{"display":"Phi-3.5-Vision 4.2b","family":"phi-3-5-vision-instruct","release_date":"2024-08-16"},"phi-3-medium-128k-instruct":{"country":"United States of America","display":"Phi-3-medium 14b","family":"phi-3-medium-128k-instruct","flop":4.032e+23,"flop_notes":"counting operations: 6×4.8×10^12 tokens × 14×10^9 parameters ≈ 4.032×10^23 FLOPS","org":"Microsoft","release_date":"2024-04-23"},"phi-3-mini-4k-instruct":{"country":"United States of America","display":"Phi-3-mini 3.8b","family":"phi-3-mini-4k-instruct","flop":7.524e+22,"flop_notes":"counting operations: 6×3.3×10^12 tokens ×3.8×10^9 parameters ≈7.524×10^22 FLOPS\nhardware estimate: 7 days ×24 hours / day×3600 sec / hour *989,000,000,000,000 FLOP/s*512 GPUs*0.3 [assumed utilization]=9.187540992×10^22","org":"Microsoft","release_date":"2024-04-23"},"phi-3-small-8k-instruct":{"country":"United States of America","display":"Phi-3-small 7b","family":"phi-3-small-8k-instruct","flop":2.1312e+23,"flop_notes":"6ND = 6 FLOP / parameter / token * 7.4B parameters * 4.8T tokens = 2.1312e+23 FLOP","org":"Microsoft","release_date":"2024-04-23"},"phi-4":{"context":16384,"country":"United States of America","created":1736489872,"display":"Phi 4","family":"phi-4","flop":9.3202015e+23,"flop_notes":"6ND = 6* 14*10^9 parameters * 10*10^12 tokens = 8.4e+23 FLOP\n\n989500000000000 FLOP / sec [assumed bf16 precision] * 1920 GPUs * 504 hours * 3600 sec / hour * 0.3 [assumed utilization] = 1.0341209e+24 FLOP\n\ngeometric mean\nsqrt(8.4e+23 * 1.0341209e+24) = 9.3202015e+23","input_modalities":["text"],"knowledge_cutoff":"2024-06-30","max_output":14745,"or_id":"microsoft/phi-4","org":"Microsoft Research","output_modalities":["text"],"price_in":0.07,"price_match":"exact","price_out":0.14,"release_date":"2024-12-12","supported_parameters":["frequency_penalty","logit_bias","max_tokens","min_p","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","top_k","top_p"]},"phi-4-mini-instruct":{"country":"United States of America","display":"Phi-4 Mini Instruct","family":"phi-4-mini-instruct","flop":9.9561596e+22,"flop_notes":"6ND = 6 FLOP / token / parameter * 3800000000 parameters * 5000000000000 tokens = 1.14e+23 FLOP\n\n512 GPUs * 312000000000000 FLOP / sec * 504 hours * 3600 sec / hour * 0.3 [assumed utilization] = 8.6951854e+22 FLOP\n\ngeometric mean: \nsqrt(1.14e+23*8.6951854e+22) = 9.9561596e+22","org":"Microsoft","release_date":"2025-02-27"},"pixtral-12b-2409":{"country":"France","display":"Pixtral 12B","family":"pixtral-12b-2409","org":"Mistral","release_date":"2024-09-17"},"pixtral-large-2411":{"country":"France","display":"Pixtral Large (November 2024)","family":"pixtral-large-2411","org":"Mistral","release_date":"2024-11-18"},"qwen-1-8b":{"display":"QWEN 1.8B","family":"qwen-1-8b","release_date":"2023-11-30"},"qwen-14b":{"country":"China","display":"QWEN 14B","family":"qwen-14b","flop":2.5e+23,"flop_notes":"3T tokens per Table 1\n\n14B*3T*6 = 2.5e23","org":"Alibaba","release_date":"2023-09-28"},"qwen-14b-chat":{"country":"China","display":"Qwen-Chat 14B","family":"qwen-14b-chat","flop":2.5e+23,"flop_notes":"3T tokens per Table 1\n\n14B*3T*6 = 2.5e23","org":"Alibaba","release_date":"2023-09-24"},"qwen-7b":{"country":"China","display":"QWEN 7B","family":"qwen-7b","flop":1.01e+23,"flop_notes":"2.4T tokens per Table 1\n\n7b*2.4T*6 = 1.01e23","org":"Alibaba","release_date":"2023-09-28"},"qwen-max-2025-01-25":{"country":"China","display":"Qwen","family":"qwen-max","org":"Alibaba","release_date":"2025-01-25"},"qwen-plus-2025-01-25":{"context":1000000,"country":"China","created":1738409840,"display":"Qwen-Plus","family":"qwen-plus","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":32768,"or_id":"qwen/qwen-plus","org":"Alibaba","output_modalities":["text"],"price_cache":0.052,"price_in":0.26,"price_match":"exact","price_out":0.78,"release_date":"2025-01-25","supported_parameters":["frequency_penalty","logprobs","max_tokens","presence_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen-plus-2025-04-28":{"country":"China","display":"Qwen Plus","family":"qwen-plus","org":"Alibaba","release_date":"2025-04-28"},"qwen-turbo-2024-11-01":{"country":"China","display":"Qwen Turbo","family":"qwen-turbo","org":"Alibaba","release_date":"2024-11-01"},"qwen-vl-chat":{"display":"Qwen-VL-Chat 9.6b","family":"qwen-vl-chat","release_date":"2023-08-20"},"qwen-vl-max":{"display":"Qwen-VL-max","family":"qwen-vl-max","open_weights":false,"org":"Alibaba","params":7000000000,"release_date":"2024-01-18"},"qwen1-5-110b-chat":{"country":"China","display":"qwen1.5-110b-chat†","family":"qwen1-5-110b-chat","flop_notes":"lower bound is taken from Qwen1.5 72B training compute estimation","org":"Alibaba","release_date":"2024-04-25"},"qwen1-5-14b":{"country":"China","display":"Qwen1.5 (14B)","family":"qwen1-5-14b","flop":3.36e+23,"flop_notes":"6 FLOP / parameter / token * 14*10^9 parameters * 4*10^12 tokens =  3.36e+23 FLOP","org":"Alibaba","release_date":"2024-02-04"},"qwen1-5-32b":{"country":"China","display":"Qwen1.5 (32B)","family":"qwen1-5-32b","flop_notes":"upper bound is taken from Qwen1.5 72B training compute estimation\n\nlower bound is taken from Qwen1.5 14B training compute estimation","org":"Alibaba","release_date":"2024-02-04"},"qwen1-5-32b-chat":{"country":"China","display":"Qwen1.5 32B Chat","family":"qwen1-5-32b-chat","flop_notes":"upper bound is taken from Qwen1.5 72B training compute estimation\n\nlower bound is taken from Qwen1.5 14B training compute estimation","org":"Alibaba","release_date":"2024-04-03"},"qwen1-5-72b-chat":{"country":"China","display":"Qwen1.5 72B Chat","family":"qwen1-5-72b-chat","flop":1.3e+24,"flop_notes":"3T training tokens: https://github.com/QwenLM/Qwen2/issues/97 \n\n6 * 72 billion * 3 trillion = ~1.3e24","org":"Alibaba","release_date":"2024-02-04"},"qwen1-5-7b":{"country":"China","display":"Qwen1.5 (7B)","family":"qwen1-5-7b","flop":1.68e+23,"flop_notes":"6 FLOP / parameter / token * 7*10^9 parameters * 4*10^12 tokens =  1.68e+23 FLOP","org":"Alibaba","release_date":"2024-02-04"},"qwen2-5-14b-instruct":{"country":"China","display":"Qwen2.5 14b instruct","family":"qwen2-5-14b-instruct","flop":1.58760000000001e+24,"flop_notes":"Training dataset size was 18 trillion\n\n6ND = 6 * 14.7 billion parameters * 18 trillion tokens = 1.59e24","org":"Alibaba","release_date":"2024-09-19"},"qwen2-5-32b-instruct":{"country":"China","display":"Qwen2.5 32B Instruct","family":"qwen2-5-32b-instruct","flop":3.51e+24,"flop_notes":"6 FLOP / parameter / token * 32.5B parameters * 18 trillion tokens = 3.51 × 10^24 FLOP","org":"Alibaba","release_date":"2024-09-17"},"qwen2-5-72b":{"country":"China","display":"Qwen2.5 72B Base","family":"qwen2-5-72b","flop":7.8e+24,"flop_notes":"Training dataset size was 18 trillion\n\n6ND = 6 * 72.7 billion parameters * 18 trillion tokens = 7.8e24","open_weights":true,"org":"Alibaba","params":72700000000,"release_date":"2024-09-19"},"qwen2-5-72b-instruct":{"country":"China","display":"qwen2.5-72b-instruct-turbo†","family":"qwen2-5-72b-instruct","flop":7.8e+24,"flop_notes":"Training dataset size was 18 trillion\n\n6ND = 6 * 72.7 billion parameters * 18 trillion tokens = 7.8e24","org":"Alibaba","release_date":"2024-09-19"},"qwen2-5-7b-instruct":{"country":"China","display":"Qwen2.5 Instruct Turbo (7B)","family":"qwen2-5-7b-instruct","flop":8.2188e+23,"flop_notes":"Training dataset size was 18 trillion\n\n6ND = 6 * 7.61 billion parameters * 18 trillion tokens = 8.2188e+23","org":"Alibaba","release_date":"2024-09-19"},"qwen2-5-coder-0-5b":{"display":"Qwen2.5-Coder-0.5B","family":"qwen2-5-coder-0-5b","release_date":"2024-09-18"},"qwen2-5-coder-1-5b":{"country":"China","display":"Qwen2.5-Coder-1.5B","family":"qwen2-5-coder-1-5b","flop":5.082e+22,"flop_notes":"6ND =  6*1540000000 parameters *5.5T tokens =5.082e+22","org":"Alibaba","release_date":"2024-09-18"},"qwen2-5-coder-14b":{"display":"Qwen2.5-Coder-14B","family":"qwen2-5-coder-14b","release_date":"2024-09-18"},"qwen2-5-coder-14b-instruct":{"display":"Qwen2.5-Coder-14B-Instruct","family":"qwen2-5-coder-14b-instruct","release_date":"2024-11-06"},"qwen2-5-coder-32b":{"country":"China","display":"Qwen2.5-Coder-32B","family":"qwen2-5-coder-32b","flop":1.0725e+24,"flop_notes":"Assuming 1 epoch\n\n6ND = 6*32.5 parameters *10^9*5.5*10^12 tokens = 1.0725e+24","org":"Alibaba","release_date":"2024-09-18"},"qwen2-5-coder-32b-instruct":{"country":"China","display":"Qwen2.5-Coder-32B-Instruct","family":"qwen2-5-coder-32b-instruct","flop":1.0725e+24,"flop_notes":"Assuming 1 epoch\n\n6ND = 6*32.5 parameters *10^9*5.5*10^12 tokens = 1.0725e+24","org":"Alibaba","release_date":"2024-11-21"},"qwen2-5-coder-3b":{"display":"Qwen2.5-Coder-3B","family":"qwen2-5-coder-3b","release_date":"2024-09-18"},"qwen2-5-coder-3b-instruct":{"display":"Qwen2.5-Coder-3B-Instruct","family":"qwen2-5-coder-3b-instruct","release_date":"2024-11-06"},"qwen2-5-coder-7b":{"country":"China","display":"Qwen2.5-Coder-7B","family":"qwen2-5-coder-7b","flop":2.5113e+23,"flop_notes":"6ND = 6 FLOP / token / parameter *7610000000 parameters *5.5T tokens =2.5113e+23 FLOP","org":"Alibaba","release_date":"2024-09-18"},"qwen2-5-coder-7b-instruct":{"country":"China","display":"Qwen2.5-Coder-7B-Instruct","family":"qwen2-5-coder-7b-instruct","flop":2.5113e+23,"flop_notes":"6ND = 6 FLOP / token / parameter *7610000000 parameters *5.5T tokens =2.5113e+23 FLOP","org":"Alibaba","release_date":"2024-09-17"},"qwen2-5-max":{"country":"China","display":"Qwen2.5","family":"qwen2-5-max","org":"Alibaba","release_date":"2025-01-28"},"qwen2-5-vl-72b-instruct":{"context":128000,"country":"China","created":1738410311,"display":"Qwen2.5 VL 72B Instruct","family":"qwen2-5-vl-72b-instruct","flop":7.8e+24,"flop_notes":"Training dataset size was 18 trillion\n\n6ND = 6 * 72.7 billion parameters * 18 trillion tokens = 7.8e24","input_modalities":["text","image"],"knowledge_cutoff":"2024-06-30","max_output":115200,"or_id":"qwen/qwen2.5-vl-72b-instruct","org":"Alibaba","output_modalities":["text"],"price_cache":0.4,"price_in":0.8,"price_match":"exact","price_out":1,"release_date":"2024-09-19","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","top_k","top_logprobs","top_p"]},"qwen2-72b-instruct":{"country":"China","display":"Qwen2 Instruct (72B)","family":"qwen2-72b-instruct","flop":3.02e+24,"flop_notes":"72 billion params, 7 trillion tokens\n\n6 * 72 billion * 7 trillion ~= 3.02e24","org":"Alibaba","release_date":"2024-06-07"},"qwen2-vl-72b-instruct":{"display":"Qwen2-VL","family":"qwen2-vl-72b-instruct","release_date":"2024-08-29"},"qwen2-vl-7b-instruct":{"display":"ReTaKe","family":"qwen2-vl-7b-instruct","release_date":"2024-08-29"},"qwen3-1-7b":{"country":"China","display":"Qwen3.1 7B","family":"qwen3-1-7b","flop":3.672e+23,"flop_notes":"6 FLOP / parameter / token * 36 * 10^12 tokens * 1.7  * 10^9 parameters = 3.672e+23 FLOP","org":"Alibaba","release_date":"2025-04-29"},"qwen3-14b":{"aa":{"agentic_index":0.9,"coding_index":13.8,"intelligence_index":null},"context":40960,"country":"China","created":1745876478,"display":"Qwen3 14B","family":"qwen3-14b","flop":3.1968e+24,"flop_notes":"6 FLOP / parameter / token * 36 * 10^12 tokens * 14.8  * 10^9 parameters = 3.1968e+24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":16384,"or_id":"qwen/qwen3-14b","org":"Alibaba","output_modalities":["text"],"price_in":0.12,"price_match":"exact","price_out":0.24,"release_date":"2025-04-29","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-235b-a22b":{"country":"China","display":"qwen3-235b-a22b-fp8-tput†","family":"qwen3-235b-a22b","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","open_weights":true,"org":"Alibaba","params":235000000000,"release_date":"2025-04-29"},"qwen3-235b-a22b-instruct-2507":{"country":"China","display":"Qwen3-235b-a22b Instruct (25/07)","family":"qwen3-235b-a22b-instruct-2507","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","org":"Alibaba","release_date":"2025-07-25"},"qwen3-235b-a22b-thinking-2507":{"aa":{"agentic_index":1.3,"coding_index":22.1,"intelligence_index":null},"context":128000,"country":"China","created":1753449557,"display":"Qwen3 235B A22B Thinking 2507","family":"qwen3-235b-a22b-thinking-2507","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-06-30","max_output":16384,"or_id":"qwen/qwen3-235b-a22b-thinking-2507","org":"Alibaba","output_modalities":["text"],"price_in":0.45,"price_match":"exact","price_out":3.5,"release_date":"2025-07-25","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","presence_penalty","reasoning","stop","temperature","tool_choice","tools","top_k","top_p"]},"qwen3-30b-a3b":{"context":40960,"country":"China","created":1745878604,"display":"Qwen3 30B A3B","family":"qwen3-30b-a3b","flop":6.48e+23,"flop_notes":"6 FLOP / parameter / token * 3*10^9 active parameters * 36000000000000 tokens = 6.48e+23 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":16384,"or_id":"qwen/qwen3-30b-a3b","org":"Alibaba","output_modalities":["text"],"price_in":0.12,"price_match":"exact","price_out":0.5,"release_date":"2025-04-29","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_p"]},"qwen3-30b-a3b-instruct-2507":{"context":262144,"country":"China","created":1753806965,"display":"Qwen3 30B A3B Instruct 2507","family":"qwen3-30b-a3b-instruct-2507","flop":6.48e+23,"flop_notes":"6 FLOP / parameter / token * 3*10^9 active parameters * 36000000000000 tokens = 6.48e+23 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-06-30","max_output":235929,"or_id":"qwen/qwen3-30b-a3b-instruct-2507","org":"Alibaba","output_modalities":["text"],"price_cache":0.1,"price_in":0.1,"price_match":"exact","price_out":0.3,"release_date":"2025-07-29","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-30b-a3b-thinking-2507":{"country":"China","display":"Qwen3 30B A3B 2507 (Reasoning)","family":"qwen3-30b-a3b-thinking-2507","flop":6.48e+23,"flop_notes":"6 FLOP / parameter / token * 3*10^9 active parameters * 36000000000000 tokens = 6.48e+23 FLOP","org":"Alibaba","release_date":"2025-07-30"},"qwen3-32b":{"aa":{"agentic_index":0.9,"coding_index":15.3,"intelligence_index":null},"context":131072,"country":"China","created":1745875945,"display":"Qwen3 32B","family":"qwen3-32b","flop":7.0848e+24,"flop_notes":"6 FLOP / parameter / token * 36 * 10^12 tokens * 32.8  * 10^9 parameters = 7.0848e+24 FLOP","input_modalities":["text"],"knowledge_cutoff":"2025-03-31","max_output":16384,"or_id":"qwen/qwen3-32b","org":"Alibaba","output_modalities":["text"],"price_in":0.08,"price_match":"exact","price_out":0.28,"release_date":"2025-04-29","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"qwen3-4b":{"country":"China","display":"Qwen3 4B","family":"qwen3-4b","flop":8.64e+23,"flop_notes":"6 FLOP / parameter / token * 36 * 10^12 tokens * 4  * 10^9 parameters = 8.64e+23 FLOP","org":"Alibaba","release_date":"2025-04-29"},"qwen3-4b-instruct-2507":{"display":"Qwen3 4B Instruct","family":"qwen3-4b-instruct-2507","release_date":"2025-08-06"},"qwen3-5-0-8b":{"country":"China","display":"Qwen3.5 0.8B (Reasoning)","family":"qwen3-5-0-8b","org":"Alibaba","release_date":"2026-02-24"},"qwen3-5-122b-a10b":{"aa":{"agentic_index":7.6,"coding_index":45.7,"intelligence_index":null},"context":262144,"country":"China","created":1772053789,"display":"Qwen3.5-122B-A10B","family":"qwen3-5-122b-a10b","input_modalities":["text","image","video"],"max_output":65536,"open_weights":true,"or_id":"qwen/qwen3.5-122b-a10b","org":"Alibaba","output_modalities":["text"],"params":122000000000,"price_in":0.26,"price_match":"exact","price_out":2.08,"release_date":"2026-02-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-5-27b":{"context":262144,"country":"China","created":1772053810,"display":"Qwen3.5-27B","family":"qwen3-5-27b","input_modalities":["text","image","video"],"max_output":81920,"or_id":"qwen/qwen3.5-27b","org":"Alibaba","output_modalities":["text"],"price_in":0.26,"price_match":"exact","price_out":2.6,"release_date":"2026-02-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-5-2b":{"country":"China","display":"Qwen3.5 2B (Non-reasoning)","family":"qwen3-5-2b","org":"Alibaba","release_date":"2026-02-24"},"qwen3-5-35b-a3b":{"aa":{"agentic_index":null,"coding_index":37,"intelligence_index":null},"context":262144,"country":"China","created":1772053822,"display":"Qwen3.5-35B-A3B","family":"qwen3-5-35b-a3b","input_modalities":["text","image","video"],"max_output":81920,"or_id":"qwen/qwen3.5-35b-a3b","org":"Alibaba","output_modalities":["text"],"price_cache":0.05,"price_in":0.14,"price_match":"exact","price_out":1,"release_date":"2026-02-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-5-397b-a17b":{"aa":{"agentic_index":8.3,"coding_index":48.2,"intelligence_index":null},"context":262144,"country":"China","created":1771223018,"display":"Qwen3.5 397B A17B","family":"qwen3-5-397b-a17b","input_modalities":["text","image","video"],"max_output":64000,"open_weights":true,"or_id":"qwen/qwen3.5-397b-a17b","org":"Alibaba","output_modalities":["text"],"params":397000000000,"price_cache":0.12,"price_in":0.6,"price_match":"exact","price_out":3.6,"release_date":"2026-02-13","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-5-4b":{"country":"China","display":"Qwen3.5 4B (Non-reasoning)","family":"qwen3-5-4b","org":"Alibaba","release_date":"2026-02-24"},"qwen3-5-9b":{"aa":{"agentic_index":1.2,"coding_index":28.7,"intelligence_index":null},"context":262144,"country":"China","created":1773152396,"display":"Qwen3.5-9B","family":"qwen3-5-9b","input_modalities":["text","image","video"],"max_output":235929,"or_id":"qwen/qwen3.5-9b","org":"Alibaba","output_modalities":["text"],"price_in":0.17,"price_match":"exact","price_out":0.25,"release_date":"2026-02-24","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_completion_tokens","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-5-flash":{"country":"China","display":"Qwen3.5 35B A3b","family":"qwen3-5-flash","org":"Alibaba","release_date":"2026-02-25"},"qwen3-5-plus":{"country":"China","display":"Qwen3.5 397B A17b","family":"qwen3-5-plus","open_weights":true,"org":"Alibaba","params":397000000000,"release_date":"2026-02-16"},"qwen3-6-27b":{"aa":{"agentic_index":18.5,"coding_index":53.7,"intelligence_index":null},"context":262144,"country":"China","created":1777255064,"display":"Qwen3.6 27B","family":"qwen3-6-27b","input_modalities":["text","image","video"],"max_output":81920,"or_id":"qwen/qwen3.6-27b","org":"Alibaba","output_modalities":["text"],"price_in":0.32,"price_match":"exact","price_out":3.2,"release_date":"2026-04-22","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-6-35b-a3b":{"aa":{"agentic_index":13.1,"coding_index":41.9,"intelligence_index":null},"context":262144,"country":"China","created":1777260255,"display":"Qwen3.6 35B A3B","family":"qwen3-6-35b-a3b","input_modalities":["text","image","video"],"max_output":235929,"or_id":"qwen/qwen3.6-35b-a3b","org":"Alibaba","output_modalities":["text"],"price_cache":0.025,"price_in":0.1,"price_match":"exact","price_out":0.9,"release_date":"2026-04-14","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-6-flash":{"context":1000000,"country":"China","created":1777261362,"display":"Qwen3.6 Flash","family":"qwen3-6-flash","input_modalities":["text","image","video"],"max_output":65536,"or_id":"qwen/qwen3.6-flash","org":"Alibaba","output_modalities":["text"],"price_in":0.1875,"price_match":"exact","price_out":1.125,"release_date":"2026-04-27","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-6-max-preview":{"country":"China","display":"Qwen3.6 Max Preview","family":"qwen3-6-max","org":"Alibaba","release_date":"2026-04-20"},"qwen3-6-plus":{"aa":{"agentic_index":null,"coding_index":54.5,"intelligence_index":null},"context":1000000,"country":"China","created":1775133557,"display":"Qwen3.6 Plus","family":"qwen3-6-plus","input_modalities":["text","image","video"],"max_output":65536,"or_id":"qwen/qwen3.6-plus","org":"Alibaba","output_modalities":["text"],"price_in":0.325,"price_match":"exact","price_out":1.95,"release_date":"2026-03-31","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-7-flash":{"context":1000000,"country":"China","created":1785190561,"display":"Qwen3.7 Flash","family":"qwen3-7-flash","input_modalities":["text","image","video"],"max_output":65536,"or_id":"qwen/qwen3.7-flash","org":"Alibaba","output_modalities":["text"],"price_cache":0.006,"price_in":0.03,"price_match":"exact","price_out":0.13,"release_date":"2026-07-27","supported_parameters":["include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","temperature","tool_choice","tools","top_logprobs","top_p"]},"qwen3-7-max":{"aa":{"agentic_index":22.5,"coding_index":66,"intelligence_index":null},"context":1000000,"country":"China","created":1779376861,"display":"Qwen3.7 Max","family":"qwen3-7-max","input_modalities":["text"],"max_output":131072,"or_id":"qwen/qwen3.7-max","org":"Alibaba","output_modalities":["text"],"price_cache":0.295,"price_in":1.475,"price_match":"exact","price_out":4.425,"release_date":"2026-05-19","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-7-plus":{"aa":{"agentic_index":17.5,"coding_index":55.9,"intelligence_index":null},"context":1000000,"country":"China","created":1780491783,"display":"Qwen3.7 Plus","family":"qwen3-7-plus","input_modalities":["text","image"],"max_output":131072,"open_weights":false,"or_id":"qwen/qwen3.7-plus","org":"Alibaba","output_modalities":["text"],"price_cache":0.064,"price_in":0.32,"price_match":"exact","price_out":1.28,"release_date":"2026-06-02","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-8-27b":{"aa":{"agentic_index":45.8,"coding_index":68.1,"intelligence_index":20.1},"context":1024000,"country":"China","created":1786722910,"display":"Qwen3.8 27B","family":"qwen3-8-27b","input_modalities":["text","image","video"],"max_output":235929,"or_id":"qwen/qwen3.8-27b","org":"Alibaba","output_modalities":["text"],"price_cache":0.0375,"price_in":0.15,"price_match":"exact","price_out":1.875,"release_date":"2026-08-14","supported_parameters":["frequency_penalty","include_reasoning","logit_bias","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_a","top_k","top_logprobs","top_p"]},"qwen3-8-flash":{"context":1000000,"country":"China","created":1787773060,"display":"Qwen3.8 Flash","family":"qwen3-8-flash","input_modalities":["text","image","video"],"max_output":131072,"or_id":"qwen/qwen3.8-flash","org":"Alibaba","output_modalities":["text"],"price_cache":0.016,"price_in":0.15,"price_match":"exact","price_out":0.47,"release_date":"2026-08-26","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-8-max":{"country":"China","display":"Qwen3.8-Max","family":"qwen3-8-max","org":"Alibaba","release_date":"2026-08-02"},"qwen3-8-max-0902":{"aa":{"agentic_index":56,"coding_index":76.2,"intelligence_index":33.7},"context":1000000,"country":"China","created":1788469704,"display":"Qwen3.8 Max (0902)","family":"qwen3-8-max-0902","input_modalities":["text","image","video"],"max_output":131072,"or_id":"qwen/qwen3.8-max-0902","org":"Alibaba","output_modalities":["text"],"price_cache":0.25,"price_in":2,"price_match":"exact","price_out":6,"release_date":"2026-09-01","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","reasoning_effort","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-8b":{"country":"China","display":"Qwen3 8B (Reasoning)","family":"qwen3-8b","flop":1.7712e+24,"flop_notes":"6 FLOP / parameter / token * 36 * 10^12 tokens * 8.2  * 10^9 parameters = 1.7712e+24 FLOP","org":"Alibaba","release_date":"2025-04-28"},"qwen3-coder-480b-a35b-instruct":{"country":"China","display":"Qwen3 Coder","family":"qwen3-coder-480b-a35b-instruct","flop":1.575e+24,"flop_notes":"6 FLOP / parameter / token * 35 * 10^9 active parameters * 7.5 * 10^12 tokens = 1.575e+24 FLOP`","org":"Alibaba","release_date":"2025-07-31"},"qwen3-coder-next":{"aa":{"agentic_index":0.9,"coding_index":36.2,"intelligence_index":null},"context":262144,"created":1770164101,"display":"Qwen3 Coder Next","family":"qwen3-coder-next","input_modalities":["text"],"max_output":235929,"open_weights":true,"or_id":"qwen/qwen3-coder-next","org":"Alibaba","output_modalities":["text"],"params":80000000000,"price_cache":0.07,"price_in":0.12,"price_match":"exact","price_out":0.8,"release_date":"2026-02-02","supported_parameters":["frequency_penalty","logit_bias","logprobs","max_tokens","presence_penalty","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"qwen3-coder-plus":{"display":"Qwen3 Coder Plus","family":"qwen3-coder-plus","release_date":"2025-09-23"},"qwen3-max-2025-09-23":{"country":"China","display":"Qwen3","family":"qwen3-max","flop":1.512e+25,"flop_notes":"6ND with:\n36T tokens is taken from the qwen3 technical report\n70B active params is based on it having \u003e1T params, and the architectures of Qwen3-235B-A22B and Qwen3-Coder-480B-A35B","org":"Alibaba","release_date":"2025-09-24"},"qwenqwen3-235b-a22b-thinking-2507":{"country":"China","display":"Qwen3 235B A22B 2507 (Reasoning)","family":"qwenqwen3-235b-a22b-thinking-2507","flop":4.752e+24,"flop_notes":"6 FLOP / parameter / token * 22*10^9 active parameters * 36000000000000 tokens = 4.752e+24 FLOP","org":"Alibaba","release_date":"2025-07-25"},"qwq-32b":{"country":"China","display":"QwQ 32B","family":"qwq-32b","flop":3.51e+24,"flop_notes":"Assuming the same dataset size as for Qwen2.5 training (18T tokens):\n\n6ND = 6 * 32500000000 parameters * 18 * 10^12 tokens =  3.51 × 10^24\n\n'Speculative' confidence","open_weights":true,"org":"Alibaba","params":32500000000,"release_date":"2025-03-05"},"qwq-32b-16k-thinking":{"country":"China","display":"QwQ 32B 16K","family":"qwq-32b-16k-thinking","flop":3.51e+24,"flop_notes":"Assuming the same dataset size as for Qwen2.5 training (18T tokens):\n\n6ND = 6 * 32500000000 parameters * 18 * 10^12 tokens =  3.51 × 10^24\n\n'Speculative' confidence","org":"Alibaba","release_date":"2025-03-05"},"qwq-32b-preview":{"country":"China","display":"QwQ 32B Preview","family":"qwq-32b","flop":3.51e+24,"flop_notes":"Assuming the same dataset size as for Qwen2.5 training (18T tokens):\n\n6ND = 6 * 32500000000 parameters * 18 * 10^12 tokens =  3.51 × 10^24\n\n'Speculative' confidence","org":"Alibaba","release_date":"2024-11-28"},"qwq-plus":{"country":"China","display":"QwQ Plus","family":"qwq-plus","org":"Alibaba","release_date":"2025-04-08"},"redpajama-incite-7b-base":{"display":"Redpajama-7B","family":"redpajama-incite-7b-base","release_date":"2023-05-04"},"reka-flash-3":{"context":65536,"country":"United States of America","created":1741812813,"display":"Reka Flash 3","family":"reka-flash-3","input_modalities":["text"],"knowledge_cutoff":"2025-01-31","max_output":58982,"or_id":"rekaai/reka-flash-3","org":"Reka AI","output_modalities":["text"],"price_in":0.1,"price_match":"exact","price_out":0.2,"release_date":"2025-03-10","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","seed","stop","structured_outputs","temperature","top_k","top_logprobs","top_p"]},"ring-2-6-1t":{"country":"China","display":"Ling 2.6 1t","family":"ring-2-6-1t","flop_notes":"9.6T tokens of continued pre-training from Ling-2.0 base checkpoint.","org":"Ant Group","release_date":"2026-05-14"},"seed-1-6":{"context":262144,"country":"China","created":1766504997,"display":"Seed 1.6","expiration_date":"2026-11-11","family":"seed-1-6","input_modalities":["image","text","video"],"max_output":32768,"or_id":"bytedance-seed/seed-1.6","org":"ByteDance","output_modalities":["text"],"price_in":0.25,"price_match":"exact","price_out":2,"release_date":"2025-06-11","supported_parameters":["frequency_penalty","include_reasoning","max_tokens","reasoning","response_format","stop","structured_outputs","temperature","tool_choice","tools","top_p"]},"seed-oss-36b-instruct":{"display":"Seed OSS 36B Instruct","family":"seed-oss-36b-instruct","release_date":"2025-08-20"},"sharegpt4video-8b":{"display":"ShareGPT4Video","family":"sharegpt4video-8b","release_date":"2024-05-27"},"slime-llama3-8b":{"display":"SliME","family":"slime-llama3-8b","release_date":"2024-06-02"},"solar-pro-3":{"country":"South Korea","display":"Solar Pro 3","family":"solar-pro-3","org":"Upstage","release_date":"2026-01-27"},"sonar":{"context":127072,"created":1738013808,"display":"Sonar","family":"sonar","input_modalities":["text","image"],"max_output":114364,"or_id":"perplexity/sonar","output_modalities":["text"],"price_in":1,"price_match":"exact","price_out":1,"supported_parameters":["frequency_penalty","max_tokens","presence_penalty","temperature","top_k","top_p","web_search_options"]},"st-llm":{"display":"ST-LLM","family":"st-llm","release_date":"2024-03-28"},"stablebeluga2":{"country":"United Kingdom","display":"StableBeluga2 70B","family":"stablebeluga2","org":"Stability AI","release_date":"2023-07-20"},"stablelm-tuned-alpha-7b":{"display":"StableLM-alpha-7B","family":"stablelm-tuned-alpha-7b","release_date":"2023-04-19"},"starcoder2-15b":{"country":"United States of America","display":"StarCoder2-15B","family":"starcoder2-15b","flop":3.87e+23,"flop_notes":"estimation is given in Table 6","org":"Hugging Face,ServiceNow,Nvidia,BigCode","release_date":"2024-02-20"},"starcoder2-3b":{"country":"United States of America","display":"StarCoder2-3B","family":"starcoder2-3b","flop":5.94e+22,"flop_notes":"estimation is given in Table 6","org":"Hugging Face,ServiceNow,Nvidia,BigCode","release_date":"2024-02-22"},"starcoder2-7b":{"country":"United States of America","display":"StarCoder2-7B","family":"starcoder2-7b","flop":1.55e+23,"flop_notes":"estimation is given in Table 6","org":"Hugging Face,ServiceNow,Nvidia,BigCode","release_date":"2024-02-20"},"step-3-7-flash":{"aa":{"agentic_index":null,"coding_index":39.6,"intelligence_index":null},"context":262144,"country":"China","created":1779985069,"display":"Step 3.7 Flash","family":"step-3-7-flash","input_modalities":["text","image","video"],"max_output":230400,"or_id":"stepfun/step-3.7-flash","org":"StepFun","output_modalities":["text"],"price_cache":0.04,"price_in":0.2,"price_match":"exact","price_out":1.15,"release_date":"2026-05-29","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"step-5-preview":{"aa":{"agentic_index":null,"coding_index":null,"intelligence_index":28.5},"context":1048576,"country":"China","created":1791462849,"display":"Step 5 Preview","family":"step-5","input_modalities":["text","image","video"],"max_output":64000,"open_weights":false,"or_id":"stepfun/step-5-preview","org":"StepFun","output_modalities":["text"],"params":600000000000,"price_cache":0.1,"price_in":1,"price_match":"exact","price_out":2.7,"release_date":"2026-09-18","supported_parameters":["frequency_penalty","include_reasoning","logprobs","max_tokens","min_p","presence_penalty","reasoning","reasoning_effort","repetition_penalty","response_format","seed","stop","structured_outputs","temperature","tool_choice","tools","top_k","top_logprobs","top_p"]},"swe-1-6":{"country":"United States of America","display":"SWE-1.6","family":"swe-1-6","open_weights":false,"org":"Cognition","release_date":"2026-04-07"},"swe-1-7":{"country":"United States of America","display":"SWE-1.7","family":"swe-1-7","org":"Cognition","release_date":"2026-07-08"},"swe-2":{"country":"United States of America","display":"SWE-2","family":"swe-2","org":"Cognition","release_date":"2026-09-10"},"switch-base":{"display":"Switch-Base","family":"switch-base"},"switch-large":{"display":"Switch-Large","family":"switch-large"},"t5-11b":{"country":"United States of America","display":"T5 (11B)","family":"t5-11b","flop":3.3e+22,"flop_notes":"https://arxiv.org/ftp/arxiv/papers/2104/2104.10350.pdf\nTable 4, 4.05e22\n\nupdate: 3.3e22 per FLAN paper from Google \nhttps://arxiv.org/pdf/2210.11416.pdf\n\n6ND rule suggests somewhat more FLOPs:\n6 * 1T * 11B = 6.6e22","open_weights":true,"org":"Google DeepMind","params":11000000000,"release_date":"2019-10-23"},"t5-3b":{"country":"United States of America","display":"T5-3B","family":"t5-3b","flop":9.0000000001e+21,"flop_notes":"Akronomicon states 1.04e+22 FLOP. Archived source: https://github.com/lightonai/akronomicon/tree/main/akrodb\nHowever, this seems dubiously high.\n\n\"We pre-train each model for 2^19 = 524,288 steps on C4 before fine-tuning.\"\n\"In total, this batch size and number of steps corresponds to pre-training on 2^35 ≈ 34B tokens.\"\n\"To compare these mixing strategies on equal footing with our baseline pre-train-then-fine-tune results, we train multi-task models for the same total number of steps: 2^19 + 2^18 = 786,432\"\nUsing the 6DN approximation gives: 6 FLOP/token/param * 2^35 pretrain tokens * (1+1/2 finetune tokens per pretrain token) * 1 iteration of training data* 2.8 billion parameters = 8.659e20 FLOP\nhttps://www.wolframalpha.com/input?i=6+*+2%5E35+*+2.8+billion+*+1.5\n\nupdate: 9.0E+21 per FLAN paper from Google \nhttps://arxiv.org/pdf/2210.11416.pdf","open_weights":true,"org":"Google DeepMind","params":2800000000,"release_date":"2019-10-23"},"t5-base":{"display":"T5-Base","family":"t5-base"},"t5-large":{"display":"T5-Large","family":"t5-large"},"t5-small":{"display":"T5-Small","family":"t5-small"},"text-ada-001":{"country":"United States of America","display":"Text Ada 001","family":"text-ada-001","org":"OpenAI"},"text-babbage-001":{"country":"United States of America","display":"Text Babbage 001","family":"text-babbage-001","org":"OpenAI"},"text-curie-001":{"country":"United States of America","display":"Text Curie 001","family":"text-curie-001","org":"OpenAI"},"text-davinci-001":{"country":"United States of America","display":"GPT-3","family":"text-davinci-001","flop":3.19181e+23,"flop_notes":"\"training our 175B PPO-ptx model requires 60 petaflops/s-days, compared to 3,640 petaflops/s-days for GPT-3 (Brown et al., 2020)\"\n\n60/3640 = +1.65% to base model compute\n\nbase model was reported 3.14e+23 FLOP\n\n3.14e+23 * 1.0165 = 319181000000000000000000","org":"OpenAI","release_date":"2022-01-27"},"text-davinci-002":{"country":"United States of America","display":"GPT-3.5","family":"text-davinci-002","flop":2.578e+24,"flop_notes":"https://colab.research.google.com/drive/1QSxa8YCWjEBQU7mrXLhw6TP1VX5oqgdW#scrollTo=Gt6Z6oZ26clI","open_weights":false,"org":"OpenAI","release_date":"2022-03-15"},"text-davinci-003":{"country":"United States of America","display":"Text Davinci 003","family":"text-davinci-003","org":"OpenAI","release_date":"2022-11-28"},"timemarker":{"display":"TimeMarker","family":"timemarker","release_date":"2024-11-27"},"tiny-recursion-model":{"display":"Tiny Recursion Model (TRM)","family":"tiny-recursion-model","release_date":"2025-10-06"},"trinity-large-thinking":{"country":"United States of America","display":"Trinity Large","family":"trinity-large-thinking","flop_notes":"Base model training compute was ~1.33e24 FLOP, we don't know enough to give a compute count for the additional parts here.","org":"Arcee AI","release_date":"2026-04-01"},"vicuna-13b-v1-1":{"display":"Vicuna-13B (v1.1)","family":"vicuna-13b-v1-1","release_date":"2023-04-12"},"vicuna-13b-v1-3":{"country":"United States of America","display":"Vicuna v1.3 (13B)","family":"vicuna-13b-v1-3","org":"Large Model Systems Organization,University of California (UC) Berkeley","release_date":"2023-06-18"},"video-ccam-7b-v1-2":{"display":"Video-CCAM","family":"video-ccam-7b-v1-2","release_date":"2024-09-29"},"video-chat2-mistral":{"display":"VideoChat2-Mistral","family":"video-chat2-mistral","release_date":"2023-11-29"},"video-llava-7b":{"display":"Video-LLaVA","family":"video-llava-7b","release_date":"2023-11-17"},"video-salmonn-2plus":{"country":"China","display":"video-SALMONN 2+","family":"video-salmonn-2plus","org":"ByteDance","release_date":"2025-06-18"},"video-xl-7b":{"display":"Video-XL","family":"video-xl-7b","release_date":"2024-10-17"},"videochat-flash-qwen2-7b-res448":{"display":"VideoChat-Flash","family":"videochat-flash-qwen2-7b-res448","release_date":"2025-01-11"},"videollama2-7b":{"display":"VideoLLaMA 2","family":"videollama2-7b","release_date":"2024-06-12"},"videollama3-7b":{"display":"VideoLLaMA 3","family":"videollama3-7b","release_date":"2024-01-22"},"vilamp-llava-qwen":{"display":"ViLAMP","family":"vilamp-llava-qwen","release_date":"2025-05-01"},"vita":{"display":"VITA","family":"vita","release_date":"2024-08-12"},"vita-1-5":{"display":"VITA 1.5","family":"vita-1-5","release_date":"2024-12-20"},"wizardlm-2-8x22b":{"country":"United States of America","display":"Wizardlm 2 8x22b","family":"wizardlm-2-8x22b","org":"Microsoft","release_date":"2024-04-15"},"xgen-7b-8k-base":{"country":"United States of America","display":"XGen-7B","family":"xgen-7b-8k-base","flop":8.02e+22,"flop_notes":"270,336 TPUv4-hours per the carbon emissions section. They must mean chip-hours, not core-hours, because they multiply by 192W, which is the max power consumption of a TPU-v4 chip.\n\nhttps://cloud.google.com/tpu/docs/system-architecture-tpu-vm#tpu_v4\n\nThis is probably the total for the whole paper? They released two versions of the base model, one of which is just trained further than the other, and they have two fine-tuned versions. If fine-tuning compute is minor, then this total is close to the largest training compute for any single model in the paper. \n\n270336 * 275 teraflop/s * 3600 * 0.3 = 8.02e22 FLOP\n\nalso, using 6ND:\n\n1484 billion tokens * 6.7 billion * 6 = 5.97e22","org":"Salesforce","release_date":"2023-06-27"},"yi-1-5-34b-chat":{"country":"China","display":"Yi 1.5 34B Chat","family":"yi-1-5-34b-chat","flop":7.344e+23,"flop_notes":"6 FLOP / parameter / token * 34*10^9 parameters * 3.6*10^12 tokens = 7.344e+23 FLOP","org":"01.AI","release_date":"2024-05-13"},"yi-34b":{"country":"China","display":"Yi 34B","family":"yi-34b","flop":6.1e+23,"flop_notes":"\"The dataset we use contains Chinese \u0026 English only. We used approximately 3T tokens\" sounds like this means it was trained on 3T tokens, not necessarily that the dataset contains 3T tokens?\n\nIf so, 34b * 3T * 6 = 6.1e23","open_weights":true,"org":"01.AI","params":34000000000,"release_date":"2023-11-02"},"yi-34b-chat":{"country":"China","display":"Yi-Chat 34B","family":"yi-34b-chat","flop":6.1e+23,"flop_notes":"\"The dataset we use contains Chinese \u0026 English only. We used approximately 3T tokens\" sounds like this means it was trained on 3T tokens, not necessarily that the dataset contains 3T tokens?\n\nIf so, 34b * 3T * 6 = 6.1e23","org":"01.AI","release_date":"2023-11-22"},"yi-6b":{"country":"China","display":"Yi 6b","family":"yi-6b","flop":1.26e+23,"flop_notes":"6*7*10^9*3*10^12 = 1.26e+23","org":"01.AI","release_date":"2023-11-02"},"yi-6b-chat":{"country":"China","display":"Yi-Chat 6B","family":"yi-6b-chat","flop":1.26e+23,"flop_notes":"6*7*10^9*3*10^12 = 1.26e+23","org":"01.AI","release_date":"2023-11-22"},"yi-9b":{"display":"Yi 9b","family":"yi-9b","release_date":"2024-03-01"},"yi-large":{"country":"China","display":"Yi Large (Preview)","family":"yi-large","flop":1.8e+24,"flop_notes":"6ND = 6*100000000000*3000000000000=1.8e+24\n\n(speculative confidence because training dataset size is very uncertain)","open_weights":false,"org":"01.AI","params":100000000000,"release_date":"2024-05-13"},"yi-lightning":{"country":"China","display":"Yi Lightning","family":"yi-lightning","flop":1.5e+24,"flop_notes":"The CEO of 01.AI tweeted that Yi-Lightning was trained for 1 month on 2000 H100s: https://x.com/kaifulee/status/1846310645849047524\nAssuming this is accurate:\n(9.9e14 * 2000) FLOP/s * 1 month * 30.5 days/month * 24hr/day * 3600 s/hr * 0.3 utilization assumption = 1.565e24","open_weights":false,"org":"01.AI","release_date":"2024-12-02"},"zai-orgglm-4-6":{"country":"China","display":"Zai Orgglm 4.6","family":"zai-orgglm-4-6","flop":4.42e+24,"flop_notes":"6 FLOP/parameter/token * 32000000000 active parameters [very likely assumption - everything else is reported to be same as at GLM 4.5] * 23000000000000 tokens = 4.42e24 FLOP","org":"Z.ai","release_date":"2025-09-30"},"zai-orgglm-4-7":{"country":"China","display":"GLM 4.7","family":"zai-orgglm-4-7","flop":4.42e+24,"flop_notes":"Training tokens not disclosed.\n\nPresuming this is a post-training update from GLM-4.5 (23T tokens) with minor additional compute, training compute is at least:\n\n6 FLOP/parameter/token * 32B active parameters * 23T tokens = 4.42e24 FLOP","open_weights":true,"org":"Z.ai","params":358000000000,"release_date":"2025-12-22"}},"price_prev":{"date":"2026-11-03T10:43:01Z","prices":{"claude-fable-5":[10,50],"claude-fable-5-1":[10,50],"claude-haiku-4-5-2025-10-01":[1,5],"claude-haiku-5-5":[0.1,0.5],"claude-opus-4-1-2025-08-05":[15,75],"claude-opus-4-6":[5,25],"claude-opus-4-7":[5,25],"claude-opus-4-8":[5,25],"claude-opus-5":[5,25],"claude-opus-5-5":[4,20],"claude-sonnet-4-5-2025-09-29":[3,15],"claude-sonnet-4-6":[3,15],"claude-sonnet-5":[2,10],"claude-sonnet-5-5":[2,10],"codestral-2508":[0.3,0.9],"command-a-plus-05-2026":[0.3,1.5],"deepseek-chat":[0.2574,1.0287],"deepseek-r1-0528":[0.5,2.15],"deepseek-v3-1-terminus":[0.27,1],"deepseek-v3-2-exp":[0.27,0.41],"deepseek-v4-1-flash":[0.045,0.5],"deepseek-v4-1-flash-1017":[0.15,0.6],"deepseek-v4-flash":[0.091,0.182],"deepseek-v4-flash-0731":[0.059,0.3],"deepseek-v4-pro":[1.6,3.2],"deepseek-v4-pro-0813":[0.115,4],"gemini-2-5-flash":[0.3,2.5],"gemini-2-5-pro":[1.25,10],"gemini-2-5-pro-preview-06-05":[1.25,10],"gemini-3-1-flash-lite":[0.25,1.5],"gemini-3-1-pro-preview":[2,12],"gemini-3-1-pro-preview-customtools":[2,12],"gemini-3-5-flash":[1.5,9],"gemini-3-5-flash-lite":[0.3,2.5],"gemini-3-6-flash":[0.75,3.75],"gemini-3-7-flash":[0.75,3.75],"gemini-3-8-flash":[0.825,4.125],"gemini-3-flash-preview":[0.5,3],"gemini-4-argon":[2,10],"gemma-2-27b-it":[0.65,0.65],"gemma-3-12b-it":[0.05,0.15],"gemma-3-27b-it":[0.08,0.35],"gemma-3-4b-it":[0.05,0.1],"gemma-4-26b-a4b-it":[0.077,0.231],"gemma-4-31b-it":[0.2,0.4],"glm-4-5":[0.6,2.2],"glm-4-5-air":[0.13,0.85],"glm-4-6":[0.5,2],"glm-4-7":[0.6,2.2],"glm-4-7-flash":[0.0605,0.4],"glm-5":[1,3.2],"glm-5-1":[1.4,4.4],"glm-5-2":[0.006,6.8],"glm-5-3":[0.08,3.6],"glm-5-3-flash":[0.0313,0.086],"gpt-3-5-turbo-0613":[1,2],"gpt-4-1-2025-04-14":[2,8],"gpt-4-1-mini-2025-04-14":[0.4,1.6],"gpt-4-1-nano-2025-04-14":[0.1,0.4],"gpt-4o-2024-05-13":[5,15],"gpt-4o-2024-08-06":[2.5,10],"gpt-4o-2024-11-20":[2.5,10],"gpt-4o-mini-2024-07-18":[0.15,0.6],"gpt-5-1-2025-11-13":[1.25,10],"gpt-5-1-codex":[1.25,10],"gpt-5-1-codex-max":[1.25,10],"gpt-5-1-codex-mini":[0.25,2],"gpt-5-2-2025-12-11":[1.75,14],"gpt-5-2-codex":[1.75,14],"gpt-5-2-pro-2025-12-11":[21,168],"gpt-5-2025-08-07":[1.25,10],"gpt-5-3-codex":[1.75,14],"gpt-5-4-2026-03-05":[2.5,15],"gpt-5-4-mini-2026-03-17":[0.75,4.5],"gpt-5-4-nano-2026-03-17":[0.2,1.25],"gpt-5-4-pro-2026-03-05":[30,180],"gpt-5-5":[5,30],"gpt-5-5-pro":[30,180],"gpt-5-6-luna":[0.2,1.2],"gpt-5-6-sol":[4,20],"gpt-5-6-terra":[2,12],"gpt-5-mini-2025-08-07":[0.25,2],"gpt-5-nano-2025-08-07":[0.05,0.4],"gpt-5-pro-2025-10-06":[15,120],"gpt-6-1-sol":[2,10],"gpt-6-1-terra":[1,5],"gpt-6-astra":[10,50],"gpt-6-luna":[0.1,0.5],"gpt-6-sol":[2,10],"gpt-oss-120b":[0.15,0.6],"gpt-oss-20b":[0.03,0.13],"grok-4-20":[1.25,2.5],"grok-4-3":[1.25,2.5],"grok-4-5":[2,6],"grok-4-6":[2,6],"grok-4-7":[2,6],"grok-build-0-1":[1,2],"hy4-preview":[0.834,2.501],"inkling":[1,4.05],"inkling-small":[0.45,1.2],"kimi-k2-5":[0.5,2.5],"kimi-k2-6":[0.65,3.41],"kimi-k2-7-code":[0.6,3.5],"kimi-k2-thinking":[0.6,2.5],"kimi-k3":[0.192,15],"kimi-k3-1-pro":[2,10],"llama-3-1-70b-instruct":[0.4,0.4],"llama-3-1-8b-instruct":[0.05,0.08],"llama-3-2-1b-instruct":[0.027,0.201],"llama-3-2-3b-instruct":[0.0509,0.335],"llama-3-3-70b-instruct":[0.2,0.52],"llama-4-maverick-17b-128e-instruct":[0.1875,0.6525],"llama-4-scout-17b-16e-instruct":[0.1,0.3],"mercury-2":[0.25,0.75],"mercury-2-5":[0.04,0.15],"mimo-v2-5":[0.4,2],"mimo-v2-5-pro":[0.48,1.8],"mimo-v2-6-flash":[0.14,0.28],"mimo-v2-6-pro":[0.435,0.87],"minimax-m2":[0.255,1.02],"minimax-m2-1":[0.3,1.2],"minimax-m2-5":[0.27,0.95],"minimax-m2-7":[0.3,1.2],"minimax-m3":[0.3,1.2],"mistral-large-2407":[2,6],"mistral-large-2512":[0.5,1.5],"mistral-large-4-0":[0.7,2.1],"mistral-small-2603":[0.15,0.6],"muse-spark-1-1":[1.25,4.25],"muse-spark-1-2":[1.25,4.25],"muse-spark-1-3":[1.25,4.25],"o3-2025-04-16":[2,8],"o3-pro-2025-06-10":[20,80],"phi-4":[0.07,0.14],"qwen-plus-2025-01-25":[0.26,0.78],"qwen2-5-vl-72b-instruct":[0.8,1],"qwen3-14b":[0.12,0.24],"qwen3-235b-a22b-thinking-2507":[0.45,3.5],"qwen3-30b-a3b":[0.12,0.5],"qwen3-30b-a3b-instruct-2507":[0.1,0.3],"qwen3-32b":[0.08,0.28],"qwen3-5-122b-a10b":[0.26,2.08],"qwen3-5-27b":[0.26,2.6],"qwen3-5-35b-a3b":[0.14,1],"qwen3-5-397b-a17b":[0.5,3.6],"qwen3-5-9b":[0.17,0.25],"qwen3-6-27b":[0.45,2.7],"qwen3-6-35b-a3b":[0.1,0.9],"qwen3-6-flash":[0.1875,1.125],"qwen3-6-plus":[0.325,1.95],"qwen3-7-flash":[0.03,0.13],"qwen3-7-max":[1.475,4.425],"qwen3-7-plus":[0.32,1.28],"qwen3-8-27b":[0.15,1.875],"qwen3-8-flash":[0.15,0.47],"qwen3-8-max-0902":[2,6],"qwen3-coder-next":[0.12,0.8],"reka-flash-3":[0.1,0.2],"sonar":[1,1],"step-3-7-flash":[0.2,1.15],"step-5-preview":[1,2.7]}},"provenance":{"epoch_etag":"\"2880633fc19f0caef8077b6b3619d2e8\"","fetched_at":"2026-11-09T10:43:01Z","latest_eval":"2026-11-06","license_note":"Benchmark data: Epoch AI, 'AI Benchmarking Hub', CC-BY 4.0. Prices: OpenRouter.","sources":{"datacurve_deepswe":"https://deepswe.datacurve.ai/artifacts/v1.1/leaderboard-live.json","epoch_benchmarks":"https://epoch.ai/data/benchmark_data.zip","epoch_models":"https://epoch.ai/data/ai_models.zip","openrouter":"https://openrouter.ai/api/v1/models"}},"quality":{"benchmarks":{"adversarial_nli":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":15,"score_col":"Score","source_rows":19,"trace_rows":0,"uncertainty_rows":0},"aider_polyglot":{"cost_rows":67,"dated_rows":61,"metrics_rows":61,"parsed_rows":72,"score_col":"Percent correct","source_rows":77,"trace_rows":0,"uncertainty_rows":0},"ale_bench":{"cost_rows":132,"dated_rows":0,"metrics_rows":132,"parsed_rows":132,"score_col":"Performance","source_rows":132,"trace_rows":0,"uncertainty_rows":0},"algotune":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":18,"score_col":"Score","source_rows":18,"trace_rows":0,"uncertainty_rows":0},"apex_agents":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":60,"score_col":"Pass@1 score","source_rows":60,"trace_rows":0,"uncertainty_rows":60},"arc_agi":{"cost_rows":159,"dated_rows":0,"metrics_rows":0,"parsed_rows":309,"score_col":"Score","source_rows":312,"trace_rows":0,"uncertainty_rows":0},"arc_agi_2":{"cost_rows":151,"dated_rows":0,"metrics_rows":0,"parsed_rows":290,"score_col":"Score","source_rows":290,"trace_rows":0,"uncertainty_rows":0},"arc_ai2":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":134,"score_col":"Challenge score","source_rows":145,"trace_rows":0,"uncertainty_rows":0},"balrog":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":41,"score_col":"Average progress","source_rows":41,"trace_rows":14,"uncertainty_rows":41},"bbh":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":88,"score_col":"Average","source_rows":92,"trace_rows":0,"uncertainty_rows":0},"blueprint_bench_2":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":35,"score_col":"Score","source_rows":35,"trace_rows":0,"uncertainty_rows":35},"bool_q":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":136,"score_col":"Score","source_rows":206,"trace_rows":0,"uncertainty_rows":0},"btf3":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":15,"score_col":"Pooled score","source_rows":15,"trace_rows":0,"uncertainty_rows":15},"cad_eval":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":15,"score_col":"Overall pass (%)","source_rows":15,"trace_rows":0,"uncertainty_rows":0},"chess_puzzles":{"cost_rows":0,"dated_rows":162,"metrics_rows":0,"parsed_rows":162,"score_col":"mean_score","source_rows":162,"trace_rows":162,"uncertainty_rows":162},"cl_bench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":23,"score_col":"Overall","source_rows":23,"trace_rows":0,"uncertainty_rows":23},"cl_bench_life":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":17,"score_col":"Overall","source_rows":17,"trace_rows":0,"uncertainty_rows":17},"common_sense_qa_2":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":6,"score_col":"Score","source_rows":10,"trace_rows":0,"uncertainty_rows":0},"critpt":{"cost_rows":11,"dated_rows":0,"metrics_rows":0,"parsed_rows":260,"score_col":"Accuracy","source_rows":260,"trace_rows":0,"uncertainty_rows":0},"cursorbench":{"cost_rows":82,"dated_rows":0,"metrics_rows":82,"parsed_rows":82,"score_col":"Score","source_rows":82,"trace_rows":0,"uncertainty_rows":0},"cybench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":22,"score_col":"Unguided % Solved","source_rows":22,"trace_rows":0,"uncertainty_rows":0},"deepresearchbench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":41,"score_col":"Average score","source_rows":41,"trace_rows":0,"uncertainty_rows":0},"deepswe":{"cost_rows":69,"dated_rows":0,"metrics_rows":69,"parsed_rows":69,"score_col":"Pass@1","source_rows":69,"trace_rows":0,"uncertainty_rows":69},"enigma_eval":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":46,"score_col":"Accuracy","source_rows":46,"trace_rows":0,"uncertainty_rows":46},"exploitbench":{"cost_rows":0,"dated_rows":0,"metrics_rows":20,"parsed_rows":20,"score_col":"Mean capability","source_rows":20,"trace_rows":0,"uncertainty_rows":0},"fictionlivebench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":42,"score_col":"120k token score","source_rows":62,"trace_rows":0,"uncertainty_rows":0},"forecastbench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":92,"score_col":"Overall score","source_rows":92,"trace_rows":0,"uncertainty_rows":92},"frontiercode":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":46,"score_col":"Main score","source_rows":51,"trace_rows":0,"uncertainty_rows":0},"frontiermath":{"cost_rows":0,"dated_rows":100,"metrics_rows":0,"parsed_rows":101,"score_col":"mean_score","source_rows":101,"trace_rows":79,"uncertainty_rows":101},"frontiermath_tier_4":{"cost_rows":0,"dated_rows":71,"metrics_rows":0,"parsed_rows":72,"score_col":"mean_score","source_rows":72,"trace_rows":46,"uncertainty_rows":72},"gbaeval":{"cost_rows":0,"dated_rows":0,"metrics_rows":23,"parsed_rows":23,"score_col":"Overall score","source_rows":23,"trace_rows":0,"uncertainty_rows":0},"gdp_pdf":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":56,"score_col":"GDP.pdf score","source_rows":59,"trace_rows":0,"uncertainty_rows":0},"gdpval":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":11,"score_col":"Win Rate (%)","source_rows":11,"trace_rows":0,"uncertainty_rows":0},"geobench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":32,"score_col":"ACW Avg Score","source_rows":32,"trace_rows":0,"uncertainty_rows":0},"gpqa_diamond":{"cost_rows":0,"dated_rows":333,"metrics_rows":0,"parsed_rows":334,"score_col":"mean_score","source_rows":334,"trace_rows":331,"uncertainty_rows":334},"gsm8k":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":170,"score_col":"EM","source_rows":235,"trace_rows":0,"uncertainty_rows":0},"gso":{"cost_rows":0,"dated_rows":36,"metrics_rows":0,"parsed_rows":42,"score_col":"Score OPT@1","source_rows":42,"trace_rows":0,"uncertainty_rows":0},"hella_swag":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":113,"score_col":"Overall accuracy","source_rows":135,"trace_rows":0,"uncertainty_rows":0},"hle":{"cost_rows":0,"dated_rows":0,"metrics_rows":54,"parsed_rows":56,"score_col":"Accuracy","source_rows":56,"trace_rows":0,"uncertainty_rows":56},"lambada":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":53,"score_col":"Score","source_rows":80,"trace_rows":0,"uncertainty_rows":0},"lech_mazur_writing":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":49,"score_col":"Mean score","source_rows":49,"trace_rows":0,"uncertainty_rows":0},"live_bench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":54,"score_col":"Global average","source_rows":64,"trace_rows":0,"uncertainty_rows":0},"math_level_5":{"cost_rows":0,"dated_rows":108,"metrics_rows":0,"parsed_rows":108,"score_col":"mean_score","source_rows":108,"trace_rows":108,"uncertainty_rows":108},"metr_time_horizons":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":49,"score_col":"Time horizon","source_rows":49,"trace_rows":0,"uncertainty_rows":49},"mindcube":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":5,"score_col":"Overall score","source_rows":5,"trace_rows":0,"uncertainty_rows":0},"mmlu":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":217,"score_col":"EM","source_rows":249,"trace_rows":0,"uncertainty_rows":0},"mystery_game_puzzles":{"cost_rows":0,"dated_rows":144,"metrics_rows":0,"parsed_rows":144,"score_col":"mean_score","source_rows":144,"trace_rows":92,"uncertainty_rows":144},"open_book_qa":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":70,"score_col":"Accuracy","source_rows":71,"trace_rows":0,"uncertainty_rows":0},"os_world":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":20,"score_col":"Score","source_rows":58,"trace_rows":0,"uncertainty_rows":0},"osworld_2":{"cost_rows":7,"dated_rows":0,"metrics_rows":0,"parsed_rows":16,"score_col":"Binary accuracy","source_rows":16,"trace_rows":0,"uncertainty_rows":0},"otis_mock_aime_2024_2025":{"cost_rows":0,"dated_rows":313,"metrics_rows":0,"parsed_rows":314,"score_col":"mean_score","source_rows":314,"trace_rows":312,"uncertainty_rows":314},"piqa":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":113,"score_col":"Score","source_rows":140,"trace_rows":0,"uncertainty_rows":0},"posttrainbench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":12,"score_col":"Average (%)","source_rows":12,"trace_rows":0,"uncertainty_rows":0},"proofbench":{"cost_rows":82,"dated_rows":0,"metrics_rows":84,"parsed_rows":84,"score_col":"Accuracy","source_rows":84,"trace_rows":0,"uncertainty_rows":57},"rli":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":18,"score_col":"Score","source_rows":18,"trace_rows":0,"uncertainty_rows":0},"scicode":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":235,"score_col":"Score","source_rows":235,"trace_rows":0,"uncertainty_rows":0},"science_qa":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":26,"score_col":"Score","source_rows":104,"trace_rows":0,"uncertainty_rows":0},"simplebench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":102,"score_col":"Score (AVG@5)","source_rows":104,"trace_rows":0,"uncertainty_rows":0},"simpleqa_verified":{"cost_rows":0,"dated_rows":101,"metrics_rows":0,"parsed_rows":101,"score_col":"mean_score","source_rows":101,"trace_rows":89,"uncertainty_rows":101},"spatialviz_bench":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":8,"score_col":"Overall score","source_rows":8,"trace_rows":0,"uncertainty_rows":0},"superglue":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":10,"score_col":"Score","source_rows":11,"trace_rows":0,"uncertainty_rows":0},"surface_evolver_bench":{"cost_rows":29,"dated_rows":0,"metrics_rows":29,"parsed_rows":29,"score_col":"Mean score","source_rows":29,"trace_rows":0,"uncertainty_rows":0},"swe_bench_verified":{"cost_rows":0,"dated_rows":35,"metrics_rows":0,"parsed_rows":35,"score_col":"mean_score","source_rows":35,"trace_rows":34,"uncertainty_rows":35},"terminalbench":{"cost_rows":0,"dated_rows":202,"metrics_rows":0,"parsed_rows":202,"score_col":"Accuracy mean","source_rows":204,"trace_rows":0,"uncertainty_rows":193},"the_agent_company":{"cost_rows":15,"dated_rows":0,"metrics_rows":15,"parsed_rows":16,"score_col":"% Score","source_rows":16,"trace_rows":0,"uncertainty_rows":0},"trivia_qa":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":115,"score_col":"EM","source_rows":115,"trace_rows":0,"uncertainty_rows":0},"vending_bench_2":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":71,"score_col":"Score","source_rows":71,"trace_rows":0,"uncertainty_rows":0},"video_mme":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":50,"score_col":"Overall (no subtitles)","source_rows":50,"trace_rows":0,"uncertainty_rows":0},"vpct":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":38,"score_col":"Correct","source_rows":38,"trace_rows":0,"uncertainty_rows":0},"webdev_arena":{"cost_rows":0,"dated_rows":0,"metrics_rows":135,"parsed_rows":135,"score_col":"Arena Score","source_rows":141,"trace_rows":0,"uncertainty_rows":135},"weirdml":{"cost_rows":173,"dated_rows":0,"metrics_rows":0,"parsed_rows":173,"score_col":"Accuracy","source_rows":174,"trace_rows":0,"uncertainty_rows":158},"wino_grande":{"cost_rows":0,"dated_rows":0,"metrics_rows":0,"parsed_rows":137,"score_col":"Accuracy","source_rows":145,"trace_rows":0,"uncertainty_rows":0}},"counts":{"benchmarks":72,"invalid_dates":0,"metadata_conflicts":1,"models":668,"openrouter_collisions":0,"quarantined":18,"scores":6107},"coverage":{"context":{"percent":51.9,"rows":3172,"total":6107},"cost":{"percent":16,"rows":977,"total":6107},"eval_date":{"percent":27.3,"rows":1666,"total":6107},"flop":{"percent":34.6,"rows":2112,"total":6107},"metrics":{"percent":11.5,"rows":704,"total":6107},"open_weights":{"percent":41.6,"rows":2539,"total":6107},"org":{"percent":95.7,"rows":5847,"total":6107},"params":{"percent":14.8,"rows":906,"total":6107},"price_in":{"percent":51.9,"rows":3172,"total":6107},"release_date":{"percent":99.1,"rows":6050,"total":6107},"trace":{"percent":20.7,"rows":1267,"total":6107},"uncertainty":{"percent":39.6,"rows":2417,"total":6107}},"gates":{"failures":[],"status":"pass"},"generated_at":"2026-11-09T10:43:01Z","status":"warn","warnings":["model metadata conflict: glm-4-5 release_date=2025-08-03/2025-08-05","quarantined benchmark_metadata: empty data or missing Model version","quarantined dtbench: no approved score schema","quarantined ebr_bench: no approved score schema","quarantined eci_scores: empty data or missing Model version","quarantined edi_scores: empty data or missing Model version","quarantined processed_data_for_eci: empty data or missing Model version","quarantined frontiermath_erdos: no approved score schema","quarantined frontiermath_tier_4_v2: no approved score schema","quarantined frontiermath_tiers_1_3_v2: no approved score schema","quarantined frontierswe: expected score column \"Dominance\" missing","quarantined furniture_assembly: no approved score schema","quarantined lmca: no approved score schema","quarantined mirrorcode: no approved score schema","quarantined model_metadata: empty data or missing Model version","quarantined spireclimb: no approved score schema","quarantined spireclimb_sts1_a20_heart: no approved score schema","quarantined spireclimb_sts2_a10: no approved score schema","quarantined weirdml_v3: no approved score schema","quarantined benchmark: benchmark_metadata","quarantined benchmark: dtbench","quarantined benchmark: ebr_bench","quarantined benchmark: eci_scores","quarantined benchmark: edi_scores","quarantined benchmark: processed_data_for_eci","quarantined benchmark: frontiermath_erdos","quarantined benchmark: frontiermath_tier_4_v2","quarantined benchmark: frontiermath_tiers_1_3_v2","quarantined benchmark: frontierswe","quarantined benchmark: furniture_assembly","quarantined benchmark: lmca","quarantined benchmark: mirrorcode","quarantined benchmark: model_metadata","quarantined benchmark: spireclimb","quarantined benchmark: spireclimb_sts1_a20_heart","quarantined benchmark: spireclimb_sts2_a10","quarantined benchmark: weirdml_v3","DeepSWE live source rejected; using Epoch fallback: generated_at is not current: 2026-09-22T06:27:15.860279+00:00"]},"schema":{"metric_units":{"calibration_error":"source score","checkpoints":"count","input_tokens":"tokens","latency_seconds":"seconds","output_tokens":"tokens","pass_at_4":"source score","runs":"runs","spend_usd":"USD","steps":"steps","total_tokens":"tokens","votes":"votes","wall_clock_hours":"hours"},"observation":{"b":"benchmark id","bv":"benchmark version","c":"reported benchmark cost","cfg":"source-published configuration","ci":"95% confidence half-width","d":"evaluation date","dn":"source display name","e":"reasoning effort","evaluated_at":"evaluation timestamp","hi":"confidence upper bound","lo":"confidence lower bound","m":"exact model revision id","metrics":"source-published resource and diagnostic metrics","notes":"source notes","oid":"stable observation id","rd":"model release date","sd":"standard deviation","se":"standard error","sid":"source row id","sm":"source model id","src":"source URL","trace":"run trace/log URL","v":"score"}},"schema_version":2,"scores":[{"b":"adversarial_nli","dn":"Phi-3-mini 3.8b","m":"phi-3-mini-4k-instruct","oid":"adversarial_nli-ce111af8d6585939","rd":"2024-04-23","sid":"recRK7mjPoMNvdB2G","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.528},{"b":"adversarial_nli","dn":"Phi-3-small 7b","m":"phi-3-small-8k-instruct","oid":"adversarial_nli-5ec1423e63aca8db","rd":"2024-04-23","sid":"reczstKSIPrFDyOvb","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.581},{"b":"adversarial_nli","dn":"Phi-3-medium 14b","m":"phi-3-medium-128k-instruct","oid":"adversarial_nli-540e6a2c02f908c2","rd":"2024-04-23","sid":"recIkuaIsFXEwVyqh","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.558},{"b":"adversarial_nli","dn":"Phi-2 2.7b","m":"phi-2","oid":"adversarial_nli-c79fd6fac9ed43c7","rd":"2023-12-12","sid":"rec56Oyo244VigZmC","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.425},{"b":"adversarial_nli","dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"adversarial_nli-f93c4a35e6970748","rd":"2023-09-27","sid":"recs6xkdcp6JUtf0U","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.471},{"b":"adversarial_nli","dn":"Gemma 7b","m":"gemma-7b","oid":"adversarial_nli-9b7716230e762a78","rd":"2024-02-21","sid":"recSFslZ5tfOvhcrj","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.487},{"b":"adversarial_nli","dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"adversarial_nli-0d99f4d2cefcbb95","rd":"2024-04-18","sid":"rec5SHmv8BEboIgY3","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.573},{"b":"adversarial_nli","dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"adversarial_nli-77cd5d1aa8f12fc7","rd":"2023-12-11","sid":"recsPgQH1jqzEBUh0","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.552},{"b":"adversarial_nli","dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"adversarial_nli-60810ebb28578577","rd":"2023-11-06","sid":"recW7PD5XPKpKzmqK","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.581},{"b":"adversarial_nli","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"adversarial_nli-5617551f0988afc7","rd":"2022-01-27","sid":"rectdKW9XkJ6YqRKY","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.354},{"b":"adversarial_nli","cfg":{"shots":"0"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"adversarial_nli-eab419feadbe6fda","rd":"2021-10-11","sid":"rec9Vgr0GSmgaNqkJ","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.366},{"b":"adversarial_nli","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"adversarial_nli-cf9d4941d82c4024","rd":"2022-01-27","sid":"recahBXKhmGNiqjFA","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.339},{"b":"adversarial_nli","cfg":{"shots":"1"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"adversarial_nli-bd825a556f2836cf","rd":"2021-10-11","sid":"recE03KGHKt3Ofp2d","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.397},{"b":"adversarial_nli","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"adversarial_nli-a74352ed4ead6ab1","rd":"2022-01-27","sid":"rec3lMHRJFfrxLZFJ","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.34},{"b":"adversarial_nli","cfg":{"shots":"few"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"adversarial_nli-3d3e92f7abe42676","rd":"2021-10-11","sid":"recRjUuV6OUDJjxbD","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.396},{"b":"aider_polyglot","c":0.0033,"cfg":{"effort":"high","format":"diff"},"e":"high","m":"gpt-oss-120b","oid":"aider_polyglot-768ec10db917253d","rd":"2025-08-05","sid":"recpcTosmykQScJV3","sm":"gpt-oss-120b_high","v":41.8},{"b":"aider_polyglot","cfg":{"format":"diff"},"m":"qwen3-235b-a22b","notes":"no cost data","oid":"aider_polyglot-efd5598614087649","rd":"2025-04-29","sid":"recPTyOmNpRhK82Et","sm":"Qwen3-235B-A22B","src":"https://aider.chat/docs/leaderboards/","v":59.6},{"b":"aider_polyglot","c":0.0039,"cfg":{"format":"diff"},"m":"deepseek-v3-2-exp","oid":"aider_polyglot-da2f97736eb4d3c3","rd":"2025-09-29","sid":"recGIpKLux7lz5yX2","sm":"DeepSeek-V3.2-Exp","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":70.2},{"b":"aider_polyglot","c":0.0058,"cfg":{"effort":"thinking","format":"diff"},"e":"thinking","m":"deepseek-v3-2-exp","oid":"aider_polyglot-3b460c604e855291","rd":"2025-09-29","sid":"recdHPvT3QOnVioE2","sm":"DeepSeek-V3.2-Exp_thinking","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":74.2},{"b":"aider_polyglot","c":10.3713,"cfg":{"effort":"low","format":"diff"},"d":"2025-08-25","e":"low","m":"gpt-5-2025-08-07","metrics":{"latency_seconds":62.4},"oid":"aider_polyglot-7dc23f5871e6aa14","rd":"2025-08-07","sid":"recUEmzrfcnLZAiii","sm":"gpt-5-2025-08-07_low","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":81.3},{"b":"aider_polyglot","c":17.693,"cfg":{"effort":"medium","format":"diff"},"d":"2025-08-25","e":"medium","m":"gpt-5-2025-08-07","metrics":{"latency_seconds":118.7},"oid":"aider_polyglot-f645d7ab8e6bed8a","rd":"2025-08-07","sid":"recESWz46G8mPRBRc","sm":"gpt-5-2025-08-07_medium","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":86.7},{"b":"aider_polyglot","c":29.0829,"cfg":{"effort":"high","format":"diff"},"d":"2025-08-23","e":"high","m":"gpt-5-2025-08-07","metrics":{"latency_seconds":194},"oid":"aider_polyglot-30952489d070a934","rd":"2025-08-07","sid":"recusk6NHwVicJ6kW","sm":"gpt-5-2025-08-07_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":88},{"b":"aider_polyglot","c":0.061,"cfg":{"effort":"medium","format":"diff"},"e":"medium","m":"o3-2025-04-16","oid":"aider_polyglot-444611a117c7bae6","rd":"2025-04-16","sid":"rec6DP0rm1WZjXNb1","sm":"o3-2025-04-16_medium","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":76.9},{"b":"aider_polyglot","c":0.7856,"cfg":{"effort":"low","format":"whole"},"d":"2025-04-10","e":"low","m":"grok-3-mini-beta","metrics":{"latency_seconds":35.1},"oid":"aider_polyglot-5250c684aa26676b","rd":"2025-04-09","sid":"recAgOnnJfaE7mIta","sm":"grok-3-mini-beta_low","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":34.7},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2025-04-19","m":"openhands-lm-32b-v0-1","metrics":{"latency_seconds":195.6},"oid":"aider_polyglot-49a988039f6cc64f","rd":"2024-03-26","sid":"recboVfDWPcwnvyJ6","sm":"openhands-lm-32b-v0.1","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":10.2},{"b":"aider_polyglot","c":0.0055,"cfg":{"format":"diff"},"m":"kimi-k2-instruct","oid":"aider_polyglot-2c582c9bff31de6c","rd":"2025-07-12","sid":"recYmlS1cYM8gEefH","sm":"Kimi-K2-Instruct","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":59.1},{"b":"aider_polyglot","c":0.094,"cfg":{"format":"diff"},"m":"grok-4-0709","oid":"aider_polyglot-8767e69978736d72","rd":"2025-07-09","sid":"reck399Bumr5Uaml4","sm":"grok-4-0709","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":79.6},{"b":"aider_polyglot","c":146.3249,"cfg":{"effort":"high","format":"diff"},"d":"2025-06-28","e":"high","m":"o3-pro-2025-06-10","metrics":{"latency_seconds":449},"oid":"aider_polyglot-0e9bff2fadc2fff9","rd":"2025-06-10","sid":"recwknvKrFOHaRtr0","sm":"o3-pro-2025-06-10_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":84.9},{"b":"aider_polyglot","c":37.4104,"cfg":{"format":"diff-fenced"},"d":"2025-05-07","m":"gemini-2-5-pro-preview-05-06","metrics":{"latency_seconds":165.3},"oid":"aider_polyglot-dd891b4cbd96dc69","rd":"2025-05-06","sid":"recQXuPMHsN1f1pXT","sm":"gemini-2.5-pro-preview-05-06","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":76.9},{"b":"aider_polyglot","c":45.5961,"cfg":{"format":"diff-fenced"},"d":"2025-06-06","m":"gemini-2-5-pro-preview-06-05","metrics":{"latency_seconds":175.2},"notes":"No thinking parameter set; default thinking token length","oid":"aider_polyglot-4b94f548742f63bb","rd":"2025-06-05","sid":"rectI2Vulkqa1ab26","sm":"gemini-2.5-pro-preview-06-05","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":79.1},{"b":"aider_polyglot","c":49.8822,"cfg":{"format":"diff-fenced"},"d":"2025-06-06","m":"gemini-2-5-pro-preview-06-05-32k","metrics":{"latency_seconds":200.3},"oid":"aider_polyglot-49c3b7df35235bd8","rd":"2025-06-05","sid":"recTdLkFf3W6Wsboo","sm":"gemini-2.5-pro-preview-06-05_32K","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":83.1},{"b":"aider_polyglot","c":8.5625,"cfg":{"format":"diff"},"d":"2025-05-25","m":"gemini-2-5-flash-preview-05-20-23k","metrics":{"latency_seconds":53.9},"oid":"aider_polyglot-e060c14e29e94c84","rd":"2025-05-20","sid":"recuekZA2IXFF4GQE","sm":"gemini-2.5-flash-preview-05-20_23K","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":55.1},{"b":"aider_polyglot","c":1.1354,"cfg":{"format":"diff"},"d":"2025-05-26","m":"gemini-2-5-flash-preview-05-20","metrics":{"latency_seconds":12.2},"oid":"aider_polyglot-901bc4e6cae4a89b","rd":"2025-05-20","sid":"recrFVqz2K5yPiBwr","sm":"gemini-2.5-flash-preview-05-20","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":44},{"b":"aider_polyglot","c":15.8155,"cfg":{"format":"diff"},"d":"2025-05-24","m":"claude-sonnet-4-2025-05-14","metrics":{"latency_seconds":29.8},"oid":"aider_polyglot-ec724f8a6786f753","rd":"2025-05-22","sid":"recPP5f3NBfwjlXKY","sm":"claude-sonnet-4-20250514","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":56.4},{"b":"aider_polyglot","c":26.5755,"cfg":{"format":"diff"},"d":"2025-05-24","m":"claude-sonnet-4-20250514-32k","metrics":{"latency_seconds":79.9},"oid":"aider_polyglot-df8db40b43da95b5","rd":"2025-05-22","sid":"rec9HZAmeUiTJn1pl","sm":"claude-sonnet-4-20250514_32K","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":61.3},{"b":"aider_polyglot","c":68.6253,"cfg":{"format":"diff"},"d":"2025-05-25","m":"claude-opus-4-2025-05-14","metrics":{"latency_seconds":42.5},"oid":"aider_polyglot-33deb3a119b2d3c3","rd":"2025-05-22","sid":"recRkLNIObleDWFSd","sm":"claude-opus-4-20250514","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":70.7},{"b":"aider_polyglot","c":65.7484,"cfg":{"format":"diff"},"d":"2025-05-25","m":"claude-opus-4-20250514-32k","metrics":{"latency_seconds":44.1},"oid":"aider_polyglot-39f2b3a6c0b8fe61","rd":"2025-05-22","sid":"recardEOKRehXXmpU","sm":"claude-opus-4-20250514_32K","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":72},{"b":"aider_polyglot","c":0.7603,"cfg":{"format":"diff"},"d":"2025-05-08","m":"qwen3-32b","metrics":{"latency_seconds":372.2},"oid":"aider_polyglot-441cdb68b9876296","rd":"2025-04-29","sid":"reca01Mic02m68RtO","sm":"Qwen3-32B","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":40},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2025-04-06","m":"llama-4-maverick-17b-128e-instruct","metrics":{"latency_seconds":20.5},"oid":"aider_polyglot-08b4ea4f70ac3189","rd":"2025-04-05","sid":"rectZuHXsjT3SSoQ8","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":15.6},{"b":"aider_polyglot","c":1.9918,"cfg":{"format":"diff"},"d":"2025-04-14","m":"gpt-4-1-mini-2025-04-14","metrics":{"latency_seconds":19.5},"oid":"aider_polyglot-5b317ade097cd388","rd":"2025-04-14","sid":"recga7XsHS6ITtdvx","sm":"gpt-4.1-mini-2025-04-14","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":32.4},{"b":"aider_polyglot","c":0.4281,"cfg":{"format":"whole"},"d":"2025-04-14","m":"gpt-4-1-nano-2025-04-14","metrics":{"latency_seconds":12},"oid":"aider_polyglot-535584a37815b5f9","rd":"2025-04-14","sid":"reciZgnWy0kynTOqe","sm":"gpt-4.1-nano-2025-04-14","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":8.9},{"b":"aider_polyglot","c":19.7416,"cfg":{"format":"diff"},"d":"2025-03-29","m":"chatgpt-4o-03-27-2025","metrics":{"latency_seconds":10.3},"oid":"aider_polyglot-26e378e0339babf7","rd":"2025-03-27","sid":"recLUsXEZABVVkGMZ","sm":"chatgpt-4o-03-27-2025","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":45.3},{"b":"aider_polyglot","c":1.8451,"cfg":{"format":"diff"},"d":"2025-04-20","m":"gemini-2-5-flash-preview-04-17","metrics":{"latency_seconds":50.1},"oid":"aider_polyglot-26dda73e89c0aea4","rd":"2025-04-17","sid":"recnt3fyqCYRO97Xc","sm":"gemini-2.5-flash-preview-04-17","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":47.1},{"b":"aider_polyglot","c":0.7346,"cfg":{"effort":"high","format":"whole"},"d":"2025-04-10","e":"high","m":"grok-3-mini-beta","metrics":{"latency_seconds":79.1},"oid":"aider_polyglot-64ea46bbf0be1c44","rd":"2025-04-09","sid":"recDFJuDElv6ujOzB","sm":"grok-3-mini-beta_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":49.3},{"b":"aider_polyglot","c":9.8556,"cfg":{"format":"diff"},"d":"2025-04-14","m":"gpt-4-1-2025-04-14","metrics":{"latency_seconds":20.5},"oid":"aider_polyglot-7ffc91eb653afa4a","rd":"2025-04-14","sid":"recLZ9wzZof5nDN7z","sm":"gpt-4.1-2025-04-14","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":52.4},{"b":"aider_polyglot","c":11.0338,"cfg":{"format":"diff"},"d":"2025-04-10","m":"grok-3-beta","metrics":{"latency_seconds":15.3},"oid":"aider_polyglot-d3ec4f9a919ff041","rd":"2025-04-09","sid":"recxcFVs5GAmiu4VS","sm":"grok-3-beta","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":53.3},{"b":"aider_polyglot","c":1.1164,"cfg":{"format":"diff"},"d":"2025-03-24","m":"deepseek-v3-0324","metrics":{"latency_seconds":290},"oid":"aider_polyglot-32db584a3aa46cbc","rd":"2025-03-24","sid":"recuYSkwNLFezoACW","sm":"DeepSeek-V3-0324","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":55.1},{"b":"aider_polyglot","c":4.8016,"cfg":{"format":"diff"},"d":"2025-06-06","m":"deepseek-r1-0528","metrics":{"latency_seconds":716.6},"oid":"aider_polyglot-d2bceafab51f86e4","rd":"2025-05-28","sid":"recRvk7ngoGy8aCj2","sm":"DeepSeek-R1-0528","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":71.4},{"b":"aider_polyglot","cfg":{"format":"diff-fenced"},"m":"gemini-2-5-pro-exp-03-25","oid":"aider_polyglot-6dbaf5e8236dd7c1","rd":"2025-03-25","sid":"recQlPsg6RDQbmeuu","sm":"gemini-2.5-pro-exp-03-25","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":72.9},{"b":"aider_polyglot","c":19.6399,"cfg":{"effort":"high","format":"diff"},"d":"2025-04-16","e":"high","m":"o4-mini-2025-04-16","metrics":{"latency_seconds":176.5},"oid":"aider_polyglot-1303dfa1797f7064","rd":"2025-04-16","sid":"recl9fv7t1P1NNdYa","sm":"o4-mini-2025-04-16_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":72},{"b":"aider_polyglot","c":21.2259,"cfg":{"effort":"high","format":"diff"},"d":"2025-06-25","e":"high","m":"o3-2025-04-16","metrics":{"latency_seconds":197.3},"oid":"aider_polyglot-6225f56897930877","rd":"2025-04-16","sid":"recxVJrYlQfsAsfER","sm":"o3-2025-04-16_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":81.3},{"b":"aider_polyglot","c":0,"cfg":{"format":"diff"},"d":"2025-01-21","m":"gemini-2-0-flash-thinking-exp-01-21","metrics":{"latency_seconds":24.2},"oid":"aider_polyglot-4683ff31fe7331a5","rd":"2025-01-21","sid":"rec00y8O2BvpWQfcl","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":18.2},{"b":"aider_polyglot","c":6.7351,"cfg":{"format":"diff"},"d":"2024-12-30","m":"gpt-4o-2024-11-20","metrics":{"latency_seconds":12.1},"oid":"aider_polyglot-a34771af6d75e4ee","rd":"2024-11-20","sid":"rec2Ms47snUu0pQTd","sm":"gpt-4o-2024-11-20","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":18.2},{"b":"aider_polyglot","c":14.4063,"cfg":{"format":"diff"},"d":"2025-01-17","m":"claude-3-5-sonnet-2024-10-22","metrics":{"latency_seconds":21.4},"oid":"aider_polyglot-f2c8a1a4a6721e1d","rd":"2024-10-22","sid":"rec57bL14FIulU6lg","sm":"claude-3-5-sonnet-20241022","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":51.6},{"b":"aider_polyglot","cfg":{"format":"diff"},"m":"qwen2-5-coder-32b-instruct","oid":"aider_polyglot-58dd48c2daabd844","rd":"2024-11-21","sid":"rec59tnsvtbsczej7","sm":"Qwen2.5-Coder-32B-Instruct","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":8},{"b":"aider_polyglot","c":6.0583,"cfg":{"format":"diff"},"d":"2024-12-21","m":"claude-3-5-haiku-2024-10-22","metrics":{"latency_seconds":31.8},"oid":"aider_polyglot-2fb8068df6133435","rd":"2024-10-22","sid":"rec9QQ4Xg51kvPPbz","sm":"claude-3-5-haiku-20241022","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":28},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2024-12-22","m":"gemini-exp-1206","metrics":{"latency_seconds":45.5},"oid":"aider_polyglot-222a4cf53d8b67e4","rd":"2024-12-06","sid":"recAV1zatcIxsAzWB","sm":"gemini-exp-1206","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":38.2},{"b":"aider_polyglot","c":5.4193,"cfg":{"format":"diff"},"d":"2025-01-20","m":"deepseek-r1","metrics":{"latency_seconds":113.7},"oid":"aider_polyglot-880d77e1f2da42b5","rd":"2025-01-20","sid":"recAxfwG5ZWZpmiDr","sm":"DeepSeek-R1","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":56.9},{"b":"aider_polyglot","c":0.001511111111,"cfg":{"format":"diff"},"m":"deepseek-v3","oid":"aider_polyglot-8a5b0be51188319c","rd":"2024-12-26","sid":"recDTQzmlxzC9g2Xv","sm":"DeepSeek-V3","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":48.4},{"b":"aider_polyglot","cfg":{"format":"diff"},"d":"2025-01-28","m":"qwen-max-2025-01-25","metrics":{"latency_seconds":39.5},"oid":"aider_polyglot-a0da50622c6e00e3","rd":"2025-01-25","sid":"recDgfyv8YjZ4PXlf","sm":"qwen-max-2025-01-25","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":21.8},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2025-02-25","m":"gemini-2-0-pro-exp-02-05","metrics":{"latency_seconds":34.8},"oid":"aider_polyglot-b81a798b0a518efe","rd":"2025-02-05","sid":"recFbazQb9nPYQlUg","sm":"gemini-2.0-pro-exp-02-05","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":35.6},{"b":"aider_polyglot","c":0,"cfg":{"format":"diff"},"d":"2025-03-06","m":"qwq-32b","metrics":{"latency_seconds":228.6},"oid":"aider_polyglot-f3cabdac9f4ad812","rd":"2025-03-05","sid":"recMjVodelAnkM0zs","sm":"QwQ-32B","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":20.9},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2025-03-14","m":"c4ai-command-a-03-2025","metrics":{"latency_seconds":85.1},"oid":"aider_polyglot-95599eb84c018956","rd":"2025-03-13","sid":"recObZ1Vigx8xfynx","sm":"c4ai-command-a-03-2025","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":12},{"b":"aider_polyglot","c":0.3236,"cfg":{"format":"whole"},"d":"2024-12-21","m":"gpt-4o-mini-2024-07-18","metrics":{"latency_seconds":17.3},"oid":"aider_polyglot-69f18fe6d7cd0fe4","rd":"2024-07-18","sid":"recS9uke58jAhuAU5","sm":"gpt-4o-mini-2024-07-18","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":3.6},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2025-03-15","m":"gemma-3-27b-it","metrics":{"latency_seconds":79.7},"oid":"aider_polyglot-2a571cad55d263c8","rd":"2025-03-12","sid":"recUtWAOufdMzGmou","sm":"gemma-3-27b-it","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":4.9},{"b":"aider_polyglot","c":183.1802,"cfg":{"format":"diff"},"d":"2025-02-27","m":"gpt-4-5-preview-2025-02-27","metrics":{"latency_seconds":113.5},"oid":"aider_polyglot-10cc1796d5c0ae1d","rd":"2025-02-27","sid":"recZJiRq9rBdLK3Oh","sm":"gpt-4.5-preview-2025-02-27","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":44.9},{"b":"aider_polyglot","c":17.7191,"cfg":{"format":"diff"},"d":"2025-02-24","m":"claude-3-7-sonnet-2025-02-19","metrics":{"latency_seconds":28.3},"oid":"aider_polyglot-d36fef1cfa22c943","rd":"2025-02-24","sid":"reckNZe7HU86uyqhi","sm":"claude-3-7-sonnet-20250219","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":60.4},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2024-12-22","m":"gemini-2-0-flash-exp","metrics":{"latency_seconds":12.2},"oid":"aider_polyglot-aa29d5933381b588","rd":"2024-12-11","sid":"reclpgowW0bcfFaLe","sm":"gemini-2.0-flash-exp","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":22.2},{"b":"aider_polyglot","c":0.5101,"cfg":{"format":"diff"},"d":"2024-12-21","m":"deepseek-v2-5","metrics":{"latency_seconds":184},"oid":"aider_polyglot-b9dc0f5e5891fae3","rd":"2024-09-06","sid":"recm8QAqvAKTqkfw5","sm":"DeepSeek-V2.5","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":17.8},{"b":"aider_polyglot","cfg":{"format":"whole"},"m":"qwen2-5-coder-32b-instruct","oid":"aider_polyglot-28c52a34e003110d","rd":"2024-11-21","sid":"recoyJG3t2fzQ52qJ","sm":"Qwen2.5-Coder-32B-Instruct","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":16.4},{"b":"aider_polyglot","c":14.3703,"cfg":{"format":"diff"},"d":"2025-02-15","m":"chatgpt-4o-01-29-2025","metrics":{"latency_seconds":12.4},"oid":"aider_polyglot-1699f029d3d44da8","rd":"2025-01-29","sid":"recp5kdfipFh2xOZX","sm":"chatgpt-4o-01-29-2025","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":27.1},{"b":"aider_polyglot","c":1.9834,"cfg":{"format":"whole"},"d":"2025-01-13","m":"codestral-2501","metrics":{"latency_seconds":9.3},"oid":"aider_polyglot-c2d045a1d0cdd695","rd":"2025-01-13","sid":"recqT3OZg5zjaI5oE","sm":"codestral-2501","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":11.1},{"b":"aider_polyglot","c":186.4958,"cfg":{"effort":"high","format":"diff"},"d":"2024-12-21","e":"high","m":"o1-2024-12-17","metrics":{"latency_seconds":133.2},"oid":"aider_polyglot-f93c1417e30664f1","rd":"2024-12-17","sid":"recsBapERP76dJ1KO","sm":"o1-2024-12-17_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":61.7},{"b":"aider_polyglot","c":8.8599,"cfg":{"effort":"medium","format":"diff"},"d":"2025-01-31","e":"medium","m":"o3-mini-2025-01-31","metrics":{"latency_seconds":47.2},"oid":"aider_polyglot-c76ac2b058caab07","rd":"2025-01-31","sid":"rectGRIeRMF8LhwRY","sm":"o3-mini-2025-01-31_medium","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":53.8},{"b":"aider_polyglot","c":0,"cfg":{"format":"whole"},"d":"2024-12-23","m":"yi-lightning","metrics":{"latency_seconds":146.7},"oid":"aider_polyglot-94712282eb6b6b60","rd":"2024-12-02","sid":"recw2nF8a1PVHJmeU","sm":"yi-lightning","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":12.9},{"b":"aider_polyglot","c":7.0286,"cfg":{"format":"diff"},"d":"2024-12-30","m":"gpt-4o-2024-08-06","metrics":{"latency_seconds":16},"oid":"aider_polyglot-62cffd6a3fb9eb1c","rd":"2024-08-06","sid":"recx22qdB0rDBivbC","sm":"gpt-4o-2024-08-06","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":23.1},{"b":"aider_polyglot","c":36.8343,"cfg":{"format":"diff"},"d":"2025-02-24","m":"claude-3-7-sonnet-20250219-32k","metrics":{"latency_seconds":105.2},"oid":"aider_polyglot-20e417a0b90cb0f2","rd":"2025-02-24","sid":"recyIxXTFMw9aOxsg","sm":"claude-3-7-sonnet-20250219_32K","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":64.9},{"b":"aider_polyglot","c":18.1584,"cfg":{"effort":"high","format":"diff"},"d":"2025-01-31","e":"high","m":"o3-mini-2025-01-31","metrics":{"latency_seconds":124.6},"oid":"aider_polyglot-0ba4f483d6ee69e8","rd":"2025-01-31","sid":"recyzvCSCQZ2WoxRz","sm":"o3-mini-2025-01-31_high","src":"https://aider.chat/docs/leaderboards/#polyglot-leaderboard","v":60.4},{"b":"aider_polyglot","c":18.577,"cfg":{"format":"whole"},"d":"2024-12-22","m":"o1-mini-2024-09-12","metrics":{"latency_seconds":34.7},"oid":"aider_polyglot-dec0f08139b23abe","rd":"2024-09-12","sid":"reck6CxUeYWlXRAHn","sm":"o1-mini-2024-09-12_unknown","src":"https://aider.chat/docs/leaderboards/","v":32.9},{"b":"aider_polyglot","c":0,"cfg":{"format":"diff-fenced"},"d":"2025-04-12","m":"gemini-2-5-pro-preview-03-25","metrics":{"latency_seconds":45.3},"oid":"aider_polyglot-19dd1ce50bc6146b","rd":"2025-03-31","sid":"recTgVaZQHy9Mnbpn","sm":"gemini-2.5-pro-preview-03-25","src":"https://aider.chat/docs/leaderboards/","v":72.9},{"b":"aider_polyglot","c":0,"cfg":{"format":"diff"},"d":"2025-05-09","m":"qwen3-235b-a22b-instruct-2507","metrics":{"latency_seconds":45.4},"oid":"aider_polyglot-2f826d921dcf5afc","rd":"2025-07-25","sid":"recYb4LclOWHSFQPO","sm":"Qwen3-235B-A22B-Instruct-2507","src":"https://aider.chat/docs/leaderboards/","v":59.6},{"b":"aider_polyglot","c":13.7517,"cfg":{"format":"diff"},"d":"2025-06-25","m":"o3-2025-04-16","metrics":{"latency_seconds":101.7},"oid":"aider_polyglot-05ebb41f14307e5a","rd":"2025-04-16","sid":"recCjydKGLLUR4gbe","sm":"o3-2025-04-16_unknown","src":"https://aider.chat/docs/leaderboards/","v":76.9},{"b":"aider_polyglot","c":59.6182,"cfg":{"effort":"high","format":"diff"},"d":"2025-07-11","e":"high","m":"grok-4-0709","metrics":{"latency_seconds":403.2},"oid":"aider_polyglot-2f7427c973dd38b6","rd":"2025-07-09","sid":"recIX9qqNMX5p7YUh","sm":"grok-4-0709_high","src":"https://aider.chat/docs/leaderboards/","v":79.6},{"b":"aider_polyglot","c":1.2357,"cfg":{"format":"diff"},"d":"2025-07-17","m":"moonshotaikimi-k2-0905","metrics":{"latency_seconds":67.6},"oid":"aider_polyglot-51cd8057192bf301","rd":"2025-09-05","sid":"rec5LVELiXbse50W1","sm":"moonshotai/kimi-k2-0905","src":"https://aider.chat/docs/leaderboards/","v":59.1},{"b":"aider_polyglot","c":0.7406,"cfg":{"effort":"high","format":"diff"},"d":"2025-08-06","e":"high","m":"openaigpt-oss-120b","metrics":{"latency_seconds":35.5},"oid":"aider_polyglot-cb6b06ad81b2777b","rd":"2025-08-05","sid":"recq6khGxdTrxTK3O","sm":"openai/gpt-oss-120b_high","src":"https://aider.chat/docs/leaderboards/","v":41.8},{"b":"aider_polyglot","c":1.3045,"cfg":{"format":"diff"},"d":"2025-10-03","m":"deepseek-reasoner","metrics":{"latency_seconds":291.2},"oid":"aider_polyglot-097f018ba64215e4","rd":"2025-12-01","sid":"reciOl0m6U21VRfmu","sm":"deepseek-reasoner","src":"https://aider.chat/docs/leaderboards/","v":74.2},{"b":"aider_polyglot","c":0.8756,"cfg":{"format":"diff"},"d":"2025-10-03","m":"deepseek-chat","metrics":{"latency_seconds":104},"oid":"aider_polyglot-96d0bd981b3e6b29","rd":"2025-12-01","sid":"rec1EYIByXDB4qe8H","sm":"deepseek-chat","src":"https://aider.chat/docs/leaderboards/","v":70.2},{"b":"ale_bench","c":2.86415,"cfg":{"effort":"max"},"dn":"gpt-6-astra-max","e":"max","m":"gpt-6-astra","metrics":{"input_tokens":2980,"output_tokens":57124.2,"total_tokens":60104.2},"oid":"ale_bench-2297612cfdebda71","rd":"2026-09-03","sid":"recie6OeSzfrEhXOW","sm":"gpt-6-astra_max","v":2951.32},{"b":"ale_bench","c":0.596828,"cfg":{"effort":"max"},"dn":"gpt-6.1-sol-max","e":"max","m":"gpt-6-1-sol","metrics":{"input_tokens":2980,"output_tokens":59568.7,"total_tokens":62548.7},"oid":"ale_bench-7d85440812e7094a","rd":"2026-09-29","sid":"recbQjMjXINIX4xjc","sm":"gpt-6.1-sol_max","v":2743.75},{"b":"ale_bench","c":0.413917,"cfg":{"effort":"max"},"dn":"gpt-6.1-terra-max","e":"max","m":"gpt-6-1-terra","metrics":{"input_tokens":2980,"output_tokens":82724.8,"total_tokens":85704.8},"oid":"ale_bench-5fa989a6f032a13e","rd":"2026-10-26","sid":"recHSwa2wuj8rGCoD","sm":"gpt-6.1-terra_max","v":2608.82},{"b":"ale_bench","c":0.843984,"cfg":{"effort":"max"},"dn":"gpt-6-sol-max","e":"max","m":"gpt-6-sol","metrics":{"input_tokens":2980,"output_tokens":84338.3,"total_tokens":87318.3},"oid":"ale_bench-36b259ac896f9634","rd":"2026-09-22","sid":"receiDDkmha5O2EgZ","sm":"gpt-6-sol_max","v":2462},{"b":"ale_bench","c":1.534078,"cfg":{"effort":"max"},"dn":"gpt-5.6-sol-max","e":"max","m":"gpt-5-6-sol","metrics":{"input_tokens":8670.8,"output_tokens":50025.6,"total_tokens":58696.4},"oid":"ale_bench-01090dc1bdcd9f38","rd":"2026-07-09","sid":"recB9LSLRE0ygr7gb","sm":"gpt-5.6-sol_max","v":2176.88},{"b":"ale_bench","c":1.890512,"cfg":{"effort":"high"},"dn":"claude-opus-5-high","e":"high","m":"claude-opus-5","metrics":{"input_tokens":4163.2,"output_tokens":74787.8,"total_tokens":78951.1},"oid":"ale_bench-29528bd668b85bdf","rd":"2026-07-24","sid":"reczXRrSMMBAUjHxZ","sm":"claude-opus-5_high","v":2164.57},{"b":"ale_bench","c":0.271178,"cfg":{"effort":"high"},"dn":"claude-opus-5.5-high","e":"high","m":"claude-opus-5-5","metrics":{"input_tokens":4165.2,"output_tokens":12725.9,"total_tokens":16891.1},"oid":"ale_bench-4154b5d1753a3d94","rd":"2026-09-22","sid":"rec2XltGq6LqiHAqJ","sm":"claude-opus-5-5_high","v":2147.38},{"b":"ale_bench","c":2.294856,"cfg":{"effort":"high"},"dn":"claude-fable-5.1-high","e":"high","m":"claude-fable-5-1","metrics":{"input_tokens":4165.2,"output_tokens":45064.1,"total_tokens":49229.3},"oid":"ale_bench-07403f336fcdcbb4","rd":"2026-09-01","sid":"recoromAv2aFG1mQC","sm":"claude-fable-5-1_high","v":2143.2},{"b":"ale_bench","c":1.949743,"cfg":{"effort":"high"},"dn":"gemini-4-argon-high","e":"high","m":"gemini-4-argon","metrics":{"input_tokens":113295.1,"output_tokens":79328.1,"total_tokens":192623.1},"oid":"ale_bench-a2fd20df52bf115a","rd":"2026-09-30","sid":"rec1Zj9NnhZWnpeWx","sm":"gemini-4-argon_high","v":2100.1},{"b":"ale_bench","c":2.108822,"cfg":{"effort":"high"},"dn":"claude-fable-5-high","e":"high","m":"claude-fable-5","metrics":{"input_tokens":4163.2,"output_tokens":41343.8,"total_tokens":45507},"oid":"ale_bench-e3e1140b71b92dae","rd":"2026-06-09","sid":"recOJYPyImXeBu5y6","sm":"claude-fable-5_high","v":2041.3},{"b":"ale_bench","c":1.254047,"cfg":{"effort":"max"},"dn":"gpt-5.6-terra-max","e":"max","m":"gpt-5-6-terra","metrics":{"input_tokens":2980,"output_tokens":83497.4,"total_tokens":86477.4},"oid":"ale_bench-916d9be69a294c09","rd":"2026-07-09","sid":"recy6tGLt0oXl5Z3t","sm":"gpt-5.6-terra_max","v":1951.42},{"b":"ale_bench","c":1.517132,"cfg":{"effort":"xhigh"},"dn":"gpt-5.5-xhigh","e":"xhigh","m":"gpt-5-5","metrics":{"input_tokens":3970.3,"output_tokens":50120.6,"total_tokens":54090.8},"oid":"ale_bench-2c99e4ce987ed2e1","rd":"2026-04-23","sid":"recvu236lk6eJfew6","sm":"gpt-5.5_xhigh","v":1942.97},{"b":"ale_bench","c":0.664453,"cfg":{"effort":"max"},"dn":"kimi-k3.1-pro-max","e":"max","m":"kimi-k3-1-pro","metrics":{"input_tokens":3011.4,"output_tokens":65883.4,"total_tokens":68894.8},"oid":"ale_bench-8113ea855f449612","rd":"2026-10-23","sid":"recQfC5gdLA6UCLLC","sm":"kimi-k3.1-pro_max","v":1932.75},{"b":"ale_bench","c":0.110921,"cfg":{"effort":"high"},"dn":"claude-sonnet-5.5-high","e":"high","m":"claude-sonnet-5-5","metrics":{"input_tokens":4165.2,"output_tokens":10259.1,"total_tokens":14424.3},"oid":"ale_bench-a8b11d1b52cfda7c","rd":"2026-09-28","sid":"recYjibuLhTTZDmIz","sm":"claude-sonnet-5-5_high","v":1819.12},{"b":"ale_bench","c":0.170538,"cfg":{"effort":"low"},"dn":"gemini-4-argon-low","e":"low","m":"gemini-4-argon","metrics":{"input_tokens":4156.3,"output_tokens":7695.6,"total_tokens":11851.9},"oid":"ale_bench-0a27cf93c0898cc0","rd":"2026-09-30","sid":"rechU0EZO7d9E7amZ","sm":"gemini-4-argon_low","v":1705.4},{"b":"ale_bench","c":0.220853,"cfg":{"effort":"xhigh"},"dn":"muse-spark-1.3-xhigh","e":"xhigh","m":"muse-spark-1-3","metrics":{"input_tokens":22158.2,"output_tokens":46065.1,"total_tokens":68223.2},"oid":"ale_bench-c964ce46a5d46057","rd":"2026-09-02","sid":"recoUMQ4ZOtgdCB0E","sm":"muse-spark-1.3_xhigh","v":1688.35},{"b":"ale_bench","c":0.359171,"cfg":{"effort":"max"},"dn":"gpt-5.6-luna-max","e":"max","m":"gpt-5-6-luna","metrics":{"input_tokens":30008.1,"output_tokens":55171.4,"total_tokens":85179.6},"oid":"ale_bench-99bfc5d7374bcd08","rd":"2026-07-09","sid":"rechpIYckpxNmorqX","sm":"gpt-5.6-luna_max","v":1667.4},{"b":"ale_bench","c":0.559265,"cfg":{"effort":"xhigh"},"dn":"gpt-5.3-codex-xhigh","e":"xhigh","m":"gpt-5-3-codex","metrics":{"input_tokens":10466.6,"output_tokens":38819.5,"total_tokens":49286.2},"oid":"ale_bench-8424c5b51f056a16","rd":"2026-02-05","sid":"recjPl07t1MvmNgDm","sm":"gpt-5.3-codex_xhigh","v":1655.22},{"b":"ale_bench","c":0.100044,"cfg":{"effort":"max"},"dn":"deepseek-v4.1-flash-1017-max","e":"max","m":"deepseek-v4-1-flash-1017","metrics":{"input_tokens":2982.2,"output_tokens":82681,"total_tokens":85663.2},"oid":"ale_bench-e078c37784aed75f","rd":"2026-10-17","sid":"recmrTBOqivIMzmGy","sm":"deepseek-v4.1-flash-1017_max","v":1622},{"b":"ale_bench","c":0.612174,"cfg":{"effort":"high"},"dn":"gpt-5.4-high","e":"high","m":"gpt-5-4-2026-03-05","metrics":{"input_tokens":5224.5,"output_tokens":40236.1,"total_tokens":45460.6},"oid":"ale_bench-76771bfafda705b8","rd":"2026-03-05","sid":"recJ9Hqsyml44GqRp","sm":"gpt-5.4-2026-03-05_high","v":1607},{"b":"ale_bench","c":0.325474,"cfg":{"effort":"xhigh"},"dn":"muse-spark-open-xhigh","e":"xhigh","m":"muse-spark-open-816b-a42b","metrics":{"input_tokens":30866.8,"output_tokens":68205.7,"total_tokens":99072.6},"oid":"ale_bench-44b97c8ea3ebae0f","rd":"2026-10-15","sid":"recpfB5OsaYt2uwRy","sm":"Muse-Spark-Open-816B-A42B_xhigh","v":1590.12},{"b":"ale_bench","c":0.328965,"cfg":{"effort":"medium"},"dn":"gpt-5.5-medium","e":"medium","m":"gpt-5-5","metrics":{"input_tokens":2980,"output_tokens":10726.1,"total_tokens":13706.1},"oid":"ale_bench-c58c05bab4d5c52b","rd":"2026-04-23","sid":"rec8dhBUXYFTQ1qEi","sm":"gpt-5.5_medium","v":1589.38},{"b":"ale_bench","c":0.02089,"cfg":{"effort":"high"},"dn":"claude-haiku-5.5-high","e":"high","m":"claude-haiku-5-5","metrics":{"input_tokens":4165.2,"output_tokens":40947,"total_tokens":45112.3},"oid":"ale_bench-7fffc0dde2c84333","rd":"2026-10-07","sid":"recxv7Egql7NoCF6X","sm":"claude-haiku-5-5_high","v":1587},{"b":"ale_bench","c":0.018045,"cfg":{"effort":"xhigh"},"dn":"gpt-6-luna-xhigh","e":"xhigh","m":"gpt-6-luna","metrics":{"input_tokens":2980,"output_tokens":36004.4,"total_tokens":38984.4},"oid":"ale_bench-2933339e1e7c98a2","rd":"2026-09-22","sid":"rec922KTi4fs6PkF3","sm":"gpt-6-luna_xhigh","v":1576.92},{"b":"ale_bench","c":0.963945,"cfg":{"effort":"high"},"dn":"claude-4.8-opus-high","e":"high","m":"claude-opus-4-8","metrics":{"input_tokens":4163.2,"output_tokens":37725.1,"total_tokens":41888.4},"oid":"ale_bench-db25750ec4c92f34","rd":"2026-05-28","sid":"rec9PMFOTHSfxdEBy","sm":"claude-opus-4-8_high","v":1563.83},{"b":"ale_bench","c":1.281554,"cfg":{"effort":"max"},"dn":"kimi-k3-max","e":"max","m":"kimi-k3","metrics":{"input_tokens":3422.2,"output_tokens":84872.3,"total_tokens":88294.4},"oid":"ale_bench-5543b6843c447a06","rd":"2026-07-16","sid":"recPTGSK6832BNjII","sm":"kimi-k3_max","v":1524.47},{"b":"ale_bench","c":0.26258,"cfg":{"effort":"medium"},"dn":"gpt-5.4-medium","e":"medium","m":"gpt-5-4-2026-03-05","metrics":{"input_tokens":7326.1,"output_tokens":16666.9,"total_tokens":23992.9},"oid":"ale_bench-46502003ae88bb45","rd":"2026-03-05","sid":"rec6y7fVydiA2X9U9","sm":"gpt-5.4-2026-03-05_medium","v":1520.72},{"b":"ale_bench","c":0.24005,"cfg":{"effort":"xhigh"},"dn":"grok-4.6-xhigh","e":"xhigh","m":"grok-4-6","metrics":{"input_tokens":3259,"output_tokens":39583.6,"total_tokens":42842.6},"oid":"ale_bench-b9ff7a035d5f0f5d","rd":"2026-08-12","sid":"recnZJup9M0H9yYwP","sm":"grok-4.6_xhigh","v":1508.08},{"b":"ale_bench","c":0.96046,"cfg":{"effort":"high"},"dn":"claude-sonnet-5-high","e":"high","m":"claude-sonnet-5","metrics":{"input_tokens":4163.2,"output_tokens":63198,"total_tokens":67361.2},"oid":"ale_bench-f134d5b7af58c8f9","rd":"2026-06-30","sid":"rec4QpFWnpOanOnNm","sm":"claude-sonnet-5_high","v":1463.12},{"b":"ale_bench","c":0.421746,"cfg":{"effort":"none"},"dn":"claude-4.8-opus-no-thinking","e":"none","m":"claude-opus-4-8","metrics":{"input_tokens":4163.2,"output_tokens":16037.2,"total_tokens":20200.4},"oid":"ale_bench-d612c2b6d2a0b4d9","rd":"2026-05-28","sid":"recw5YfULeyAfAYgR","sm":"claude-opus-4-8_none","v":1411.83},{"b":"ale_bench","c":0.072764,"cfg":{"effort":"max"},"dn":"deepseek-v4-pro-0813-max","e":"max","m":"deepseek-v4-pro-0813","metrics":{"input_tokens":3048.2,"output_tokens":82223.8,"total_tokens":85272},"oid":"ale_bench-241a492387449648","rd":"2026-08-13","sid":"receNOu1SaASh6NZL","sm":"deepseek-v4-pro-0813_max","v":1403.15},{"b":"ale_bench","c":0.091067,"dn":"gemini-3-flash-preview-high","m":"gemini-3-flash-preview","metrics":{"input_tokens":3260.3,"output_tokens":29987.6,"total_tokens":33247.9},"oid":"ale_bench-1eb3fd2232390a0a","rd":"2025-12-17","sid":"recfMGg7Kvqpid4qZ","sm":"gemini-3-flash-preview","v":1367.2},{"b":"ale_bench","c":1.770753,"cfg":{"effort":"medium"},"dn":"claude-4.6-sonnet-medium","e":"medium","m":"claude-sonnet-4-6","metrics":{"input_tokens":55201,"output_tokens":107010,"total_tokens":162211},"oid":"ale_bench-6d46a5f3da54a387","rd":"2026-02-17","sid":"rec79auRfHPVtkKyP","sm":"claude-sonnet-4-6_medium","v":1327.3},{"b":"ale_bench","c":0.224549,"dn":"claude-4.7-opus-no-thinking","m":"claude-opus-4-7","metrics":{"input_tokens":4168.2,"output_tokens":8148.3,"total_tokens":12316.5},"oid":"ale_bench-4ea505c06a4793ec","rd":"2026-04-16","sid":"recy14NQpCEvMvUAx","sm":"claude-opus-4-7","v":1323.05},{"b":"ale_bench","c":0.284001,"cfg":{"effort":"high"},"dn":"glm-5.3-high","e":"high","m":"glm-5-3","metrics":{"input_tokens":3404.2,"output_tokens":63718.3,"total_tokens":67122.5},"oid":"ale_bench-6f8165dc865c3cb5","rd":"2026-08-14","sid":"recNCRjM56K1cVX2R","sm":"glm-5.3_high","v":1317.4},{"b":"ale_bench","c":0.176767,"cfg":{"effort":"high"},"dn":"grok-4.5-high","e":"high","m":"grok-4-5","metrics":{"input_tokens":3477.2,"output_tokens":28869.3,"total_tokens":32346.5},"oid":"ale_bench-55381937a6055bb8","rd":"2026-07-08","sid":"recTr2zSEcDvzbwvB","sm":"grok-4.5_high","v":1309.28},{"b":"ale_bench","c":0.019526,"cfg":{"effort":"max"},"dn":"deepseek-v4-flash-0731-max","e":"max","m":"deepseek-v4-flash-0731","metrics":{"input_tokens":3048.2,"output_tokens":68383.5,"total_tokens":71431.7},"oid":"ale_bench-8ac63f1e19a767e5","rd":"2026-07-31","sid":"recuPYbCCpYAt7G1G","sm":"deepseek-v4-flash-0731_max","v":1306.08},{"b":"ale_bench","c":0.587753,"dn":"gpt-5.2-codex-xhigh","m":"gpt-5-2-codex","metrics":{"input_tokens":4434.3,"output_tokens":41699.5,"total_tokens":46133.8},"oid":"ale_bench-b14fa36c22f97341","rd":"2025-12-18","sid":"rec76peITUi0G9hWI","sm":"gpt-5.2-codex","v":1299.9},{"b":"ale_bench","c":0.296865,"cfg":{"effort":"high"},"dn":"gpt-5.2-high","e":"high","m":"gpt-5-2-2025-12-11","metrics":{"input_tokens":2980,"output_tokens":21050.7,"total_tokens":24030.7},"oid":"ale_bench-0a558a82ddc8b33a","rd":"2025-12-11","sid":"recbx03MbsAIPxPvj","sm":"gpt-5.2-2025-12-11_high","v":1293.55},{"b":"ale_bench","c":0.344275,"cfg":{"effort":"high"},"dn":"gemini-3.8-flash-high","e":"high","m":"gemini-3-8-flash","metrics":{"input_tokens":3260.3,"output_tokens":45251.3,"total_tokens":48511.7},"oid":"ale_bench-bdc8fbb9338d6105","rd":"2026-09-02","sid":"recdGDUA9gtoZhnsC","sm":"gemini-3.8-flash_high","v":1269.72},{"b":"ale_bench","c":0.142947,"cfg":{"effort":"medium"},"dn":"gpt-5.2-medium","e":"medium","m":"gpt-5-2-2025-12-11","metrics":{"input_tokens":2980,"output_tokens":10094,"total_tokens":13073.9},"oid":"ale_bench-aef863c78366fe0c","rd":"2025-12-11","sid":"recVnMSgaT9EDtGvh","sm":"gpt-5.2-2025-12-11_medium","v":1249.83},{"b":"ale_bench","c":0.512093,"dn":"gpt-5.1-codex-high","m":"gpt-5-1-codex","metrics":{"input_tokens":2980,"output_tokens":51030.5,"total_tokens":54010.5},"oid":"ale_bench-dd666bb6fb453078","rd":"2025-11-12","sid":"recAh1GsKfoTx69bz","sm":"gpt-5.1-codex","v":1244.92},{"b":"ale_bench","c":0.03528,"dn":"mimo-v2.6-flash","m":"mimo-v2-6-flash","metrics":{"input_tokens":27215.9,"output_tokens":115534.1,"total_tokens":142750},"oid":"ale_bench-7b2bea2d23b0325c","rd":"2026-09-21","sid":"recvIP49qkTEq798L","sm":"mimo-v2.6-flash","v":1244.2},{"b":"ale_bench","c":0.456946,"dn":"mistral-large-4","m":"mistral-large-4-0","metrics":{"input_tokens":17621.4,"output_tokens":103987.6,"total_tokens":121609},"oid":"ale_bench-55560f49f5684316","rd":"2026-10-06","sid":"recyIQCyCKeiINnL2","sm":"mistral-large-4-0_unknown","v":1213.15},{"b":"ale_bench","c":0.253186,"dn":"gpt-5.1-codex-max-high","m":"gpt-5-1-codex-max","metrics":{"input_tokens":2980,"output_tokens":25138.3,"total_tokens":28118.3},"oid":"ale_bench-b7ddc94cc700c58e","rd":"2025-11-19","sid":"rechIkCR6I2vgX5ia","sm":"gpt-5.1-codex-max","v":1208.83},{"b":"ale_bench","c":0.315236,"cfg":{"effort":"high"},"dn":"gpt-5.1-thinking","e":"high","m":"gpt-5-1-2025-11-13","metrics":{"input_tokens":2980,"output_tokens":31365.3,"total_tokens":34345.3},"oid":"ale_bench-4849a19234f0d58b","rd":"2025-11-13","sid":"recLNNj0GfqyDx4nk","sm":"gpt-5.1-2025-11-13_high","v":1192.15},{"b":"ale_bench","c":0.3845,"dn":"qwen3.7-max","m":"qwen3-7-max","metrics":{"input_tokens":3253.8,"output_tokens":50182.1,"total_tokens":53435.9},"oid":"ale_bench-9157d329a79f92ee","rd":"2026-05-19","sid":"recA9I9t2L4RQrsQs","sm":"qwen3.7-max","v":1189.4},{"b":"ale_bench","c":0.271408,"cfg":{"effort":"high"},"dn":"gpt-5.4-mini-high","e":"high","m":"gpt-5-4-mini-2026-03-17","metrics":{"input_tokens":14858,"output_tokens":58093.8,"total_tokens":72951.9},"oid":"ale_bench-7f2cdf9fecab2907","rd":"2026-03-17","sid":"recJwrUEXk9e8HLtF","sm":"gpt-5.4-mini-2026-03-17_high","v":1188.58},{"b":"ale_bench","c":0.259385,"dn":"gemini-3-pro-preview-high","m":"gemini-3-pro-preview","metrics":{"input_tokens":3260.3,"output_tokens":21301,"total_tokens":24561.3},"oid":"ale_bench-69b7f7984d920aee","rd":"2025-11-18","sid":"recPGiw7I44Wc9L5u","sm":"gemini-3-pro-preview","v":1176.75},{"b":"ale_bench","c":0.244948,"cfg":{"effort":"high"},"dn":"gpt-5-thinking","e":"high","m":"gpt-5-2025-08-07","metrics":{"input_tokens":2980,"output_tokens":24147.5,"total_tokens":27127.5},"oid":"ale_bench-26690d139db35ab3","rd":"2025-08-07","sid":"rec6WynYilo1cLLBN","sm":"gpt-5-2025-08-07_high","v":1162.45},{"b":"ale_bench","c":0.388403,"dn":"gemini-3.1-pro-preview-high","m":"gemini-3-1-pro-preview","metrics":{"input_tokens":3260.3,"output_tokens":31823.5,"total_tokens":35083.8},"oid":"ale_bench-bfd314fc766e26b0","rd":"2026-02-19","sid":"recMLdPe4oTagwHaL","sm":"gemini-3.1-pro-preview","v":1160.6},{"b":"ale_bench","c":0.108379,"dn":"mimo-v2.6-pro","m":"mimo-v2-6-pro","metrics":{"input_tokens":49449.9,"output_tokens":103073.2,"total_tokens":152523},"oid":"ale_bench-31d8da5072a3f8ca","rd":"2026-09-21","sid":"recj8xTSS8LtuRsA1","sm":"mimo-v2.6-pro","v":1158.1},{"b":"ale_bench","c":0.13838,"dn":"grok-4.20-beta","m":"grok-4-20","metrics":{"input_tokens":3013.2,"output_tokens":22619,"total_tokens":25632.3},"oid":"ale_bench-27522ad817d319d4","rd":"2026-02-17","sid":"recyOlTS8wLrTAP1n","sm":"grok-4-20","v":1150.28},{"b":"ale_bench","c":0.073484,"cfg":{"effort":"none"},"dn":"gpt-5.5-none","e":"none","m":"gpt-5-5","metrics":{"input_tokens":2980,"output_tokens":2208.2,"total_tokens":5188.1},"oid":"ale_bench-f8cbdfbc19f892ef","rd":"2026-04-23","sid":"rec9k7gkUezxdYAtu","sm":"gpt-5.5_none","v":1127.58},{"b":"ale_bench","c":0.196861,"dn":"kimi-k2.6","m":"kimi-k2-6","metrics":{"input_tokens":2927,"output_tokens":48986.8,"total_tokens":51913.8},"oid":"ale_bench-ad4f6f645080bf90","rd":"2026-04-20","sid":"recgKppN84MLEgXWf","sm":"kimi-k2.6","v":1092.67},{"b":"ale_bench","c":0.081663,"cfg":{"effort":"max"},"dn":"deepseek-v4.1-flash-max","e":"max","m":"deepseek-v4-1-flash","metrics":{"input_tokens":2982.2,"output_tokens":67312.6,"total_tokens":70294.7},"oid":"ale_bench-cb1c45ca34733a27","rd":"2026-09-09","sid":"recgLH5qA0YS3ftkG","sm":"deepseek-v4.1-flash_max","v":1092.35},{"b":"ale_bench","c":0.052314,"cfg":{"effort":"none"},"dn":"gpt-5.4-none","e":"none","m":"gpt-5-4-2026-03-05","metrics":{"input_tokens":2980,"output_tokens":3225.2,"total_tokens":6205.2},"oid":"ale_bench-7e8a8994b6c93737","rd":"2026-03-05","sid":"recMra8s8Fm1RC85F","sm":"gpt-5.4-2026-03-05_none","v":1086.03},{"b":"ale_bench","c":0.157666,"cfg":{"effort":"high"},"dn":"glm-5.2-high","e":"high","m":"glm-5-2","metrics":{"input_tokens":3062.9,"output_tokens":35035.6,"total_tokens":38098.5},"oid":"ale_bench-fc460839234ad1e1","rd":"2026-06-16","sid":"reczG9I4ZtXmjGGEg","sm":"glm-5.2_high","v":1046.95},{"b":"ale_bench","c":0.292547,"dn":"claude-4.5-opus","m":"claude-opus-4-5-20251101-16k","metrics":{"input_tokens":3386.2,"output_tokens":11024.6,"total_tokens":14410.8},"oid":"ale_bench-0a5ea199de117687","rd":"2025-11-24","sid":"recsmwFHMqdNiVKhZ","sm":"claude-opus-4-5-20251101_16K","v":1025.38},{"b":"ale_bench","c":0.148957,"cfg":{"effort":"max"},"dn":"glm-5.2-max","e":"max","m":"glm-5-2","metrics":{"input_tokens":3009.8,"output_tokens":33097.8,"total_tokens":36107.6},"oid":"ale_bench-7265ecf44b35fa1c","rd":"2026-06-16","sid":"recjoKzgMKFtXvQGM","sm":"glm-5.2_max","v":1010.17},{"b":"ale_bench","c":0.148062,"cfg":{"effort":"high"},"dn":"deepseek-v4-pro-high","e":"high","m":"deepseek-v4-pro","metrics":{"input_tokens":2956.2,"output_tokens":42362.1,"total_tokens":45318.2},"oid":"ale_bench-701f3d28ee505ecf","rd":"2026-04-24","sid":"recRdiZQ9HF3Kug6B","sm":"deepseek-v4-pro_high","v":1006.08},{"b":"ale_bench","c":0.027084,"cfg":{"effort":"high"},"dn":"gpt-5.4-nano-high","e":"high","m":"gpt-5-4-nano-2026-03-17","metrics":{"input_tokens":2980,"output_tokens":21398.1,"total_tokens":24378.1},"oid":"ale_bench-7b512bda197d98c1","rd":"2026-03-17","sid":"rec9SvDfW1FDfMZ6h","sm":"gpt-5.4-nano-2026-03-17_high","v":1004.52},{"b":"ale_bench","c":0.061688,"dn":"claude-4.6-opus-no-thinking","m":"claude-opus-4-6","metrics":{"input_tokens":3357.2,"output_tokens":1796.1,"total_tokens":5153.3},"oid":"ale_bench-324433690fd43f75","rd":"2026-02-05","sid":"recYZMZozZDvmw4y5","sm":"claude-opus-4-6","v":996.5},{"b":"ale_bench","c":0.202548,"dn":"inkling-max","m":"inkling","metrics":{"input_tokens":7217.1,"output_tokens":48499.4,"total_tokens":55716.5},"oid":"ale_bench-18deb5db17c16988","rd":"2026-07-15","sid":"rec9Usn4fwoVZrJbE","sm":"Inkling","v":946},{"b":"ale_bench","c":0.055799,"dn":"grok-4.3","m":"grok-4-3","metrics":{"input_tokens":3013.2,"output_tokens":21052.6,"total_tokens":24065.8},"oid":"ale_bench-1784a9a9f1eb429a","rd":"2026-04-17","sid":"reczLlezoqIBH2cwe","sm":"grok-4.3_unknown","v":944.17},{"b":"ale_bench","c":0.103474,"cfg":{"effort":"high"},"dn":"o3-high","e":"high","m":"o3-2025-04-16","metrics":{"input_tokens":2980,"output_tokens":12303.9,"total_tokens":15283.9},"oid":"ale_bench-04681c72fbbafd87","rd":"2025-04-16","sid":"recxPLRj9LORsZ4hq","sm":"o3-2025-04-16_high","v":933.55},{"b":"ale_bench","c":0.009725,"dn":"gemma-4-26b-a4b-it","m":"gemma-4-26b-a4b","metrics":{"input_tokens":4500.4,"output_tokens":22849.9,"total_tokens":27350.3},"oid":"ale_bench-2a076fda4c597f15","rd":"2026-04-02","sid":"recatyI2kjiCX3JZJ","sm":"gemma-4-26b-a4b","v":927.17},{"b":"ale_bench","c":0.00475,"dn":"gemma-4-31b-it","m":"gemma-4-31b-it","metrics":{"input_tokens":3327.7,"output_tokens":10709.5,"total_tokens":14037.1},"oid":"ale_bench-f05319b00b2bb862","rd":"2026-04-02","sid":"recRBJmJYtgEGyP5M","sm":"gemma-4-31b-it","v":925.5},{"b":"ale_bench","c":0.441921,"cfg":{"effort":"high"},"dn":"gemini-3.5-flash-high","e":"high","m":"gemini-3-5-flash","metrics":{"input_tokens":3260.3,"output_tokens":48620.1,"total_tokens":51880.4},"oid":"ale_bench-6ee35221672ea4c8","rd":"2026-05-19","sid":"recqmNjSqCngkNswY","sm":"gemini-3.5-flash_high","v":911.02},{"b":"ale_bench","c":0.243628,"cfg":{"effort":"high"},"dn":"gemini-3.7-flash-high","e":"high","m":"gemini-3-7-flash","metrics":{"input_tokens":3260.3,"output_tokens":31850,"total_tokens":35110.3},"oid":"ale_bench-30fda85d26fdc279","rd":"2026-08-13","sid":"recocjfk7wUzbsucJ","sm":"gemini-3.7-flash_high","v":904.3},{"b":"ale_bench","c":0.102977,"dn":"mimo-v2.5-pro","m":"mimo-v2-5-pro","metrics":{"input_tokens":3144.1,"output_tokens":34020.6,"total_tokens":37164.7},"oid":"ale_bench-0ddc5b8aba072513","sid":"recregiKTmVaqrBkO","sm":"mimo-v2.5-pro","v":899.8},{"b":"ale_bench","c":0.12937,"dn":"glm-5.1","m":"glm-5-1","metrics":{"input_tokens":3002.8,"output_tokens":28925.6,"total_tokens":31928.4},"oid":"ale_bench-ebcaa7d355388dfc","rd":"2026-04-07","sid":"recZnRHVnR4hKTTVy","sm":"glm-5.1","v":887.1},{"b":"ale_bench","c":0.184743,"dn":"kimi-k2.7-code","m":"kimi-k2-7-code","metrics":{"input_tokens":2926.4,"output_tokens":45506.1,"total_tokens":48432.6},"oid":"ale_bench-3a68a904c203e2b7","rd":"2026-06-12","sid":"rec4wsb1ZD4xGsic4","sm":"kimi-k2.7-code","v":886.23},{"b":"ale_bench","c":0.044719,"cfg":{"effort":"high"},"dn":"o4-mini-high","e":"high","m":"o4-mini-2025-04-16","metrics":{"input_tokens":2980,"output_tokens":9563.7,"total_tokens":12543.7},"oid":"ale_bench-beddd8dc35d6a66f","rd":"2025-04-16","sid":"recGp1XiFoZzUBgf9","sm":"o4-mini-2025-04-16_high","v":826.17},{"b":"ale_bench","c":0.047458,"dn":"kimi-k2.5","m":"kimi-k2-5","metrics":{"input_tokens":3172.4,"output_tokens":15508.2,"total_tokens":18680.6},"oid":"ale_bench-6cd0aa07fbb36864","rd":"2026-01-27","sid":"recan8UplurA5tsd5","sm":"kimi-k2.5","v":821.65},{"b":"ale_bench","c":0.043744,"cfg":{"effort":"minimal"},"dn":"gpt-5","e":"minimal","m":"gpt-5-2025-08-07","metrics":{"input_tokens":2980,"output_tokens":4027.1,"total_tokens":7007.1},"oid":"ale_bench-4ee92c54aafdf885","rd":"2025-08-07","sid":"rec39nSc7yf62OCDn","sm":"gpt-5-2025-08-07_minimal","v":807.65},{"b":"ale_bench","c":0.043794,"dn":"deepseek-r1-0528","m":"deepseek-r1-0528","metrics":{"input_tokens":2958.2,"output_tokens":20931.4,"total_tokens":23889.5},"oid":"ale_bench-9ac0cd9ffb4d9600","rd":"2025-05-28","sid":"recvNA7WMkiG702yk","sm":"DeepSeek-R1-0528","v":804.12},{"b":"ale_bench","c":0.020554,"cfg":{"effort":"high"},"dn":"gpt-5-mini-thinking","e":"high","m":"gpt-5-mini-2025-08-07","metrics":{"input_tokens":2980,"output_tokens":9959.6,"total_tokens":12939.6},"oid":"ale_bench-d979fa3bbd12167a","rd":"2025-08-07","sid":"recF8zlQux77xZ6g3","sm":"gpt-5-mini-2025-08-07_high","v":799.77},{"b":"ale_bench","c":0.025178,"dn":"gemini-3.1-flash-lite-preview-high","m":"gemini-3-1-flash-lite","metrics":{"input_tokens":3260.3,"output_tokens":16242,"total_tokens":19502.3},"oid":"ale_bench-25a0d1e23e92d8a5","rd":"2026-03-03","sid":"recEPv2aRJrEaOAFr","sm":"gemini-3.1-flash-lite","v":797.73},{"b":"ale_bench","c":0.151558,"dn":"claude-4.5-sonnet","m":"claude-sonnet-4-5-20250929-32k","metrics":{"input_tokens":3385.2,"output_tokens":9426.8,"total_tokens":12812},"oid":"ale_bench-36e67b9b3a557429","rd":"2025-09-29","sid":"recP4QoLV0cIM4QEg","sm":"claude-sonnet-4-5-20250929_32K","v":796.15},{"b":"ale_bench","c":0.007498,"dn":"mercury-2","m":"mercury-2","metrics":{"input_tokens":2919.1,"output_tokens":9229.4,"total_tokens":12148.5},"oid":"ale_bench-7e6efc61b6084303","sid":"recUDbBV9Gp87FjIG","sm":"mercury-2","v":785.58},{"b":"ale_bench","c":0.26988,"dn":"gemini-2.5-pro-thinking","m":"gemini-2-5-pro-32k","metrics":{"input_tokens":3260.3,"output_tokens":26618.8,"total_tokens":29879.2},"oid":"ale_bench-c21524de07ae654f","rd":"2025-06-17","sid":"recbAsFUHmHBAu3vT","sm":"gemini-2.5-pro_32K","v":785.52},{"b":"ale_bench","c":0.076718,"dn":"mimo-v2-pro","m":"mimo-v2-pro","metrics":{"input_tokens":3144.1,"output_tokens":24935.8,"total_tokens":28079.9},"oid":"ale_bench-cb03774d7aaebb3b","sid":"rec2NM5401b4oz68n","sm":"mimo-v2-pro","v":785.17},{"b":"ale_bench","c":0.073284,"dn":"glm-5","m":"glm-5","metrics":{"input_tokens":2998.3,"output_tokens":22200.7,"total_tokens":25199.1},"oid":"ale_bench-f2a510663027debb","rd":"2026-02-11","sid":"rec1dnhYFrGJo63GA","sm":"glm-5","v":765.62},{"b":"ale_bench","c":0.045726,"cfg":{"effort":"high"},"dn":"gemini-3.5-flash-lite-high","e":"high","m":"gemini-3-5-flash-lite","metrics":{"input_tokens":3260.3,"output_tokens":17899.3,"total_tokens":21159.7},"oid":"ale_bench-fa1db75552e7589c","rd":"2026-07-21","sid":"recQXW49hzii6cT10","sm":"gemini-3.5-flash-lite_high","v":765.27},{"b":"ale_bench","c":0.272986,"dn":"mistral-medium-3.5","m":"mistral-medium-2604","metrics":{"input_tokens":3131.5,"output_tokens":35774.6,"total_tokens":38906.1},"oid":"ale_bench-10da1be26090dd88","rd":"2026-04-28","sid":"reca0mbRGHCs7WXD9","sm":"mistral-medium-2604","v":763.98},{"b":"ale_bench","c":0.01883,"dn":"deepseek-v3.1-terminus","m":"deepseek-v3-1-terminus","metrics":{"input_tokens":2956.2,"output_tokens":18032,"total_tokens":20988.2},"oid":"ale_bench-58fdbf011a443bd2","rd":"2025-09-22","sid":"recg5MtftYSapOSmp","sm":"DeepSeek-V3.1-Terminus","v":745.17},{"b":"ale_bench","c":0.006839,"dn":"mimo-v2-flash","m":"mimo-v2-flash","metrics":{"input_tokens":3144.1,"output_tokens":21749.7,"total_tokens":24893.8},"oid":"ale_bench-067cc90e5c2a76e8","sid":"recFoFuXbB7FjjKsy","sm":"mimo-v2-flash","v":737.95},{"b":"ale_bench","c":0.007194,"cfg":{"effort":"high"},"dn":"gpt-5-nano-thinking","e":"high","m":"gpt-5-nano-2025-08-07","metrics":{"input_tokens":2980,"output_tokens":17654.2,"total_tokens":20634.2},"oid":"ale_bench-bcf308c3ecd8f6e5","rd":"2025-08-07","sid":"reca7nu2VsTuwErjy","sm":"gpt-5-nano-2025-08-07_high","v":718.67},{"b":"ale_bench","c":0.244468,"cfg":{"effort":"high"},"dn":"gemini-3.6-flash-high","e":"high","m":"gemini-3-6-flash","metrics":{"input_tokens":3260.3,"output_tokens":31943.7,"total_tokens":35204},"oid":"ale_bench-094fe8d301c096b7","rd":"2026-07-21","sid":"recjoXKmlx7gYuFVy","sm":"gemini-3.6-flash_high","v":715.52},{"b":"ale_bench","c":0.032705,"dn":"step-3.7-flash-medium","m":"step-3-7-flash","metrics":{"input_tokens":2974.2,"output_tokens":28039.8,"total_tokens":31014},"oid":"ale_bench-65ccc8984acfd077","rd":"2026-05-29","sid":"recagy7r3WEm2iRqt","sm":"step-3.7-flash","v":694.12},{"b":"ale_bench","c":0.005179,"cfg":{"effort":"high"},"dn":"deepseek-v4-flash-high","e":"high","m":"deepseek-v4-flash","metrics":{"input_tokens":2956.2,"output_tokens":18163.3,"total_tokens":21119.4},"oid":"ale_bench-e697416ba505a347","rd":"2026-04-24","sid":"recYO1qmn1ztCHit3","sm":"deepseek-v4-flash_high","v":678.2},{"b":"ale_bench","c":0.434347,"dn":"claude-4.1-opus","m":"claude-opus-4-1-20250805-16k","metrics":{"input_tokens":3385.2,"output_tokens":5114.3,"total_tokens":8499.5},"oid":"ale_bench-720f29be42e7b3fc","rd":"2025-08-05","sid":"recC0TKO3FdEEKkCQ","sm":"claude-opus-4-1-20250805_16K","v":674.77},{"b":"ale_bench","c":0.057069,"dn":"qwen3.6-plus","m":"qwen3-6-plus","metrics":{"input_tokens":3253.8,"output_tokens":18480.7,"total_tokens":21734.5},"oid":"ale_bench-624be09ca97ea26b","rd":"2026-03-31","sid":"rec6zR5hjTMdLrqle","sm":"qwen3.6-plus","v":670.15},{"b":"ale_bench","c":0.068155,"dn":"gemini-2.5-flash-thinking","m":"gemini-2-5-flash","metrics":{"input_tokens":3260.3,"output_tokens":26902.7,"total_tokens":30163},"oid":"ale_bench-37d23515d13b0dad","rd":"2025-06-17","sid":"reczvWjUbMeknlk0h","sm":"gemini-2.5-flash","v":661.88},{"b":"ale_bench","c":0.185869,"dn":"claude-4-sonnet","m":"claude-sonnet-4-20250514-32k","metrics":{"input_tokens":3385.2,"output_tokens":11714.2,"total_tokens":15099.5},"oid":"ale_bench-810760c94e8ba35a","rd":"2025-05-22","sid":"recIYwm1vA9ARngD5","sm":"claude-sonnet-4-20250514_32K","v":655.35},{"b":"ale_bench","c":0.071831,"dn":"claude-4.5-haiku","m":"claude-haiku-4-5-20251001-32k","metrics":{"input_tokens":3385.2,"output_tokens":13689.2,"total_tokens":17074.4},"oid":"ale_bench-14a3928dd951bc04","rd":"2025-10-15","sid":"recbKb04w1qoUQ2ul","sm":"claude-haiku-4-5-20251001_32K","v":653.48},{"b":"ale_bench","c":0.111764,"dn":"minimax-m3","m":"minimax-m3","metrics":{"input_tokens":11345.1,"output_tokens":43775.2,"total_tokens":55120.2},"oid":"ale_bench-bdf20de4fba3b53c","rd":"2026-06-01","sid":"recYfSDgNqvnJ1jjt","sm":"MiniMax-M3","v":640.02},{"b":"ale_bench","c":0.05359,"dn":"minimax-m2.1","m":"minimax-m2-1","metrics":{"input_tokens":3591.4,"output_tokens":44296.8,"total_tokens":47888.2},"oid":"ale_bench-0b2c463ffbc68c35","rd":"2025-12-23","sid":"recUmR7BX9AlkUNlP","sm":"MiniMax-M2.1","v":623.83},{"b":"ale_bench","c":0.03981,"dn":"qwen3.5-397b-a17b","m":"qwen3-5-plus","metrics":{"input_tokens":3254.6,"output_tokens":10516,"total_tokens":13770.6},"oid":"ale_bench-701b67397a541972","rd":"2026-02-16","sid":"recnZ9X6dzou95O2Z","sm":"qwen3.5-plus","v":621.92},{"b":"ale_bench","c":0.036658,"dn":"minimax-m2.5","m":"minimax-m2-5","metrics":{"input_tokens":2915.3,"output_tokens":30193.3,"total_tokens":33108.6},"oid":"ale_bench-dbaf7531c9b42971","rd":"2026-02-12","sid":"rec42FtxIr02i8ymQ","sm":"MiniMax-M2.5","v":618.17},{"b":"ale_bench","c":0.047177,"dn":"minimax-m2.7","m":"minimax-m2-7","metrics":{"input_tokens":3028.3,"output_tokens":38883.5,"total_tokens":41911.8},"oid":"ale_bench-2f601e7ac58ba35a","rd":"2026-03-18","sid":"recxlSP1YCTC8EWDT","sm":"MiniMax-M2.7","v":599.25},{"b":"ale_bench","c":0.087819,"dn":"kimi-k2-thinking","m":"kimi-k2-thinking","metrics":{"input_tokens":20969.9,"output_tokens":30386.6,"total_tokens":51356.5},"oid":"ale_bench-57672b8c379cff0a","rd":"2025-11-06","sid":"rec12bf4Di9CdFcWo","sm":"kimi-k2-thinking","v":597.5},{"b":"ale_bench","c":0.008947,"dn":"grok-code-fast-1","m":"grok-code-fast-1","metrics":{"input_tokens":3090.2,"output_tokens":5819.6,"total_tokens":8909.8},"oid":"ale_bench-831504980b3a5f61","rd":"2025-08-28","sid":"recbNjBcKr8QDUSq1","sm":"grok-code-fast-1","v":587.73},{"b":"ale_bench","c":0.001303,"dn":"gpt-oss-120b","m":"gpt-oss-120b","metrics":{"input_tokens":3044,"output_tokens":1997,"total_tokens":5041.1},"oid":"ale_bench-c24fc895304372e0","rd":"2025-08-05","sid":"rect7OMebdGAo6fqi","sm":"gpt-oss-120b","v":575.62},{"b":"ale_bench","c":0.000717,"dn":"gpt-oss-20b","m":"gpt-oss-20b","metrics":{"input_tokens":3045,"output_tokens":2905.6,"total_tokens":5950.6},"oid":"ale_bench-88b67391511c3f09","rd":"2025-08-05","sid":"recnLZ6QnnjGuQZTM","sm":"gpt-oss-20b","v":566.05},{"b":"ale_bench","c":0.026917,"dn":"gpt-4.1","m":"gpt-4-1-2025-04-14","metrics":{"input_tokens":2981,"output_tokens":2669.8,"total_tokens":5650.8},"oid":"ale_bench-f18b823b08d049a6","rd":"2025-04-14","sid":"recEWPIQaBwBDVE3l","sm":"gpt-4.1-2025-04-14","v":558.1},{"b":"ale_bench","c":0.0166,"cfg":{"effort":"none"},"dn":"deepseek-v4-pro-no-thinking","e":"none","m":"deepseek-v4-pro","metrics":{"input_tokens":2956.2,"output_tokens":4198.3,"total_tokens":7154.5},"oid":"ale_bench-e8b1e4c2e83e646c","rd":"2026-04-24","sid":"recgLipv7iMGBxsY3","sm":"deepseek-v4-pro_none","v":521.67},{"b":"ale_bench","c":0.037558,"dn":"mimo-v2.5","m":"mimo-v2-5","metrics":{"input_tokens":3144.1,"output_tokens":18430.7,"total_tokens":21574.8},"oid":"ale_bench-c8f3ad6346a103ca","sid":"recQ87IVme56eUHvg","sm":"mimo-v2.5","v":513.95},{"b":"ale_bench","c":0.015587,"dn":"mistral-small-4","m":"mistral-small-2603","metrics":{"input_tokens":3223.8,"output_tokens":25335.8,"total_tokens":28559.6},"oid":"ale_bench-49a0b9b63f93517b","rd":"2026-03-16","sid":"recbi3hiT1WWlf37V","sm":"mistral-small-2603","v":497.62},{"b":"ale_bench","c":0.003677,"dn":"qwen3-coder","m":"qwen3-coder-480b-a35b-instruct","metrics":{"input_tokens":3146.1,"output_tokens":2303.5,"total_tokens":5449.6},"oid":"ale_bench-d5082e20416eccca","rd":"2025-07-31","sid":"rectfaG2yA6jknnC0","sm":"Qwen3-Coder-480B-A35B-Instruct","v":461.45},{"b":"ale_bench","c":0.013921,"dn":"qwen3-coder-plus","m":"qwen3-coder-plus","metrics":{"input_tokens":3146.1,"output_tokens":2388.7,"total_tokens":5534.8},"oid":"ale_bench-bbcc0cd320058c65","rd":"2025-09-23","sid":"recwjAf9rCgzMWw6A","sm":"qwen3-coder-plus","v":456.5},{"b":"ale_bench","c":0.007552,"dn":"ling-2.6-1t","m":"ring-2-6-1t","metrics":{"input_tokens":3252.7,"output_tokens":2648.8,"total_tokens":5901.5},"oid":"ale_bench-6867b6bbfc3d8417","rd":"2026-05-14","sid":"recXl9L72hDOd17ck","sm":"Ring-2.6-1T","v":432.57},{"b":"ale_bench","c":0.043156,"dn":"glm-4.7","m":"glm-4-7","metrics":{"input_tokens":3047.3,"output_tokens":18952.5,"total_tokens":21999.8},"oid":"ale_bench-87175952142f921d","rd":"2025-12-22","sid":"reca1Sqghwxvyrjw1","sm":"glm-4.7","v":399.48},{"b":"ale_bench","c":0.009221,"dn":"grok-4.1-fast","m":"grok-4-1-fast-reasoning","metrics":{"input_tokens":3151.5,"output_tokens":17412.8,"total_tokens":20564.3},"oid":"ale_bench-e2244a9e6c4928ee","rd":"2025-11-19","sid":"recplTud0DFTSx8os","sm":"grok-4-1-fast-reasoning","v":394.93},{"b":"ale_bench","c":0.022125,"dn":"qwen3-max","m":"qwen3-max-2025-09-23","metrics":{"input_tokens":3146.1,"output_tokens":3058.3,"total_tokens":6204.4},"oid":"ale_bench-96c9c67aca0480f5","rd":"2025-09-24","sid":"recTOuWlGawMHCGL6","sm":"qwen3-max-2025-09-23","v":370.45},{"b":"ale_bench","c":0.041945,"dn":"qwen3.5-27b","m":"qwen3-5-27b","metrics":{"input_tokens":3253.8,"output_tokens":17070.4,"total_tokens":20324.2},"oid":"ale_bench-e77ca7c08334c203","rd":"2026-02-24","sid":"reco2WcUdeq6ajJnn","sm":"qwen3.5-27B","v":349.45},{"b":"ale_bench","c":0.030077,"dn":"glm-4.5","m":"glm-4-5","metrics":{"input_tokens":3003.8,"output_tokens":13478.6,"total_tokens":16482.4},"oid":"ale_bench-c75822ffb18ca893","rd":"2025-08-03","sid":"recZ1Gqky82AyyVl4","sm":"glm-4.5","v":344.82},{"b":"ale_bench","c":0.018794,"dn":"glm-4.6","m":"glm-4-6","metrics":{"input_tokens":3005.8,"output_tokens":7877.3,"total_tokens":10883.1},"oid":"ale_bench-be01fef0f72081f6","rd":"2025-09-30","sid":"recVYZwA3NeTHdxxh","sm":"glm-4.6","v":340.82},{"b":"ale_bench","c":0.035628,"dn":"qwen3.6-flash","m":"qwen3-6-flash","metrics":{"input_tokens":3253.8,"output_tokens":23209.4,"total_tokens":26463.2},"oid":"ale_bench-19000b05a1aa9169","rd":"2026-04-27","sid":"recfF5fHwXAQjWfmf","sm":"qwen3.6-flash","v":326.4},{"b":"ale_bench","c":0.011091,"dn":"gemini-2.5-flash-lite-thinking","m":"gemini-2-5-flash-lite-preview-06-17-thinking","metrics":{"input_tokens":3260.3,"output_tokens":27106.2,"total_tokens":30366.5},"oid":"ale_bench-02b6aca8aa7f5c35","rd":"2025-06-17","sid":"rec8Y7bVTallu536A","sm":"gemini-2.5-flash-lite-preview-06-17-thinking","v":325.9},{"b":"ale_bench","c":0.000882,"cfg":{"effort":"none"},"dn":"deepseek-v4-flash-no-thinking","e":"none","m":"deepseek-v4-flash","metrics":{"input_tokens":2956.2,"output_tokens":2257.2,"total_tokens":5213.4},"oid":"ale_bench-a6a7701af8713977","rd":"2026-04-24","sid":"recWsgZQjBL5E6mbZ","sm":"deepseek-v4-flash_none","v":324.98},{"b":"ale_bench","c":0.010869,"cfg":{"effort":"high"},"dn":"glm-5.3-flash-high","e":"high","m":"glm-5-3-flash","metrics":{"input_tokens":3009.8,"output_tokens":20965,"total_tokens":23974.8},"oid":"ale_bench-83bfced9c9cc25fe","rd":"2026-08-20","sid":"recuPsUOAayyoYdDf","sm":"glm-5.3-flash_high","v":303.55},{"b":"ale_bench","c":0.006234,"cfg":{"effort":"high"},"dn":"mercury-2.5-high","e":"high","m":"mercury-2-5","metrics":{"input_tokens":3170.7,"output_tokens":7539.3,"total_tokens":10710},"oid":"ale_bench-66d2baca45d202ac","rd":"2026-09-08","sid":"recSByU5Ezps9QbPn","sm":"mercury-2.5_high","v":301.65},{"b":"ale_bench","c":0.008051,"dn":"kimi-k2-0905","m":"moonshotaikimi-k2-0905","metrics":{"input_tokens":2926.4,"output_tokens":2517.9,"total_tokens":5444.3},"oid":"ale_bench-21aaf68016586e82","rd":"2025-09-05","sid":"rec7ES6bAyOQJGeOi","sm":"moonshotai/kimi-k2-0905","v":267.12},{"b":"ale_bench","c":0.005104,"dn":"mistral-large-3-2512","m":"mistral-large-2512","metrics":{"input_tokens":3119.5,"output_tokens":2362.8,"total_tokens":5482.3},"oid":"ale_bench-98e1118504725001","rd":"2025-12-02","sid":"recyERFlruRqxpyzN","sm":"mistral-large-2512","v":264.7},{"b":"ale_bench","c":0.004156,"dn":"nova-2-lite-v1","m":"amazon-nova-lite-v10","metrics":{"input_tokens":2906.3,"output_tokens":1313.6,"total_tokens":4219.9},"oid":"ale_bench-0ff1b3641133a85e","rd":"2024-12-03","sid":"recDn1v3tPnzJqBcR","sm":"amazon.nova-lite-v1:0","v":236.25},{"b":"ale_bench","c":0.03787,"dn":"qwen3.5-35b-a3b","m":"qwen3-5-flash","metrics":{"input_tokens":3253.8,"output_tokens":18528.2,"total_tokens":21782},"oid":"ale_bench-d8be64e8a6f99f0d","rd":"2026-02-25","sid":"recEk656rx6DJXFA2","sm":"qwen3.5-flash","v":221.8},{"b":"ale_bench","c":0.008207,"dn":"nemotron-3-super","m":"nemotron-3-super","metrics":{"input_tokens":3130.5,"output_tokens":15788,"total_tokens":18918.5},"oid":"ale_bench-07ba68d4664a52f1","rd":"2026-03-11","sid":"rec81MFJjUipO1wMs","sm":"nemotron-3-super","v":213.9},{"b":"ale_bench","c":0.007372,"dn":"mistral-medium-3.1","m":"mistral-medium-2508","metrics":{"input_tokens":3119.5,"output_tokens":3062.2,"total_tokens":6181.7},"oid":"ale_bench-8d422868d75018af","rd":"2025-08-12","sid":"recSrNJEdMR2lieIW","sm":"mistral-medium-2508","v":210.18},{"b":"ale_bench","c":0.001369,"dn":"llama-4-maverick","m":"llama-4-maverick-17b-128e-instruct-fp8","metrics":{"input_tokens":2920,"output_tokens":1405.8,"total_tokens":4325.7},"oid":"ale_bench-6d1b91703cf0c28d","rd":"2025-04-05","sid":"reczDwnVCF3skLR5s","sm":"Llama-4-Maverick-17B-128E-Instruct-FP8","v":172.97},{"b":"ale_bench","c":0.00278,"dn":"codestral-2508","m":"codestral-2508","metrics":{"input_tokens":3119.5,"output_tokens":2049.1,"total_tokens":5168.5},"oid":"ale_bench-071231cfca22ab44","rd":"2025-07-30","sid":"recX9dYbmB4f2h19A","sm":"codestral-2508","v":137.78},{"b":"algotune","cfg":{"effort":"medium"},"dn":"GPT-5.2 (high)","e":"medium","m":"gpt-5-2-2025-12-11","oid":"algotune-96dcccaa48e0a676","rd":"2025-12-11","sid":"recmOTqcaDOeGrw0X","sm":"gpt-5.2-2025-12-11_medium","v":2.05},{"b":"algotune","cfg":{"effort":"high"},"dn":"Gemini 3.1 Pro Preview (high)","e":"high","m":"gemini-3-1-pro-preview","oid":"algotune-7efb9a31ebe9e64d","rd":"2026-02-19","sid":"recAScSfvSJwGYgPe","sm":"gemini-3.1-pro-preview","v":2.02},{"b":"algotune","cfg":{"effort":"high"},"dn":"GPT-5.4 (high)","e":"high","m":"gpt-5-4-2026-03-05","oid":"algotune-1b3f96ce7fee7943","rd":"2026-03-05","sid":"recJlBSVOYqi6zcbx","sm":"gpt-5.4-2026-03-05_high","v":1.85},{"b":"algotune","cfg":{"effort":"high"},"dn":"Gemini 3 Pro Preview (high)","e":"high","m":"gemini-3-pro-preview","oid":"algotune-b44f77de15e2519d","rd":"2025-11-18","sid":"rec9Ywmj1VE3j4ICv","sm":"gemini-3-pro-preview","v":1.83},{"b":"algotune","dn":"Claude Opus 4.5 (high)","m":"claude-opus-4-5-2025-11-01","oid":"algotune-c69d584912a8db5e","rd":"2025-11-24","sid":"rec9M5y5wsXfBJbRe","sm":"claude-opus-4-5-20251101_unknown","v":1.77},{"b":"algotune","cfg":{"effort":"high"},"dn":"o4-mini (high)","e":"high","m":"o4-mini-2025-04-16","oid":"algotune-aaf15d52ca222ff6","rd":"2025-04-16","sid":"recShU7tWqeNxYW6v","sm":"o4-mini-2025-04-16_high","v":1.72},{"b":"algotune","cfg":{"effort":"high"},"dn":"DeepSeek R1 (high)","e":"high","m":"deepseek-r1","oid":"algotune-528ebd8e695cd95f","rd":"2025-01-20","sid":"recZ2SREKLJZZNFXX","sm":"DeepSeek-R1","v":1.7},{"b":"algotune","cfg":{"effort":"high"},"dn":"GPT-5 (high)","e":"high","m":"gpt-5-2025-08-07","oid":"algotune-59fb0c719ac3e887","rd":"2025-08-07","sid":"recrVKyfGfV9ZzFp3","sm":"gpt-5-2025-08-07_high","v":1.67},{"b":"algotune","cfg":{"effort":"thinking"},"dn":"GLM-4.5 (high)","e":"thinking","m":"glm-4-5","oid":"algotune-f1730bcdf290ee5d","rd":"2025-08-05","sid":"recJ2Ik2vP4dZOVnP","sm":"glm-4.5_thinking","v":1.52},{"b":"algotune","dn":"Claude Sonnet 4.5 (high)","m":"claude-sonnet-4-5-2025-09-29","oid":"algotune-2cbb237e84d2a2b0","rd":"2025-09-29","sid":"recjoFts6Mliie0dX","sm":"claude-sonnet-4-5-20250929_unknown","v":1.52},{"b":"algotune","cfg":{"effort":"high"},"dn":"Gemini 2.5 Pro (high)","e":"high","m":"gemini-2-5-pro","oid":"algotune-53347558214853e6","rd":"2025-06-17","sid":"recSo4CBg6ggAqzOZ","sm":"gemini-2.5-pro","v":1.51},{"b":"algotune","cfg":{"effort":"high"},"dn":"Claude Opus 4.6 (high)","e":"high","m":"claude-opus-4-6","oid":"algotune-ecaae43a4e4221e9","rd":"2026-02-05","sid":"recnaW71BcUPjCQyi","sm":"claude-opus-4-6_high","v":1.47},{"b":"algotune","cfg":{"effort":"high"},"dn":"Qwen3 Coder (high)","e":"high","m":"qwen3-coder-480b-a35b-instruct","oid":"algotune-d141a686c3b9f4ec","rd":"2025-07-31","sid":"recjIn0fy7wmd3eDJ","sm":"Qwen3-Coder-480B-A35B-Instruct","v":1.44},{"b":"algotune","cfg":{"effort":"high"},"dn":"gpt-oss-120b (high)","e":"high","m":"openaigpt-oss-120b","oid":"algotune-d16bba1d6266df9c","rd":"2025-08-05","sid":"recm2TzBFSajymm65","sm":"openai/gpt-oss-120b_high","v":1.41},{"b":"algotune","cfg":{"effort":"high"},"dn":"GPT-5 Mini (high)","e":"high","m":"gpt-5-mini-2025-08-07","oid":"algotune-946391e0474c1d83","rd":"2025-08-07","sid":"recBWU1GTOFUQ2rwv","sm":"gpt-5-mini-2025-08-07_high","v":1.38},{"b":"algotune","dn":"Claude Opus 4.1 (high)","m":"claude-opus-4-1-2025-08-05","oid":"algotune-99b8e293ef7f3c8f","rd":"2025-08-05","sid":"recDC5tvNBdXf2D8H","sm":"claude-opus-4-1-20250805_unknown","v":1.34},{"b":"algotune","dn":"Claude Opus 4 (high)","m":"claude-opus-4-2025-05-14","oid":"algotune-be886fdc0beba199","rd":"2025-05-22","sid":"recBBUj2dIBUqY2tR","sm":"claude-opus-4-20250514_unknown","v":1.33},{"b":"algotune","cfg":{"effort":"medium"},"dn":"GPT-5 Pro (medium)","e":"medium","m":"gpt-5-pro-2025-10-06","oid":"algotune-3603b561016f9805","rd":"2025-10-07","sid":"recISd3dRCHYjetpt","sm":"gpt-5-pro-2025-10-06_medium","v":1.31},{"b":"apex_agents","dn":"Gemini 4 Argon","m":"gemini-4-argon","oid":"apex_agents-15b79d17cde03313","rd":"2026-09-30","se":4.3,"sid":"recythOL2ifqR8nCk","sm":"gemini-4-argon_unknown","v":0.8240000000000001},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Sonnet 5.5 [claude-sonnet-5-5]","e":"max","m":"claude-sonnet-5-5","oid":"apex_agents-0620f5f547d6f4c8","rd":"2026-09-28","se":4.7,"sid":"recwsmBXqLEgJjkrE","sm":"claude-sonnet-5-5_max","v":0.755},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Opus 5.5 [claude-opus-5-5]","e":"max","m":"claude-opus-5-5","oid":"apex_agents-a4dd5eaceef99356","rd":"2026-09-22","se":4.9,"sid":"rec6TErVzXnuNRUoV","sm":"claude-opus-5-5_max","v":0.735},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"DeepSeek-V4.1-Flash-1017","e":"max","m":"deepseek-v4-1-flash-1017","oid":"apex_agents-5bf745058edf8d11","rd":"2026-10-17","se":4.9,"sid":"rec4h0kAzxLz00RFT","sm":"deepseek-v4.1-flash-1017_max","v":0.698},{"b":"apex_agents","dn":"Fable 5.1 [claude-fable-5.1]","m":"claude-fable-5-1","oid":"apex_agents-40ace82e6b3dfdaf","rd":"2026-09-01","se":4.9,"sid":"reciHu2R0cgIa9tz3","sm":"claude-fable-5-1_unknown","v":0.6859999999999999},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"GPT-6.1 Terra","e":"max","m":"gpt-6-1-terra","oid":"apex_agents-fcde2bd53bf3525e","rd":"2026-10-26","se":5.2,"sid":"recbZ5jYnCHQNhXed","sm":"gpt-6.1-terra_max","v":0.6829999999999999},{"b":"apex_agents","dn":"Gemini 3.7 Flash","m":"gemini-3-7-flash","oid":"apex_agents-40343911ea109fbc","rd":"2026-08-13","se":5.1,"sid":"recW38cKLjAptrWAQ","sm":"gemini-3.7-flash_unknown","v":0.6779999999999999},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Muse Wonder 1.0","e":"max","m":"muse-wonder-1-0","oid":"apex_agents-5848bd045a86052d","rd":"2026-10-12","se":5.3,"sid":"recJ1sZSqwFwfdlZ4","sm":"muse-wonder-1.0_max","v":0.6609999999999999},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Opus 5","e":"max","m":"claude-opus-5","oid":"apex_agents-f2c8a906f003dfd4","rd":"2026-07-24","se":5.1,"sid":"rec1AxRjtBjTtTCHM","sm":"claude-opus-5_max","v":0.658},{"b":"apex_agents","dn":"Grok 4.6","m":"grok-4-6","oid":"apex_agents-abdff807a07ee74a","rd":"2026-08-12","se":5.2,"sid":"recqbm6QR4O0FYZGg","sm":"grok-4.6_unknown","v":0.653},{"b":"apex_agents","dn":"GPT-6 Astra","m":"gpt-6-astra","oid":"apex_agents-999317947910f339","rd":"2026-09-03","se":5.6,"sid":"rec8tVCWwIBymY5yY","sm":"gpt-6-astra_unknown","v":0.647},{"b":"apex_agents","dn":"Gemini 3.8 Flash","m":"gemini-3-8-flash","oid":"apex_agents-8abc18a28da02091","rd":"2026-09-02","se":5.2,"sid":"recsEbJSODxwEhWql","sm":"gemini-3.8-flash_unknown","v":0.643},{"b":"apex_agents","dn":"Fable 5","m":"claude-fable-5","oid":"apex_agents-8260a228d7733b44","rd":"2026-06-09","se":5.3,"sid":"reckaD0wz9XOvArFA","sm":"claude-fable-5","v":0.636},{"b":"apex_agents","dn":"Qwen3.8-Max","m":"qwen3-8-max","oid":"apex_agents-4099ccf171ea2d48","rd":"2026-08-02","se":4.9,"sid":"reclUBETu0CNAWUZa","sm":"qwen3.8-max_unknown","v":0.633},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"GPT-6.1 Sol","e":"max","m":"gpt-6-1-sol","oid":"apex_agents-d799f075465f860c","rd":"2026-09-29","se":5.6,"sid":"reciOAOiuYHbbtCcU","sm":"gpt-6.1-sol_max","v":0.6},{"b":"apex_agents","cfg":{"effort":"high"},"dn":"Fable 5.1 (High)","e":"high","m":"claude-fable-5-1","oid":"apex_agents-03cd966ce63996ba","rd":"2026-09-01","se":5.6,"sid":"recNL6V1sB2zRy2vL","sm":"claude-fable-5-1_high","v":0.5970000000000001},{"b":"apex_agents","dn":"Muse Spark 1.3 [muse-spark-1-3]","m":"muse-spark-1-3","oid":"apex_agents-3b95c7c5a709e4a7","rd":"2026-09-02","se":5.5,"sid":"reczj4OgICoiocuwT","sm":"muse-spark-1.3_unknown","v":0.5860000000000001},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"GPT-5.6 Terra","e":"max","m":"gpt-5-6-terra","oid":"apex_agents-931b3c2689ff891a","rd":"2026-07-09","se":5.3,"sid":"recIl7G6eKw5AFuMv","sm":"gpt-5.6-terra_max","v":0.5820000000000001},{"b":"apex_agents","dn":"GLM-5.3","m":"glm-5-3","oid":"apex_agents-dd2eff15e9403dee","rd":"2026-08-14","se":5.1,"sid":"recXcmZluX6NVmdiN","sm":"glm-5.3_unknown","v":0.5660000000000001},{"b":"apex_agents","dn":"Grok 4.5","m":"grok-4-5","oid":"apex_agents-15d8a2b5b85fd2a8","rd":"2026-07-08","se":5.4,"sid":"rececsj9bbRs6VG4W","sm":"grok-4.5_unknown","v":0.562},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Haiku 5.5","e":"max","m":"claude-haiku-5-5","oid":"apex_agents-29d78b338ca3f8ad","rd":"2026-10-07","se":5.2,"sid":"recuIVFmW80Eh5kgM","sm":"claude-haiku-5-5_max","v":0.561},{"b":"apex_agents","dn":"GPT-5.5","m":"gpt-5-5","oid":"apex_agents-0b31eb1f4197565d","rd":"2026-04-23","se":5.5,"sid":"recAtyT31eNSS0yck","sm":"gpt-5.5_unknown","v":0.551},{"b":"apex_agents","dn":"Grok 4.7","m":"grok-4-7","oid":"apex_agents-8d53b228e3c9aee2","rd":"2026-09-21","se":5.1,"sid":"recMu2uHBeJH8IVe8","sm":"grok-4.7_unknown","v":0.546},{"b":"apex_agents","dn":"Sonnet 5","m":"claude-sonnet-5","oid":"apex_agents-e10c0125ef704989","rd":"2026-06-30","se":5.5,"sid":"recDITjv4rt440qU6","sm":"claude-sonnet-5_unknown","v":0.545},{"b":"apex_agents","dn":"MiMo V2.6 Pro","m":"mimo-v2-6-pro","oid":"apex_agents-8a5f57454e02d21a","rd":"2026-09-21","se":5.2,"sid":"recXC689WvlAiyNVG","sm":"mimo-v2.6-pro","v":0.544},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"GPT-6 Sol","e":"max","m":"gpt-6-sol","oid":"apex_agents-5e90ebc14cb16603","rd":"2026-09-22","se":5.6,"sid":"recLBjIuBBmwMbz6o","sm":"gpt-6-sol_max","v":0.543},{"b":"apex_agents","dn":"GLM-5.3-Flash","m":"glm-5-3-flash","oid":"apex_agents-b15d6784eebf6269","rd":"2026-08-20","se":5.3,"sid":"recoUnuvNXMuPUypv","sm":"glm-5.3-flash_unknown","v":0.528},{"b":"apex_agents","cfg":{"effort":"medium"},"dn":"Opus 5.5 [opus-5.5]","e":"medium","m":"claude-opus-5-5","oid":"apex_agents-2b5a3b2bd8913aa2","rd":"2026-09-22","se":5.6,"sid":"recjdN7zdajrdYE2h","sm":"claude-opus-5-5_medium","v":0.525},{"b":"apex_agents","dn":"GPT-5.4","m":"gpt-5-4-2026-03-05","oid":"apex_agents-22064d5325078c76","rd":"2026-03-05","se":5.3,"sid":"recRf8ng4aMSZ2eaV","sm":"gpt-5.4-2026-03-05_unknown","v":0.524},{"b":"apex_agents","cfg":{"effort":"high"},"dn":"Mistral Large 4","e":"high","m":"mistral-large-4-0","oid":"apex_agents-cb9350381f188790","rd":"2026-10-06","se":5.2,"sid":"recUDFi77FVGk9jHw","sm":"mistral-large-4-0_high","v":0.516},{"b":"apex_agents","cfg":{"effort":"promax"},"dn":"GPT-5.6 Sol","e":"promax","m":"gpt-5-6-sol","oid":"apex_agents-2a25c9a6b1ec4d4d","rd":"2026-07-09","se":5.5,"sid":"recG6d1VEmPidhR3T","sm":"gpt-5.6-sol_promax","v":0.514},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Opus 4.7","e":"max","m":"claude-opus-4-7","oid":"apex_agents-e2f514edf6c4a45b","rd":"2026-04-16","se":5.4,"sid":"recDYoVbtdEHFX5Vv","sm":"claude-opus-4-7_max","v":0.49200000000000005},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Opus 4.8","e":"max","m":"claude-opus-4-8","oid":"apex_agents-801a047973882177","rd":"2026-05-28","se":5.5,"sid":"reckFRpJRQsQ8Z4Sm","sm":"claude-opus-4-8_max","v":0.489},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Beam","e":"max","m":"beam-1-0","oid":"apex_agents-c45685bf1cfd3e4b","rd":"2026-10-05","se":5.1,"sid":"rechVxuQGmTJGrEhy","sm":"beam-1.0_max","v":0.488},{"b":"apex_agents","dn":"Kimi K3","m":"kimi-k3","oid":"apex_agents-edc5188947aa1521","rd":"2026-07-16","se":5.2,"sid":"rec69p8O5WoeewHzP","sm":"kimi-k3_unknown","v":0.478},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Muse Spark 1.3 (Max)","e":"max","m":"muse-spark-1-3","oid":"apex_agents-bcd3ecba849908aa","rd":"2026-09-02","se":5.7,"sid":"recXul8NIwqqTNIrA","sm":"muse-spark-1.3_max","v":0.47600000000000003},{"b":"apex_agents","dn":"Qwen 3.8 27B","m":"qwen3-8-27b","oid":"apex_agents-3c42965ffafc29c6","rd":"2026-08-14","se":4.9,"sid":"recfGdLMX8buaQjoL","sm":"qwen3.8-27b","v":0.47500000000000003},{"b":"apex_agents","dn":"DeepSeek-V4-Pro-0813","m":"deepseek-v4-pro-0813","oid":"apex_agents-53cba23651a231f4","rd":"2026-08-13","se":5.6,"sid":"recZWJ893wBDvmzR7","sm":"deepseek-v4-pro-0813_unknown","v":0.473},{"b":"apex_agents","dn":"Gemini 3.6 Flash","m":"gemini-3-6-flash","oid":"apex_agents-c59498b2158bec12","rd":"2026-07-21","se":5.2,"sid":"recrKr7XmbXXZO8qD","sm":"gemini-3.6-flash_unknown","v":0.469},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"Opus 4.6","e":"max","m":"claude-opus-4-6","oid":"apex_agents-b393fe77049a23cf","rd":"2026-02-05","se":5.5,"sid":"recfKa2RdTGFZh3gQ","sm":"claude-opus-4-6_max","v":0.46299999999999997},{"b":"apex_agents","dn":"Muse Spark 1.2","m":"muse-spark-1-2","oid":"apex_agents-adc36cc726cc3812","rd":"2026-08-05","se":5,"sid":"recu1kljfh6r4IOhx","sm":"muse-spark-1.2","v":0.454},{"b":"apex_agents","dn":"GLM-5.2","m":"glm-5-2","oid":"apex_agents-24eb2033a715a4a8","rd":"2026-06-16","se":5.7,"sid":"rec7GciBovlZrZmaF","sm":"glm-5.2_unknown","v":0.452},{"b":"apex_agents","cfg":{"effort":"medium"},"dn":"Sonnet 5.5 (Medium)","e":"medium","m":"claude-sonnet-5-5","oid":"apex_agents-9b8fe36c9b635329","rd":"2026-09-28","se":5.6,"sid":"recWY8vFxRRaVvJK3","sm":"claude-sonnet-5-5_medium","v":0.446},{"b":"apex_agents","cfg":{"effort":"max"},"dn":"GPT-6 Luna","e":"max","m":"gpt-6-luna","oid":"apex_agents-f12f330b442579de","rd":"2026-09-22","se":5.5,"sid":"rec1U23OkfwoEOepV","sm":"gpt-6-luna_max","v":0.443},{"b":"apex_agents","cfg":{"effort":"high"},"dn":"Sonnet 4.6","e":"high","m":"claude-sonnet-4-6","oid":"apex_agents-ce41ae9c012f8964","rd":"2026-02-17","se":5.5,"sid":"recNKXGqIBBG6Q12j","sm":"claude-sonnet-4-6_high","v":0.43},{"b":"apex_agents","dn":"GPT-5.6 Luna","m":"gpt-5-6-luna","oid":"apex_agents-13711e689bd01152","rd":"2026-07-09","se":5.4,"sid":"recjiuLSNycx2dtn8","sm":"gpt-5.6-luna_unknown","v":0.43},{"b":"apex_agents","dn":"GLM-5.1","m":"glm-5-1","oid":"apex_agents-2a67f7df63e9d403","rd":"2026-04-07","se":5.4,"sid":"recB78P8U9128Kwmw","sm":"glm-5.1","v":0.409},{"b":"apex_agents","dn":"DeepSeek-V4.1-Flash","m":"deepseek-v4-1-flash","oid":"apex_agents-34bfa458cb65a2a8","rd":"2026-09-09","se":4.9,"sid":"recnwdMkLlog5SyR5","sm":"deepseek-v4.1-flash_unknown","v":0.395},{"b":"apex_agents","dn":"MiniMax-M3","m":"minimax-m3","oid":"apex_agents-f505e0998c660844","rd":"2026-06-01","se":4.9,"sid":"recddMEKJx52F4GnA","sm":"MiniMax-M3","v":0.37700000000000006},{"b":"apex_agents","dn":"Kimi K2.7 Code","m":"kimi-k2-7-code","oid":"apex_agents-13ebbde0ac9348db","rd":"2026-06-12","se":4.8,"sid":"recc9OBbGet6i61HC","sm":"kimi-k2.7-code","v":0.376},{"b":"apex_agents","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"apex_agents-529377f6ed1d9077","rd":"2026-02-19","se":5,"sid":"recPuGr6uc4AJrsxK","sm":"gemini-3.1-pro-preview","v":0.35100000000000003},{"b":"apex_agents","dn":"Inkling","m":"inkling","oid":"apex_agents-7f69701ce6a02fec","rd":"2026-07-15","se":4.9,"sid":"recc5ZC9fNHvAvhao","sm":"Inkling","v":0.33799999999999997},{"b":"apex_agents","dn":"Muse Spark Open","m":"muse-spark-open-816b-a42b","oid":"apex_agents-9fca58a8fde7763a","rd":"2026-10-15","se":5.3,"sid":"recMOZJL02wkyStrJ","sm":"Muse-Spark-Open-816B-A42B_unknown","v":0.334},{"b":"apex_agents","dn":"Muse Spark 1.1","m":"muse-spark-1-1","oid":"apex_agents-1b9e2c8064548055","rd":"2026-07-09","se":5.1,"sid":"recf1h0VzvdEIHDhj","sm":"muse-spark-1.1","v":0.318},{"b":"apex_agents","dn":"Gemini 3.5 Flash Lite","m":"gemini-3-5-flash-lite","oid":"apex_agents-756cce7d81b355f0","rd":"2026-07-21","se":4.9,"sid":"recWGYASTIVksDKsa","sm":"gemini-3.5-flash-lite","v":0.29300000000000004},{"b":"apex_agents","dn":"Gemini 3.5 Flash","m":"gemini-3-5-flash","oid":"apex_agents-99f256b3bcf06c8a","rd":"2026-05-19","se":5,"sid":"rec1aJTfBd33EL5vQ","sm":"gemini-3.5-flash_unknown","v":0.275},{"b":"apex_agents","dn":"Qwen3.5","m":"qwen3-5-397b-a17b","oid":"apex_agents-cb08aad3fd5dfa35","rd":"2026-02-13","se":4.7,"sid":"recooiGyFYUY9K58B","sm":"qwen3.5-397b-a17b","v":0.249},{"b":"apex_agents","dn":"Nemotron 3 Ultra","m":"nemotron-3-ultra","oid":"apex_agents-131554287519938c","rd":"2026-06-04","se":4.5,"sid":"recHN2QJQtb3Ee8Ir","sm":"nemotron-3-ultra","v":0.227},{"b":"apex_agents","dn":"DeepSeek-V3.2","m":"deepseekdeepseek-v3-2","oid":"apex_agents-c4751389ee2c0149","rd":"2025-12-01","se":4.1,"sid":"rec38Wcf41rxdUK44","sm":"deepseek/deepseek-v3.2","v":0.21300000000000002},{"b":"apex_agents","dn":"GPT-OSS-120B","m":"gpt-oss-120b","oid":"apex_agents-8dc6ca29e4566507","rd":"2025-08-05","se":2,"sid":"rec9zkde8jhxtGWhs","sm":"gpt-oss-120b","v":0.044000000000000004},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"GPT-6 Astra (Max)","e":"max","m":"gpt-6-astra","oid":"arc_agi_2-1589e4caad290ff9","rd":"2026-09-03","sid":"recqWVwwNamfxF1zu","sm":"gpt-6-astra_max","v":0.9500000000000001},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"GPT-6.1 Sol (Max)","e":"max","m":"gpt-6-1-sol","oid":"arc_agi_2-d21e817e906f9b95","rd":"2026-09-29","sid":"recEn87cPCL4zBDX9","sm":"gpt-6.1-sol_max","v":0.9416666666666668},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-6 Astra (XHigh)","e":"xhigh","m":"gpt-6-astra","oid":"arc_agi_2-fbe1b008c68aeebb","rd":"2026-09-03","sid":"recuO4X9wY6J0GPRP","sm":"gpt-6-astra_xhigh","v":0.9333333333333333},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Opus 5.5 (High)","e":"high","m":"claude-opus-5-5","oid":"arc_agi_2-96d2bd19fae7b3fb","rd":"2026-09-22","sid":"rec0XmjLEfbQPNFj3","sm":"claude-opus-5-5_high","v":0.9333333333333333},{"b":"arc_agi_2","c":1.44,"cfg":{"effort":"max"},"dn":"GPT-5.6 Sol (Max)","e":"max","m":"gpt-5-6-sol","oid":"arc_agi_2-c1d7a506983ceee2","rd":"2026-07-09","sid":"recIQjZYogw8GYgLW","sm":"gpt-5.6-sol_max","v":0.925},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Claude Opus 5.5 (XHigh)","e":"xhigh","m":"claude-opus-5-5","oid":"arc_agi_2-5d5015df65670858","rd":"2026-09-22","sid":"recjdCxN2bD3djTOO","sm":"claude-opus-5-5_xhigh","v":0.925},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"GPT-6 Astra (High)","e":"high","m":"gpt-6-astra","oid":"arc_agi_2-7a935a166987ba22","rd":"2026-09-03","sid":"rec2IrmPOhpgWZ3Zl","sm":"gpt-6-astra_high","v":0.9208333333333333},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"GPT-6 Astra (Medium)","e":"medium","m":"gpt-6-astra","oid":"arc_agi_2-e3a54ad9d2fc1962","rd":"2026-09-03","sid":"recQLii7vQCYRgYaO","sm":"gpt-6-astra_medium","v":0.9208333333333333},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude Opus 5.5 (Max)","e":"max","m":"claude-opus-5-5","oid":"arc_agi_2-7332df34f735990a","rd":"2026-09-22","sid":"reciGvWCfQIQzEmnO","sm":"claude-opus-5-5_max","v":0.9166666666666666},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-6.1 Sol (XHigh)","e":"xhigh","m":"gpt-6-1-sol","oid":"arc_agi_2-00d01a9a41568bc0","rd":"2026-09-29","sid":"recA2qz6EjG8Yd9O7","sm":"gpt-6.1-sol_xhigh","v":0.9166666666666666},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"GPT-6.1 Sol (High)","e":"high","m":"gpt-6-1-sol","oid":"arc_agi_2-65913b4db309e939","rd":"2026-09-29","sid":"recPqdeFjOGTEXsBg","sm":"gpt-6.1-sol_high","v":0.9166666666666666},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 4 Argon (High)","e":"high","m":"gemini-4-argon","oid":"arc_agi_2-c6ebb256abdd7c0e","rd":"2026-09-30","sid":"rec9xEpWuQctZx5Yy","sm":"gemini-4-argon_high","v":0.9125},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude Opus 5 (Max)","e":"max","m":"claude-opus-5","oid":"arc_agi_2-f084aaaaf587133b","rd":"2026-07-24","sid":"recRyRMLEzUUt9Acr","sm":"claude-opus-5_max","v":0.9042},{"b":"arc_agi_2","c":1.04,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol (xHigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"arc_agi_2-691632cfb66c5027","rd":"2026-07-09","sid":"recjEDHJMdFvdEoxY","sm":"gpt-5.6-sol_xhigh","v":0.9},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol (XHigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"arc_agi_2-95d3777082deb363","rd":"2026-07-09","sid":"rechqnvfszeSZZcTx","sm":"gpt-5.6-sol_xhigh","v":0.9},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude Fable 5.1 (Max)","e":"max","m":"claude-fable-5-1","oid":"arc_agi_2-00e74096dfb51a8b","rd":"2026-09-01","sid":"recI68BcNmOFzBN1F","sm":"claude-fable-5-1_max","v":0.9},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Claude Fable 5.1 (XHigh)","e":"xhigh","m":"claude-fable-5-1","oid":"arc_agi_2-c22064b30a028d83","rd":"2026-09-01","sid":"recZtv97M9zVWp43I","sm":"claude-fable-5-1_xhigh","v":0.9},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"GPT-6 Sol (Max)","e":"max","m":"gpt-6-sol","oid":"arc_agi_2-e0480933a394c87d","rd":"2026-09-22","sid":"reclgf5ruPfGJvCNf","sm":"gpt-6-sol_max","v":0.8958333333333335},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude Fable 5 (Max)","e":"max","m":"claude-fable-5","oid":"arc_agi_2-99e2a12611212de4","rd":"2026-06-09","sid":"recu2ZdK004X3AWeO","sm":"claude-fable-5_max","v":0.8916666666666667},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 3.8 Flash (High)","e":"high","m":"gemini-3-8-flash","oid":"arc_agi_2-8db85ee9205fd296","rd":"2026-09-02","sid":"recEpF6e453CCZJ72","sm":"gemini-3.8-flash_high","v":0.8916666666666667},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Fable 5.1 (High)","e":"high","m":"claude-fable-5-1","oid":"arc_agi_2-0e11045b34ff61e2","rd":"2026-09-01","sid":"recHs8jTkcWuXJSC9","sm":"claude-fable-5-1_high","v":0.8875000000000001},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Claude Fable 5 (XHigh)","e":"xhigh","m":"claude-fable-5","oid":"arc_agi_2-89ee4dd1e2d05d6e","rd":"2026-06-09","sid":"rectyLB4NFarEdw0u","sm":"claude-fable-5_xhigh","v":0.8833333333333333},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude Opus 5 (High)","e":"max","m":"claude-opus-5","oid":"arc_agi_2-59f04504ef0c0a5a","rd":"2026-07-24","sid":"recKlOZaCgYdWkVtG","sm":"claude-opus-5_max","v":0.8833},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Opus 5 (High)","e":"high","m":"claude-opus-5","oid":"arc_agi_2-3b2caeab13a3a327","rd":"2026-07-24","sid":"recB3Hc7Wjn7FZ2vt","sm":"claude-opus-5_high","v":0.8833},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Fable 5 (High)","e":"high","m":"claude-fable-5","oid":"arc_agi_2-f66332235d49749f","rd":"2026-06-09","sid":"recoGNnJQf5s62aEC","sm":"claude-fable-5_high","v":0.875},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Claude Opus 5.5 (Medium)","e":"medium","m":"claude-opus-5-5","oid":"arc_agi_2-66c4ab89877768b1","rd":"2026-09-22","sid":"recPthl3n22wvWvIU","sm":"claude-opus-5-5_medium","v":0.875},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 4 Argon (Medium)","e":"medium","m":"gemini-4-argon","oid":"arc_agi_2-5fba513034591e99","rd":"2026-09-30","sid":"recOA2kl6GfLzAubk","sm":"gemini-4-argon_medium","v":0.8708333333333333},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"GPT-6.1 Sol (Medium)","e":"medium","m":"gpt-6-1-sol","oid":"arc_agi_2-be1ab9366357c255","rd":"2026-09-29","sid":"recYSQ5ByKpGp0LKP","sm":"gpt-6.1-sol_medium","v":0.8666666666666667},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Claude Fable 5.1 (Medium)","e":"medium","m":"claude-fable-5-1","oid":"arc_agi_2-04a4172a2ff1e1e4","rd":"2026-09-01","sid":"recT6vtsKxK14oaiK","sm":"claude-fable-5-1_medium","v":0.8625},{"b":"arc_agi_2","c":0.74,"cfg":{"effort":"high"},"dn":"GPT-5.6 Sol (High)","e":"high","m":"gpt-5-6-sol","oid":"arc_agi_2-87a0c1b69333d64f","rd":"2026-07-09","sid":"recLS6d8WS2n1rGSY","sm":"gpt-5.6-sol_high","v":0.8542000000000001},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"GPT-6 Astra (Low)","e":"low","m":"gpt-6-astra","oid":"arc_agi_2-fd22e52812dea408","rd":"2026-09-03","sid":"recG9zf5LfCTMpwnb","sm":"gpt-6-astra_low","v":0.8541666666666666},{"b":"arc_agi_2","c":1.87,"cfg":{"effort":"xhigh"},"dn":"GPT-5.5 (xHigh)","e":"xhigh","m":"gpt-5-5","oid":"arc_agi_2-59fc45d43e7b50a2","rd":"2026-04-23","sid":"recPMkvFz7bdsGYig","sm":"gpt-5.5_xhigh","v":0.85},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 (XHigh)","e":"xhigh","m":"gpt-5-5","oid":"arc_agi_2-803fb312d5ba015f","rd":"2026-04-23","sid":"recQa67QhUbEC1Z8v","sm":"gpt-5.5_xhigh","v":0.85},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 3.7 Flash (High)","e":"high","m":"gemini-3-7-flash","oid":"arc_agi_2-f1d25b5e73b44acc","rd":"2026-08-13","sid":"reczv0UYeDcnI7XYG","sm":"gemini-3.7-flash_high","v":0.8458333333333333},{"b":"arc_agi_2","dn":"Gemini 3 Deep Think (2/26)","m":"gemini-3-deep-think-preview","oid":"arc_agi_2-3e7154e683bd408c","rd":"2026-02-12","sid":"recMlHyQpZ45OKxQ0","sm":"gemini-3-deep-think-preview","v":0.8458},{"b":"arc_agi_2","c":10.51,"cfg":{"effort":"high"},"dn":"GPT-5.5 Pro (High)","e":"high","m":"gpt-5-5-pro","oid":"arc_agi_2-7a601e04333a680d","rd":"2026-04-23","sid":"recE8W46kbiuOPGxo","sm":"gpt-5.5-pro_high","v":0.8458},{"b":"arc_agi_2","c":10.76,"cfg":{"effort":"xhigh"},"dn":"GPT-5.5 Pro (xHigh)","e":"xhigh","m":"gpt-5-5-pro","oid":"arc_agi_2-1f4ff55ccacc6cd0","rd":"2026-04-23","sid":"recgSWEl9GTw04NMC","sm":"gpt-5.5-pro_xhigh","v":0.8416},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 Pro (XHigh)","e":"xhigh","m":"gpt-5-5-pro","oid":"arc_agi_2-5a19b48e17d5bbb0","rd":"2026-04-23","sid":"recCsqWAVFHQVM3mx","sm":"gpt-5.5-pro_xhigh","v":0.8416},{"b":"arc_agi_2","c":1.09,"cfg":{"effort":"max"},"dn":"GPT-5.6 Terra (Max)","e":"max","m":"gpt-5-6-terra","oid":"arc_agi_2-ad3bc02c9b0fecbe","rd":"2026-07-09","sid":"rec6orAgSUYUy5B5x","sm":"gpt-5.6-terra_max","v":0.839},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Claude Sonnet 5.5 (XHigh)","e":"xhigh","m":"claude-sonnet-5-5","oid":"arc_agi_2-4c8697eeb032a3d8","rd":"2026-09-28","sid":"reclwthLdzhvIpMOa","sm":"claude-sonnet-5-5_xhigh","v":0.8388888888888889},{"b":"arc_agi_2","c":16.41,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Pro (xHigh)","e":"xhigh","m":"gpt-5-4-pro-2026-03-05","oid":"arc_agi_2-7bee7658917429ef","rd":"2026-03-05","sid":"recQTn93SRyeXe4j7","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.8333},{"b":"arc_agi_2","c":1.45,"cfg":{"effort":"high"},"dn":"GPT-5.5 (High)","e":"high","m":"gpt-5-5","oid":"arc_agi_2-13adce895dadc324","rd":"2026-04-23","sid":"reclzZXNkcRC5jiKS","sm":"gpt-5.5_high","v":0.8333},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Pro (XHigh)","e":"xhigh","m":"gpt-5-4-pro-2026-03-05","oid":"arc_agi_2-0851211807a7fb52","rd":"2026-03-05","sid":"recoggJGwWvf6rzzm","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.8333},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 3.8 Flash (Medium)","e":"medium","m":"gemini-3-8-flash","oid":"arc_agi_2-733607667b48e3e1","rd":"2026-09-02","sid":"recSQW22neBgoxnk8","sm":"gemini-3.8-flash_medium","v":0.8291666666666667},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Claude Fable 5 (Medium)","e":"medium","m":"claude-fable-5","oid":"arc_agi_2-9ad80abfa8b57fc4","rd":"2026-06-09","sid":"recOj0wdwD2kU9r8l","sm":"claude-fable-5_medium","v":0.8250000000000001},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Claude Fable 5.1 (Low)","e":"low","m":"claude-fable-5-1","oid":"arc_agi_2-4c8611d93d02f6a9","rd":"2026-09-01","sid":"recgGXbNz9PP5hyy2","sm":"claude-fable-5-1_low","v":0.7833333333333333},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-6 Sol (XHigh)","e":"xhigh","m":"gpt-6-sol","oid":"arc_agi_2-2aa7f047e5cb8460","rd":"2026-09-22","sid":"recaxpOe7WYYOhEHv","sm":"gpt-6-sol_xhigh","v":0.7805555555555556},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash 1017 (Max)","e":"max","m":"deepseek-v4-1-flash-1017","oid":"arc_agi_2-91dfa2cdb58429ad","rd":"2026-10-17","sid":"rec9NCKuywR6qBoJK","sm":"deepseek-v4.1-flash-1017_max","v":0.7805555555555556},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 3.8 Flash (Low)","e":"low","m":"gemini-3-8-flash","oid":"arc_agi_2-5f61ecf280f5f735","rd":"2026-09-02","sid":"rec2uPyplc1lZ8CVH","sm":"gemini-3.8-flash_low","v":0.775},{"b":"arc_agi_2","c":0.9622,"dn":"Gemini 3.1 Pro (Preview","m":"gemini-3-1-pro-preview","oid":"arc_agi_2-e22aab2302318e29","rd":"2026-02-19","sid":"recqiypjdc93Zi185","sm":"gemini-3.1-pro-preview","v":0.771},{"b":"arc_agi_2","c":0.9622,"dn":"Gemini 3.1 Pro (Preview)","m":"gemini-3-1-pro-preview","oid":"arc_agi_2-861feb2541151bc1","rd":"2026-02-19","sid":"recbE2tiH9D8MeBaF","sm":"gemini-3.1-pro-preview","v":0.7708},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Claude Fable 5 (Low)","e":"low","m":"claude-fable-5","oid":"arc_agi_2-ce9ff8996130149a","rd":"2026-06-09","sid":"rech4ukMMdk3W0Q5a","sm":"claude-fable-5_low","v":0.7680555555555555},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"GPT-6.1 Sol (Low)","e":"low","m":"gpt-6-1-sol","oid":"arc_agi_2-a10433d456c0aaec","rd":"2026-09-29","sid":"rec1jlUUUwswKuq62","sm":"gpt-6.1-sol_low","v":0.7666666666666667},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"Claude 4.7 (Max)","e":"max","m":"claude-opus-4-7","oid":"arc_agi_2-7b53172e78383125","rd":"2026-04-16","sid":"rectnLtWG36YhOCLr","sm":"claude-opus-4-7_max","v":0.7583},{"b":"arc_agi_2","c":7.43,"cfg":{"effort":"max"},"dn":"Claude Opus 4.7 (Max)","e":"max","m":"claude-opus-4-7","oid":"arc_agi_2-282084994e8d07d6","rd":"2026-04-16","sid":"reckzAUlEOn9DIazK","sm":"claude-opus-4-7_max","v":0.758},{"b":"arc_agi_2","c":0.69,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra (xHigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"arc_agi_2-d00cd0654e7ea9e1","rd":"2026-07-09","sid":"rec86VcvrMgRUOXNK","sm":"gpt-5.6-terra_xhigh","v":0.7417},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra (XHigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"arc_agi_2-e3e26d3624765459","rd":"2026-07-09","sid":"recU58jwAC2hZugaR","sm":"gpt-5.6-terra_xhigh","v":0.7417},{"b":"arc_agi_2","c":1.52,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (xHigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"arc_agi_2-f0d08756700f59d9","rd":"2026-03-05","sid":"receOCJkifwiw4KJD","sm":"gpt-5.4-2026-03-05_xhigh","v":0.7395},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (XHigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"arc_agi_2-e7897858b71ebbee","rd":"2026-03-05","sid":"recfpKIv4dsidnQCJ","sm":"gpt-5.4-2026-03-05_xhigh","v":0.7395},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"DeepSeek V4.1 Flash 1017 (Low)","e":"low","m":"deepseek-v4-1-flash-1017","oid":"arc_agi_2-4178d9b10c204565","rd":"2026-10-17","sid":"recD4KDkVs2d7BZCu","sm":"deepseek-v4.1-flash-1017_low","v":0.7361111111111112},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash (Max)","e":"max","m":"deepseek-v4-1-flash","oid":"arc_agi_2-7be7da37787254a7","rd":"2026-09-09","sid":"rec8JvXW85spVMfgN","sm":"deepseek-v4.1-flash_max","v":0.7291666666666666},{"b":"arc_agi_2","c":0.8503,"cfg":{"effort":"high"},"dn":"Gemini 3.5 Flash (High)","e":"high","m":"gemini-3-5-flash","oid":"arc_agi_2-85f2c89cb650cab1","rd":"2026-05-19","sid":"recTdKZ1IRjGsKQDv","sm":"gemini-3.5-flash_high","v":0.7208},{"b":"arc_agi_2","c":2.7425,"cfg":{"effort":"high"},"dn":"Claude Opus 4.8 (High)","e":"high","m":"claude-opus-4-8","oid":"arc_agi_2-29428c7d36c48e5e","rd":"2026-05-28","sid":"recyZR7wTddhJC0iQ","sm":"claude-opus-4-8_high","v":0.7208},{"b":"arc_agi_2","c":2.3896,"cfg":{"effort":"medium"},"dn":"Claude Opus 4.8 (Medium)","e":"medium","m":"claude-opus-4-8","oid":"arc_agi_2-627d193c786119f8","rd":"2026-05-28","sid":"recmL22XFCsheVQDp","sm":"claude-opus-4-8_medium","v":0.7167},{"b":"arc_agi_2","c":0.86,"cfg":{"effort":"medium"},"dn":"GPT-5.5 (Medium)","e":"medium","m":"gpt-5-5","oid":"arc_agi_2-264962adf46daf2e","rd":"2026-04-23","sid":"recdw1w2Kk0j96ky7","sm":"gpt-5.5_medium","v":0.7042},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Claude Opus 5.5 (Low)","e":"low","m":"claude-opus-5-5","oid":"arc_agi_2-a8d98e87a61a1d44","rd":"2026-09-22","sid":"recxXUOv2ybbXF8OX","sm":"claude-opus-5-5_low","v":0.7013888888888891},{"b":"arc_agi_2","dn":"Claude Opus 4.6 (120K, High)","m":"claude-opus-4-6-120k","oid":"arc_agi_2-5eeda326051cecb8","rd":"2026-02-05","sid":"recudCqZO2yzH6CbV","sm":"claude-opus-4-6_120K","v":0.6917},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"GPT-6 Sol (High)","e":"high","m":"gpt-6-sol","oid":"arc_agi_2-cd5ba392ff5656c5","rd":"2026-09-22","sid":"recd2qURgAkummpTb","sm":"gpt-6-sol_high","v":0.6888888888888888},{"b":"arc_agi_2","dn":"Claude Opus 4.6 (120K, Max)","m":"claude-opus-4-6-120k","oid":"arc_agi_2-3219825b712f2ded","rd":"2026-02-05","sid":"recZU5GRhR5s7GfZQ","sm":"claude-opus-4-6_120K","v":0.6875},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"DeepSeek V4.1 Flash 1017 (High)","e":"high","m":"deepseek-v4-1-flash-1017","oid":"arc_agi_2-589c2c8e8be86373","rd":"2026-10-17","sid":"recY4v2mzQ9GJNQlK","sm":"deepseek-v4.1-flash-1017_high","v":0.6861111111111113},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude 4.7 (High)","e":"high","m":"claude-opus-4-7","oid":"arc_agi_2-a5e0652da78e759a","rd":"2026-04-16","sid":"recvk5AeE5Go58i7M","sm":"claude-opus-4-7_high","v":0.6833},{"b":"arc_agi_2","c":3.17,"cfg":{"effort":"high"},"dn":"Claude Opus 4.7 (High)","e":"high","m":"claude-opus-4-7","oid":"arc_agi_2-05a9260fdeccee82","rd":"2026-04-16","sid":"recPv91tJnBXcqcnw","sm":"claude-opus-4-7_high","v":0.683},{"b":"arc_agi_2","c":1.02,"cfg":{"effort":"high"},"dn":"GPT-5.4 (High)","e":"high","m":"gpt-5-4-2026-03-05","oid":"arc_agi_2-5a39a23aa94e67e1","rd":"2026-03-05","sid":"recSSPRQPLhYElikJ","sm":"gpt-5.4-2026-03-05_high","v":0.675},{"b":"arc_agi_2","c":2.96,"cfg":{"effort":"medium"},"dn":"Claude 4.7 (Medium)","e":"medium","m":"claude-opus-4-7","oid":"arc_agi_2-e4d7edb2d8bab4d5","rd":"2026-04-16","sid":"recCucGVHjJSEdw24","sm":"claude-opus-4-7_medium","v":0.675},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"DeepSeek V4.1 Flash (High)","e":"high","m":"deepseek-v4-1-flash","oid":"arc_agi_2-b1ce8f0ec44ccd94","rd":"2026-09-09","sid":"rechNq43hqFRhQO5y","sm":"deepseek-v4.1-flash_high","v":0.675},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Sonnet 5.5 (High)","e":"high","m":"claude-sonnet-5-5","oid":"arc_agi_2-ef83bc99c699d49b","rd":"2026-09-28","sid":"recKZkTgAwN7tZDPL","sm":"claude-sonnet-5-5_high","v":0.6736111111111112},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Grok 4.6 (XHigh)","e":"xhigh","m":"grok-4-6","oid":"arc_agi_2-04f804e294ec9d95","rd":"2026-08-12","sid":"recnfCzGM2VEgFNR4","sm":"grok-4.6_xhigh","v":0.6708333333333333},{"b":"arc_agi_2","c":0.47,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Sol (Medium)","e":"medium","m":"gpt-5-6-sol","oid":"arc_agi_2-61be15dec4753264","rd":"2026-07-09","sid":"rec8lDUeLifBHj8WC","sm":"gpt-5.6-sol_medium","v":0.6708},{"b":"arc_agi_2","c":0.55,"cfg":{"effort":"high"},"dn":"GPT-5.6 Terra (High)","e":"high","m":"gpt-5-6-terra","oid":"arc_agi_2-80eb95fa25365c92","rd":"2026-07-09","sid":"recxzIevuyQumJson","sm":"gpt-5.6-terra_high","v":0.6708},{"b":"arc_agi_2","dn":"Claude Opus 4.6 (120K, Medium)","m":"claude-opus-4-6-120k","oid":"arc_agi_2-62742ab8f740cae6","rd":"2026-02-05","sid":"recsGnPYUYFhuwrgg","sm":"claude-opus-4-6_120K","v":0.6625},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"GLM-5.3-Flash (Max)","e":"max","m":"glm-5-3-flash","oid":"arc_agi_2-59907a5a8006d3c1","rd":"2026-08-20","sid":"recY6FdDmud74DNlv","sm":"glm-5.3-flash_max","v":0.6583333333333333},{"b":"arc_agi_2","c":0.92,"dn":"Grok 4.20 (Reasoning)","m":"grok-4-20","oid":"arc_agi_2-c6a69f7cc377dfef","rd":"2026-02-17","sid":"rec0q8XRFqTb9j85f","sm":"grok-4-20","v":0.6514},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Grok 4.6 (High)","e":"high","m":"grok-4-6","oid":"arc_agi_2-6f8ee177527718f7","rd":"2026-08-12","sid":"recN0KfJ7ZaHfvhAX","sm":"grok-4.6_high","v":0.6513888888888889},{"b":"arc_agi_2","dn":"Claude Opus 4.6 (120K, Low)","m":"claude-opus-4-6-120k","oid":"arc_agi_2-78dce3e5e9bacaf9","rd":"2026-02-05","sid":"rec0H4L8zJfLACTG4","sm":"claude-opus-4-6_120K","v":0.6458},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 3.7 Flash (Medium)","e":"medium","m":"gemini-3-7-flash","oid":"arc_agi_2-497788ae7be24a9f","rd":"2026-08-13","sid":"rec2wVAL92Ucyu4qv","sm":"gemini-3.7-flash_medium","v":0.6375},{"b":"arc_agi_2","c":1.6818,"cfg":{"effort":"low"},"dn":"Claude Opus 4.8 (Low)","e":"low","m":"claude-opus-4-8","oid":"arc_agi_2-039a97746975d8c2","rd":"2026-05-28","sid":"recs1YmhMRL5aoCXZ","sm":"claude-opus-4-8_low","v":0.6222},{"b":"arc_agi_2","c":2.38,"cfg":{"effort":"low"},"dn":"Claude 4.7 (Low)","e":"low","m":"claude-opus-4-7","oid":"arc_agi_2-41433323e0921bd2","rd":"2026-04-16","sid":"recyXuHnrwdkJwc9k","sm":"claude-opus-4-7_low","v":0.6208},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"DeepSeek V4 Flash 0731 (Max)","e":"max","m":"deepseek-v4-flash-0731","oid":"arc_agi_2-074a2d848b87acf8","rd":"2026-07-31","sid":"recqcmcc7Kh55NAHM","sm":"deepseek-v4-flash-0731_max","v":0.6138888888888888},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Grok 4.7 (XHigh)","e":"xhigh","m":"grok-4-7","oid":"arc_agi_2-3619006da50a7bef","rd":"2026-09-21","sid":"recmQfENMjU9yADbb","sm":"grok-4.7_xhigh","v":0.6138888888888888},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Grok 4.6 (Medium)","e":"medium","m":"grok-4-6","oid":"arc_agi_2-08a9095c19a48ed5","rd":"2026-08-12","sid":"reckZ7gdV3Ubjjm8X","sm":"grok-4.6_medium","v":0.6125},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"DeepSeek V4 Pro 0813 (Max)","e":"max","m":"deepseek-v4-pro-0813","oid":"arc_agi_2-b314d9a8137f08d7","rd":"2026-08-13","sid":"recqVgMSHWE1dbgLk","sm":"deepseek-v4-pro-0813_max","v":0.6125},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Claude Haiku 5.5 (XHigh)","e":"xhigh","m":"claude-haiku-5-5","oid":"arc_agi_2-12f6ebda723815be","rd":"2026-10-07","sid":"recpECnHyLpmC6g0G","sm":"claude-haiku-5-5_xhigh","v":0.6125},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"DeepSeek V4.1 Flash (Low)","e":"low","m":"deepseek-v4-1-flash","oid":"arc_agi_2-25ac8a7c77a30208","rd":"2026-09-09","sid":"recVrJ0ki4HapKQCh","sm":"deepseek-v4.1-flash_low","v":0.6055555555555555},{"b":"arc_agi_2","c":2.6966,"cfg":{"effort":"high"},"dn":"Claude Sonnet 4.6 (High)","e":"high","m":"claude-sonnet-4-6","oid":"arc_agi_2-e9a07616f4065b4a","rd":"2026-02-17","sid":"recIlUapCqVFHs6qj","sm":"claude-sonnet-4-6_high","v":0.6042},{"b":"arc_agi_2","c":1.5926940250000003,"cfg":{"effort":"max"},"dn":"Kimi K3 (Max)","e":"max","m":"kimi-k3","oid":"arc_agi_2-4607f4a08426e979","rd":"2026-07-16","sid":"rec4wHjT7MjgYLj9Q","sm":"kimi-k3_max","v":0.6041666666666666},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 3.6 Flash (High)","e":"high","m":"gemini-3-6-flash","oid":"arc_agi_2-54e2adb51b86fc54","rd":"2026-07-21","sid":"recbJYchGoqumGxjl","sm":"gemini-3.6-flash_high","v":0.6041666666666666},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"DeepSeek V4 Pro 0813 (High)","e":"high","m":"deepseek-v4-pro-0813","oid":"arc_agi_2-7d10671a78b07068","rd":"2026-08-13","sid":"recOlpjOWSD4HNdzV","sm":"deepseek-v4-pro-0813_high","v":0.5972222222222221},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"GPT-6 Astra (None)","e":"none","m":"gpt-6-astra","oid":"arc_agi_2-4036ff19880a10ad","rd":"2026-09-03","sid":"recrfBvJzhO8K1XGJ","sm":"gpt-6-astra_none","v":0.5958333333333333},{"b":"arc_agi_2","c":0.67,"cfg":{"effort":"max"},"dn":"GPT-5.6 Luna (Max)","e":"max","m":"gpt-5-6-luna","oid":"arc_agi_2-2fba80bfeba8e10f","rd":"2026-07-09","sid":"rec0Or1yRuToQETpO","sm":"gpt-5.6-luna_max","v":0.5954},{"b":"arc_agi_2","cfg":{"effort":"max"},"dn":"GPT-6 Luna (Max)","e":"max","m":"gpt-6-luna","oid":"arc_agi_2-b86feed2b7e06c22","rd":"2026-09-22","sid":"rec1kIxgw33n3cMDe","sm":"gpt-6-luna_max","v":0.5930555555555556},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Grok 4.7 (Medium)","e":"medium","m":"grok-4-7","oid":"arc_agi_2-c750cd402fcaf4c6","rd":"2026-09-21","sid":"recIqQeuB67uhmyqw","sm":"grok-4.7_medium","v":0.5875},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Grok 4.7 (High)","e":"high","m":"grok-4-7","oid":"arc_agi_2-60f5a1c2e56b90d5","rd":"2026-09-21","sid":"rec75TfdEwYfJHmkf","sm":"grok-4.7_high","v":0.5833333333333334},{"b":"arc_agi_2","c":2.7179,"cfg":{"effort":"max"},"dn":"Claude Sonnet 4.6 (Max)","e":"max","m":"claude-sonnet-4-6","oid":"arc_agi_2-361ecdb1391e4ad9","rd":"2026-02-17","sid":"reco6ICSvrm675Zqs","sm":"claude-sonnet-4-6_max","v":0.5833},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"GPT-6 Sol (Medium)","e":"medium","m":"gpt-6-sol","oid":"arc_agi_2-2d779a96a0ae053d","rd":"2026-09-22","sid":"recAivokOQtVjz9ub","sm":"gpt-6-sol_medium","v":0.5777777777777778},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Muse Spark Open (XHigh)","e":"xhigh","m":"muse-spark-open-816b-a42b","oid":"arc_agi_2-e2504897f09c0378","rd":"2026-10-15","sid":"receKD8XFzWvRgbbE","sm":"Muse-Spark-Open-816B-A42B_xhigh","v":0.575},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"DeepSeek V4 Pro 0813 (Low)","e":"low","m":"deepseek-v4-pro-0813","oid":"arc_agi_2-93c7b9d701f5d11d","rd":"2026-08-13","sid":"recnvqDqZzZ6fLyzg","sm":"deepseek-v4-pro-0813_low","v":0.5625},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"DeepSeek V4 Flash 0731 (High)","e":"high","m":"deepseek-v4-flash-0731","oid":"arc_agi_2-f0d4664571bbeb2f","rd":"2026-07-31","sid":"rec8CN4idKsciI46Q","sm":"deepseek-v4-flash-0731_high","v":0.5597222222222221},{"b":"arc_agi_2","c":0.68,"cfg":{"effort":"medium"},"dn":"GPT-5.4 (Medium)","e":"medium","m":"gpt-5-4-2026-03-05","oid":"arc_agi_2-0cf3d2caf8a55dcf","rd":"2026-03-05","sid":"recMec1xlzVYbvboI","sm":"gpt-5.4-2026-03-05_medium","v":0.5542},{"b":"arc_agi_2","c":0.9472229249999998,"cfg":{"effort":"high"},"dn":"Kimi K3 (High)","e":"high","m":"kimi-k3","oid":"arc_agi_2-76034d0e214ff272","rd":"2026-07-16","sid":"receUA2QktYdPoRHt","sm":"kimi-k3_high","v":0.55},{"b":"arc_agi_2","c":15.721,"cfg":{"effort":"high"},"dn":"GPT-5.2 Pro (High)","e":"high","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi_2-3706626ddea26dcd","rd":"2025-12-11","sid":"recC6V59wjvT4mX12","sm":"gpt-5.2-pro-2025-12-11_high","v":0.5416},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 3.7 Flash (Low)","e":"low","m":"gemini-3-7-flash","oid":"arc_agi_2-ec6bc86114825f77","rd":"2026-08-13","sid":"rec9BSK1rciwNOFqp","sm":"gemini-3.7-flash_low","v":0.5291666666666667},{"b":"arc_agi_2","c":1.8982,"cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (X-High)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-90ae4e4d82a7f310","rd":"2025-12-11","sid":"recS9jrqUC4LBQDMg","sm":"gpt-5.2-2025-12-11_xhigh","v":0.5291},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (xHigh)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-9b7cfbc82964cb13","rd":"2025-12-11","sid":"recaVCKBQjIjHS5cs","sm":"gpt-5.2-2025-12-11_xhigh","v":0.5291},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (XHigh)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-5055cfb687d5bdc9","rd":"2025-12-11","sid":"recKOxOLmel1LyZ4u","sm":"gpt-5.2-2025-12-11_xhigh","v":0.5291},{"b":"arc_agi_2","c":0.78,"cfg":{"effort":"high"},"dn":"Grok 4.5 (High)","e":"high","m":"grok-4-5","oid":"arc_agi_2-0caa6edf33d1914e","rd":"2026-07-08","sid":"rec9nxVNWAW6hkK3O","sm":"grok-4.5_high","v":0.5264},{"b":"arc_agi_2","c":0.75,"cfg":{"effort":"medium"},"dn":"Grok 4.5 (Medium)","e":"medium","m":"grok-4-5","oid":"arc_agi_2-9842ebf539334ed8","rd":"2026-07-08","sid":"recuhP5poLz9p1THE","sm":"grok-4.5_medium","v":0.5264},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 3.6 Flash (Medium)","e":"medium","m":"gemini-3-6-flash","oid":"arc_agi_2-6f78a32de0169fb9","rd":"2026-07-21","sid":"recBXB0b72NG9lA0o","sm":"gemini-3.6-flash_medium","v":0.5041666666666667},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"GLM-5.3-Flash (High)","e":"high","m":"glm-5-3-flash","oid":"arc_agi_2-ddf553d2e55ba352","rd":"2026-08-20","sid":"recXM4pZFYPQu0P58","sm":"glm-5.3-flash_high","v":0.5013888888888889},{"b":"arc_agi_2","c":0.54,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna (xHigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"arc_agi_2-3efd0d881960cdbe","rd":"2026-07-09","sid":"recUz4W8immjSHPf0","sm":"gpt-5.6-luna_xhigh","v":0.4764},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna (XHigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"arc_agi_2-27b3dcedf0155f85","rd":"2026-07-09","sid":"rec5u9d3UJgddydsq","sm":"gpt-5.6-luna_xhigh","v":0.4764},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Muse Spark Open (High)","e":"high","m":"muse-spark-open-816b-a42b","oid":"arc_agi_2-f0f2ffd89fea2af3","rd":"2026-10-15","sid":"reciN9YfqbtP1yXws","sm":"Muse-Spark-Open-816B-A42B_high","v":0.47500000000000003},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"DeepSeek V4 Flash 0731 (Low)","e":"low","m":"deepseek-v4-flash-0731","oid":"arc_agi_2-b26793ada09f0a55","rd":"2026-07-31","sid":"rec6M7tXqk2R8OtRT","sm":"deepseek-v4-flash-0731_low","v":0.4597222222222222},{"b":"arc_agi_2","dn":"Gemini 3 Deep Think (Preview) ²","m":"gemini-3-deep-think-preview","oid":"arc_agi_2-6902785f7a43885a","rd":"2026-02-12","sid":"recQNM7laEXnYaZlu","sm":"gemini-3-deep-think-preview","v":0.4514},{"b":"arc_agi_2","c":1.393,"cfg":{"effort":"high"},"dn":"GPT-5.2 (High)","e":"high","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-2fa168d6ced97470","rd":"2025-12-11","sid":"recZgpSoatzOtn3rb","sm":"gpt-5.2-2025-12-11_high","v":0.4333},{"b":"arc_agi_2","c":0.32,"cfg":{"effort":"low"},"dn":"GPT-5.6 Sol (Low)","e":"low","m":"gpt-5-6-sol","oid":"arc_agi_2-5d53c651f45c0d3a","rd":"2026-07-09","sid":"recYYBTJBos18GflF","sm":"gpt-5.6-sol_low","v":0.425},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Qwen3.8-27B (XHigh)","e":"xhigh","m":"qwen3-8-27b","oid":"arc_agi_2-9320d830041ab995","rd":"2026-08-14","sid":"recaJwwU0sNCfsH5d","sm":"qwen3.8-27b_xhigh","v":0.4236111111111111},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 4 Argon (Low)","e":"low","m":"gemini-4-argon","oid":"arc_agi_2-ea93033211a44d08","rd":"2026-09-30","sid":"rechhE5adomMk7wyX","sm":"gemini-4-argon_low","v":0.4208333333333334},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-6 Luna (XHigh)","e":"xhigh","m":"gpt-6-luna","oid":"arc_agi_2-c56ebe7a530eca7f","rd":"2026-09-22","sid":"rec02FpVp3L8dwBgu","sm":"gpt-6-luna_xhigh","v":0.41944444444444445},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Inkling Small (XHigh)","e":"xhigh","m":"inkling-small","oid":"arc_agi_2-0795c4fcd086d1aa","rd":"2026-07-15","sid":"recsPDzpQmPtMA77v","sm":"inkling-small_xhigh","v":0.40138888888888885},{"b":"arc_agi_2","c":8.9928,"cfg":{"effort":"medium"},"dn":"GPT-5.2 Pro (Medium)","e":"medium","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi_2-d4366faef16cc5f2","rd":"2025-12-11","sid":"recNfd2GOThl8JJsR","sm":"gpt-5.2-pro-2025-12-11_medium","v":0.3847},{"b":"arc_agi_2","c":2.4,"dn":"Opus 4.5 (Thinking, 64K)","m":"claude-opus-4-5-20251101-64k","oid":"arc_agi_2-de09543e192c1790","rd":"2025-11-24","sid":"recv5sJsLP2dDw5bN","sm":"claude-opus-4-5-20251101_64K","v":0.3764},{"b":"arc_agi_2","c":0.27,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Terra (Medium)","e":"medium","m":"gpt-5-6-terra","oid":"arc_agi_2-f176c394f8fa509b","rd":"2026-07-09","sid":"recxR9NtkO4IkDlEG","sm":"gpt-5.6-terra_medium","v":0.375},{"b":"arc_agi_2","c":0.6420766277310923,"dn":"Inkling","m":"inkling","oid":"arc_agi_2-bf38102612ed959e","rd":"2026-07-15","sid":"recAYJP4QCBvL0DVU","sm":"Inkling","v":0.3652777777777778},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 3 Flash Preview (High)","e":"high","m":"gemini-3-flash-preview","oid":"arc_agi_2-8612bbac0df81ef2","rd":"2025-12-17","sid":"recatuxrQ5J2dsdHg","sm":"gemini-3-flash-preview","v":0.3361},{"b":"arc_agi_2","c":0.35,"cfg":{"effort":"low"},"dn":"GPT-5.5 (Low)","e":"low","m":"gpt-5-5","oid":"arc_agi_2-211aecc22c0ec3f6","rd":"2026-04-23","sid":"recIDNd1cnFpANX8V","sm":"gpt-5.5_low","v":0.3333},{"b":"arc_agi_2","c":0.4,"cfg":{"effort":"low"},"dn":"Grok 4.5 (Low)","e":"low","m":"grok-4-5","oid":"arc_agi_2-4da271b9ab9d0215","rd":"2026-07-08","sid":"recyCgaKaJtUJErQt","sm":"grok-4.5_low","v":0.3306},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Inkling Small (High)","e":"high","m":"inkling-small","oid":"arc_agi_2-fcbea566a2519f80","rd":"2026-07-15","sid":"reck4VZ8pzUebO0an","sm":"inkling-small_high","v":0.3305555555555556},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Claude Sonnet 5.5 (Medium)","e":"medium","m":"claude-sonnet-5-5","oid":"arc_agi_2-458486402fe10589","rd":"2026-09-28","sid":"recXy5Z6bajUACSLt","sm":"claude-sonnet-5-5_medium","v":0.3277777777777777},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"GPT-6 Sol (Low)","e":"low","m":"gpt-6-sol","oid":"arc_agi_2-94cf23da5b1068e7","rd":"2026-09-22","sid":"recM1EPIRyZXTR6Eu","sm":"gpt-6-sol_low","v":0.3152777777777777},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"GPT-6 Luna (High)","e":"high","m":"gpt-6-luna","oid":"arc_agi_2-ea92af6f6b081add","rd":"2026-09-22","sid":"recx0dGdcO2qqWceL","sm":"gpt-6-luna_high","v":0.3138888888888889},{"b":"arc_agi_2","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"arc_agi_2-45e228fd024e7cbe","rd":"2025-11-18","sid":"recpfIvjpQ9j5qjkw","sm":"gemini-3-pro-preview","v":0.3111},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Claude Haiku 5.5 (High)","e":"high","m":"claude-haiku-5-5","oid":"arc_agi_2-12548132d3e72cd7","rd":"2026-10-07","sid":"rec7VBMJpz87c7ISk","sm":"claude-haiku-5-5_high","v":0.3069444444444444},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 3.6 Flash (Low)","e":"low","m":"gemini-3-6-flash","oid":"arc_agi_2-2da9488e6a2d9d80","rd":"2026-07-21","sid":"rec3i0ahAD3ypm5Zn","sm":"gemini-3.6-flash_low","v":0.30416666666666664},{"b":"arc_agi_2","c":0.29,"cfg":{"effort":"high"},"dn":"GPT-5.6 Luna (High)","e":"high","m":"gpt-5-6-luna","oid":"arc_agi_2-6ecaca2920cc88b7","rd":"2026-07-09","sid":"recPiIGgsFkyUNMRY","sm":"gpt-5.6-luna_high","v":0.2931},{"b":"arc_agi_2","c":0.27,"cfg":{"effort":"low"},"dn":"GPT-5.4 (Low)","e":"low","m":"gpt-5-4-2026-03-05","oid":"arc_agi_2-e983abf0ffb2c98a","rd":"2026-03-05","sid":"recKYmHa8qFLZepys","sm":"gpt-5.4-2026-03-05_low","v":0.2917},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"GLM-5.3-Flash (Low)","e":"low","m":"glm-5-3-flash","oid":"arc_agi_2-6b46563c7aecd6c3","rd":"2026-08-20","sid":"recCk2LBnZJz3WMjM","sm":"glm-5.3-flash_low","v":0.2791666666666667},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Grok 4.6 (Low)","e":"low","m":"grok-4-6","oid":"arc_agi_2-4ae16e051880a36a","rd":"2026-08-12","sid":"recKiwALxu32UoPq6","sm":"grok-4.6_low","v":0.2763888888888889},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Muse Spark Open (Medium)","e":"medium","m":"muse-spark-open-816b-a42b","oid":"arc_agi_2-f4f3069486ae5d94","rd":"2026-10-15","sid":"rec3TfkvV51jpeJSr","sm":"Muse-Spark-Open-816B-A42B_medium","v":0.275},{"b":"arc_agi_2","c":0.7594,"cfg":{"effort":"medium"},"dn":"GPT-5.2 (Medium)","e":"medium","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-854482227645da92","rd":"2025-12-11","sid":"recC0T6EG6UZ9vBNr","sm":"gpt-5.2-2025-12-11_medium","v":0.2667},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Claude Sonnet 5.5 (Low)","e":"low","m":"claude-sonnet-5-5","oid":"arc_agi_2-2fc54b7a4523d8ab","rd":"2026-09-28","sid":"recEbAfD1hPSWfxTP","sm":"claude-sonnet-5-5_low","v":0.24027777777777776},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"Beam 1.0 (XHigh)","e":"xhigh","m":"beam-1-0","oid":"arc_agi_2-642dfc4b800b3187","rd":"2026-10-05","sid":"rec6ext6dkgRLfGFd","sm":"beam-1.0_xhigh","v":0.23472222222222222},{"b":"arc_agi_2","c":0.79,"dn":"Opus 4.5 (Thinking, 16K)","m":"claude-opus-4-5-20251101-16k","oid":"arc_agi_2-7b31d48bafb777ee","rd":"2025-11-24","sid":"rec75TsP1Xy1ROEez","sm":"claude-opus-4-5-20251101_16K","v":0.2278},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Qwen3.8-27B (Low)","e":"low","m":"qwen3-8-27b","oid":"arc_agi_2-9e3d2fd00490edb1","rd":"2026-08-14","sid":"rec6bR67M9UBlhFO6","sm":"qwen3.8-27b_low","v":0.2277777777777778},{"b":"arc_agi_2","c":0.2501,"dn":"GLM-5.2","m":"glm-5-2","oid":"arc_agi_2-ab0eba2a23ef81cf","rd":"2026-06-16","sid":"recMTvejGGEdgJHsM","sm":"glm-5.2_unknown","v":0.22775000000000004},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Beam 1.0 (High)","e":"high","m":"beam-1-0","oid":"arc_agi_2-ad0826ddeacdb75c","rd":"2026-10-05","sid":"rechROYywaNPBi7MT","sm":"beam-1.0_high","v":0.2},{"b":"arc_agi_2","c":0.75,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Mini (xHigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi_2-e2d82158284cbb36","rd":"2026-03-17","sid":"recTo7b6iwUks0bgH","sm":"gpt-5.4-mini-2026-03-17_xhigh","v":0.189},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Mini (XHigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi_2-6d9f1eec88d596c9","rd":"2026-03-17","sid":"recOyW5fmk9px98go","sm":"gpt-5.4-mini-2026-03-17_xhigh","v":0.189},{"b":"arc_agi_2","c":0.17,"cfg":{"effort":"low"},"dn":"GPT-5.6 Terra (Low)","e":"low","m":"gpt-5-6-terra","oid":"arc_agi_2-b0ab828ad1854105","rd":"2026-07-09","sid":"recBmdT4RIrE6PjQC","sm":"gpt-5.6-terra_low","v":0.1875},{"b":"arc_agi_2","c":7.1432,"dn":"GPT-5 Pro","m":"gpt-5-pro-2025-10-06","oid":"arc_agi_2-f2a0638e282b299b","rd":"2025-10-07","sid":"recEX7lSg0Egk0RAh","sm":"gpt-5-pro-2025-10-06_unknown","v":0.1833},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"GPT-6 Luna (Medium)","e":"medium","m":"gpt-6-luna","oid":"arc_agi_2-7e0bbf62eac4d545","rd":"2026-09-22","sid":"reciIrQlX6rvHdQv3","sm":"gpt-6-luna_medium","v":0.18055555555555555},{"b":"arc_agi_2","c":1.1704,"cfg":{"effort":"high"},"dn":"GPT-5.1 (Thinking, High)","e":"high","m":"gpt-5-1-2025-11-13","oid":"arc_agi_2-6c1c79b2b1bfc46f","rd":"2025-11-13","sid":"rec0q3XOoL5ig2rmC","sm":"gpt-5.1-2025-11-13_high","v":0.1764},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Grok 4.7 (Low)","e":"low","m":"grok-4-7","oid":"arc_agi_2-ae3d30e1bdd82348","rd":"2026-09-21","sid":"recNL8awhVSpj73i0","sm":"grok-4.7_low","v":0.16666666666666666},{"b":"arc_agi_2","cfg":{"effort":"thinking"},"dn":"Grok 4 (Thinking)","e":"thinking","m":"grok-4-0709","oid":"arc_agi_2-75bf265fb489fa4a","rd":"2025-07-09","sid":"recejdgNxM9Ez5aMB","sm":"grok-4-0709","v":0.15975},{"b":"arc_agi_2","c":0.48,"dn":"Opus 4.5 (Thinking, 8K)","m":"claude-opus-4-5-20251101-8k","oid":"arc_agi_2-b094590d1247fc85","rd":"2025-11-24","sid":"recz5urlvGWID1WRk","sm":"claude-opus-4-5-20251101_8K","v":0.1389},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Inkling Small (Medium)","e":"medium","m":"inkling-small","oid":"arc_agi_2-0d289c9f6f0d8d55","rd":"2026-07-15","sid":"recLQJEf1lasl3iLq","sm":"inkling-small_medium","v":0.13611111111111113},{"b":"arc_agi_2","c":0.7589,"dn":"Claude Sonnet 4.5 (Thinking 32K)","m":"claude-sonnet-4-5-20250929-32k","oid":"arc_agi_2-ad7bcb30552fe4cb","rd":"2025-09-29","sid":"recBx0qeevdxXEZ8W","sm":"claude-sonnet-4-5-20250929_32K","v":0.1361},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Qwen3.8-27B (Medium)","e":"medium","m":"qwen3-8-27b","oid":"arc_agi_2-e07a2d8a31eba0ba","rd":"2026-08-14","sid":"recJ7JYZpijk4LvpG","sm":"qwen3.8-27b_medium","v":0.13194444444444445},{"b":"arc_agi_2","c":0.56,"cfg":{"effort":"high"},"dn":"GPT-5.4 Mini (High)","e":"high","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi_2-8519a92a1353ed21","rd":"2026-03-17","sid":"rec9Usyz2ygQJzgfc","sm":"gpt-5.4-mini-2026-03-17_high","v":0.1319},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 3 Flash Preview (Medium)","e":"medium","m":"gemini-3-flash-preview","oid":"arc_agi_2-3ffc8a5c4eac043e","rd":"2025-12-17","sid":"recbLiiMWEmhWHfWb","sm":"gemini-3-flash-preview","v":0.1278},{"b":"arc_agi_2","c":0.2541386750000001,"cfg":{"effort":"low"},"dn":"Kimi K3 (Low)","e":"low","m":"kimi-k3","oid":"arc_agi_2-e9054cbb1f53273e","rd":"2026-07-16","sid":"recH5KE0OlSAl8M1N","sm":"kimi-k3_low","v":0.12361111111111112},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Muse Spark Open (Low)","e":"low","m":"muse-spark-open-816b-a42b","oid":"arc_agi_2-1810fc00a88840dd","rd":"2026-10-15","sid":"recblITzcTiuRARQi","sm":"Muse-Spark-Open-816B-A42B_low","v":0.12222222222222222},{"b":"arc_agi_2","c":0.28,"dn":"Kimi K2.5","m":"kimi-k2-5","oid":"arc_agi_2-bfa0225a7585990c","rd":"2026-01-27","sid":"rec04h17vuGSX2tFu","sm":"kimi-k2.5","v":0.11810000000000001},{"b":"arc_agi_2","cfg":{"effort":"high"},"dn":"Gemini 3.5 Flash-Lite (High)","e":"high","m":"gemini-3-5-flash-lite","oid":"arc_agi_2-9cb10ab6b8aaa206","rd":"2026-07-21","sid":"reclIbIS7tm9flLtQ","sm":"gemini-3.5-flash-lite_high","v":0.10277777777777779},{"b":"arc_agi_2","c":0.7302,"cfg":{"effort":"high"},"dn":"GPT-5 (High)","e":"high","m":"gpt-5-2025-08-07","oid":"arc_agi_2-344429283e814ef9","rd":"2025-08-07","sid":"rec4cuWugsakV5GLa","sm":"gpt-5-2025-08-07_high","v":0.0986},{"b":"arc_agi_2","c":0.2642,"cfg":{"effort":"low"},"dn":"GPT-5.2 (Low)","e":"low","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-b896e0be5153a192","rd":"2025-12-11","sid":"recCKI1LVFbETuKYX","sm":"gpt-5.2-2025-12-11_low","v":0.0972},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Claude Haiku 5.5 (Medium)","e":"medium","m":"claude-haiku-5-5","oid":"arc_agi_2-b6050dacfba3e376","rd":"2026-10-07","sid":"rec8Rwdgo19iGJvYB","sm":"claude-haiku-5-5_medium","v":0.09444444444444443},{"b":"arc_agi_2","c":0.1074,"cfg":{"effort":"minimal"},"dn":"Gemini 3.5 Flash (Minimal)","e":"minimal","m":"gemini-3-5-flash","oid":"arc_agi_2-c1c299f2bea8ea71","rd":"2026-05-19","sid":"recADvSyiT3Ze8RTf","sm":"gemini-3.5-flash_minimal","v":0.0889},{"b":"arc_agi_2","c":1.9284,"dn":"Claude Opus 4 (Thinking 16K)","m":"claude-opus-4-20250514-16k","oid":"arc_agi_2-68fbbc60de8b8ab4","rd":"2025-05-22","sid":"recPD5WSzdGAFCyf4","sm":"claude-opus-4-20250514_16K","v":0.0861},{"b":"arc_agi_2","c":0.22,"dn":"Opus 4.5 (Thinking, None)","m":"claude-opus-4-5-2025-11-01","oid":"arc_agi_2-9d139a3d8bc0be08","rd":"2025-11-24","sid":"recS34qcEATpV9BeX","sm":"claude-opus-4-5-20251101","v":0.0778},{"b":"arc_agi_2","c":0.4486,"cfg":{"effort":"medium"},"dn":"GPT-5 (Medium)","e":"medium","m":"gpt-5-2025-08-07","oid":"arc_agi_2-3be01a4cb2c495a1","rd":"2025-08-07","sid":"recpm3DPVRPZRfhTi","sm":"gpt-5-2025-08-07_medium","v":0.0749},{"b":"arc_agi_2","c":0.11,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Luna (Medium)","e":"medium","m":"gpt-5-6-luna","oid":"arc_agi_2-5ae78274a52c95fd","rd":"2026-07-09","sid":"recMYrbV5q1DoQno8","sm":"gpt-5.6-luna_medium","v":0.0736},{"b":"arc_agi_2","c":0.3499,"dn":"Claude Sonnet 4.5 (Thinking 16K)","m":"claude-sonnet-4-5-20250929-16k","oid":"arc_agi_2-9097bcba923bccd2","rd":"2025-09-29","sid":"recxP8m3sCHQpBdki","sm":"claude-sonnet-4-5-20250929_16K","v":0.0694},{"b":"arc_agi_2","c":0.2349,"dn":"Claude Sonnet 4.5 (Thinking 8K)","m":"claude-sonnet-4-5-20250929-8k","oid":"arc_agi_2-0a8f9a7d87948ff9","rd":"2025-09-29","sid":"recRCNsLfgxjg8Syp","sm":"claude-sonnet-4-5-20250929_8K","v":0.0694},{"b":"arc_agi_2","c":0.8339,"cfg":{"effort":"high"},"dn":"o3 (High)","e":"high","m":"o3-2025-04-16","oid":"arc_agi_2-8fac9e29b4b6f7d3","rd":"2025-04-16","sid":"recuw1OMe7R6TbHZM","sm":"o3-2025-04-16_high","v":0.0653},{"b":"arc_agi_2","c":0.4206,"cfg":{"effort":"medium"},"dn":"GPT-5.1 (Thinking, Medium)","e":"medium","m":"gpt-5-1-2025-11-13","oid":"arc_agi_2-e92b249d437bbb68","rd":"2025-11-13","sid":"recNeyhzsD2NO7yJf","sm":"gpt-5.1-2025-11-13_medium","v":0.0653},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Beam 1.0 (Medium)","e":"medium","m":"beam-1-0","oid":"arc_agi_2-ff47afa97641f3b5","rd":"2026-10-05","sid":"recZSHuEj5N8Hjl4J","sm":"beam-1.0_medium","v":0.0638888888888889},{"b":"arc_agi_2","c":2.1,"dn":"Tiny Recursion Model (TRM)","m":"tiny-recursion-model","oid":"arc_agi_2-a6bdb0e60aac7ea3","rd":"2025-10-06","sid":"rec8J2syeKkuAoziw","sm":"tiny-recursion-model","v":0.0625},{"b":"arc_agi_2","c":0.856,"cfg":{"effort":"high"},"dn":"o4-mini (High)","e":"high","m":"o4-mini-2025-04-16","oid":"arc_agi_2-8d04a035a41eda7f","rd":"2025-04-16","sid":"recP5Py2i3sdybPd9","sm":"o4-mini-2025-04-16_high","v":0.0611},{"b":"arc_agi_2","c":0.4857,"dn":"Claude Sonnet 4 (Thinking 16K)","m":"claude-sonnet-4-20250514-16k","oid":"arc_agi_2-c03d8dae3d7b4528","rd":"2025-05-22","sid":"recRLtrNpStDOz6At","sm":"claude-sonnet-4-20250514_16K","v":0.0593},{"b":"arc_agi_2","c":0.1424,"dn":"Claude Sonnet 4.5 (Thinking 1K)","m":"claude-sonnet-4-5-20250929-1k","oid":"arc_agi_2-989675cf7511c6d7","rd":"2025-09-29","sid":"recrn6MqgrBWyFiCl","sm":"claude-sonnet-4-5-20250929_1K","v":0.058300000000000005},{"b":"arc_agi_2","c":0.16,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Nano (xHigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi_2-c70a670e97f61e2c","rd":"2026-03-17","sid":"recDffGQ5x3Y5ZmgX","sm":"gpt-5.4-nano-2026-03-17_xhigh","v":0.0569},{"b":"arc_agi_2","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Nano (XHigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi_2-256d66d1d998be99","rd":"2026-03-17","sid":"rec673d2Y0YYbsyCN","sm":"gpt-5.4-nano-2026-03-17_xhigh","v":0.0569},{"b":"arc_agi_2","c":0.061,"dn":"Grok 4 (Fast Reasoning)","m":"grok-4-fast","oid":"arc_agi_2-66abcad4b01707ce","rd":"2025-09-19","sid":"recdEUBeSozsdeDTq","sm":"grok-4-fast","v":0.05280000000000001},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Gemini 3.5 Flash-Lite (Medium)","e":"medium","m":"gemini-3-5-flash-lite","oid":"arc_agi_2-89054d70b6d6d7d0","rd":"2026-07-21","sid":"recFK2geZNpxgTR6S","sm":"gemini-3.5-flash-lite_medium","v":0.05277777777777778},{"b":"arc_agi_2","c":0.07,"cfg":{"effort":"low"},"dn":"GPT-5.6 Luna (Low)","e":"low","m":"gpt-5-6-luna","oid":"arc_agi_2-42ab7f9744ae1543","rd":"2026-07-09","sid":"recBsVVGrHfOLtNQV","sm":"gpt-5.6-luna_low","v":0.05140000000000001},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Inkling Small (Low)","e":"low","m":"inkling-small","oid":"arc_agi_2-ef94aa69cd9094ce","rd":"2026-07-15","sid":"recTGtS1poOZmyDqT","sm":"inkling-small_low","v":0.04861111111111112},{"b":"arc_agi_2","c":7.5516,"cfg":{"effort":"high"},"dn":"o3-Pro (High)","e":"high","m":"o3-pro-2025-06-10","oid":"arc_agi_2-4caa1d5c4e272e2f","rd":"2025-06-10","sid":"recNnknL5K4qDifiQ","sm":"o3-pro-2025-06-10_high","v":0.0486},{"b":"arc_agi_2","c":0.757,"dn":"Gemini 2.5 Pro (Thinking 32K)","m":"gemini-2-5-pro-32k","oid":"arc_agi_2-25f3a4a7129b184a","rd":"2025-06-17","sid":"recwTBxP1gSo6ob15","sm":"gemini-2.5-pro_32K","v":0.0486},{"b":"arc_agi_2","c":0.17,"dn":"Minimax M2.5","m":"minimax-m2-5","oid":"arc_agi_2-1e37012183a3070a","rd":"2026-02-12","sid":"recvWNVGtC7Qa9iB1","sm":"MiniMax-M2.5","v":0.0486},{"b":"arc_agi_2","c":0.27,"dn":"GLM-5","m":"glm-5","oid":"arc_agi_2-cec3247ee736ab87","rd":"2026-02-11","sid":"recqRJ9q9MWn4yawx","sm":"glm-5","v":0.0486},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"GPT-6 Luna (Low)","e":"low","m":"gpt-6-luna","oid":"arc_agi_2-ab9622f51a0262ce","rd":"2026-09-22","sid":"recnMvDSNo5pUrqE6","sm":"gpt-6-luna_low","v":0.04583333333333333},{"b":"arc_agi_2","c":1.1569,"dn":"Claude Opus 4 (Thinking 8K)","m":"claude-opus-4-20250514-8k","oid":"arc_agi_2-c162c83ac6372e66","rd":"2025-05-22","sid":"recgJTyUSx4UkgSdV","sm":"claude-opus-4-20250514_8K","v":0.0452},{"b":"arc_agi_2","c":0.1977,"cfg":{"effort":"high"},"dn":"GPT-5 Mini (High)","e":"high","m":"gpt-5-mini-2025-08-07","oid":"arc_agi_2-363fa81a05099dbe","rd":"2025-08-07","sid":"recMKGnQgFOoIKBTQ","sm":"gpt-5-mini-2025-08-07_high","v":0.0444},{"b":"arc_agi_2","c":0.33,"cfg":{"effort":"medium"},"dn":"GPT-5.4 Mini (Medium)","e":"medium","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi_2-812f104fefa2ebd2","rd":"2026-03-17","sid":"recFStDaPJA1WgZ7p","sm":"gpt-5.4-mini-2026-03-17_medium","v":0.0444},{"b":"arc_agi_2","c":0.7145,"dn":"Gemini 2.5 Pro (Thinking 16K)","m":"gemini-2-5-pro-16k","oid":"arc_agi_2-f3ecf33430712295","rd":"2025-06-17","sid":"recssQ9dWplBUSA3l","sm":"gemini-2.5-pro_16K","v":0.0403},{"b":"arc_agi_2","c":0.0629,"cfg":{"effort":"medium"},"dn":"GPT-5 Mini (Medium)","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"arc_agi_2-5a3056d34303aab6","rd":"2025-08-07","sid":"recMk9nDIaqlCGFqp","sm":"gpt-5-mini-2025-08-07_medium","v":0.0403},{"b":"arc_agi_2","c":0.3766,"dn":"Claude Haiku 4.5 (Thinking 32K)","m":"claude-haiku-4-5-20251001-32k","oid":"arc_agi_2-b9f5b7443b7da2d4","rd":"2025-10-15","sid":"rec0kK1dRykoyqY0K","sm":"claude-haiku-4-5-20251001_32K","v":0.0403},{"b":"arc_agi_2","c":0.12,"dn":"Deepseek V3.2","m":"deepseekdeepseek-v3-2","oid":"arc_agi_2-3b09959bc8c3d793","rd":"2025-12-01","sid":"recxNNssasIdr4hNl","sm":"deepseek/deepseek-v3.2","v":0.0403},{"b":"arc_agi_2","cfg":{"effort":"minimal"},"dn":"Inkling Small (Minimal)","e":"minimal","m":"inkling-small","oid":"arc_agi_2-71c6fe860c36cdf2","rd":"2026-07-15","sid":"recShkDWJemDJ83cK","sm":"inkling-small_minimal","v":0.04027777777777778},{"b":"arc_agi_2","c":0.1295,"dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"arc_agi_2-46c71949553c1043","rd":"2025-09-29","sid":"recRp5W7nPR9PibPQ","sm":"claude-sonnet-4-5-20250929","v":0.0375},{"b":"arc_agi_2","c":0.13,"cfg":{"effort":"high"},"dn":"GPT-5.4 Nano (High)","e":"high","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi_2-8658ff3b032f103a","rd":"2026-03-17","sid":"recUj2nMwaabqY4xy","sm":"gpt-5.4-nano-2026-03-17_high","v":0.0361},{"b":"arc_agi_2","cfg":{"effort":"minimal"},"dn":"Gemini 3 Flash Preview (Minimal)","e":"minimal","m":"gemini-3-flash-preview","oid":"arc_agi_2-f004d44a009d2694","rd":"2025-12-17","sid":"rec4D7ffvN5bl5c10","sm":"gemini-3-flash-preview","v":0.0333},{"b":"arc_agi_2","c":0.5472,"cfg":{"effort":"high"},"dn":"o3-mini (High)","e":"high","m":"o3-mini-2025-01-31","oid":"arc_agi_2-bf5832c455f0e979","rd":"2025-01-31","sid":"recIVyGy3qnEsUXSz","sm":"o3-mini-2025-01-31_high","v":0.0299},{"b":"arc_agi_2","c":0.4787,"cfg":{"effort":"medium"},"dn":"o3 (Medium)","e":"medium","m":"o3-2025-04-16","oid":"arc_agi_2-63e1c2795c7e32ca","rd":"2025-04-16","sid":"reckVnQ5Dgul9PJ0n","sm":"o3-2025-04-16_medium","v":0.0298},{"b":"arc_agi_2","c":0.4439,"dn":"Gemini 2.5 Pro (Thinking 8K)","m":"gemini-2-5-pro-8k","oid":"arc_agi_2-688ec2425a52cf53","rd":"2025-06-17","sid":"recpBihogJ0mcYOfS","sm":"gemini-2.5-pro_8K","v":0.0292},{"b":"arc_agi_2","c":0.1392,"dn":"Claude Haiku 4.5 (Thinking 16K)","m":"claude-haiku-4-5-20251001-16k","oid":"arc_agi_2-e35cff083c1dcbe7","rd":"2025-10-15","sid":"recm2RWPI31nRlRMe","sm":"claude-haiku-4-5-20251001_16K","v":0.0278},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Beam 1.0 (Low)","e":"low","m":"beam-1-0","oid":"arc_agi_2-34994f309b8704e2","rd":"2026-10-05","sid":"recHN8q6LmbeiXGPa","sm":"beam-1.0_low","v":0.027777777777777783},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Claude Haiku 5.5 (Low)","e":"low","m":"claude-haiku-5-5","oid":"arc_agi_2-3ee4cd2f06b2ed17","rd":"2026-10-07","sid":"rec1rxzST2H3xPb5b","sm":"claude-haiku-5-5_low","v":0.027777777777777776},{"b":"arc_agi_2","cfg":{"effort":"minimal"},"dn":"Gemini 3.6 Flash (Minimal)","e":"minimal","m":"gemini-3-6-flash","oid":"arc_agi_2-4759b101e694211e","rd":"2026-07-21","sid":"reczuYhRHn1Q64mB2","sm":"gemini-3.6-flash_minimal","v":0.02638888888888889},{"b":"arc_agi_2","c":0.0295,"cfg":{"effort":"high"},"dn":"GPT-5 Nano (High)","e":"high","m":"gpt-5-nano-2025-08-07","oid":"arc_agi_2-959345f9db94e869","rd":"2025-08-07","sid":"recupFdrzUCkiIOJx","sm":"gpt-5-nano-2025-08-07_high","v":0.026100000000000005},{"b":"arc_agi_2","c":0.3191,"dn":"Gemini 2.5 Flash (Preview) (Thinking 24K)","m":"gemini-2-5-flash-preview-05-20-23k","oid":"arc_agi_2-460a7e1b63890988","rd":"2025-05-20","sid":"recX5WZPpdxH99Q0W","sm":"gemini-2.5-flash-preview-05-20_23K","v":0.025400000000000002},{"b":"arc_agi_2","c":0.2311,"cfg":{"effort":"medium"},"dn":"o4-mini (Medium)","e":"medium","m":"o4-mini-2025-04-16","oid":"arc_agi_2-f3bb62f2c4dc6ba8","rd":"2025-04-16","sid":"recL7H56qJnD1JaLX","sm":"o4-mini-2025-04-16_medium","v":0.0236},{"b":"arc_agi_2","c":0.0302,"dn":"Gemini 2.5 Flash (Preview) (Thinking 1K)","m":"gemini-2-5-flash-preview-05-20-1k","oid":"arc_agi_2-4a1dd344a578d0bf","rd":"2025-05-20","sid":"recx8bMgTPWFjSce5","sm":"gemini-2.5-flash-preview-05-20_1K","v":0.0216},{"b":"arc_agi_2","c":0.1994,"dn":"Gemini 2.5 Flash (Preview) (Thinking 8K)","m":"gemini-2-5-flash-preview-05-20-8k","oid":"arc_agi_2-6e38c3681f11be4b","rd":"2025-05-20","sid":"recc2zuUdVClpEfo3","sm":"gemini-2.5-flash-preview-05-20_8K","v":0.0212},{"b":"arc_agi_2","c":0.2654,"dn":"Claude Sonnet 4 (Thinking 8K)","m":"claude-sonnet-4-20250514-8k","oid":"arc_agi_2-f41ff26d2a5e4827","rd":"2025-05-22","sid":"rec9yHZPujDE8MdDK","sm":"claude-sonnet-4-20250514_8K","v":0.0212},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"DeepSeek V4 Flash 0731 (None)","e":"none","m":"deepseek-v4-flash-0731","oid":"arc_agi_2-a2bfbdf89bae97f1","rd":"2026-07-31","sid":"recNIbGpTiROaJXNN","sm":"deepseek-v4-flash-0731_none","v":0.020833333333333332},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash 1017 (None)","e":"none","m":"deepseek-v4-1-flash-1017","oid":"arc_agi_2-9e02baa613a35cda","rd":"2026-10-17","sid":"reclJO3ciVSnMtOx9","sm":"deepseek-v4.1-flash-1017_none","v":0.020833333333333332},{"b":"arc_agi_2","c":0.2843,"cfg":{"effort":"medium"},"dn":"o3-mini (Medium)","e":"medium","m":"o3-mini-2025-01-31","oid":"arc_agi_2-f588f0665a00c500","rd":"2025-01-31","sid":"rec3DBzjmVooH06Du","sm":"o3-mini-2025-01-31_medium","v":0.020800000000000003},{"b":"arc_agi_2","c":2.2293,"cfg":{"effort":"low"},"dn":"o3-Pro (Low)","e":"low","m":"o3-pro-2025-06-10","oid":"arc_agi_2-7997daa9e65d7a7f","rd":"2025-06-10","sid":"recOPdwHvGY12QF3a","sm":"o3-pro-2025-06-10_low","v":0.020500000000000004},{"b":"arc_agi_2","c":0.2343,"cfg":{"effort":"low"},"dn":"o3 (Low)","e":"low","m":"o3-2025-04-16","oid":"arc_agi_2-aa978ae5e1c4a293","rd":"2025-04-16","sid":"recQRWOdhT4RaKHDF","sm":"o3-2025-04-16_low","v":0.0199},{"b":"arc_agi_2","c":0.3173,"dn":"Gemini 2.5 Flash (Preview) (Thinking 16K)","m":"gemini-2-5-flash-preview-05-20-16k","oid":"arc_agi_2-5fc2235250536ca8","rd":"2025-05-20","sid":"recPzm2d30ETKcuf6","sm":"gemini-2.5-flash-preview-05-20_16K","v":0.0198},{"b":"arc_agi_2","c":4.7441,"cfg":{"effort":"medium"},"dn":"o3-Pro (Medium)","e":"medium","m":"o3-pro-2025-06-10","oid":"arc_agi_2-7a972d2e0a2982ff","rd":"2025-06-10","sid":"recPRCqGgqp2fVU13","sm":"o3-pro-2025-06-10_medium","v":0.0194},{"b":"arc_agi_2","c":0.1896,"cfg":{"effort":"low"},"dn":"GPT-5 (Low)","e":"low","m":"gpt-5-2025-08-07","oid":"arc_agi_2-3e087348b2891b83","rd":"2025-08-07","sid":"rec66wbMAhLTeT1Mv","sm":"gpt-5-2025-08-07_low","v":0.0194},{"b":"arc_agi_2","c":0.1896,"cfg":{"effort":"low"},"dn":"GPT-5 (Low)","e":"low","m":"gpt-5-2025-08-07","oid":"arc_agi_2-de8acb1fe8364c8a","rd":"2025-08-07","sid":"recIgLhVAZcjtBEEr","sm":"gpt-5-2025-08-07_low","v":0.0194},{"b":"arc_agi_2","c":0.129,"cfg":{"effort":"low"},"dn":"GPT-5.1 (Thinking, Low)","e":"low","m":"gpt-5-1-2025-11-13","oid":"arc_agi_2-6c641e673e9fe7c7","rd":"2025-11-13","sid":"rec5P1fa26x1HY993","sm":"gpt-5.1-2025-11-13_low","v":0.0194},{"b":"arc_agi_2","c":0.06,"cfg":{"effort":"medium"},"dn":"GPT-5.4 Nano (Medium)","e":"medium","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi_2-c07c039198694635","rd":"2026-03-17","sid":"recL6ss3SiXe5h3I1","sm":"gpt-5.4-nano-2026-03-17_medium","v":0.0194},{"b":"arc_agi_2","c":0.057,"dn":"Gemini 2.5 Flash (Preview)","m":"gemini-2-5-flash-preview-05-20","oid":"arc_agi_2-bb61b14c1aa6e8bb","rd":"2025-05-20","sid":"recBr7W43spzTlYWg","sm":"gemini-2.5-flash-preview-05-20","v":0.0169},{"b":"arc_agi_2","c":0.05,"cfg":{"effort":"low"},"dn":"o4-mini (Low)","e":"low","m":"o4-mini-2025-04-16","oid":"arc_agi_2-e9e4e930a7820448","rd":"2025-04-16","sid":"recooai9u1xla5nXC","sm":"o4-mini-2025-04-16_low","v":0.0167},{"b":"arc_agi_2","c":0.0094,"cfg":{"effort":"minimal"},"dn":"GPT-5 Mini (Minimal)","e":"minimal","m":"gpt-5-mini-2025-08-07","oid":"arc_agi_2-8191bbb367b254bc","rd":"2025-08-07","sid":"recIEWFxMvibFys75","sm":"gpt-5-mini-2025-08-07_minimal","v":0.0167},{"b":"arc_agi_2","c":0.0909,"dn":"Claude Haiku 4.5 (Thinking 8K)","m":"claude-haiku-4-5-20251001-8k","oid":"arc_agi_2-b46f3635107aba2e","rd":"2025-10-15","sid":"rec3Bb2paOld3N4s0","sm":"claude-haiku-4-5-20251001_8K","v":0.0167},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"GPT-6 Sol (None)","e":"none","m":"gpt-6-sol","oid":"arc_agi_2-9f2116da22547811","rd":"2026-09-22","sid":"recAdUSYrL4QnHxKf","sm":"gpt-6-sol_none","v":0.016666666666666666},{"b":"arc_agi_2","c":0.01,"cfg":{"effort":"low"},"dn":"GPT-5.4 Nano (Low)","e":"low","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi_2-7e08701f7fb97110","rd":"2026-03-17","sid":"rec62ZrX5hl15IxAl","sm":"gpt-5.4-nano-2026-03-17_low","v":0.015300000000000001},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 3.5 Flash-Lite (Low)","e":"low","m":"gemini-3-5-flash-lite","oid":"arc_agi_2-9019ba0472c91ba8","rd":"2026-07-21","sid":"reccKj7kqdUYGxFcd","sm":"gemini-3.5-flash-lite_low","v":0.015277777777777777},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"Qwen3.8-27B (None)","e":"none","m":"qwen3-8-27b","oid":"arc_agi_2-fa95ed362370cafd","rd":"2026-08-14","sid":"recFLcFnmWI3m9kv1","sm":"qwen3.8-27b_none","v":0.015277777777777777},{"b":"arc_agi_2","c":0.004,"dn":"Gemini 2.0 Flash","m":"gemini-2-0-flash-001","oid":"arc_agi_2-9d73c57e7c315c90","rd":"2025-02-05","sid":"recmtA3PiKSki7Pff","sm":"gemini-2.0-flash-001","v":0.013000000000000001},{"b":"arc_agi_2","c":0.08,"dn":"Deepseek R1","m":"deepseek-r1","oid":"arc_agi_2-c968e0b48a850cc0","rd":"2025-01-20","sid":"recSe6e45cOKfp8SF","sm":"DeepSeek-R1","v":0.013000000000000001},{"b":"arc_agi_2","c":0.1272,"dn":"Claude Sonnet 4","m":"claude-sonnet-4-2025-05-14","oid":"arc_agi_2-4f534c53e00b13c3","rd":"2025-05-22","sid":"rec5UCwOgvD2BGMyd","sm":"claude-sonnet-4-20250514","v":0.012700000000000001},{"b":"arc_agi_2","c":0.23,"dn":"Codex Mini (Latest)","m":"codex-mini-2025-05-16","oid":"arc_agi_2-9ec4fe6261587414","rd":"2025-05-16","sid":"rec5tJjCzoBNRSY9a","sm":"codex-mini-2025-05-16","v":0.012700000000000001},{"b":"arc_agi_2","c":0.6388,"dn":"Claude Opus 4","m":"claude-opus-4-2025-05-14","oid":"arc_agi_2-07db8adb190ce4c3","rd":"2025-05-22","sid":"recIqYwza9VtPYCLm","sm":"claude-opus-4-20250514","v":0.012700000000000001},{"b":"arc_agi_2","c":0.0044,"dn":"Qwen3-235b-a22b Instruct (25/07)","m":"qwen3-235b-a22b-instruct-2507","oid":"arc_agi_2-b2ac4059447dc917","rd":"2025-07-25","sid":"recwktfYzggCeGG9v","sm":"Qwen3-235B-A22B-Instruct-2507","v":0.0125},{"b":"arc_agi_2","c":0.0426,"dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"arc_agi_2-53cf7e557f161827","rd":"2025-10-15","sid":"recSdlPGq90xI0GOZ","sm":"claude-haiku-4-5-20251001","v":0.0125},{"b":"arc_agi_2","cfg":{"effort":"low"},"dn":"Gemini 3 Flash Preview (Low)","e":"low","m":"gemini-3-flash-preview","oid":"arc_agi_2-0bae246bcbb7223b","rd":"2025-12-17","sid":"recBFyJPjiMxVqayO","sm":"gemini-3-flash-preview","v":0.0125},{"b":"arc_agi_2","c":0.0471,"dn":"Claude Haiku 4.5 (Thinking 1K)","m":"claude-haiku-4-5-20251001-1k","oid":"arc_agi_2-7691b5fa31917b82","rd":"2025-10-15","sid":"recjklNjhC70JPIUW","sm":"claude-haiku-4-5-20251001_1K","v":0.0125},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash (None)","e":"none","m":"deepseek-v4-1-flash","oid":"arc_agi_2-2283bd82b6c027f3","rd":"2026-09-09","sid":"recxp4Wlnj91gJfkP","sm":"deepseek-v4.1-flash_none","v":0.0125},{"b":"arc_agi_2","c":0.0527,"dn":"Deepseek R1 (05/28)","m":"deepseek-r1-0528","oid":"arc_agi_2-d275ef2c1ffedf24","rd":"2025-05-28","sid":"rectThfd9Zm2iI4QV","sm":"DeepSeek-R1-0528","v":0.0112},{"b":"arc_agi_2","c":0.06,"cfg":{"effort":"low"},"dn":"GPT-5.4 Mini (Low)","e":"low","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi_2-dcb0722efcfc8bff","rd":"2026-03-17","sid":"recw8aOAONIT3ypKX","sm":"gpt-5.4-mini-2026-03-17_low","v":0.0111},{"b":"arc_agi_2","c":0.36,"dn":"Claude 3.7 (8K)","m":"claude-3-7-sonnet-20250219-8k","oid":"arc_agi_2-cf9ab8922c4927a0","rd":"2025-02-24","sid":"recKX0Qa3ulDrS9Bz","sm":"claude-3-7-sonnet-20250219_8K","v":0.009},{"b":"arc_agi_2","c":0.0137,"cfg":{"effort":"medium"},"dn":"GPT-5 Nano (Medium)","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"arc_agi_2-a489f58344a44ca9","rd":"2025-08-07","sid":"recT6rpJqwTwbDf70","sm":"gpt-5-nano-2025-08-07_medium","v":0.0088},{"b":"arc_agi_2","c":0.1425,"dn":"Claude Sonnet 4 (Thinking 1K)","m":"claude-sonnet-4-20250514-1k","oid":"arc_agi_2-09937901795e1711","rd":"2025-05-22","sid":"recbHOUFE4mIkwxSV","sm":"claude-sonnet-4-20250514_1K","v":0.0085},{"b":"arc_agi_2","cfg":{"effort":"minimal"},"dn":"Gemini 3.5 Flash-Lite (Minimal)","e":"minimal","m":"gemini-3-5-flash-lite","oid":"arc_agi_2-d66cb10aa28ead0a","rd":"2026-07-21","sid":"rec7LQvTu0JSmrK2q","sm":"gemini-3.5-flash-lite_minimal","v":0.008333333333333333},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"DeepSeek V4 Pro 0813 (None)","e":"none","m":"deepseek-v4-pro-0813","oid":"arc_agi_2-e1213b6f209f0e8c","rd":"2026-08-13","sid":"recvl7EONLls7UXBK","sm":"deepseek-v4-pro-0813_none","v":0.008333333333333333},{"b":"arc_agi_2","c":0.1907,"dn":"o1-mini","m":"o1-mini-2024-09-12","oid":"arc_agi_2-65a40e09c91e6a6c","rd":"2024-09-12","sid":"recrnQ8qOsjBQtr5f","sm":"o1-mini-2024-09-12_unknown","v":0.0083},{"b":"arc_agi_2","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"arc_agi_2-f40372b9d8663380","rd":"2025-12-11","sid":"recPo5Agh7dkNgnn7","sm":"gpt-5.2-2025-12-11_unknown","v":0.0083},{"b":"arc_agi_2","c":0.0189,"cfg":{"effort":"low"},"dn":"GPT-5 Mini (Low)","e":"low","m":"gpt-5-mini-2025-08-07","oid":"arc_agi_2-f47967e0499aaf04","rd":"2025-08-07","sid":"recxiCwTDG3hY2wmo","sm":"gpt-5-mini-2025-08-07_low","v":0.0083},{"b":"arc_agi_2","c":2.1,"dn":"GPT-4.5","m":"gpt-4-5-preview-2025-02-27","oid":"arc_agi_2-1de7218cf9853cd7","rd":"2025-02-27","sid":"recl8hoiV46IoCWb2","sm":"gpt-4.5-preview-2025-02-27","v":0.008},{"b":"arc_agi_2","c":0.04,"dn":"Gemini 1.5 Pro","m":"gemini-1-5-pro-002","oid":"arc_agi_2-131f41b6228a3fc1","rd":"2024-09-24","sid":"recqMImpcYCvdizAV","sm":"gemini-1.5-pro-002","v":0.008},{"b":"arc_agi_2","c":0.51,"dn":"Claude 3.7 (16K)","m":"claude-3-7-sonnet-20250219-16k","oid":"arc_agi_2-ce551915387fa34e","rd":"2025-02-24","sid":"recvxEFt4mLttTnjm","sm":"claude-3-7-sonnet-20250219_16K","v":0.007000000000000001},{"b":"arc_agi_2","c":0.0691,"dn":"GPT-4.1","m":"gpt-4-1-2025-04-14","oid":"arc_agi_2-6b82383ac9bbec02","rd":"2025-04-14","sid":"recMVaawKzYg6YFHo","sm":"gpt-4.1-2025-04-14","v":0.0042},{"b":"arc_agi_2","c":0.0131,"cfg":{"effort":"low"},"dn":"Grok 3 Mini (Low)","e":"low","m":"grok-3-mini","oid":"arc_agi_2-95ece3f12f132215","rd":"2025-06-24","sid":"recZYdwW6KKeyEeof","sm":"grok-3-mini_low","v":0.0042},{"b":"arc_agi_2","c":0.0581,"cfg":{"effort":"none"},"dn":"GPT-5.1 (Thinking, None)","e":"none","m":"gpt-5-1-2025-11-13","oid":"arc_agi_2-de69d58a24958194","rd":"2025-11-13","sid":"recW0MLjEwpTkwj5d","sm":"gpt-5.1-2025-11-13_none","v":0.0042},{"b":"arc_agi_2","c":0.14,"dn":"Claude 3.7 (1K)","m":"claude-3-7-sonnet-20250219-1k","oid":"arc_agi_2-cb2c7999d463ffcb","rd":"2025-02-24","sid":"rece595ML3KUNIT1d","sm":"claude-3-7-sonnet-20250219_1K","v":0.004},{"b":"arc_agi_2","c":0.12,"dn":"Claude 3.7","m":"claude-3-7-sonnet-2025-02-19","oid":"arc_agi_2-13d3f3dcf10b145b","rd":"2025-02-24","sid":"recJXJI55BlVB71Qc","sm":"claude-3-7-sonnet-20250219","v":0},{"b":"arc_agi_2","c":0.01,"dn":"GPT-4o-mini","m":"gpt-4o-mini-2024-07-18","oid":"arc_agi_2-03c070835248b971","rd":"2024-07-18","sid":"recc7I0Ztw0vTRwWh","sm":"gpt-4o-mini-2024-07-18","v":0},{"b":"arc_agi_2","c":0.0036,"dn":"GPT-4.1-Nano","m":"gpt-4-1-nano-2025-04-14","oid":"arc_agi_2-abf5fd81d450df3f","rd":"2025-04-14","sid":"recS5hpSeASTkWd5s","sm":"gpt-4.1-nano-2025-04-14","v":0},{"b":"arc_agi_2","c":0.0062,"dn":"Llama 4 Scout","m":"llama-4-scout-17b-16e-instruct","oid":"arc_agi_2-d8bf8765574f0178","rd":"2025-04-05","sid":"recxnrmYCOKiqBoXB","sm":"Llama-4-Scout-17B-16E-Instruct","v":0},{"b":"arc_agi_2","c":0.0121,"dn":"Llama 4 Maverick","m":"llama-4-maverick-17b-128e-instruct","oid":"arc_agi_2-601d753ed58707eb","rd":"2025-04-05","sid":"reczNMgFqv1ZlJEFV","sm":"Llama-4-Maverick-17B-128E-Instruct","v":0},{"b":"arc_agi_2","c":0.0623,"cfg":{"effort":"low"},"dn":"o3-mini (Low)","e":"low","m":"o3-mini-2025-01-31","oid":"arc_agi_2-e0325aca2689070b","rd":"2025-01-31","sid":"rec0Bj1MbQ11bEqWS","sm":"o3-mini-2025-01-31_low","v":0},{"b":"arc_agi_2","c":0.0139,"dn":"GPT-4.1-Mini","m":"gpt-4-1-mini-2025-04-14","oid":"arc_agi_2-bf8210ea0c049180","rd":"2025-04-14","sid":"recAgj8f9SDr9AZyr","sm":"gpt-4.1-mini-2025-04-14","v":0},{"b":"arc_agi_2","c":0.1421,"dn":"Grok 3","m":"grok-3","oid":"arc_agi_2-3182915920a7385e","rd":"2025-04-09","sid":"recmCmq0lWBA7qzvR","sm":"grok-3","v":0},{"b":"arc_agi_2","c":0.7503,"dn":"Claude Opus 4 (Thinking 1K)","m":"claude-opus-4-20250514-1k","oid":"arc_agi_2-f6daa615d3376722","rd":"2025-05-22","sid":"recrcQGgMO3pWfmku","sm":"claude-opus-4-20250514_1K","v":0},{"b":"arc_agi_2","c":0.0488,"dn":"Magistral Small","m":"magistral-small-2506","oid":"arc_agi_2-72d8cb095e455169","rd":"2025-06-10","sid":"recOuF80RXt6b2WX8","sm":"magistral-small-2506","v":0},{"b":"arc_agi_2","cfg":{"effort":"medium"},"dn":"Magistral Medium","e":"medium","m":"magistral-medium-2506","oid":"arc_agi_2-790861723b9e7880","rd":"2025-06-10","sid":"recfC45QI8NicBzwM","sm":"magistral-medium-2506","v":0},{"b":"arc_agi_2","c":0.0885,"dn":"Gemini 2.5 Pro (Thinking 1K)","m":"gemini-2-5-pro-preview-06-05-1k","oid":"arc_agi_2-944c15ed9e912a53","rd":"2025-06-05","sid":"reczqe8FyQd5h61ZE","sm":"gemini-2.5-pro-preview-06-05_1K","v":0},{"b":"arc_agi_2","c":0.0562,"cfg":{"effort":"minimal"},"dn":"GPT-5 (Minimal)","e":"minimal","m":"gpt-5-2025-08-07","oid":"arc_agi_2-cad205bafb887eb8","rd":"2025-08-07","sid":"recLswC6YRLQnH9FQ","sm":"gpt-5-2025-08-07_minimal","v":0},{"b":"arc_agi_2","c":0.0033,"cfg":{"effort":"low"},"dn":"GPT-5 Nano (Low)","e":"low","m":"gpt-5-nano-2025-08-07","oid":"arc_agi_2-5b6761c42559c9ae","rd":"2025-08-07","sid":"recZf3o0zbmGDIYvX","sm":"gpt-5-nano-2025-08-07_low","v":0},{"b":"arc_agi_2","c":0.08,"dn":"GPT-4o","m":"gpt-4o-2024-11-20","oid":"arc_agi_2-58cff665d22e4f98","rd":"2024-11-20","sid":"recXQ8nDyFM2FrQuM","sm":"gpt-4o-2024-11-20","v":0},{"b":"arc_agi_2","c":0.0025,"cfg":{"effort":"minimal"},"dn":"GPT-5 Nano (Minimal)","e":"minimal","m":"gpt-5-nano-2025-08-07","oid":"arc_agi_2-12b4e298561d567a","rd":"2025-08-07","sid":"recvfjus4uMwZPSYy","sm":"gpt-5-nano-2025-08-07_minimal","v":0},{"b":"arc_agi_2","cfg":{"effort":"thinking"},"dn":"Magistral Medium (Thinking)","e":"thinking","m":"magistral-medium-2506","oid":"arc_agi_2-cd7163a6f2a4f51d","rd":"2025-06-10","sid":"recIb0Zl4pxpR9ELk","sm":"magistral-medium-2506","v":0},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"Inkling Small (None)","e":"none","m":"inkling-small","oid":"arc_agi_2-ec894c63633c4e81","rd":"2026-07-15","sid":"recovw8MqEuXF8DWr","sm":"inkling-small_none","v":0},{"b":"arc_agi_2","cfg":{"effort":"none"},"dn":"GPT-6 Luna (None)","e":"none","m":"gpt-6-luna","oid":"arc_agi_2-a341c2b94f07626e","rd":"2026-09-22","sid":"reciGPGUnAyc9ZsrU","sm":"gpt-6-luna_none","v":0},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Claude Fable 5 (XHigh)","e":"xhigh","m":"claude-fable-5","oid":"arc_agi-39f9f538378a7585","rd":"2026-06-09","sid":"recRjJaF7m8Y15119","sm":"claude-fable-5_xhigh","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Fable 5 (Max)","e":"max","m":"claude-fable-5","oid":"arc_agi-333fa28c12fc7aff","rd":"2026-06-09","sid":"recr0Gvn97QjInOIz","sm":"claude-fable-5_max","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-6 Astra (XHigh)","e":"xhigh","m":"gpt-6-astra","oid":"arc_agi-f101218ecb35e592","rd":"2026-09-03","sid":"recyEDXXOmAiUIkgE","sm":"gpt-6-astra_xhigh","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-6 Astra (High)","e":"high","m":"gpt-6-astra","oid":"arc_agi-ecdb9d43ca42becd","rd":"2026-09-03","sid":"reczM3BItCjOcxd45","sm":"gpt-6-astra_high","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Opus 5.5 (High)","e":"high","m":"claude-opus-5-5","oid":"arc_agi-890a93cb67fe8fe5","rd":"2026-09-22","sid":"recysyJljY45NZlLc","sm":"claude-opus-5-5_high","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-6.1 Sol (XHigh)","e":"xhigh","m":"gpt-6-1-sol","oid":"arc_agi-373946f0e144d200","rd":"2026-09-29","sid":"recMW30wY6ZEg280e","sm":"gpt-6.1-sol_xhigh","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-6.1 Sol (High)","e":"high","m":"gpt-6-1-sol","oid":"arc_agi-6a11430f221a0f5c","rd":"2026-09-29","sid":"recaJC1yARFk9jQsG","sm":"gpt-6.1-sol_high","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Gemini 3.8 Flash (High)","e":"high","m":"gemini-3-8-flash","oid":"arc_agi-c98f94ff195cbec3","rd":"2026-09-02","sid":"recUS52JUahz03XaX","sm":"gemini-3.8-flash_high","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Gemini 4 Argon (Medium)","e":"medium","m":"gemini-4-argon","oid":"arc_agi-133f8b924555b33f","rd":"2026-09-30","sid":"recE0CZhrJwWrmDM5","sm":"gemini-4-argon_medium","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Gemini 4 Argon (High)","e":"high","m":"gemini-4-argon","oid":"arc_agi-171f6b7f9d65abeb","rd":"2026-09-30","sid":"recguygHYW2l1K4Gz","sm":"gemini-4-argon_high","src":"https://arcprize.org/leaderboard","v":0.985},{"b":"arc_agi","c":0.5223,"dn":"Gemini 3.1 Pro (Preview)","m":"gemini-3-1-pro-preview","oid":"arc_agi-46907d1b109f772a","rd":"2026-02-19","sid":"recVzc3uOYbFmIWtV","sm":"gemini-3.1-pro-preview","src":"https://arcprize.org/leaderboard","v":0.98},{"b":"arc_agi","c":0.4,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol (xHigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"arc_agi-afd39aecf819f7c6","rd":"2026-07-09","sid":"recrNPlnXllOt0Hvg","sm":"gpt-5.6-sol_xhigh","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Opus 5 (High)","e":"max","m":"claude-opus-5","oid":"arc_agi-54e3936e32de323d","rd":"2026-07-24","sid":"recSq1VCkt6IwLBGs","sm":"claude-opus-5_max","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Opus 5 (High)","e":"high","m":"claude-opus-5","oid":"arc_agi-618bb6d9611f1d44","rd":"2026-07-24","sid":"recyQEeaf1aLYbe8E","sm":"claude-opus-5_high","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Opus 5 (Max)","e":"max","m":"claude-opus-5","oid":"arc_agi-fad6326de20a92b8","rd":"2026-07-24","sid":"rechlxvtvt0jfACe1","sm":"claude-opus-5_max","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol (XHigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"arc_agi-61cb1abb002b656f","rd":"2026-07-09","sid":"recP0shtwkpwoQk7S","sm":"gpt-5.6-sol_xhigh","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Fable 5.1 (Max)","e":"max","m":"claude-fable-5-1","oid":"arc_agi-22fac8b691a332c4","rd":"2026-09-01","sid":"recvaiu98Swpidp4x","sm":"claude-fable-5-1_max","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"GPT-6 Astra (Max)","e":"max","m":"gpt-6-astra","oid":"arc_agi-90124efd627fcaf6","rd":"2026-09-03","sid":"rec6WjXtyqxLyETqC","sm":"gpt-6-astra_max","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"GPT-6 Astra (Medium)","e":"medium","m":"gpt-6-astra","oid":"arc_agi-87255205cc7cd919","rd":"2026-09-03","sid":"recNKa28y2D8Phzla","sm":"gpt-6-astra_medium","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Opus 5.5 (Max)","e":"max","m":"claude-opus-5-5","oid":"arc_agi-4f6ecd69a7e7afdd","rd":"2026-09-22","sid":"rec1IeqwhlQmodx2I","sm":"claude-opus-5-5_max","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Claude Opus 5.5 (XHigh)","e":"xhigh","m":"claude-opus-5-5","oid":"arc_agi-1b7f73d14b332277","rd":"2026-09-22","sid":"recZsJ27udDHU9Av1","sm":"claude-opus-5-5_xhigh","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Claude Opus 5.5 (Medium)","e":"medium","m":"claude-opus-5-5","oid":"arc_agi-ebf7fa86a8b6d678","rd":"2026-09-22","sid":"recjHEJE0LM6Uq948","sm":"claude-opus-5-5_medium","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Gemini 3.8 Flash (Medium)","e":"medium","m":"gemini-3-8-flash","oid":"arc_agi-2a9210b3cb8a16cb","rd":"2026-09-02","sid":"recnv8lLg5ADoP5fY","sm":"gemini-3.8-flash_medium","src":"https://arcprize.org/leaderboard","v":0.975},{"b":"arc_agi","c":0.3,"cfg":{"effort":"high"},"dn":"GPT-5.6 Sol (High)","e":"high","m":"gpt-5-6-sol","oid":"arc_agi-4e9cffdc3b392409","rd":"2026-07-09","sid":"recCdwY3IG6HNCGGJ","sm":"gpt-5.6-sol_high","src":"https://arcprize.org/leaderboard","v":0.97},{"b":"arc_agi","c":4.53,"cfg":{"effort":"high"},"dn":"GPT-5.5 Pro (High)","e":"high","m":"gpt-5-5-pro","oid":"arc_agi-b1f095ca96dff018","rd":"2026-04-23","sid":"recl8K2yYdGuM18Vy","sm":"gpt-5.5-pro_high","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","c":0.54,"cfg":{"effort":"max"},"dn":"GPT-5.6 Sol (Max)","e":"max","m":"gpt-5-6-sol","oid":"arc_agi-e3b29097c1995247","rd":"2026-07-09","sid":"recQ2TSIxubhoK4nZ","sm":"gpt-5.6-sol_max","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","c":0.55,"cfg":{"effort":"max"},"dn":"GPT-5.6 Terra (Max)","e":"max","m":"gpt-5-6-terra","oid":"arc_agi-b94d0f1a6b4b116d","rd":"2026-07-09","sid":"recabQE8jIZy7XTnk","sm":"gpt-5.6-terra_max","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Claude Fable 5.1 (XHigh)","e":"xhigh","m":"claude-fable-5-1","oid":"arc_agi-c490a6c6beed3446","rd":"2026-09-01","sid":"rece2Ph5U0ubgtxBs","sm":"claude-fable-5-1_xhigh","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"GPT-6 Astra (Low)","e":"low","m":"gpt-6-astra","oid":"arc_agi-be68b7a1e50e0606","rd":"2026-09-03","sid":"recaEMrIaVJltKYO2","sm":"gpt-6-astra_low","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"GPT-6.1 Sol (Max)","e":"max","m":"gpt-6-1-sol","oid":"arc_agi-8ffb0f5abdd8a598","rd":"2026-09-29","sid":"recXdSM65dkne27Oz","sm":"gpt-6.1-sol_max","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Sonnet 5.5 (Max)","e":"max","m":"claude-sonnet-5-5","oid":"arc_agi-39d1406acbbfcdea","rd":"2026-09-28","sid":"reciXbOKaf1TyHNnH","sm":"claude-sonnet-5-5_max","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Claude Sonnet 5.5 (XHigh)","e":"xhigh","m":"claude-sonnet-5-5","oid":"arc_agi-292714d1bca40d98","rd":"2026-09-28","sid":"recuY51ujbKuqjEOd","sm":"claude-sonnet-5-5_xhigh","src":"https://arcprize.org/leaderboard","v":0.965},{"b":"arc_agi","dn":"Gemini 3 Deep Think (2/26)","m":"gemini-3-deep-think-preview","oid":"arc_agi-354f4f24608dec87","rd":"2026-02-12","sid":"recB6y5fhltz222iF","sm":"gemini-3-deep-think-preview","src":"https://arcprize.org/leaderboard","v":0.96},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Fable 5.1 (High)","e":"high","m":"claude-fable-5-1","oid":"arc_agi-e2c71316fb590618","rd":"2026-09-01","sid":"rec6sj6eIQNzR0HJq","sm":"claude-fable-5-1_high","src":"https://arcprize.org/leaderboard","v":0.96},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Fable 5 (High)","e":"high","m":"claude-fable-5","oid":"arc_agi-1254d89a613fcdda","rd":"2026-06-09","sid":"recgy0nKpQ48MWzVu","sm":"claude-fable-5_high","src":"https://arcprize.org/leaderboard","v":0.9550000000000001},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Gemini 3.7 Flash (High)","e":"high","m":"gemini-3-7-flash","oid":"arc_agi-75a57fd15054f498","rd":"2026-08-13","sid":"recsY80edOvxj4oOR","sm":"gemini-3.7-flash_high","src":"https://arcprize.org/leaderboard","v":0.9550000000000001},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"GPT-6 Sol (Max)","e":"max","m":"gpt-6-sol","oid":"arc_agi-fc179a04194f86da","rd":"2026-09-22","sid":"rectIHDdOk1NRtKSn","sm":"gpt-6-sol_max","src":"https://arcprize.org/leaderboard","v":0.9550000000000001},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"GPT-6.1 Sol (Medium)","e":"medium","m":"gpt-6-1-sol","oid":"arc_agi-3b69adfa1203eafb","rd":"2026-09-29","sid":"recrBpf66NQy8mN4T","sm":"gpt-6.1-sol_medium","src":"https://arcprize.org/leaderboard","v":0.9550000000000001},{"b":"arc_agi","c":0.73,"cfg":{"effort":"xhigh"},"dn":"GPT-5.5 (xHigh)","e":"xhigh","m":"gpt-5-5","oid":"arc_agi-a6ddbc273ba7e201","rd":"2026-04-23","sid":"recAosypqreZ8jmIT","sm":"gpt-5.5_xhigh","src":"https://arcprize.org/leaderboard","v":0.9500000000000001},{"b":"arc_agi","c":4.52,"cfg":{"effort":"xhigh"},"dn":"GPT-5.5 Pro (xHigh)","e":"xhigh","m":"gpt-5-5-pro","oid":"arc_agi-3430b9eb9ab1c831","rd":"2026-04-23","sid":"recrLBN8ncVRXcY2G","sm":"gpt-5.5-pro_xhigh","src":"https://arcprize.org/leaderboard","v":0.9500000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 Pro (XHigh)","e":"xhigh","m":"gpt-5-5-pro","oid":"arc_agi-071f44a54e805b0e","rd":"2026-04-23","sid":"rec9O8ndiJV4IzmHu","sm":"gpt-5.5-pro_xhigh","src":"https://arcprize.org/leaderboard","v":0.9500000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 (XHigh)","e":"xhigh","m":"gpt-5-5","oid":"arc_agi-465dadd670357c6c","rd":"2026-04-23","sid":"recqSzsMcRfH9WyxG","sm":"gpt-5.5_xhigh","src":"https://arcprize.org/leaderboard","v":0.9500000000000001},{"b":"arc_agi","c":7.21,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Pro (xHigh)","e":"xhigh","m":"gpt-5-4-pro-2026-03-05","oid":"arc_agi-01bd3c43e2bacae3","rd":"2026-03-05","sid":"recoac8q5cQOYtiEr","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.9450000000000001},{"b":"arc_agi","c":0.56,"cfg":{"effort":"high"},"dn":"GPT-5.5 (High)","e":"high","m":"gpt-5-5","oid":"arc_agi-78306e0974ca993f","rd":"2026-04-23","sid":"rechGGHK5NkpuRbvZ","sm":"gpt-5.5_high","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","c":0.7704815999999995,"cfg":{"effort":"max"},"dn":"Kimi K3 (Max)","e":"max","m":"kimi-k3","oid":"arc_agi-d13a866ce01bb4af","rd":"2026-07-16","sid":"recSadJjXBVay9MJG","sm":"kimi-k3_max","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Pro (XHigh)","e":"xhigh","m":"gpt-5-4-pro-2026-03-05","oid":"arc_agi-d250bf84eca46bd5","rd":"2026-03-05","sid":"recXEd6tFFq0OodXL","sm":"gpt-5.4-pro-2026-03-05_xhigh","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Claude Fable 5.1 (Medium)","e":"medium","m":"claude-fable-5-1","oid":"arc_agi-be6364486f38d804","rd":"2026-09-01","sid":"rec3AzR8j5g4enMwx","sm":"claude-fable-5-1_medium","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash (Max)","e":"max","m":"deepseek-v4-1-flash","oid":"arc_agi-5380ee87b78169ba","rd":"2026-09-09","sid":"rec03PZkQa0xvTemg","sm":"deepseek-v4.1-flash_max","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"DeepSeek V4.1 Flash 1017 (High)","e":"high","m":"deepseek-v4-1-flash-1017","oid":"arc_agi-5b22cfe98ab75252","rd":"2026-10-17","sid":"recekcid5JP3pqa9K","sm":"deepseek-v4.1-flash-1017_high","src":"https://arcprize.org/leaderboard","v":0.9450000000000001},{"b":"arc_agi","c":0.26,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra (xHigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"arc_agi-58dd731c8304bc04","rd":"2026-07-09","sid":"recZAbgf19TPU7sN1","sm":"gpt-5.6-terra_xhigh","src":"https://arcprize.org/leaderboard","v":0.9400000000000001},{"b":"arc_agi","dn":"Claude Opus 4.6 (120K, High)","m":"claude-opus-4-6-120k","oid":"arc_agi-5b4e7f5d5a77ed93","rd":"2026-02-05","sid":"recazrzN9UYUSV1iu","sm":"claude-opus-4-6_120K","src":"https://arcprize.org/leaderboard","v":0.9400000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra (XHigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"arc_agi-0b9ef5b79f4bb623","rd":"2026-07-09","sid":"recUgeEiFE9fXQL3Z","sm":"gpt-5.6-terra_xhigh","src":"https://arcprize.org/leaderboard","v":0.9400000000000001},{"b":"arc_agi","dn":"Claude Opus 4.6 (High, 120k thinking)","m":"claude-opus-4-6-120k","oid":"arc_agi-3110ac09a859fea8","rd":"2026-02-05","sid":"recZTnD93GgPevLI2","sm":"claude-opus-4-6_120K","src":"https://arcprize.org/leaderboard","v":0.94},{"b":"arc_agi","c":0.62,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (xHigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"arc_agi-e636629cfceba799","rd":"2026-03-05","sid":"recYZZpkF3X0HOhG6","sm":"gpt-5.4-2026-03-05_xhigh","v":0.9367000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (XHigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"arc_agi-d9ef55f272b16b53","rd":"2026-03-05","sid":"recWiLBMjYySlXFw8","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://arcprize.org/leaderboard","v":0.9367000000000001},{"b":"arc_agi","c":1.41,"cfg":{"effort":"high"},"dn":"Claude Opus 4.7 (High)","e":"high","m":"claude-opus-4-7","oid":"arc_agi-5168716c6e2c22df","rd":"2026-04-16","sid":"recsAZge7JFpmZUCV","sm":"claude-opus-4-7_high","v":0.935},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude 4.7 (High)","e":"high","m":"claude-opus-4-7","oid":"arc_agi-3ca6c2344a7d3475","rd":"2026-04-16","sid":"recGPGem5qYI7gRl4","sm":"claude-opus-4-7_high","src":"https://arcprize.org/leaderboard","v":0.935},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"GPT-6.1 Sol (Low)","e":"low","m":"gpt-6-1-sol","oid":"arc_agi-e025ba977cb07c3d","rd":"2026-09-29","sid":"recumwttgXcWDbT5w","sm":"gpt-6.1-sol_low","src":"https://arcprize.org/leaderboard","v":0.935},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash 1017 (Max)","e":"max","m":"deepseek-v4-1-flash-1017","oid":"arc_agi-9bd39812e4531e4d","rd":"2026-10-17","sid":"rectlq2FfZ6wgFgwo","sm":"deepseek-v4.1-flash-1017_max","src":"https://arcprize.org/leaderboard","v":0.935},{"b":"arc_agi","dn":"Claude Opus 4.6 (120K, Max)","m":"claude-opus-4-6-120k","oid":"arc_agi-3138ca639c5ef421","rd":"2026-02-05","sid":"recM43lOIaRtUf1mn","sm":"claude-opus-4-6_120K","src":"https://arcprize.org/leaderboard","v":0.93},{"b":"arc_agi","c":0.37,"cfg":{"effort":"high"},"dn":"GPT-5.4 (High)","e":"high","m":"gpt-5-4-2026-03-05","oid":"arc_agi-66a9c9e7a1b9439e","rd":"2026-03-05","sid":"recggD3Y6D88kSVaH","sm":"gpt-5.4-2026-03-05_high","v":0.9267000000000001},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-6 Sol (XHigh)","e":"xhigh","m":"gpt-6-sol","oid":"arc_agi-c7c0dac5427b4e9c","rd":"2026-09-22","sid":"recsCTOwUeAFAOimV","sm":"gpt-6-sol_xhigh","src":"https://arcprize.org/leaderboard","v":0.9266666666666666},{"b":"arc_agi","c":0.4278,"cfg":{"effort":"high"},"dn":"Gemini 3.5 Flash (High)","e":"high","m":"gemini-3-5-flash","oid":"arc_agi-12800ce46108659b","rd":"2026-05-19","sid":"recVnYyaVQjDvDtup","sm":"gemini-3.5-flash_high","src":"https://arcprize.org/leaderboard","v":0.925},{"b":"arc_agi","c":2.332,"cfg":{"effort":"max"},"dn":"Claude Opus 4.8 (Max)","e":"max","m":"claude-opus-4-8","oid":"arc_agi-98678412957fd25a","rd":"2026-05-28","sid":"recX5Vpu2SxT2jvtJ","sm":"claude-opus-4-8_max","src":"https://arcprize.org/leaderboard","v":0.925},{"b":"arc_agi","c":0.22,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Sol (Medium)","e":"medium","m":"gpt-5-6-sol","oid":"arc_agi-b9e5471e497986f5","rd":"2026-07-09","sid":"recBnEMNkjbANkBPV","sm":"gpt-5.6-sol_medium","src":"https://arcprize.org/leaderboard","v":0.925},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Claude Fable 5 (Medium)","e":"medium","m":"claude-fable-5","oid":"arc_agi-b8d5386cbfb029dd","rd":"2026-06-09","sid":"recnnva4EmGtevmRJ","sm":"claude-fable-5_medium","src":"https://arcprize.org/leaderboard","v":0.925},{"b":"arc_agi","c":0.39,"cfg":{"effort":"medium"},"dn":"GPT-5.5 (Medium)","e":"medium","m":"gpt-5-5","oid":"arc_agi-d1b8ad82c6dfa1f3","rd":"2026-04-23","sid":"rec3T7XZVcuk74DEh","sm":"gpt-5.5_medium","src":"https://arcprize.org/leaderboard","v":0.9217000000000001},{"b":"arc_agi","c":2.58,"cfg":{"effort":"max"},"dn":"Claude Opus 4.7 (Max)","e":"max","m":"claude-opus-4-7","oid":"arc_agi-ab304a3848a51998","rd":"2026-04-16","sid":"recBvnwhpAjJYVBYf","sm":"claude-opus-4-7_max","v":0.92},{"b":"arc_agi","c":1.0416,"cfg":{"effort":"high"},"dn":"Claude Opus 4.8 (High)","e":"high","m":"claude-opus-4-8","oid":"arc_agi-52c924bc098e2cd0","rd":"2026-05-28","sid":"recYbC1GbUCLyPSIp","sm":"claude-opus-4-8_high","src":"https://arcprize.org/leaderboard","v":0.92},{"b":"arc_agi","c":0.19,"cfg":{"effort":"high"},"dn":"GPT-5.6 Terra (High)","e":"high","m":"gpt-5-6-terra","oid":"arc_agi-c809dc82e9d1ea71","rd":"2026-07-09","sid":"recgED1Md3qIP7P7j","sm":"gpt-5.6-terra_high","src":"https://arcprize.org/leaderboard","v":0.92},{"b":"arc_agi","dn":"Claude Opus 4.6 (120K, Medium)","m":"claude-opus-4-6-120k","oid":"arc_agi-c92fcec4886d2f7c","rd":"2026-02-05","sid":"recoN4YV9cS1BcESr","sm":"claude-opus-4-6_120K","src":"https://arcprize.org/leaderboard","v":0.92},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude 4.7 (Max)","e":"max","m":"claude-opus-4-7","oid":"arc_agi-43d94f5a02037861","rd":"2026-04-16","sid":"rec0Zi0OrdC9mEo7f","sm":"claude-opus-4-7_max","src":"https://arcprize.org/leaderboard","v":0.92},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Sonnet 5.5 (High)","e":"high","m":"claude-sonnet-5-5","oid":"arc_agi-1ec3506cd0f989af","rd":"2026-09-28","sid":"rec4YxsA76vMi6HeI","sm":"claude-sonnet-5-5_high","src":"https://arcprize.org/leaderboard","v":0.92},{"b":"arc_agi","c":0.9115,"cfg":{"effort":"medium"},"dn":"Claude Opus 4.8 (Medium)","e":"medium","m":"claude-opus-4-8","oid":"arc_agi-46726337304f5513","rd":"2026-05-28","sid":"recOGBDeWlb8DCkFv","sm":"claude-opus-4-8_medium","src":"https://arcprize.org/leaderboard","v":0.915},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"DeepSeek V4.1 Flash 1017 (Low)","e":"low","m":"deepseek-v4-1-flash-1017","oid":"arc_agi-ce9d68b6e7bd29f3","rd":"2026-10-17","sid":"recs4bRzWCI5sqJNL","sm":"deepseek-v4.1-flash-1017_low","src":"https://arcprize.org/leaderboard","v":0.915},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Gemini 3.6 Flash (High)","e":"high","m":"gemini-3-6-flash","oid":"arc_agi-7b310f1998966cfc","rd":"2026-07-21","sid":"recwFpri94sXuU76A","sm":"gemini-3.6-flash_high","src":"https://arcprize.org/leaderboard","v":0.9116666666666666},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Gemini 3.7 Flash (Medium)","e":"medium","m":"gemini-3-7-flash","oid":"arc_agi-15fea6efb0818578","rd":"2026-08-13","sid":"recfGdSHF9DQEv10q","sm":"gemini-3.7-flash_medium","src":"https://arcprize.org/leaderboard","v":0.9116666666666666},{"b":"arc_agi","c":0.76,"cfg":{"effort":"low"},"dn":"Claude 4.7 (Low)","e":"low","m":"claude-opus-4-7","oid":"arc_agi-e41d6af84a1bb5ef","rd":"2026-04-16","sid":"recQXzhqc6WDcJWp6","sm":"claude-opus-4-7_low","src":"https://arcprize.org/leaderboard","v":0.91},{"b":"arc_agi","c":1.04,"cfg":{"effort":"medium"},"dn":"Claude 4.7 (Medium)","e":"medium","m":"claude-opus-4-7","oid":"arc_agi-66fb79b40c04e528","rd":"2026-04-16","sid":"recBBFNgJ4AHLvHWQ","sm":"claude-opus-4-7_medium","src":"https://arcprize.org/leaderboard","v":0.91},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-6 Sol (High)","e":"high","m":"gpt-6-sol","oid":"arc_agi-23af39b3cc2c0af9","rd":"2026-09-22","sid":"rec9VMdTpPjpBgVjh","sm":"gpt-6-sol_high","src":"https://arcprize.org/leaderboard","v":0.91},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"GLM-5.3-Flash (Max)","e":"max","m":"glm-5-3-flash","oid":"arc_agi-3f4a505c88ac425f","rd":"2026-08-20","sid":"recvofHSjBKUgoxSb","sm":"glm-5.3-flash_max","src":"https://arcprize.org/leaderboard","v":0.91},{"b":"arc_agi","c":11.6542,"cfg":{"effort":"xhigh"},"dn":"GPT-5.2 Pro (X-High)","e":"xhigh","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi-d39755e91a5b6457","rd":"2025-12-11","sid":"recT4SRr3jv5zIy1J","sm":"gpt-5.2-pro-2025-12-11_xhigh","v":0.905},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 Pro (xHigh)","e":"xhigh","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi-a198e815b4baa9bd","rd":"2025-12-11","sid":"recKxtWWM4vXnlEV3","sm":"gpt-5.2-pro-2025-12-11_xhigh","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 Pro (XHigh)","e":"xhigh","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi-a88c64981278d34f","rd":"2025-12-11","sid":"recgKflccwPhpkeuw","sm":"gpt-5.2-pro-2025-12-11_xhigh","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Claude Fable 5 (Low)","e":"low","m":"claude-fable-5","oid":"arc_agi-80b5949708a65629","rd":"2026-06-09","sid":"recicAFlVclfoLIh1","sm":"claude-fable-5_low","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"DeepSeek V4 Pro 0813 (Low)","e":"low","m":"deepseek-v4-pro-0813","oid":"arc_agi-cb0a17765039a9c9","rd":"2026-08-13","sid":"rec0JM4Fs9lGdZZjO","sm":"deepseek-v4-pro-0813_low","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Gemini 3.8 Flash (Low)","e":"low","m":"gemini-3-8-flash","oid":"arc_agi-fb51172b44ec2c68","rd":"2026-09-02","sid":"rec7TCZLb1chGzFNe","sm":"gemini-3.8-flash_low","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"DeepSeek V4.1 Flash (Low)","e":"low","m":"deepseek-v4-1-flash","oid":"arc_agi-8594023324d70129","rd":"2026-09-09","sid":"recTRRlP1cf0g3Akw","sm":"deepseek-v4.1-flash_low","src":"https://arcprize.org/leaderboard","v":0.905},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Grok 4.7 (High)","e":"high","m":"grok-4-7","oid":"arc_agi-c05b7cc1dad7ebf6","rd":"2026-09-21","sid":"recRXbA47zGQ6cvM9","sm":"grok-4.7_high","src":"https://arcprize.org/leaderboard","v":0.9016666666666666},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"DeepSeek V4 Pro 0813 (Max)","e":"max","m":"deepseek-v4-pro-0813","oid":"arc_agi-29712895093892ae","rd":"2026-08-13","sid":"reci9sB7ercpQZT3r","sm":"deepseek-v4-pro-0813_max","src":"https://arcprize.org/leaderboard","v":0.9},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Claude Fable 5.1 (Low)","e":"low","m":"claude-fable-5-1","oid":"arc_agi-0dd54cfd13e283dc","rd":"2026-09-01","sid":"rec3yIZZxWevjB7wc","sm":"claude-fable-5-1_low","src":"https://arcprize.org/leaderboard","v":0.9},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Grok 4.7 (Medium)","e":"medium","m":"grok-4-7","oid":"arc_agi-ab9a0e282ae7970d","rd":"2026-09-21","sid":"rec1jjTAcJ9jiqyhZ","sm":"grok-4.7_medium","src":"https://arcprize.org/leaderboard","v":0.9},{"b":"arc_agi","c":0.35,"dn":"Grok 4.20 (Reasoning)","m":"grok-4-20","oid":"arc_agi-ab1da071283c9087","rd":"2026-02-17","sid":"rec1UyvWkk0dqISpu","sm":"grok-4-20","src":"https://arcprize.org/leaderboard","v":0.895},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Grok 4.7 (XHigh)","e":"xhigh","m":"grok-4-7","oid":"arc_agi-64d44b336f88462a","rd":"2026-09-21","sid":"recRul5pqB3jxI0pP","sm":"grok-4.7_xhigh","src":"https://arcprize.org/leaderboard","v":0.895},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"DeepSeek V4 Flash 0731 (Max)","e":"max","m":"deepseek-v4-flash-0731","oid":"arc_agi-ea6c2870cd047685","rd":"2026-07-31","sid":"rece26MXZZVoiMLqB","sm":"deepseek-v4-flash-0731_max","src":"https://arcprize.org/leaderboard","v":0.89},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"Claude Haiku 5.5 (Max)","e":"max","m":"claude-haiku-5-5","oid":"arc_agi-ecc52db4ab215ec7","rd":"2026-10-07","sid":"rec3yL31Vz5VXApWU","sm":"claude-haiku-5-5_max","src":"https://arcprize.org/leaderboard","v":0.89},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Muse Spark Open (XHigh)","e":"xhigh","m":"muse-spark-open-816b-a42b","oid":"arc_agi-a23f1f1b15dd4fc0","rd":"2026-10-15","sid":"recTcufRzrYPvwKbL","sm":"Muse-Spark-Open-816B-A42B_xhigh","src":"https://arcprize.org/leaderboard","v":0.89},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Claude Opus 5.5 (Low)","e":"low","m":"claude-opus-5-5","oid":"arc_agi-10e68245acd84c97","rd":"2026-09-22","sid":"recwmHB4kGboOp7TZ","sm":"claude-opus-5-5_low","src":"https://arcprize.org/leaderboard","v":0.885},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"DeepSeek V4.1 Flash (High)","e":"high","m":"deepseek-v4-1-flash","oid":"arc_agi-4360160c62648041","rd":"2026-09-09","sid":"rec8UYQFgWOyu1KPm","sm":"deepseek-v4.1-flash_high","src":"https://arcprize.org/leaderboard","v":0.885},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Muse Spark Open (High)","e":"high","m":"muse-spark-open-816b-a42b","oid":"arc_agi-9c3ed9146efd60ac","rd":"2026-10-15","sid":"recEpdByen0ab9LEq","sm":"Muse-Spark-Open-816B-A42B_high","src":"https://arcprize.org/leaderboard","v":0.885},{"b":"arc_agi","c":0.6711,"cfg":{"effort":"low"},"dn":"Claude Opus 4.8 (Low)","e":"low","m":"claude-opus-4-8","oid":"arc_agi-5157d8ba97daee85","rd":"2026-05-28","sid":"rec3CNglZ0AQBUIRC","sm":"claude-opus-4-8_low","src":"https://arcprize.org/leaderboard","v":0.88},{"b":"arc_agi","c":0.32,"cfg":{"effort":"max"},"dn":"GPT-5.6 Luna (Max)","e":"max","m":"gpt-5-6-luna","oid":"arc_agi-6a1dd27aa187b536","rd":"2026-07-09","sid":"recYjOYezFdRMvEGW","sm":"gpt-5.6-luna_max","src":"https://arcprize.org/leaderboard","v":0.88},{"b":"arc_agi","c":0.21,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna (xHigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"arc_agi-9ba52f2ebd3c06ee","rd":"2026-07-09","sid":"recN52OtrmxlFYCZs","sm":"gpt-5.6-luna_xhigh","src":"https://arcprize.org/leaderboard","v":0.8767},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna (XHigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"arc_agi-8f120442772b4165","rd":"2026-07-09","sid":"recqLDFFeV5jU45oY","sm":"gpt-5.6-luna_xhigh","src":"https://arcprize.org/leaderboard","v":0.8767},{"b":"arc_agi","c":77.16,"dn":"Gemini 3 Deep Think (Preview)","m":"gemini-3-deep-think-preview","oid":"arc_agi-3d1198440fd8d5bd","rd":"2026-02-12","sid":"recpC5nib10TXfNiG","sm":"gemini-3-deep-think-preview","v":0.875},{"b":"arc_agi","dn":"Gemini 3 Deep Think (Preview) ²","m":"gemini-3-deep-think-preview","oid":"arc_agi-5c116f821aa39cba","rd":"2026-02-12","sid":"recvkCnt2z3iBicgq","sm":"gemini-3-deep-think-preview","src":"https://arcprize.org/leaderboard","v":0.875},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Grok 4.6 (Medium)","e":"medium","m":"grok-4-6","oid":"arc_agi-43b8e01fc2b645df","rd":"2026-08-12","sid":"recIu38h0FzayjCH9","sm":"grok-4.6_medium","src":"https://arcprize.org/leaderboard","v":0.875},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Qwen3.8-27B (XHigh)","e":"xhigh","m":"qwen3-8-27b","oid":"arc_agi-aefce40dc947fc11","rd":"2026-08-14","sid":"recFN8VP5x8favCm9","sm":"qwen3.8-27b_xhigh","src":"https://arcprize.org/leaderboard","v":0.875},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Claude Haiku 5.5 (XHigh)","e":"xhigh","m":"claude-haiku-5-5","oid":"arc_agi-923b247599781d89","rd":"2026-10-07","sid":"recasQQqwOasXIfD0","sm":"claude-haiku-5-5_xhigh","src":"https://arcprize.org/leaderboard","v":0.875},{"b":"arc_agi","c":0.31,"cfg":{"effort":"medium"},"dn":"Grok 4.5 (Medium)","e":"medium","m":"grok-4-5","oid":"arc_agi-fdd8a91b8b37db44","rd":"2026-07-08","sid":"recaxlUnEyQPIo6dH","sm":"grok-4.5_medium","src":"https://arcprize.org/leaderboard","v":0.8717},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"DeepSeek V4 Pro 0813 (High)","e":"high","m":"deepseek-v4-pro-0813","oid":"arc_agi-12eccda1c034eb49","rd":"2026-08-13","sid":"rechTprNycUD7yIEb","sm":"deepseek-v4-pro-0813_high","src":"https://arcprize.org/leaderboard","v":0.8716666666666666},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"DeepSeek V4 Flash 0731 (High)","e":"high","m":"deepseek-v4-flash-0731","oid":"arc_agi-2fc645d4352ad503","rd":"2026-07-31","sid":"rec3H90U0FFYQCSXb","sm":"deepseek-v4-flash-0731_high","src":"https://arcprize.org/leaderboard","v":0.87},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Grok 4.6 (XHigh)","e":"xhigh","m":"grok-4-6","oid":"arc_agi-782a9ee07e1f85f0","rd":"2026-08-12","sid":"recs712JyZ2vRzQvO","sm":"grok-4.6_xhigh","src":"https://arcprize.org/leaderboard","v":0.87},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Grok 4.6 (High)","e":"high","m":"grok-4-6","oid":"arc_agi-48e4d89092eda76d","rd":"2026-08-12","sid":"recwvQT7NkltiBYPm","sm":"grok-4.6_high","src":"https://arcprize.org/leaderboard","v":0.87},{"b":"arc_agi","cfg":{"effort":"max"},"dn":"GPT-6 Luna (Max)","e":"max","m":"gpt-6-luna","oid":"arc_agi-3c8251faccc8ac17","rd":"2026-09-22","sid":"recZNbMuT1pjom7Jn","sm":"gpt-6-luna_max","src":"https://arcprize.org/leaderboard","v":0.8666666666666667},{"b":"arc_agi","c":0.47977101000000005,"cfg":{"effort":"high"},"dn":"Kimi K3 (High)","e":"high","m":"kimi-k3","oid":"arc_agi-304fc1ae77d10317","rd":"2026-07-16","sid":"recqGzK15OabAHffc","sm":"kimi-k3_high","src":"https://arcprize.org/leaderboard","v":0.8666666666666666},{"b":"arc_agi","c":1.1908,"cfg":{"effort":"high"},"dn":"Claude Sonnet 4.6 (High)","e":"high","m":"claude-sonnet-4-6","oid":"arc_agi-cd626b7c80d0d1fb","rd":"2026-02-17","sid":"recVQARdNCk8I5YS2","sm":"claude-sonnet-4-6_high","v":0.865},{"b":"arc_agi","c":0.96,"cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (X-High)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi-934a2b9b85f7dfb0","rd":"2025-12-11","sid":"recgxHD7jV9nk6VyZ","sm":"gpt-5.2-2025-12-11_xhigh","src":"https://arcprize.org/leaderboard","v":0.862},{"b":"arc_agi","c":0.25,"cfg":{"effort":"medium"},"dn":"GPT-5.4 (Medium)","e":"medium","m":"gpt-5-4-2026-03-05","oid":"arc_agi-2753580faf5cf91d","rd":"2026-03-05","sid":"recJZdb4rBpTeaaTI","sm":"gpt-5.4-2026-03-05_medium","v":0.8617},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (xHigh)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi-60906f11b0a5ba24","rd":"2025-12-11","sid":"recKpS4EjVfqnm3Ow","sm":"gpt-5.2-2025-12-11_xhigh","src":"https://arcprize.org/leaderboard","v":0.8617},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.2 (XHigh)","e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"arc_agi-8ed507c6b4d48b5e","rd":"2025-12-11","sid":"recoRUIJo45xau1mW","sm":"gpt-5.2-2025-12-11_xhigh","src":"https://arcprize.org/leaderboard","v":0.8617},{"b":"arc_agi","c":1.452,"cfg":{"effort":"max"},"dn":"Claude Sonnet 4.6 (Max)","e":"max","m":"claude-sonnet-4-6","oid":"arc_agi-5f7951b763cf6f20","rd":"2026-02-17","sid":"recsU65UVdKsRlUJk","sm":"claude-sonnet-4-6_max","v":0.86},{"b":"arc_agi","dn":"Claude Opus 4.6 (120K, Low)","m":"claude-opus-4-6-120k","oid":"arc_agi-1a7a1ca15ddf5c52","rd":"2026-02-05","sid":"recAgfLcp2ceTsir9","sm":"claude-opus-4-6_120K","src":"https://arcprize.org/leaderboard","v":0.86},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"GPT-6 Astra (None)","e":"none","m":"gpt-6-astra","oid":"arc_agi-f244407321ab12b7","rd":"2026-09-03","sid":"recIdwsWZBFa80dKN","sm":"gpt-6-astra_none","src":"https://arcprize.org/leaderboard","v":0.86},{"b":"arc_agi","c":5.8694,"cfg":{"effort":"high"},"dn":"GPT-5.2 Pro (High)","e":"high","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi-0ab4f2146321b2db","rd":"2025-12-11","sid":"recKYn0z5TIlRsr82","sm":"gpt-5.2-pro-2025-12-11_high","v":0.8567},{"b":"arc_agi","c":0.33,"cfg":{"effort":"high"},"dn":"Grok 4.5 (High)","e":"high","m":"grok-4-5","oid":"arc_agi-5e161fd4644031ba","rd":"2026-07-08","sid":"recmSEXUPS7bh0iUI","sm":"grok-4.5_high","src":"https://arcprize.org/leaderboard","v":0.8567},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Gemini 3.7 Flash (Low)","e":"low","m":"gemini-3-7-flash","oid":"arc_agi-ae7248fceb381c39","rd":"2026-08-13","sid":"recIL73XKtvNVD7Om","sm":"gemini-3.7-flash_low","src":"https://arcprize.org/leaderboard","v":0.8516666666666666},{"b":"arc_agi","c":0.1743,"cfg":{"effort":"high"},"dn":"Gemini 3 Flash Preview (High)","e":"high","m":"gemini-3-flash-preview","oid":"arc_agi-f35dbc33c8222888","rd":"2025-12-17","sid":"recc51QLAwOR4KjmP","sm":"gemini-3-flash-preview","src":"https://arcprize.org/leaderboard","v":0.8467},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Inkling Small (XHigh)","e":"xhigh","m":"inkling-small","oid":"arc_agi-1a2d206ff78714f4","rd":"2026-07-15","sid":"rec19KlLfCM7UpqBJ","sm":"inkling-small_xhigh","src":"https://arcprize.org/leaderboard","v":0.84},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"DeepSeek V4 Flash 0731 (Low)","e":"low","m":"deepseek-v4-flash-0731","oid":"arc_agi-8e9f4ba933cdc8e4","rd":"2026-07-31","sid":"recgsyaqI99Ziz4t5","sm":"deepseek-v4-flash-0731_low","src":"https://arcprize.org/leaderboard","v":0.84},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"GPT-6 Sol (Medium)","e":"medium","m":"gpt-6-sol","oid":"arc_agi-aca7d04b2b0f6aab","rd":"2026-09-22","sid":"recQm0cv8CM03fnzk","sm":"gpt-6-sol_medium","src":"https://arcprize.org/leaderboard","v":0.8366666666666666},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Gemini 3.6 Flash (Medium)","e":"medium","m":"gemini-3-6-flash","oid":"arc_agi-2a19a2bd5e529a41","rd":"2026-07-21","sid":"reczMZaXZ861qN6Hg","sm":"gemini-3.6-flash_medium","src":"https://arcprize.org/leaderboard","v":0.8316666666666666},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Claude Haiku 5.5 (High)","e":"high","m":"claude-haiku-5-5","oid":"arc_agi-3b968cd97a4399ca","rd":"2026-10-07","sid":"reczdLUkMttxTvtM9","sm":"claude-haiku-5-5_high","src":"https://arcprize.org/leaderboard","v":0.8150000000000001},{"b":"arc_agi","c":3.9774,"cfg":{"effort":"medium"},"dn":"GPT-5.2 Pro (Medium)","e":"medium","m":"gpt-5-2-pro-2025-12-11","oid":"arc_agi-39fca2528b2532e0","rd":"2025-12-11","sid":"recAdSbvORoPGa4Qi","sm":"gpt-5.2-pro-2025-12-11_medium","v":0.8117000000000001},{"b":"arc_agi","c":1.47,"dn":"Claude Opus 4.5 (64k thinking)","m":"claude-opus-4-5-20251101-64k","oid":"arc_agi-b767343b543b0188","rd":"2025-11-24","sid":"recXBzrtTETaQuvxw","sm":"claude-opus-4-5-20251101_64K","src":"https://arcprize.org/leaderboard","v":0.8},{"b":"arc_agi","dn":"Opus 4.5 (Thinking, 64K)","m":"claude-opus-4-5-20251101-64k","oid":"arc_agi-00fd31b50f3d42dc","rd":"2025-11-24","sid":"recDG3Rbrpu9Cjze2","sm":"claude-opus-4-5-20251101_64K","src":"https://arcprize.org/leaderboard","v":0.8},{"b":"arc_agi","c":0.30047416199999993,"dn":"Inkling","m":"inkling","oid":"arc_agi-932a41ecf4e8a368","rd":"2026-07-15","sid":"recMea5hfj7XIOzYH","sm":"Inkling","src":"https://arcprize.org/leaderboard","v":0.795},{"b":"arc_agi","c":0.18,"cfg":{"effort":"low"},"dn":"Grok 4.5 (Low)","e":"low","m":"grok-4-5","oid":"arc_agi-5177107a8107bf38","rd":"2026-07-08","sid":"recTjmRwwXkhXuf3X","sm":"grok-4.5_low","src":"https://arcprize.org/leaderboard","v":0.7917000000000001},{"b":"arc_agi","c":0.5196,"cfg":{"effort":"high"},"dn":"GPT-5.2 (High)","e":"high","m":"gpt-5-2-2025-12-11","oid":"arc_agi-20cc7eb407698079","rd":"2025-12-11","sid":"recG0lrSvCvamnnBm","sm":"gpt-5.2-2025-12-11_high","src":"https://arcprize.org/leaderboard","v":0.7867000000000001},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Inkling Small (High)","e":"high","m":"inkling-small","oid":"arc_agi-7546abf648a0bacb","rd":"2026-07-15","sid":"recgRmEW1KZcdGKZm","sm":"inkling-small_high","src":"https://arcprize.org/leaderboard","v":0.78},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Muse Spark Open (Medium)","e":"medium","m":"muse-spark-open-816b-a42b","oid":"arc_agi-0cce5c676d45b2b1","rd":"2026-10-15","sid":"rec32FqkBKpZn9dVr","sm":"Muse-Spark-Open-816B-A42B_medium","src":"https://arcprize.org/leaderboard","v":0.7766666666666666},{"b":"arc_agi","c":0.1916,"dn":"GLM-5.2","m":"glm-5-2","oid":"arc_agi-8d35b4f8f79ddfba","rd":"2026-06-16","sid":"rec783bNfpoKvxStC","sm":"glm-5.2_unknown","src":"https://arcprize.org/leaderboard","v":0.77},{"b":"arc_agi","c":0.13,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Terra (Medium)","e":"medium","m":"gpt-5-6-terra","oid":"arc_agi-19e0b94353b84bf0","rd":"2026-07-09","sid":"rec2gOLjWGIo3EDZc","sm":"gpt-5.6-terra_medium","src":"https://arcprize.org/leaderboard","v":0.77},{"b":"arc_agi","c":0.13,"cfg":{"effort":"high"},"dn":"GPT-5.6 Luna (High)","e":"high","m":"gpt-5-6-luna","oid":"arc_agi-5cbd47d0b59d881b","rd":"2026-07-09","sid":"recZ2UaRzTx3PmoSP","sm":"gpt-5.6-luna_high","src":"https://arcprize.org/leaderboard","v":0.765},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Gemini 3.6 Flash (Low)","e":"low","m":"gemini-3-6-flash","oid":"arc_agi-ede0b7e642a12261","rd":"2026-07-21","sid":"recC6P2Ozh3zZKLqc","sm":"gemini-3.6-flash_low","src":"https://arcprize.org/leaderboard","v":0.765},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Gemini 4 Argon (Low)","e":"low","m":"gemini-4-argon","oid":"arc_agi-8f128f7ee9d463da","rd":"2026-09-30","sid":"recpkijjheBxEhBoY","sm":"gemini-4-argon_low","src":"https://arcprize.org/leaderboard","v":0.765},{"b":"arc_agi","c":0.2,"cfg":{"effort":"low"},"dn":"GPT-5.5 (Low)","e":"low","m":"gpt-5-5","oid":"arc_agi-e61b52ea6c4ca8e7","rd":"2026-04-23","sid":"rec7WJ0gbABZ60vSH","sm":"gpt-5.5_low","src":"https://arcprize.org/leaderboard","v":0.7617},{"b":"arc_agi","dn":"Opus 4.5 (Thinking, 32K)","m":"claude-opus-4-5-20251101-32k","oid":"arc_agi-8ac3d53eddccacc4","rd":"2025-11-24","sid":"recQaSIdYDM01Js2n","sm":"claude-opus-4-5-20251101_32K","src":"https://arcprize.org/leaderboard","v":0.7583},{"b":"arc_agi","c":0.95,"dn":"Claude Opus 4.5 (32k thinking)","m":"claude-opus-4-5-20251101-32k","oid":"arc_agi-f06374c8819e86d5","rd":"2025-11-24","sid":"recn5WFdr0nWTOt7b","sm":"claude-opus-4-5-20251101_32K","src":"https://arcprize.org/leaderboard","v":0.758},{"b":"arc_agi","c":0.4926,"dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"arc_agi-375be84114255894","rd":"2025-11-18","sid":"recJT3M7RL0TgeABi","sm":"gemini-3-pro-preview","src":"https://arcprize.org/leaderboard","v":0.75},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Grok 4.6 (Low)","e":"low","m":"grok-4-6","oid":"arc_agi-22022fabd85377ab","rd":"2026-08-12","sid":"recFcvwZPnbu9Wk1z","sm":"grok-4.6_low","src":"https://arcprize.org/leaderboard","v":0.7483333333333334},{"b":"arc_agi","c":0.17,"cfg":{"effort":"low"},"dn":"GPT-5.6 Sol (Low)","e":"low","m":"gpt-5-6-sol","oid":"arc_agi-3516ecc7dd618eb7","rd":"2026-07-09","sid":"recFnbBRcRFSkkzd1","sm":"gpt-5.6-sol_low","src":"https://arcprize.org/leaderboard","v":0.745},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"Beam 1.0 (XHigh)","e":"xhigh","m":"beam-1-0","oid":"arc_agi-c73ff76bb946d6ee","rd":"2026-10-05","sid":"recofsNJjBkM3nej6","sm":"beam-1.0_xhigh","src":"https://arcprize.org/leaderboard","v":0.74},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-6 Luna (XHigh)","e":"xhigh","m":"gpt-6-luna","oid":"arc_agi-46a9e9c7c387701b","rd":"2026-09-22","sid":"recezdwwug8BWPqxw","sm":"gpt-6-luna_xhigh","src":"https://arcprize.org/leaderboard","v":0.73},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-5.1 (Thinking, High)","e":"high","m":"gpt-5-1-2025-11-13","oid":"arc_agi-b875ee6a122fdc80","rd":"2025-11-13","sid":"recGheKChbMIg5rp8","sm":"gpt-5.1-2025-11-13_high","src":"https://arcprize.org/leaderboard","v":0.7283},{"b":"arc_agi","c":0.674,"cfg":{"effort":"high"},"dn":"GPT-5.1 (High)","e":"high","m":"gpt-5-1-2025-11-13","oid":"arc_agi-a275d2b3d531e230","rd":"2025-11-13","sid":"rec8sLsUfujiSrlk4","sm":"gpt-5.1-2025-11-13_high","src":"https://arcprize.org/leaderboard","v":0.728},{"b":"arc_agi","c":0.3448,"cfg":{"effort":"medium"},"dn":"GPT-5.2 (Medium)","e":"medium","m":"gpt-5-2-2025-12-11","oid":"arc_agi-c9a42d02f7547bce","rd":"2025-12-11","sid":"recgYU9vtVcXAzyE7","sm":"gpt-5.2-2025-12-11_medium","v":0.7267},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"GPT-6 Sol (Low)","e":"low","m":"gpt-6-sol","oid":"arc_agi-54289e51bb5142fe","rd":"2026-09-22","sid":"recquQQLYJxHUiNIt","sm":"gpt-6-sol_low","src":"https://arcprize.org/leaderboard","v":0.7216666666666666},{"b":"arc_agi","c":0.62,"dn":"Opus 4.5 (Thinking, 16K)","m":"claude-opus-4-5-20251101-16k","oid":"arc_agi-4754e0a48bf6dd5b","rd":"2025-11-24","sid":"recPFxyFbtk2zgGrK","sm":"claude-opus-4-5-20251101_16K","v":0.72},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GLM-5.3-Flash (High)","e":"high","m":"glm-5-3-flash","oid":"arc_agi-53e2e223122e6226","rd":"2026-08-20","sid":"recrAwVmg00zQ9oxw","sm":"glm-5.3-flash_high","src":"https://arcprize.org/leaderboard","v":0.7183333333333335},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-6 Luna (High)","e":"high","m":"gpt-6-luna","oid":"arc_agi-5b33712bf4626592","rd":"2026-09-22","sid":"recUpnHQuxxXxgJsV","sm":"gpt-6-luna_high","src":"https://arcprize.org/leaderboard","v":0.7033333333333333},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"GPT-5 Pro","e":"high","m":"gpt-5-pro-2025-10-06","oid":"arc_agi-4d13c49fbafa85f9","rd":"2025-10-07","sid":"rec4jlS7emdBeDgkx","sm":"gpt-5-pro-2025-10-06_high","src":"https://arcprize.org/leaderboard","v":0.702},{"b":"arc_agi","c":4.7785,"dn":"GPT-5 Pro","m":"gpt-5-pro-2025-10-06","oid":"arc_agi-1b12826bae90d225","rd":"2025-10-07","sid":"recS53Gln4vftmgkx","sm":"gpt-5-pro-2025-10-06_unknown","v":0.7017},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Claude Sonnet 5.5 (Medium)","e":"medium","m":"claude-sonnet-5-5","oid":"arc_agi-e4c54caf91c0cd54","rd":"2026-09-28","sid":"rec0E2Grx4QvjqoI4","sm":"claude-sonnet-5-5_medium","src":"https://arcprize.org/leaderboard","v":0.7000000000000001},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Qwen3.8-27B (Low)","e":"low","m":"qwen3-8-27b","oid":"arc_agi-2b6394f3ea694c48","rd":"2026-08-14","sid":"reckBty9F3wXHROL2","sm":"qwen3.8-27b_low","src":"https://arcprize.org/leaderboard","v":0.6916666666666665},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Qwen3.8-27B (Medium)","e":"medium","m":"qwen3-8-27b","oid":"arc_agi-30aa8be19e8b379d","rd":"2026-08-14","sid":"recWKN7wHP9ggifO4","sm":"qwen3.8-27b_medium","src":"https://arcprize.org/leaderboard","v":0.6866666666666665},{"b":"arc_agi","c":0.15,"cfg":{"effort":"low"},"dn":"GPT-5.4 (Low)","e":"low","m":"gpt-5-4-2026-03-05","oid":"arc_agi-97d24607d5f98f2e","rd":"2026-03-05","sid":"recmvkkmWAsDklsPA","sm":"gpt-5.4-2026-03-05_low","v":0.6817000000000001},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Beam 1.0 (High)","e":"high","m":"beam-1-0","oid":"arc_agi-c048f55914bf97ff","rd":"2026-10-05","sid":"recTvl3IejyaivWi0","sm":"beam-1.0_high","src":"https://arcprize.org/leaderboard","v":0.6766666666666666},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Inkling Small (Medium)","e":"medium","m":"inkling-small","oid":"arc_agi-2ed987e508a93f9c","rd":"2026-07-15","sid":"recqGEDOjI6CKQgg0","sm":"inkling-small_medium","src":"https://arcprize.org/leaderboard","v":0.67},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Claude Sonnet 5.5 (Low)","e":"low","m":"claude-sonnet-5-5","oid":"arc_agi-351f30dcc3cbcd68","rd":"2026-09-28","sid":"recJfzfmmzTM6kuEO","sm":"claude-sonnet-5-5_low","src":"https://arcprize.org/leaderboard","v":0.67},{"b":"arc_agi","c":2.17,"cfg":{"effort":"thinking"},"dn":"Grok 4 (Thinking)","e":"thinking","m":"grok-4-0709","oid":"arc_agi-58da8460757265f9","rd":"2025-07-09","sid":"rec2xTKJLWfMMxIvS","sm":"grok-4-0709","v":0.6667000000000001},{"b":"arc_agi","c":0.5087,"cfg":{"effort":"high"},"dn":"GPT-5 (High)","e":"high","m":"gpt-5-2025-08-07","oid":"arc_agi-6b224d1932341612","rd":"2025-08-07","sid":"recy0dyn1Ex4jNC4H","sm":"gpt-5-2025-08-07_high","src":"https://arcprize.org/leaderboard","v":0.6567000000000001},{"b":"arc_agi","c":0.18061866,"cfg":{"effort":"low"},"dn":"Kimi K3 (Low)","e":"low","m":"kimi-k3","oid":"arc_agi-61390f89df69b04a","rd":"2026-07-16","sid":"rec5cJ247ekdZACxN","sm":"kimi-k3_low","src":"https://arcprize.org/leaderboard","v":0.6566666666666666},{"b":"arc_agi","c":0.15,"dn":"Kimi K2.5","m":"kimi-k2-5","oid":"arc_agi-9d353da849ce51b3","rd":"2026-01-27","sid":"reckd4ffXh86QXwwY","sm":"kimi-k2.5","v":0.6533},{"b":"arc_agi","c":0.5193,"dn":"Claude Sonnet 4.5 (Thinking 32K)","m":"claude-sonnet-4-5-20250929-32k","oid":"arc_agi-a90b4880c078224b","rd":"2025-09-29","sid":"rec17BpugSGZTB9s8","sm":"claude-sonnet-4-5-20250929_32K","v":0.6367},{"b":"arc_agi","c":0.07,"dn":"Minimax M2.5","m":"minimax-m2-5","oid":"arc_agi-7964c8f662e10cbe","rd":"2026-02-12","sid":"receB8OOl8dtDUVr6","sm":"MiniMax-M2.5","v":0.6367},{"b":"arc_agi","c":0.51,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Mini (xHigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi-1465a1829449b478","rd":"2026-03-17","sid":"rec05QEcCMeKiXIgI","sm":"gpt-5.4-mini-2026-03-17_xhigh","src":"https://arcprize.org/leaderboard","v":0.6367},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Mini (XHigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi-73dd79a327fbe4c0","rd":"2026-03-17","sid":"recg1uxrFjyxqvPTs","sm":"gpt-5.4-mini-2026-03-17_xhigh","src":"https://arcprize.org/leaderboard","v":0.6367},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Grok 4.7 (Low)","e":"low","m":"grok-4-7","oid":"arc_agi-8e939f6266cfccaf","rd":"2026-09-21","sid":"recvGYk9xsBiIFYQz","sm":"grok-4.7_low","src":"https://arcprize.org/leaderboard","v":0.6333333333333333},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"GPT-6 Luna (Medium)","e":"medium","m":"gpt-6-luna","oid":"arc_agi-75da5b2375909dec","rd":"2026-09-22","sid":"recENgw6kT7CkuaUg","sm":"gpt-6-luna_medium","src":"https://arcprize.org/leaderboard","v":0.61},{"b":"arc_agi","c":0.5002,"cfg":{"effort":"high"},"dn":"o3 (High)","e":"high","m":"o3-2025-04-16","oid":"arc_agi-a3fb856c86205a65","rd":"2025-04-16","sid":"recyX2f3amNA4ZPUQ","sm":"o3-2025-04-16_high","src":"https://arcprize.org/leaderboard","v":0.6083},{"b":"arc_agi","c":0.09,"cfg":{"effort":"low"},"dn":"GPT-5.6 Terra (Low)","e":"low","m":"gpt-5-6-terra","oid":"arc_agi-6977d9a63ae6c23b","rd":"2026-07-09","sid":"recHsvqtEj8sE4t1S","sm":"gpt-5.6-terra_low","src":"https://arcprize.org/leaderboard","v":0.6017},{"b":"arc_agi","c":4.16,"cfg":{"effort":"high"},"dn":"o3-Pro (High)","e":"high","m":"o3-pro-2025-06-10","oid":"arc_agi-b420b29e0c9a825c","rd":"2025-06-10","sid":"recgEreMuMqYTWJuX","sm":"o3-pro-2025-06-10_high","src":"https://arcprize.org/leaderboard","v":0.5933},{"b":"arc_agi","c":0.4058,"cfg":{"effort":"high"},"dn":"o4-mini (High)","e":"high","m":"o4-mini-2025-04-16","oid":"arc_agi-0336cf886796ecbd","rd":"2025-04-16","sid":"receup1qngSep4blO","sm":"o4-mini-2025-04-16_high","src":"https://arcprize.org/leaderboard","v":0.5867},{"b":"arc_agi","c":0.38,"dn":"Opus 4.5 (Thinking, 8K)","m":"claude-opus-4-5-20251101-8k","oid":"arc_agi-e6728d2c378691dd","rd":"2025-11-24","sid":"recnTJS3iYYQ2jc7n","sm":"claude-opus-4-5-20251101_8K","v":0.5867},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Muse Spark Open (Low)","e":"low","m":"muse-spark-open-816b-a42b","oid":"arc_agi-35c898b41756d276","rd":"2026-10-15","sid":"reclYInB9ng2Mq8Ty","sm":"Muse-Spark-Open-816B-A42B_low","src":"https://arcprize.org/leaderboard","v":0.585},{"b":"arc_agi","c":0.27,"cfg":{"effort":"high"},"dn":"GPT-5.4 Mini (High)","e":"high","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi-0c50e315d8bc0d99","rd":"2026-03-17","sid":"recQuMohacriH1ZXn","sm":"gpt-5.4-mini-2026-03-17_high","src":"https://arcprize.org/leaderboard","v":0.58},{"b":"arc_agi","c":0.3149,"cfg":{"effort":"medium"},"dn":"GPT-5.1 (Medium)","e":"medium","m":"gpt-5-1-2025-11-13","oid":"arc_agi-a0f0911e0b738ac5","rd":"2025-11-13","sid":"reccC3Lgebztl4MAm","sm":"gpt-5.1-2025-11-13_medium","src":"https://arcprize.org/leaderboard","v":0.577},{"b":"arc_agi","c":0.0612,"cfg":{"effort":"medium"},"dn":"Gemini 3 Flash Preview (Medium)","e":"medium","m":"gemini-3-flash-preview","oid":"arc_agi-bc41ddd46aac557a","rd":"2025-12-17","sid":"recuhSCKFRclgYZzv","sm":"gemini-3-flash-preview","src":"https://arcprize.org/leaderboard","v":0.5767},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"GPT-5.1 (Thinking, Medium)","e":"medium","m":"gpt-5-1-2025-11-13","oid":"arc_agi-170a27465fa77b8d","rd":"2025-11-13","sid":"recfBsUpGwiKhprnF","sm":"gpt-5.1-2025-11-13_medium","src":"https://arcprize.org/leaderboard","v":0.5767},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Claude Haiku 5.5 (Medium)","e":"medium","m":"claude-haiku-5-5","oid":"arc_agi-e401240fc1714d3e","rd":"2026-10-07","sid":"recYwW67J8PONyVOE","sm":"claude-haiku-5-5_medium","src":"https://arcprize.org/leaderboard","v":0.5733333333333333},{"b":"arc_agi","c":3.1766,"cfg":{"effort":"medium"},"dn":"o3-Pro (Medium)","e":"medium","m":"o3-pro-2025-06-10","oid":"arc_agi-012065b061697dc3","rd":"2025-06-10","sid":"reco8WbS3E6S6DSwq","sm":"o3-pro-2025-06-10_medium","src":"https://arcprize.org/leaderboard","v":0.57},{"b":"arc_agi","c":0.08,"dn":"Deepseek V3.2","m":"deepseekdeepseek-v3-2","oid":"arc_agi-729fcf323336bb19","rd":"2025-12-01","sid":"recexUBOP4rLbxskl","sm":"deepseek/deepseek-v3.2","v":0.57},{"b":"arc_agi","c":0.06,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Luna (Medium)","e":"medium","m":"gpt-5-6-luna","oid":"arc_agi-29c7d2d9d1594ba8","rd":"2026-07-09","sid":"recUw0ZVlPFxA8ETK","sm":"gpt-5.6-luna_medium","src":"https://arcprize.org/leaderboard","v":0.565},{"b":"arc_agi","c":0.3301,"cfg":{"effort":"medium"},"dn":"GPT-5 (Medium)","e":"medium","m":"gpt-5-2025-08-07","oid":"arc_agi-d0b4453b09f275a3","rd":"2025-08-07","sid":"recfuBzPqyWo5BEjp","sm":"gpt-5-2025-08-07_medium","src":"https://arcprize.org/leaderboard","v":0.5617},{"b":"arc_agi","c":0.1657,"cfg":{"effort":"low"},"dn":"GPT-5.2 (Low)","e":"low","m":"gpt-5-2-2025-12-11","oid":"arc_agi-dbaccad4445e419a","rd":"2025-12-11","sid":"rectoL9U5nR5I823g","sm":"gpt-5.2-2025-12-11_low","v":0.5567},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Beam 1.0 (Medium)","e":"medium","m":"beam-1-0","oid":"arc_agi-e485cb6ba0204fb8","rd":"2026-10-05","sid":"recPGmhOziQ7sVoSO","sm":"beam-1.0_medium","src":"https://arcprize.org/leaderboard","v":0.5566666666666666},{"b":"arc_agi","c":0.116,"cfg":{"effort":"high"},"dn":"GPT-5 Mini (High)","e":"high","m":"gpt-5-mini-2025-08-07","oid":"arc_agi-b0cb0386939919f6","rd":"2025-08-07","sid":"recYLj0L3xmejkpDR","sm":"gpt-5-mini-2025-08-07_high","v":0.5433},{"b":"arc_agi","c":0.2882,"cfg":{"effort":"medium"},"dn":"o3 (Medium)","e":"medium","m":"o3-2025-04-16","oid":"arc_agi-f144d7dabb80aaff","rd":"2025-04-16","sid":"reczsOK58YJa6GAAz","sm":"o3-2025-04-16_medium","src":"https://arcprize.org/leaderboard","v":0.5383},{"b":"arc_agi","cfg":{"effort":"high"},"dn":"Gemini 3.5 Flash-Lite (High)","e":"high","m":"gemini-3-5-flash-lite","oid":"arc_agi-fc9075b427d412c8","rd":"2026-07-21","sid":"rec3MeJ4tl0mWlnxT","sm":"gemini-3.5-flash-lite_high","src":"https://arcprize.org/leaderboard","v":0.535},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Inkling Small (Low)","e":"low","m":"inkling-small","oid":"arc_agi-1efe95607c2d0c74","rd":"2026-07-15","sid":"recG1iu3ZYYWAELeh","sm":"inkling-small_low","src":"https://arcprize.org/leaderboard","v":0.525},{"b":"arc_agi","c":0.13,"cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Nano (xHigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi-4edeef491c0f8c81","rd":"2026-03-17","sid":"recNETiFSCWWQlZLG","sm":"gpt-5.4-nano-2026-03-17_xhigh","src":"https://arcprize.org/leaderboard","v":0.515},{"b":"arc_agi","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Nano (XHigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi-4acd6e84d5ccabad","rd":"2026-03-17","sid":"recXcgwRs2pY4JXLA","sm":"gpt-5.4-nano-2026-03-17_xhigh","src":"https://arcprize.org/leaderboard","v":0.515},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Claude Haiku 5.5 (Low)","e":"low","m":"claude-haiku-5-5","oid":"arc_agi-0cc0cb0ca7a22c67","rd":"2026-10-07","sid":"reciaCfSQDiho6p0D","sm":"claude-haiku-5-5_low","src":"https://arcprize.org/leaderboard","v":0.5033333333333334},{"b":"arc_agi","c":0.0645,"cfg":{"effort":"minimal"},"dn":"Gemini 3.5 Flash (Minimal)","e":"minimal","m":"gemini-3-5-flash","oid":"arc_agi-06051f3d0cf4c4ec","rd":"2026-05-19","sid":"recnkvbCp6BqU3GuM","sm":"gemini-3.5-flash_minimal","src":"https://arcprize.org/leaderboard","v":0.4883},{"b":"arc_agi","c":0.0307,"dn":"Grok 4 (Fast Reasoning)","m":"grok-4-fast","oid":"arc_agi-448df771cd275b01","rd":"2025-09-19","sid":"recNDgnXdjMNbLBuO","sm":"grok-4-fast","v":0.485},{"b":"arc_agi","cfg":{"effort":"minimal"},"dn":"Inkling Small (Minimal)","e":"minimal","m":"inkling-small","oid":"arc_agi-e4168f61895e4612","rd":"2026-07-15","sid":"recPyBo0rj5J9H8TR","sm":"inkling-small_minimal","src":"https://arcprize.org/leaderboard","v":0.485},{"b":"arc_agi","c":0.2683,"dn":"Claude Sonnet 4.5 (Thinking 16K)","m":"claude-sonnet-4-5-20250929-16k","oid":"arc_agi-49ba96760afbd3f6","rd":"2025-09-29","sid":"recBS5Nb7JpfywLTo","sm":"claude-sonnet-4-5-20250929_16K","v":0.4833},{"b":"arc_agi","c":0.2558,"dn":"Claude Haiku 4.5 (Thinking 32K)","m":"claude-haiku-4-5-20251001-32k","oid":"arc_agi-e18d5eb173ae3886","rd":"2025-10-15","sid":"recSjmMESs5Bif5dG","sm":"claude-haiku-4-5-20251001_32K","v":0.4767},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"GLM-5.3-Flash (Low)","e":"low","m":"glm-5-3-flash","oid":"arc_agi-e02a0fa4e452f5b3","rd":"2026-08-20","sid":"rec9UZOsyeu6hmc6q","sm":"glm-5.3-flash_low","src":"https://arcprize.org/leaderboard","v":0.4699999999999999},{"b":"arc_agi","c":0.1776,"dn":"Claude Sonnet 4.5 (Thinking 8K)","m":"claude-sonnet-4-5-20250929-8k","oid":"arc_agi-e2586de9010fcde6","rd":"2025-09-29","sid":"recm8p4NdctHUJDq3","sm":"claude-sonnet-4-5-20250929_8K","v":0.465},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Beam 1.0 (Low)","e":"low","m":"beam-1-0","oid":"arc_agi-f5ebfe41e91f18b1","rd":"2026-10-05","sid":"reca1iDdShyLJmdMn","sm":"beam-1.0_low","src":"https://arcprize.org/leaderboard","v":0.45},{"b":"arc_agi","c":0.17,"dn":"GLM-5","m":"glm-5","oid":"arc_agi-0bc30d06101651e4","rd":"2026-02-11","sid":"reczebBHCblQzBn5N","sm":"glm-5","v":0.44670000000000004},{"b":"arc_agi","c":1.6382,"cfg":{"effort":"low"},"dn":"o3-Pro (Low)","e":"low","m":"o3-pro-2025-06-10","oid":"arc_agi-468fbe3e15dd14d8","rd":"2025-06-10","sid":"rec2DTU08MgXm0hds","sm":"o3-pro-2025-06-10_low","src":"https://arcprize.org/leaderboard","v":0.4433000000000001},{"b":"arc_agi","c":0.1531,"cfg":{"effort":"low"},"dn":"GPT-5 (Low)","e":"low","m":"gpt-5-2025-08-07","oid":"arc_agi-a976756082c79d95","rd":"2025-08-07","sid":"recyhP0MFpu4yP9Fs","sm":"gpt-5-2025-08-07_low","src":"https://arcprize.org/leaderboard","v":0.44},{"b":"arc_agi","c":0.15,"cfg":{"effort":"medium"},"dn":"o4-mini (Medium)","e":"medium","m":"o4-mini-2025-04-16","oid":"arc_agi-0e51f7c7b27811d7","rd":"2025-04-16","sid":"recFnX3IAcQd2NffG","sm":"o4-mini-2025-04-16_medium","src":"https://arcprize.org/leaderboard","v":0.4183},{"b":"arc_agi","c":0.1764,"cfg":{"effort":"low"},"dn":"o3 (Low)","e":"low","m":"o3-2025-04-16","oid":"arc_agi-932f27045ea1d1e4","rd":"2025-04-16","sid":"recmynZQA4oogWrzw","sm":"o3-2025-04-16_low","src":"https://arcprize.org/leaderboard","v":0.415},{"b":"arc_agi","c":0.4839,"dn":"Gemini 2.5 Pro (Thinking 16K)","m":"gemini-2-5-pro-16k","oid":"arc_agi-af634adbc89f9ad8","rd":"2025-06-17","sid":"recMuDqJwbLT2EDuY","sm":"gemini-2.5-pro_16K","v":0.41000000000000003},{"b":"arc_agi","c":0.16,"cfg":{"effort":"medium"},"dn":"GPT-5.4 Mini (Medium)","e":"medium","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi-91eb75eb4f99de69","rd":"2026-03-17","sid":"recmn6AVhsYVjq4DI","sm":"gpt-5.4-mini-2026-03-17_medium","src":"https://arcprize.org/leaderboard","v":0.4083},{"b":"arc_agi","c":0.3658,"dn":"Claude Sonnet 4 (Thinking 16K)","m":"claude-sonnet-4-20250514-16k","oid":"arc_agi-19a286b502b3b7eb","rd":"2025-05-22","sid":"rec00IX4WEZ6XvDQd","sm":"claude-sonnet-4-20250514_16K","src":"https://arcprize.org/leaderboard","v":0.4},{"b":"arc_agi","c":1.7638,"dn":"Tiny Recursion Model (TRM)","m":"tiny-recursion-model","oid":"arc_agi-23ae2769dceae6aa","rd":"2025-10-06","sid":"recFIWEjevgWkejR0","sm":"tiny-recursion-model","v":0.4},{"b":"arc_agi","c":0.13,"dn":"Opus 4.5 (Thinking, None)","m":"claude-opus-4-5-2025-11-01","oid":"arc_agi-7b6427c6ea55271a","rd":"2025-11-24","sid":"recQS3YkzKXSzapvO","sm":"claude-opus-4-5-20251101","v":0.4},{"b":"arc_agi","c":0.07,"cfg":{"effort":"high"},"dn":"GPT-5.4 Nano (High)","e":"high","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi-2186f7d8588afbb3","rd":"2026-03-17","sid":"recKvmcoxVuVU3rEP","sm":"gpt-5.4-nano-2026-03-17_high","src":"https://arcprize.org/leaderboard","v":0.38170000000000004},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"GPT-6 Luna (Low)","e":"low","m":"gpt-6-luna","oid":"arc_agi-c141db2cd91a9404","rd":"2026-09-22","sid":"recV4eMYYvT9zGHJr","sm":"gpt-6-luna_low","src":"https://arcprize.org/leaderboard","v":0.37666666666666665},{"b":"arc_agi","c":0.0401,"cfg":{"effort":"medium"},"dn":"GPT-5 Mini (Medium)","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"arc_agi-2cfabfbc7703138f","rd":"2025-08-07","sid":"rec5TE9pAmGVT4dTe","sm":"gpt-5-mini-2025-08-07_medium","v":0.3733000000000001},{"b":"arc_agi","c":0.1042,"dn":"Claude Haiku 4.5 (Thinking 16K)","m":"claude-haiku-4-5-20251001-16k","oid":"arc_agi-87e66e21df9878df","rd":"2025-10-15","sid":"rec66qqPrc9KEuz6G","sm":"claude-haiku-4-5-20251001_16K","v":0.3733000000000001},{"b":"arc_agi","c":0.5123,"dn":"Gemini 2.5 Pro (Thinking 32K)","m":"gemini-2-5-pro-32k","oid":"arc_agi-01d1e83be8717fd6","rd":"2025-06-17","sid":"recgu9IZuwPFiUJYg","sm":"gemini-2.5-pro_32K","v":0.37},{"b":"arc_agi","c":1.2496,"dn":"Claude Opus 4 (Thinking 16K)","m":"claude-opus-4-20250514-16k","oid":"arc_agi-dbd238eee31aac1a","rd":"2025-05-22","sid":"recpPgRxRuNQ6MJbU","sm":"claude-opus-4-20250514_16K","src":"https://arcprize.org/leaderboard","v":0.3567},{"b":"arc_agi","cfg":{"effort":"minimal"},"dn":"Gemini 3.6 Flash (Minimal)","e":"minimal","m":"gemini-3-6-flash","oid":"arc_agi-9bbc3a4368ef06e5","rd":"2026-07-21","sid":"recydzNjNPevATSzW","sm":"gemini-3.6-flash_minimal","src":"https://arcprize.org/leaderboard","v":0.34500000000000003},{"b":"arc_agi","c":0.3989,"cfg":{"effort":"high"},"dn":"o3-mini (High)","e":"high","m":"o3-mini-2025-01-31","oid":"arc_agi-33b935b641eff64b","rd":"2025-01-31","sid":"recsslT51vOvwD7Hw","sm":"o3-mini-2025-01-31_high","src":"https://arcprize.org/leaderboard","v":0.345},{"b":"arc_agi","c":0.04,"cfg":{"effort":"low"},"dn":"GPT-5.6 Luna (Low)","e":"low","m":"gpt-5-6-luna","oid":"arc_agi-8ee7425e296670c4","rd":"2026-07-09","sid":"recABKYEfZ7PyiB5S","sm":"gpt-5.6-luna_low","src":"https://arcprize.org/leaderboard","v":0.3417},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"Qwen3.8-27B (None)","e":"none","m":"qwen3-8-27b","oid":"arc_agi-e9943cf108a5d6d5","rd":"2026-08-14","sid":"recTAjz1rP5gg1LDy","sm":"qwen3.8-27b_none","src":"https://arcprize.org/leaderboard","v":0.34},{"b":"arc_agi","c":0.2134,"dn":"Gemini 2.5 Flash (Preview) (Thinking 16K)","m":"gemini-2-5-flash-preview-05-20-16k","oid":"arc_agi-0581aa8e793512f5","rd":"2025-05-20","sid":"recPvvngATSJ9tzRl","sm":"gemini-2.5-flash-preview-05-20_16K","src":"https://arcprize.org/leaderboard","v":0.3333},{"b":"arc_agi","c":0.0371,"dn":"Gemini 2.5 Flash (Preview)","m":"gemini-2-5-flash-preview-05-20","oid":"arc_agi-59a9ca1c9d4d2938","rd":"2025-05-20","sid":"recRaEsoFzvHSY2ds","sm":"gemini-2.5-flash-preview-05-20","src":"https://arcprize.org/leaderboard","v":0.3333},{"b":"arc_agi","c":0.1,"cfg":{"effort":"low"},"dn":"GPT-5.1 (Thinking, Low)","e":"low","m":"gpt-5-1-2025-11-13","oid":"arc_agi-297ab0935cfa87c9","rd":"2025-11-13","sid":"recwxEKhGNXOQtDaL","sm":"gpt-5.1-2025-11-13_low","v":0.33170000000000005},{"b":"arc_agi","dn":"Gemini 2.5 Pro (Preview)","m":"gemini-2-5-pro-preview-03-25","oid":"arc_agi-79cee73ca6186ab7","rd":"2025-03-31","sid":"recGhSAAXTywTt6ks","sm":"gemini-2.5-pro-preview-03-25","src":"https://arcprize.org/leaderboard","v":0.33},{"b":"arc_agi","c":0.04,"cfg":{"effort":"medium"},"dn":"GPT-5.4 Nano (Medium)","e":"medium","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi-54b47bee8103b82a","rd":"2026-03-17","sid":"recjUSAkzsZXxB1Bq","sm":"gpt-5.4-nano-2026-03-17_medium","src":"https://arcprize.org/leaderboard","v":0.33},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"Gemini 3.5 Flash-Lite (Medium)","e":"medium","m":"gemini-3-5-flash-lite","oid":"arc_agi-1a58b0142f787e25","rd":"2026-07-21","sid":"recmVW0vB0fyGe6YR","sm":"gemini-3.5-flash-lite_medium","src":"https://arcprize.org/leaderboard","v":0.3233333333333333},{"b":"arc_agi","c":0.1971,"dn":"Gemini 2.5 Flash (Preview) (Thinking 24K)","m":"gemini-2-5-flash-preview-05-20-23k","oid":"arc_agi-30cb48a0f28a915c","rd":"2025-05-20","sid":"recrGgsi1mnHqQ5CS","sm":"gemini-2.5-flash-preview-05-20_23K","v":0.3233},{"b":"arc_agi","dn":"Gemini 2.5 Flash (Preview) (Thinking 24K)","m":"gemini-2-5-flash-preview-04-17-24k-thinking","oid":"arc_agi-e52e53344c04662d","rd":"2025-04-17","sid":"rec9VDig3HkReY0oP","sm":"gemini-2.5-flash-preview-04-17 (24K thinking)","src":"https://arcprize.org/leaderboard","v":0.323},{"b":"arc_agi","c":0.0573,"dn":"Gemini 2.5 Pro (Preview, Thinking 1K)","m":"gemini-2-5-pro-preview-06-05-1k","notes":"Thinking budgets have only been supported since 06-05","oid":"arc_agi-8ff30779e6de1b47","rd":"2025-06-05","sid":"recdEFBjW5RECOz15","sm":"gemini-2.5-pro-preview-06-05_1K","src":"https://arcprize.org/leaderboard","v":0.313},{"b":"arc_agi","c":0.0928,"dn":"Claude Sonnet 4.5 (Thinking 1K)","m":"claude-sonnet-4-5-20250929-1k","oid":"arc_agi-695b3d1349a95942","rd":"2025-09-29","sid":"recZuRgDEIeVMg4pB","sm":"claude-sonnet-4-5-20250929_1K","v":0.31},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"o1 (Medium)","e":"medium","m":"o1-2024-12-17","oid":"arc_agi-1dbeb6b9c14b6aeb","rd":"2024-12-17","sid":"rec3bVHAcLI4a7u7z","sm":"o1-2024-12-17_medium","src":"https://arcprize.org/leaderboard","v":0.307},{"b":"arc_agi","c":0.7408,"dn":"Claude Opus 4 (Thinking 8K)","m":"claude-opus-4-20250514-8k","oid":"arc_agi-aea1ff1e9725493e","rd":"2025-05-22","sid":"rec5eQ6jRbh3n8fQc","sm":"claude-opus-4-20250514_8K","src":"https://arcprize.org/leaderboard","v":0.3067},{"b":"arc_agi","c":0.2947,"dn":"Gemini 2.5 Pro (Thinking 8K)","m":"gemini-2-5-pro-8k","oid":"arc_agi-e6ce4978c0b04253","rd":"2025-06-17","sid":"recRQcZH0KSrPBUgQ","sm":"gemini-2.5-pro_8K","v":0.295},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"GPT-6 Sol (None)","e":"none","m":"gpt-6-sol","oid":"arc_agi-5b0cc7d95a3d8a6f","rd":"2026-09-22","sid":"recWLIKodNGblRCPh","sm":"gpt-6-sol_none","src":"https://arcprize.org/leaderboard","v":0.29333333333333333},{"b":"arc_agi","c":0.1952,"dn":"Claude Sonnet 4 (Thinking 8K)","m":"claude-sonnet-4-20250514-8k","oid":"arc_agi-d69292238b5ab7e3","rd":"2025-05-22","sid":"recd23dJHC8IgD2IC","sm":"claude-sonnet-4-20250514_8K","src":"https://arcprize.org/leaderboard","v":0.29},{"b":"arc_agi","c":0.0163,"cfg":{"effort":"low"},"dn":"Gemini 3 Flash Preview (Low)","e":"low","m":"gemini-3-flash-preview","oid":"arc_agi-e15b1a286e0e676b","rd":"2025-12-17","sid":"recYOvqwUmS3YeSk7","sm":"gemini-3-flash-preview","src":"https://arcprize.org/leaderboard","v":0.29},{"b":"arc_agi","c":0.33,"dn":"Claude 3.7 (16K)","m":"claude-3-7-sonnet-20250219-16k","oid":"arc_agi-b7a5ed5ae70b94da","rd":"2025-02-24","sid":"recgR391dmSWyZjKu","sm":"claude-3-7-sonnet-20250219_16K","src":"https://arcprize.org/leaderboard","v":0.286},{"b":"arc_agi","c":0.0937,"dn":"Claude Sonnet 4 (Thinking 1K)","m":"claude-sonnet-4-20250514-1k","oid":"arc_agi-ffbf586632d7c08a","rd":"2025-05-22","sid":"recs4iQhoJgUrLWrd","sm":"claude-sonnet-4-20250514_1K","src":"https://arcprize.org/leaderboard","v":0.28},{"b":"arc_agi","c":0.1597,"dn":"Codex Mini (Latest)","m":"codex-mini-2025-05-16","oid":"arc_agi-84defb2f135b8443","rd":"2025-05-16","sid":"recBLorgitaes8FMx","sm":"codex-mini-2025-05-16","src":"https://arcprize.org/leaderboard","v":0.2733},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"o1 (Low)","e":"low","m":"o1-2024-12-17","oid":"arc_agi-6e22b9f64ca3c3b7","rd":"2024-12-17","sid":"recxLLEulVCEVpIcG","sm":"o1-2024-12-17_low","src":"https://arcprize.org/leaderboard","v":0.272},{"b":"arc_agi","c":0.5021,"dn":"Claude Opus 4 (Thinking 1K)","m":"claude-opus-4-20250514-1k","oid":"arc_agi-2157f6a8c11fcf84","rd":"2025-05-22","sid":"recOlznt9pRz5uq9J","sm":"claude-opus-4-20250514_1K","src":"https://arcprize.org/leaderboard","v":0.27},{"b":"arc_agi","c":0.0135,"cfg":{"effort":"low"},"dn":"GPT-5 Mini (Low)","e":"low","m":"gpt-5-mini-2025-08-07","oid":"arc_agi-8fb34b559fbc307e","rd":"2025-08-07","sid":"recPVPAR1lQ3MnEGK","sm":"gpt-5-mini-2025-08-07_low","v":0.2633},{"b":"arc_agi","c":0.1344,"dn":"Gemini 2.5 Flash (Preview) (Thinking 8K)","m":"gemini-2-5-flash-preview-05-20-8k","oid":"arc_agi-23f90adb427d65a2","rd":"2025-05-20","sid":"recy1lONU8pakDYKr","sm":"gemini-2.5-flash-preview-05-20_8K","src":"https://arcprize.org/leaderboard","v":0.2583},{"b":"arc_agi","c":0.0808,"dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"arc_agi-48ec475366d6459d","rd":"2025-09-29","sid":"recfm25PSdUjoU7HO","sm":"claude-sonnet-4-5-20250929","v":0.255},{"b":"arc_agi","c":0.0687,"dn":"Claude Haiku 4.5 (Thinking 8K)","m":"claude-haiku-4-5-20251001-8k","oid":"arc_agi-5e119cc32094e72b","rd":"2025-10-15","sid":"rechpW8H5iHdEgOUM","sm":"claude-haiku-4-5-20251001_8K","v":0.255},{"b":"arc_agi","c":0.0806,"dn":"Claude Sonnet 4","m":"claude-sonnet-4-2025-05-14","oid":"arc_agi-9bbb6eaa0e24fb4f","rd":"2025-05-22","sid":"recXVePhVjOhYoNSC","sm":"claude-sonnet-4-20250514","src":"https://arcprize.org/leaderboard","v":0.2383},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"o1-pro (Low)","e":"low","m":"o1-pro-2025-03-19","oid":"arc_agi-bfad063519f193e6","rd":"2025-03-19","sid":"recW7rLYsNpIjAlPE","sm":"o1-pro-2025-03-19_low","src":"https://arcprize.org/leaderboard","v":0.233},{"b":"arc_agi","c":0.4036,"dn":"Claude Opus 4","m":"claude-opus-4-2025-05-14","oid":"arc_agi-ee8f99a25cdfe52e","rd":"2025-05-22","sid":"recU2yBBdPh7FDErC","sm":"claude-opus-4-20250514","src":"https://arcprize.org/leaderboard","v":0.225},{"b":"arc_agi","c":0.1907,"cfg":{"effort":"medium"},"dn":"o3-mini (Medium)","e":"medium","m":"o3-mini-2025-01-31","oid":"arc_agi-17e54c39582edc09","rd":"2025-01-31","sid":"recKkBnHhDmmhkCSp","sm":"o3-mini-2025-01-31_medium","src":"https://arcprize.org/leaderboard","v":0.2233},{"b":"arc_agi","c":0.012,"cfg":{"effort":"minimal"},"dn":"Gemini 3 Flash Preview (Minimal)","e":"minimal","m":"gemini-3-flash-preview","oid":"arc_agi-f131f5ce83382a15","rd":"2025-12-17","sid":"recUb9rzqKIPGj4Ll","sm":"gemini-3-flash-preview","v":0.215},{"b":"arc_agi","c":0.0406,"cfg":{"effort":"low"},"dn":"o4-mini (Low)","e":"low","m":"o4-mini-2025-04-16","oid":"arc_agi-939f6274be51485a","rd":"2025-04-16","sid":"reci1Fl7wFPBUAxIi","sm":"o4-mini-2025-04-16_low","src":"https://arcprize.org/leaderboard","v":0.2133},{"b":"arc_agi","c":0.0464,"dn":"Deepseek R1 (05/28)","m":"deepseek-r1-0528","oid":"arc_agi-91230ab064d2223a","rd":"2025-05-28","sid":"recsscdqM4qvGHGT3","sm":"DeepSeek-R1-0528","src":"https://arcprize.org/leaderboard","v":0.2121},{"b":"arc_agi","c":0.21,"dn":"Claude 3.7 (8K)","m":"claude-3-7-sonnet-20250219-8k","oid":"arc_agi-45eb32f242f90545","rd":"2025-02-24","sid":"reckQgsdCUEH1BmYZ","sm":"claude-3-7-sonnet-20250219_8K","src":"https://arcprize.org/leaderboard","v":0.212},{"b":"arc_agi","c":0.0124,"cfg":{"effort":"medium"},"dn":"GPT-5 Nano (Medium)","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"arc_agi-ef61a8d169bd84fc","rd":"2025-08-07","sid":"reccPnp1eWrOMDX51","sm":"gpt-5-nano-2025-08-07_medium","v":0.2071},{"b":"arc_agi","c":0.01,"cfg":{"effort":"low"},"dn":"GPT-5.4 Nano (Low)","e":"low","m":"gpt-5-4-nano-2026-03-17","oid":"arc_agi-16bf90e4ca03ce22","rd":"2026-03-17","sid":"recfphgegRzZ4mOYq","sm":"gpt-5.4-nano-2026-03-17_low","src":"https://arcprize.org/leaderboard","v":0.1833},{"b":"arc_agi","dn":"o1-preview","m":"o1-preview-2024-09-12","oid":"arc_agi-0af46e68d2727288","rd":"2024-09-12","sid":"reclttZQlaXCYpTDb","sm":"o1-preview-2024-09-12","src":"https://arcprize.org/leaderboard","v":0.18},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Gemini 3.5 Flash-Lite (Low)","e":"low","m":"gemini-3-5-flash-lite","oid":"arc_agi-714b472406da4e40","rd":"2026-07-21","sid":"recnf60khv2UIdIGX","sm":"gemini-3.5-flash-lite_low","src":"https://arcprize.org/leaderboard","v":0.17},{"b":"arc_agi","c":0.0306,"dn":"Claude Haiku 4.5 (Thinking 1K)","m":"claude-haiku-4-5-20251001-1k","oid":"arc_agi-b7bf041354e636b2","rd":"2025-10-15","sid":"rechYdBGiz7Q83lQN","sm":"claude-haiku-4-5-20251001_1K","v":0.16830000000000003},{"b":"arc_agi","c":0.0292,"cfg":{"effort":"high"},"dn":"GPT-5 Nano (High)","e":"high","m":"gpt-5-nano-2025-08-07","oid":"arc_agi-3b02f12751910d13","rd":"2025-08-07","sid":"rec5XtpFjG24wKtGF","sm":"gpt-5-nano-2025-08-07_high","v":0.1667},{"b":"arc_agi","cfg":{"effort":"low"},"dn":"Grok 3 Mini (Low)","e":"low","m":"grok-3-mini-beta","oid":"arc_agi-9263793d143e510b","rd":"2025-04-09","sid":"recUL8yDJO8uotk8q","sm":"grok-3-mini-beta_low","src":"https://arcprize.org/leaderboard","v":0.165},{"b":"arc_agi","c":0.0099,"cfg":{"effort":"low"},"dn":"Grok 3 Mini (Low)","e":"low","m":"grok-3-mini","oid":"arc_agi-cd5a1f98e3b0f660","rd":"2025-06-24","sid":"recmB2931TiCLxLua","sm":"grok-3-mini_low","v":0.165},{"b":"arc_agi","c":0.0356,"dn":"Gemini 2.5 Flash (Preview) (Thinking 1K)","m":"gemini-2-5-flash-preview-05-20-1k","oid":"arc_agi-2bffe6be44c1ec45","rd":"2025-05-20","sid":"recxGd7FrJeq7ZyUy","sm":"gemini-2.5-flash-preview-05-20_1K","src":"https://arcprize.org/leaderboard","v":0.16},{"b":"arc_agi","dn":"Gemini 2.5 Pro (Thinking 1K)","m":"gemini-2-5-pro-preview-06-05-1k","oid":"arc_agi-039913d831b287fd","rd":"2025-06-05","sid":"rec2WZOsDxvKPGFle","sm":"gemini-2.5-pro-preview-06-05_1K","src":"https://arcprize.org/leaderboard","v":0.16},{"b":"arc_agi","c":0.06,"dn":"Deepseek R1","m":"deepseek-r1","oid":"arc_agi-6b115709413e72c0","rd":"2025-01-20","sid":"rec9Em7ivae72BNBO","sm":"DeepSeek-R1","src":"https://arcprize.org/leaderboard","v":0.158},{"b":"arc_agi","c":0.0519,"cfg":{"effort":"low"},"dn":"o3-mini (Low)","e":"low","m":"o3-mini-2025-01-31","oid":"arc_agi-9142f05d564def23","rd":"2025-01-31","sid":"recoKTXJX6Wtyzmyn","sm":"o3-mini-2025-01-31_low","src":"https://arcprize.org/leaderboard","v":0.145},{"b":"arc_agi","c":0.0263,"dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"arc_agi-38f87777564e5b20","rd":"2025-10-15","sid":"recO8w0BTuGH9P0y9","sm":"claude-haiku-4-5-20251001","v":0.1433},{"b":"arc_agi","cfg":{"effort":"medium"},"dn":"o1-mini","e":"medium","m":"o1-mini-2024-09-12","oid":"arc_agi-9052b5e85186850f","rd":"2024-09-12","sid":"recjBMo8w3uRVYaQN","sm":"o1-mini-2024-09-12_medium","src":"https://arcprize.org/leaderboard","v":0.14},{"b":"arc_agi","c":0.135,"dn":"o1-mini","m":"o1-mini-2024-09-12","oid":"arc_agi-579d9bbdcc59232e","rd":"2024-09-12","sid":"rec5TYrCBulg0P7BN","sm":"o1-mini-2024-09-12_unknown","v":0.14},{"b":"arc_agi","c":0.058,"dn":"Claude 3.7","m":"claude-3-7-sonnet-2025-02-19","oid":"arc_agi-4a3efe159a721129","rd":"2025-02-24","sid":"recDy7yrZGrBGUAKD","sm":"claude-3-7-sonnet-20250219","src":"https://arcprize.org/leaderboard","v":0.136},{"b":"arc_agi","c":0.04,"cfg":{"effort":"low"},"dn":"GPT-5.4 Mini (Low)","e":"low","m":"gpt-5-4-mini-2026-03-17","oid":"arc_agi-82e2948234cd2157","rd":"2026-03-17","sid":"recwxgmyNfideG5hT","sm":"gpt-5.4-mini-2026-03-17_low","src":"https://arcprize.org/leaderboard","v":0.13},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"DeepSeek V4 Pro 0813 (None)","e":"none","m":"deepseek-v4-pro-0813","oid":"arc_agi-3f44d312b36f793d","rd":"2026-08-13","sid":"rec5EN1pnHdlwhDyR","sm":"deepseek-v4-pro-0813_none","src":"https://arcprize.org/leaderboard","v":0.13},{"b":"arc_agi","c":0.082,"cfg":{"effort":"none"},"dn":"GPT-5.2","e":"none","m":"gpt-5-2-2025-12-11","oid":"arc_agi-e78d5946ad8640ff","rd":"2025-12-11","sid":"reckceJM0xScUaZLy","sm":"gpt-5.2-2025-12-11_none","v":0.1233},{"b":"arc_agi","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"arc_agi-e7e89abd93336af1","rd":"2025-12-11","sid":"recBUAKMpjE17mSDr","sm":"gpt-5.2-2025-12-11_unknown","src":"https://arcprize.org/leaderboard","v":0.1233},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"DeepSeek V4 Flash 0731 (None)","e":"none","m":"deepseek-v4-flash-0731","oid":"arc_agi-95be20b2280a0a6d","rd":"2026-07-31","sid":"rec5A1zniiTY6ZtUV","sm":"deepseek-v4-flash-0731_none","src":"https://arcprize.org/leaderboard","v":0.11833333333333335},{"b":"arc_agi","c":0.07,"dn":"Claude 3.7 (1K)","m":"claude-3-7-sonnet-20250219-1k","oid":"arc_agi-eca7ef4e7f93bb8d","rd":"2025-02-24","sid":"rec96fPyndWwzLTzI","sm":"claude-3-7-sonnet-20250219_1K","src":"https://arcprize.org/leaderboard","v":0.116},{"b":"arc_agi","c":0.0025,"dn":"Qwen3-235b-a22b Instruct (25/07)","m":"qwen3-235b-a22b-instruct-2507","oid":"arc_agi-1a309e755a8f4bfa","rd":"2025-07-25","sid":"recXyCIvzq1Y2Fu9B","sm":"Qwen3-235B-A22B-Instruct-2507","v":0.11},{"b":"arc_agi","c":0.29,"dn":"GPT-4.5","m":"gpt-4-5-preview-2025-02-27","oid":"arc_agi-4a2f6332eb063618","rd":"2025-02-27","sid":"recqWXrYnfvx1C2KT","sm":"gpt-4.5-preview-2025-02-27","src":"https://arcprize.org/leaderboard","v":0.103},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"GPT-6 Luna (None)","e":"none","m":"gpt-6-luna","oid":"arc_agi-176d6dd84019245a","rd":"2026-09-22","sid":"recZg4KhlBaBZWwiy","sm":"gpt-6-luna_none","src":"https://arcprize.org/leaderboard","v":0.08833333333333335},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash (None)","e":"none","m":"deepseek-v4-1-flash","oid":"arc_agi-02b50fb1da2c75f0","rd":"2026-09-09","sid":"recPdtmPC4SMDUm3t","sm":"deepseek-v4.1-flash_none","src":"https://arcprize.org/leaderboard","v":0.07833333333333332},{"b":"arc_agi","cfg":{"effort":"minimal"},"dn":"Gemini 3.5 Flash-Lite (Minimal)","e":"minimal","m":"gemini-3-5-flash-lite","oid":"arc_agi-c12115a003c9037b","rd":"2026-07-21","sid":"recGGYJXXvy8DSVWD","sm":"gemini-3.5-flash-lite_minimal","src":"https://arcprize.org/leaderboard","v":0.075},{"b":"arc_agi","c":0.123,"cfg":{"effort":"thinking"},"dn":"Magistral Medium (Thinking)","e":"thinking","m":"magistral-medium-2506","oid":"arc_agi-868067f6ea1f65ff","rd":"2025-06-10","sid":"recNqBbmZH3t0BZnD","sm":"magistral-medium-2506","src":"https://arcprize.org/leaderboard","v":0.061200000000000004},{"b":"arc_agi","c":0.0335,"cfg":{"effort":"minimal"},"dn":"GPT-5 (Minimal)","e":"minimal","m":"gpt-5-2025-08-07","oid":"arc_agi-61dd10b838e67901","rd":"2025-08-07","sid":"rec7EWmQhvfrKlA5M","sm":"gpt-5-2025-08-07_minimal","v":0.06},{"b":"arc_agi","c":0.108,"cfg":{"effort":"medium"},"dn":"Magistral Medium","e":"medium","m":"magistral-medium-2506","oid":"arc_agi-d0590efd00066878","rd":"2025-06-10","sid":"recq9HDi3QWg3tymY","sm":"magistral-medium-2506","v":0.0591},{"b":"arc_agi","c":0.0338,"cfg":{"effort":"none"},"dn":"GPT-5.1 (Thinking, None)","e":"none","m":"gpt-5-1-2025-11-13","oid":"arc_agi-1b4f610089342451","rd":"2025-11-13","sid":"recvFAXe18z3JQjOe","sm":"gpt-5.1-2025-11-13_none","v":0.058300000000000005},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash 1017 (None)","e":"none","m":"deepseek-v4-1-flash-1017","oid":"arc_agi-54879879eea16c1f","rd":"2026-10-17","sid":"recFmnjnet9YbEfpj","sm":"deepseek-v4.1-flash-1017_none","src":"https://arcprize.org/leaderboard","v":0.05666666666666667},{"b":"arc_agi","c":0.039,"dn":"GPT-4.1","m":"gpt-4-1-2025-04-14","oid":"arc_agi-cff4faeb6e3508fb","rd":"2025-04-14","sid":"rec3x5nkdnWzgTzeC","sm":"gpt-4.1-2025-04-14","src":"https://arcprize.org/leaderboard","v":0.055},{"b":"arc_agi","c":0.0931,"dn":"Grok 3","m":"grok-3","oid":"arc_agi-9b5be48db7b1fef6","rd":"2025-04-09","sid":"rec6qqzTzHa8ARTi6","sm":"grok-3","src":"https://arcprize.org/leaderboard","v":0.055},{"b":"arc_agi","cfg":{"effort":"none"},"dn":"Inkling Small (None)","e":"none","m":"inkling-small","oid":"arc_agi-09ba9e30f0f11f2d","rd":"2026-07-15","sid":"recwdFt5kxgB2Wsps","sm":"inkling-small_none","src":"https://arcprize.org/leaderboard","v":0.055},{"b":"arc_agi","c":0.0057,"cfg":{"effort":"minimal"},"dn":"GPT-5 Mini (Minimal)","e":"minimal","m":"gpt-5-mini-2025-08-07","oid":"arc_agi-75dd72cff2a50565","rd":"2025-08-07","sid":"recjlR2PzTtBVfJjg","sm":"gpt-5-mini-2025-08-07_minimal","v":0.0533},{"b":"arc_agi","c":0.0399,"dn":"Magistral Small","m":"magistral-small-2506","oid":"arc_agi-ee33e9823f1f63b5","rd":"2025-06-10","sid":"recitQEg9v6tfnwI9","sm":"magistral-small-2506","v":0.05},{"b":"arc_agi","c":0.05,"dn":"GPT-4o","m":"gpt-4o-2024-11-20","oid":"arc_agi-97143306bb78abe5","rd":"2024-11-20","sid":"recEtb28sN0f3OFf5","sm":"gpt-4o-2024-11-20","src":"https://arcprize.org/leaderboard","v":0.045},{"b":"arc_agi","c":0.0078,"dn":"Llama 4 Maverick","m":"llama-4-maverick-17b-128e-instruct","oid":"arc_agi-48b5f5bdccfea1a6","rd":"2025-04-05","sid":"rec0EiReUxsja9Iui","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://arcprize.org/leaderboard","v":0.0438},{"b":"arc_agi","c":0.0033,"cfg":{"effort":"low"},"dn":"GPT-5 Nano (Low)","e":"low","m":"gpt-5-nano-2025-08-07","oid":"arc_agi-b7aebeb7ab1f6acc","rd":"2025-08-07","sid":"recrZHnvXhnIH3CXM","sm":"gpt-5-nano-2025-08-07_low","v":0.0404},{"b":"arc_agi","c":0.0078,"dn":"GPT-4.1-Mini","m":"gpt-4-1-mini-2025-04-14","oid":"arc_agi-941d55097ea0b23e","rd":"2025-04-14","sid":"recFVGAte0Uy8kc22","sm":"gpt-4.1-mini-2025-04-14","src":"https://arcprize.org/leaderboard","v":0.035},{"b":"arc_agi","c":0.0015,"cfg":{"effort":"minimal"},"dn":"GPT-5 Nano (Minimal)","e":"minimal","m":"gpt-5-nano-2025-08-07","oid":"arc_agi-eed05a8bf2287f73","rd":"2025-08-07","sid":"rec2bl4JRg9TTqfnc","sm":"gpt-5-nano-2025-08-07_minimal","v":0.015},{"b":"arc_agi","c":0.0041,"dn":"Llama 4 Scout","m":"llama-4-scout-17b-16e-instruct","oid":"arc_agi-56965018f3763773","rd":"2025-04-05","sid":"recNN11X9GLUAxRYd","sm":"Llama-4-Scout-17B-16E-Instruct","src":"https://arcprize.org/leaderboard","v":0.005},{"b":"arc_agi","c":0.0021,"dn":"GPT-4.1-Nano","m":"gpt-4-1-nano-2025-04-14","oid":"arc_agi-9694b30780d2c580","rd":"2025-04-14","sid":"recNs9X6RPBPUfcRN","sm":"gpt-4.1-nano-2025-04-14","src":"https://arcprize.org/leaderboard","v":0},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"Yi 6b","m":"yi-6b","oid":"arc_ai2-377c976b0842eee8","rd":"2023-11-02","sid":"recBDxFxuSKRTNMCm","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.503},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"Yi 9b","m":"yi-9b","oid":"arc_ai2-067f0422bd0108ac","rd":"2024-03-01","sid":"recqf9Jh0xFM1IXnC","sm":"Yi-9B","src":"http://arxiv.org/abs/2403.04652","v":0.556},{"b":"arc_ai2","cfg":{"shots":"2-shot"},"dn":"PaLM","m":"palm-540b","oid":"arc_ai2-7f88fc22f7ed47ba","rd":"2022-04-04","sid":"recbeIMXP2c5ukqiJ","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.601},{"b":"arc_ai2","cfg":{"shots":"2-shot"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"arc_ai2-b489fb0a58544e88","rd":"2023-05-17","sid":"recekhdpsUQa4s2Hm","sm":"PaLM 2-S","src":"http://arxiv.org/abs/2311.16867","v":0.596},{"b":"arc_ai2","cfg":{"shots":"2-shot"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"arc_ai2-0523ca677cbd1cfa","rd":"2023-05-17","sid":"recuxZiJTrnM847mA","sm":"PaLM 2-M","src":"http://arxiv.org/abs/2311.16867","v":0.649},{"b":"arc_ai2","cfg":{"shots":"2-shot"},"dn":"PaLM 2-L","m":"palm-2-l","oid":"arc_ai2-3e3b6d8f72fb2ef3","rd":"2023-05-17","sid":"recWEx4BOQTar2SBJ","sm":"PaLM 2-L","src":"http://arxiv.org/abs/2311.16867","v":0.692},{"b":"arc_ai2","cfg":{"shots":"2-shot"},"dn":"Falcon 180B","m":"falcon-180b","oid":"arc_ai2-3b4fd256215eb2cd","rd":"2023-09-06","sid":"recyedY0eMSNaxyzQ","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.678},{"b":"arc_ai2","dn":"GPT-3","m":"text-davinci-001","oid":"arc_ai2-485744f842b305d6","rd":"2022-01-27","sid":"recon4Hh3kTsbzIUH","sm":"text-davinci-001","src":"http://arxiv.org/abs/2311.16867","v":0.514},{"b":"arc_ai2","dn":"PaLM","m":"palm-540b","oid":"arc_ai2-5a8f26a13fd25c81","rd":"2022-04-04","sid":"recPA0ga3kjdYrNvX","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.53},{"b":"arc_ai2","dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"arc_ai2-943d71b32c93c78e","rd":"2023-07-18","sid":"recMgIDIjScfAKlAK","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2311.16867","v":0.459},{"b":"arc_ai2","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"arc_ai2-2d52997d46db2a27","rd":"2023-07-18","sid":"reccokU2BaZ83v8sC","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2311.16867","v":0.494},{"b":"arc_ai2","dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"arc_ai2-5fafcb204bdc4ff2","rd":"2023-07-18","sid":"recCQAtSkG6833htq","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2311.16867","v":0.545},{"b":"arc_ai2","dn":"LLaMA-2 70B","m":"llama-2-70b-hf","oid":"arc_ai2-c322ea198095086c","rd":"2023-07-18","sid":"recWxdEsOORwjod7s","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2311.16867","v":0.574},{"b":"arc_ai2","dn":"Falcon 7B","m":"falcon-7b","oid":"arc_ai2-ad3f1b7f2e5cf410","rd":"2023-04-24","sid":"recTQceLSvuKmSReV","sm":"falcon-7b","src":"http://arxiv.org/abs/2311.16867","v":0.445},{"b":"arc_ai2","dn":"Falcon 40B","m":"falcon-40b","oid":"arc_ai2-104c7de50fef3b16","rd":"2023-03-15","sid":"recTUiYeh3GLxrG9E","sm":"falcon-40b","src":"http://arxiv.org/abs/2311.16867","v":0.567},{"b":"arc_ai2","dn":"Falcon 180B","m":"falcon-180b","oid":"arc_ai2-50b16b0378ef5d3e","rd":"2023-09-06","sid":"rec7VFstStJ24o6LM","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.637},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"MPT 7B","m":"mpt-7b","oid":"arc_ai2-de627a591e5b0550","rd":"2023-05-05","sid":"recpTlvsgRj6K0TU9","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.426},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Falcon 7B","m":"falcon-7b","oid":"arc_ai2-7a1afe44a7a4f98c","rd":"2023-04-24","sid":"recx3YFpGEIXLrY5E","sm":"falcon-7b","src":"https://arxiv.org/abs/2309.16609","v":0.424},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"ChatGLM2 6B","m":"chatglm2-6b","oid":"arc_ai2-94385bdaad2163c4","rd":"2023-06-24","sid":"recXAARbHOX9VJjNY","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.61},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"InternLM 7B","m":"internlm-7b","oid":"arc_ai2-5401c85bcb66a772","rd":"2023-07-05","sid":"rec1k3Qvpm4E2qFw2","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.695},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"InternLM 20B","m":"internlm-20b","oid":"arc_ai2-3f23e4db33378a9b","rd":"2023-09-18","sid":"recAO3NVZ8A3ac9a3","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.817},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Baichuan2 7B","m":"baichuan-2-7b-base","oid":"arc_ai2-727d5ecf2d3c2315","rd":"2023-09-20","sid":"rec9IABwNbrfCP4NR","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.325},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"arc_ai2-55a90568db3bf6df","rd":"2023-09-06","sid":"recpRCQEqtiQDGRiB","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.38},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 7B","m":"llama-7b","oid":"arc_ai2-d99d3c17e776bb29","rd":"2023-02-24","sid":"recNvbTr84ONFah46","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.476},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 13B","m":"llama-13b","oid":"arc_ai2-4dc09f1ac3c0d007","rd":"2023-02-24","sid":"recOtuOh1YwJucfwl","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.527},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 33B","m":"llama-33b","oid":"arc_ai2-31edeebd329da2fd","rd":"2023-02-24","sid":"recoTYayUgTAlaU57","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.675},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 65B","m":"llama-65b","oid":"arc_ai2-893e34e858229119","rd":"2023-02-24","sid":"reclOJSyiTKHZfjyb","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.695},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"arc_ai2-568056d1be2bc1ca","rd":"2023-07-18","sid":"recuJuGMJDq55ts5K","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.459},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"arc_ai2-680e4cf23f4a7c05","rd":"2023-07-18","sid":"recF3fmFjfDHG8vQC","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.603},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"arc_ai2-919b63390e834c94","rd":"2023-07-18","sid":"recsMnkbkzX4mZGhQ","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.783},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"StableBeluga2 70B","m":"stablebeluga2","oid":"arc_ai2-1162b175a0c8e494","rd":"2023-07-20","sid":"reca6kz5FN4kqYoLL","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.861},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"QWEN 1.8B","m":"qwen-1-8b","oid":"arc_ai2-1d93b517d71cbe33","rd":"2023-11-30","sid":"recTH4MZO44wHEljJ","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.532},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"QWEN 7B","m":"qwen-7b","oid":"arc_ai2-07c7aa94aba31704","rd":"2023-09-28","sid":"recGHH7EGCn0ZpKha","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.753},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"QWEN 14B","m":"qwen-14b","oid":"arc_ai2-bf064086414b0ade","rd":"2023-09-28","sid":"recb3dKzs466t5yxc","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.844},{"b":"arc_ai2","cfg":{"shots":"1-shot"},"dn":"PaLM","m":"palm-540b","oid":"arc_ai2-b1668068f2406e82","rd":"2022-04-04","sid":"recoZOE4lE6gjLj0D","sm":"PaLM 540B","src":"http://arxiv.org/abs/2305.10403","v":0.601},{"b":"arc_ai2","cfg":{"shots":"1-shot"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"arc_ai2-b0c01218ab25ddb4","rd":"2023-05-17","sid":"rec6sY7Z0wdEAbFCL","sm":"PaLM 2-S","src":"http://arxiv.org/abs/2305.10403","v":0.596},{"b":"arc_ai2","cfg":{"shots":"1-shot"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"arc_ai2-398d30047dbe375b","rd":"2023-05-17","sid":"recr4wf65U2XCfXQt","sm":"PaLM 2-M","src":"http://arxiv.org/abs/2305.10403","v":0.649},{"b":"arc_ai2","cfg":{"shots":"1-shot"},"dn":"PaLM 2-L","m":"palm-2-l","oid":"arc_ai2-603fcd9aac3093f4","rd":"2023-05-17","sid":"recojP6HTbGGTVbzk","sm":"PaLM 2-L","src":"http://arxiv.org/abs/2305.10403","v":0.692},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"arc_ai2-4a41b74423b15bce","rd":"2023-07-18","sid":"recvbbV1ndfleEaFI","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.494},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"arc_ai2-e373d7f4358d2724","rd":"2023-07-18","sid":"recE3lvgrGObdJyUb","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.545},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Qwen 14B","m":"qwen-14b","oid":"arc_ai2-0caba79e387b5453","rd":"2023-09-28","sid":"recPlkIgHY3AIp4VA","sm":"Qwen-14B","src":"http://arxiv.org/abs/2402.16819","v":0.844},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"arc_ai2-52d740f63ce2b2f1","rd":"2023-09-27","sid":"recywSGJNH0NxjYpT","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.555},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Gemma 7B","m":"gemma-7b","oid":"arc_ai2-f153f96618e368e9","rd":"2024-02-21","sid":"recnxE0Ob3qfuAUIo","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.532},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"arc_ai2-f8a6cbe235732f9d","rd":"2024-02-26","sid":"recKztdCp2jstcKfc","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.555},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Falcon 7B","m":"falcon-7b","oid":"arc_ai2-a13c41d741771596","rd":"2023-04-24","sid":"recS3RjfIU600rOkX","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.4343},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"Falcon 7B","m":"falcon-7b","oid":"arc_ai2-6dab857c801935ee","rd":"2023-04-24","sid":"recP3Y7YbGtlvVoLz","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.4786},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Falcon 40B","m":"falcon-40b","oid":"arc_ai2-5e442ad17f3bbd93","rd":"2023-03-15","sid":"recJlXiNWhl3fA6kY","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.5469},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"Falcon 40B","m":"falcon-40b","oid":"arc_ai2-386ffa394863fe32","rd":"2023-03-15","sid":"recct3WooWBjSgf7i","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.6186},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"MPT 7B","m":"mpt-7b","oid":"arc_ai2-ad95b3925080c9e3","rd":"2023-05-05","sid":"recIwq5o7M0MzTSOQ","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.426},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"MPT 30B","m":"mpt-30b","oid":"arc_ai2-8c1a8ce4d21218bc","rd":"2023-06-22","sid":"receG6c5LTBxKHwK3","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.506},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Falcon 7B","m":"falcon-7b","oid":"arc_ai2-df7d496a0c040729","rd":"2023-04-24","sid":"recLxwQkk6elP9rvJ","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.424},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Falcon 40B","m":"falcon-40b","oid":"arc_ai2-70829ba4db863c52","rd":"2023-03-15","sid":"recyiJ5QhobNkgTh8","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.545},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"arc_ai2-9f15a5e73421a435","rd":"2023-02-24","sid":"recfekftkllVQjldP","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.476},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"arc_ai2-da42343f71c90693","rd":"2023-02-24","sid":"recTN8LK88YmYWoJe","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.527},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"arc_ai2-3e5037627d47e496","rd":"2023-02-24","sid":"rec5EHyTp1YUNUlIV","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.578},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"arc_ai2-58cbd984a71115cc","rd":"2023-02-24","sid":"recBTk09y2SlTwbse","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.56},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"arc_ai2-1e770131145fecee","rd":"2023-07-18","sid":"recMlVcLfzfIeByw5","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.459},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"arc_ai2-a9cd11a9aa82f416","rd":"2023-07-18","sid":"rec4odRrMQ8MiCOBG","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.494},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"arc_ai2-6c2d96166c4babba","rd":"2023-07-18","sid":"recdrH9ADacP1cq1N","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.545},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"arc_ai2-a14427d4bd5d7635","rd":"2023-07-18","sid":"recCunGxbV5MxvAIQ","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.574},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"GPT-3 175B","m":"text-davinci-001","oid":"arc_ai2-ce212a0813c25c1c","rd":"2022-01-27","sid":"recHNTbNvpxyoh4wP","sm":"text-davinci-001","src":"http://arxiv.org/abs/2302.13971","v":0.514},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"PaLM 62B","m":"palm-62b","oid":"arc_ai2-8ebb7265e0102798","rd":"2022-04-04","sid":"recL1vBgiMBZ5R4DU","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.525},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"PaLM 540B","m":"palm-540b","oid":"arc_ai2-791e65701ab6a87b","rd":"2022-04-04","sid":"reccXZcXgV6vDCk2y","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.53},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 7B","m":"llama-7b","oid":"arc_ai2-e77e9646df5e861c","rd":"2023-02-24","sid":"recG8mIav5orjHdDC","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.476},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 13B","m":"llama-13b","oid":"arc_ai2-38b52290eb1195fa","rd":"2023-02-24","sid":"recCVeGPJ1aSElADj","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.527},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 33B","m":"llama-33b","oid":"arc_ai2-747db7341fc7afe5","rd":"2023-02-24","sid":"recIDEycNEywpwdkf","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.578},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 65B","m":"llama-65b","oid":"arc_ai2-693f491ef327a695","rd":"2023-02-24","sid":"rec1kNbH3eKsesiQn","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.56},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"arc_ai2-3740b10825585e23","rd":"2024-05-07","sid":"rech3PGmSJM5UVUj1","sm":"DeepSeek-V2","src":"http://arxiv.org/abs/2412.19437","v":0.922},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"arc_ai2-4a2b041ead51ccb5","rd":"2024-09-19","sid":"rec0eZFhwFqf505ep","sm":"Qwen2.5-72B","src":"http://arxiv.org/abs/2412.19437","v":0.945},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"arc_ai2-7981c55524732f01","rd":"2024-07-23","sid":"recqQq5QZQFh9wVBf","sm":"Llama-3.1-405B","src":"http://arxiv.org/abs/2412.19437","v":0.953},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"arc_ai2-d9370dda1721db44","rd":"2024-12-26","sid":"recJdOZPC2C12w8XY","sm":"DeepSeek-V3","src":"http://arxiv.org/abs/2412.19437","v":0.953},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"arc_ai2-f548266ca511900d","rd":"2023-07-18","sid":"recCVgWGnaJ4pJcjb","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2401.04088","v":0.432},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"arc_ai2-acc098f46fb6e84e","rd":"2023-07-18","sid":"recxm6VEU7XA7OetW","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2401.04088","v":0.488},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 1 33B","m":"llama-33b","oid":"arc_ai2-17714eebef6deb5c","rd":"2023-02-24","sid":"recRd4cheNnjH84Mj","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2401.04088","v":0.544},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"arc_ai2-5f4cce5a51bf1b4c","rd":"2023-07-18","sid":"recZnTdIuD44Xt6sD","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2401.04088","v":0.565},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"arc_ai2-d594a7fbccefc984","rd":"2023-09-27","sid":"recc1svpFIcg7vAL9","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.549},{"b":"arc_ai2","cfg":{"shots":"0-shot"},"dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"arc_ai2-8e684f6f730a1f1b","rd":"2023-12-11","sid":"recwv4JdA8LfWCnVJ","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.597},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Phi-3-mini 3.8b","m":"phi-3-mini-4k-instruct","oid":"arc_ai2-a65bc4a06e876dce","rd":"2024-04-23","sid":"recFlvyWCd6NoUbI5","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.849},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Phi-3-small 7b","m":"phi-3-small-8k-instruct","oid":"arc_ai2-ae5c2505e8ab05e2","rd":"2024-04-23","sid":"recxwZXkfAY9VYS4M","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.907},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Phi-3-medium 14b","m":"phi-3-medium-128k-instruct","oid":"arc_ai2-c2cf5b18ff6f6d52","rd":"2024-04-23","sid":"recnLrwmDVsPnimhW","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.916},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Phi-2 2.7b","m":"phi-2","oid":"arc_ai2-32e05e7b244598aa","rd":"2023-12-12","sid":"rec9o6whbdGoRj4EL","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.759},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"arc_ai2-2acd3d0a317fdc23","rd":"2023-09-27","sid":"recm8JgtAr9cYvI3c","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.786},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Gemma 7b","m":"gemma-7b","oid":"arc_ai2-edea417791e2a418","rd":"2024-02-21","sid":"reccsBnBbU3vLIlXL","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.783},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"arc_ai2-26aa0a2775a65149","rd":"2024-04-18","sid":"recyK5toDoO0k8HO4","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.828},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"arc_ai2-a6cd490f5b499218","rd":"2023-12-11","sid":"recMzqmrpgMj3B6cS","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.873},{"b":"arc_ai2","cfg":{"shots":"10-shot"},"dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"arc_ai2-7056945685e59198","rd":"2023-11-06","sid":"recfRXMnx6rHuGIBt","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.874},{"b":"arc_ai2","dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"arc_ai2-a99ce21e8d844c97","rd":"2023-07-18","sid":"recYond491moEZsn0","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2403.08295","v":0.459},{"b":"arc_ai2","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"arc_ai2-62e2e7950313a0ba","rd":"2023-07-18","sid":"recZUkL7VensCpe7G","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2403.08295","v":0.494},{"b":"arc_ai2","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"arc_ai2-ea411fc2e8af0268","rd":"2023-09-27","sid":"recyVBqHxwXmkfQPN","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2403.08295","v":0.549},{"b":"arc_ai2","dn":"Gemma 2B","m":"gemma-2b","oid":"arc_ai2-7912abebe9760417","rd":"2024-02-21","sid":"recQD5p42nBSC7DhU","sm":"gemma-2b","src":"http://arxiv.org/abs/2403.08295","v":0.421},{"b":"arc_ai2","dn":"Gemma 7B","m":"gemma-7b","oid":"arc_ai2-d50bdac5f5023d3e","rd":"2024-02-21","sid":"rec9mwOSlc5EhlJyP","sm":"gemma-7b","src":"http://arxiv.org/abs/2403.08295","v":0.532},{"b":"arc_ai2","dn":"Qwen2.5-Coder-0.5B","m":"qwen2-5-coder-0-5b","oid":"arc_ai2-ba1282aea9b717d0","rd":"2024-09-18","sid":"recj6fIwM4g5QhZPo","sm":"Qwen2.5-Coder-0.5B","src":"http://arxiv.org/abs/2409.12186","v":0.344},{"b":"arc_ai2","dn":"DS-Coder-1.3B-Base","m":"deepseek-coder-1-3b-base","oid":"arc_ai2-3fbb866e36cd8f8a","rd":"2023-11-02","sid":"recHGOaX4WRoiifCr","sm":"deepseek-coder-1.3b-base","src":"http://arxiv.org/abs/2409.12186","v":0.254},{"b":"arc_ai2","dn":"Qwen2.5-Coder-1.5B","m":"qwen2-5-coder-1-5b","oid":"arc_ai2-e7b920875e827477","rd":"2024-09-18","sid":"rec7OCOx9wHGEaQoJ","sm":"Qwen2.5-Coder-1.5B","src":"http://arxiv.org/abs/2409.12186","v":0.452},{"b":"arc_ai2","dn":"StarCoder2-3B","m":"starcoder2-3b","oid":"arc_ai2-34e2f2cf7b473741","rd":"2024-02-22","sid":"recxa9NmUOx3zyxMY","sm":"starcoder2-3b","src":"http://arxiv.org/abs/2409.12186","v":0.342},{"b":"arc_ai2","dn":"Qwen2.5-Coder-3B","m":"qwen2-5-coder-3b","oid":"arc_ai2-ac9522c9a75c3aad","rd":"2024-09-18","sid":"rec50W2CfGuH4w5Uy","sm":"Qwen2.5-Coder-3B","src":"http://arxiv.org/abs/2409.12186","v":0.529},{"b":"arc_ai2","dn":"StarCoder2-7B","m":"starcoder2-7b","oid":"arc_ai2-9eec1429b30001bc","rd":"2024-02-20","sid":"recmivd1RvFYMTrow","sm":"starcoder2-7b","src":"http://arxiv.org/abs/2409.12186","v":0.387},{"b":"arc_ai2","dn":"DS-Coder-6.7B-Base","m":"deepseek-coder-6-7b-base","oid":"arc_ai2-4670efbbd51dfc7d","rd":"2023-11-02","sid":"rec5apbI4aqTcGOt2","sm":"deepseek-coder-6.7b-base","src":"http://arxiv.org/abs/2409.12186","v":0.364},{"b":"arc_ai2","dn":"DS-Coder-V2-Lite-Base","m":"deepseek-coder-v2-lite-base","oid":"arc_ai2-2d208fa82ac1582c","rd":"2024-06-13","sid":"reck06B6EFq2sdA9w","sm":"DeepSeek-Coder-V2-Lite-Base","src":"http://arxiv.org/abs/2409.12186","v":0.573},{"b":"arc_ai2","dn":"CodeQwen1.5-7B","m":"codeqwen1-5-7b","oid":"arc_ai2-9476df0b880ae218","rd":"2024-04-15","sid":"recpJSkO0X2NX3uo4","sm":"CodeQwen1.5-7B","src":"http://arxiv.org/abs/2409.12186","v":0.357},{"b":"arc_ai2","dn":"Qwen2.5-Coder-7B","m":"qwen2-5-coder-7b","oid":"arc_ai2-a3793f268aa38baa","rd":"2024-09-18","sid":"recXIKMqsRclf2Qql","sm":"Qwen2.5-Coder-7B","src":"http://arxiv.org/abs/2409.12186","v":0.609},{"b":"arc_ai2","dn":"StarCoder2-15B","m":"starcoder2-15b","oid":"arc_ai2-9ef007cec9a5e59f","rd":"2024-02-20","sid":"recZRpnfj4C3Md7l1","sm":"starcoder2-15b","src":"http://arxiv.org/abs/2409.12186","v":0.472},{"b":"arc_ai2","dn":"Qwen2.5-Coder-14B","m":"qwen2-5-coder-14b","oid":"arc_ai2-02a1da111222ed51","rd":"2024-09-18","sid":"recoOOluDbZTCxEFb","sm":"Qwen2.5-Coder-14B","src":"http://arxiv.org/abs/2409.12186","v":0.66},{"b":"arc_ai2","dn":"DS-Coder-33B-Base","m":"deepseek-coder-33b-base","oid":"arc_ai2-2810c6c5a7cfd418","rd":"2023-11-02","sid":"recpeuK5Fs5GpJMEj","sm":"deepseek-coder-33b-base","src":"http://arxiv.org/abs/2409.12186","v":0.422},{"b":"arc_ai2","dn":"DS-Coder-V2-Base","m":"deepseek-coder-v2-base","oid":"arc_ai2-a0b0a55a5c843d99","rd":"2024-06-17","sid":"recZW2JY1PtSj2NeX","sm":"DeepSeek-Coder-V2-Base","src":"http://arxiv.org/abs/2409.12186","v":0.643},{"b":"arc_ai2","dn":"Qwen2.5-Coder-32B","m":"qwen2-5-coder-32b","oid":"arc_ai2-c75d845dc10a13db","rd":"2024-09-18","sid":"rec3ZJZr9FPPsNCST","sm":"Qwen2.5-Coder-32B","src":"http://arxiv.org/abs/2409.12186","v":0.705},{"b":"arc_ai2","dn":"Vicuna-13B (v1.1)","m":"vicuna-13b-v1-1","oid":"arc_ai2-4045f955e89b7575","rd":"2023-04-12","sid":"recoGOGv6pY2Z77AB","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.432},{"b":"arc_ai2","dn":"Llama2-7B","m":"llama-2-7b","oid":"arc_ai2-cd56bfc6dac2e475","rd":"2023-07-18","sid":"recSNV2YipA1bob60","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.434},{"b":"arc_ai2","dn":"Llama-7B","m":"llama-7b","oid":"arc_ai2-2db9031d7e975842","rd":"2023-02-24","sid":"recH3a6zzXMsqvy9U","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.385},{"b":"arc_ai2","dn":"MPT-7B","m":"mpt-7b","oid":"arc_ai2-27404531bb4f1554","rd":"2023-05-05","sid":"rec81EbC7EomkLKBX","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.405},{"b":"arc_ai2","dn":"Falcon-7B","m":"falcon-7b","oid":"arc_ai2-d88751d67db0fb9c","rd":"2023-04-24","sid":"recDO0Z46FYYeRAj8","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.363},{"b":"arc_ai2","dn":"OPT-1.3B","m":"opt-1-3b","oid":"arc_ai2-bef7646d3c4275cf","rd":"2022-05-11","sid":"recY7BYwRaSw0WvJH","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.232},{"b":"arc_ai2","dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"arc_ai2-2e6b9423516ffaf4","rd":"2019-11-05","sid":"recKlotp4q8bnKtwd","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.25},{"b":"arc_ai2","dn":"phi-1.5 (1.3B)","m":"phi-1-5","oid":"arc_ai2-d9bc5e5aaf442269","rd":"2023-09-11","sid":"recoj6GK2WTBxH06V","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.444},{"b":"arc_ai2","cfg":{"shots":"25-shot"},"dn":"GPT-3.5","m":"text-davinci-002","oid":"arc_ai2-55f0c3503c2ef251","rd":"2022-03-15","sid":"recGUeMteustGGfMg","sm":"text-davinci-002","src":"http://arxiv.org/abs/2303.08774","v":0.852},{"b":"arc_ai2","cfg":{"shots":"8-shot"},"dn":"8-shot PaLM","m":"palm-540b","oid":"arc_ai2-1b9697726cfbcda9","rd":"2022-04-04","sid":"rechw4k0fmZIGuIGv","sm":"PaLM 540B","src":"http://arxiv.org/abs/2303.08774","v":0.852},{"b":"arc_ai2","dn":"GPT-3 (175B) (Zero-shot)","m":"text-davinci-001","oid":"arc_ai2-7988a7b5c6c8b9fc","rd":"2022-01-27","sid":"recsaQHxqvzRAnRqF","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.514},{"b":"arc_ai2","dn":"GPT-3 (175B) (One-shot)","m":"text-davinci-001","oid":"arc_ai2-aa22c696119d8ea2","rd":"2022-01-27","sid":"reco3SamtEdKFjE4P","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.532},{"b":"arc_ai2","dn":"GPT-3 (175B) (50-shot)","m":"text-davinci-001","oid":"arc_ai2-23d9396b6e0bdc88","rd":"2022-01-27","sid":"recH2Cy2QrGCCajNe","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.515},{"b":"arc_ai2","dn":"Claude Instant 1.1","m":"claude-instant-1-1","oid":"arc_ai2-450b8b712cab9050","sid":"recNlveRZDstMlvIa","sm":"claude-instant-1.1","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.857},{"b":"arc_ai2","dn":"Claude Instant 1.2","m":"claude-instant-1-2","oid":"arc_ai2-c7f9a22f036e93a4","rd":"2023-08-09","sid":"recCzRTq1cFC8vOhO","sm":"claude-instant-1.2","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.863},{"b":"arc_ai2","dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"arc_ai2-912be40c52f4e6e7","rd":"2023-06-27","sid":"recDdVbzbODl8KRDE","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.412},{"b":"arc_ai2","dn":"LLaMA-7B","m":"llama-7b","oid":"arc_ai2-3915f143ceeddda8","rd":"2023-02-24","sid":"rec96oGAR33iueEEJ","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.448},{"b":"arc_ai2","dn":"Falcon-7B","m":"falcon-7b","oid":"arc_ai2-eab346ade3646beb","rd":"2023-04-24","sid":"recIJQRtQygxkLIBx","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.434},{"b":"arc_ai2","dn":"MPT-7B","m":"mpt-7b","oid":"arc_ai2-5022b04489cd2152","rd":"2023-05-05","sid":"recbWNpk3h1b9y7PR","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.417},{"b":"arc_ai2","dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"arc_ai2-29454b3ac486ff9b","rd":"2023-06-07","sid":"recdgUh5eB5jQ4oyE","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.387},{"b":"arc_ai2","dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"arc_ai2-577043fd6c3e6b41","rd":"2023-05-04","sid":"rec8cEStkqtkCyGmJ","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.391},{"b":"arc_ai2","dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"arc_ai2-21454baf48bb4d05","rd":"2022-04-07","sid":"recSYskjlkTSJPrPU","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.411},{"b":"arc_ai2","dn":"OPT-13B","m":"opt-13b","oid":"arc_ai2-0c671f0d032d7fee","rd":"2022-05-11","sid":"recsLxqhuH0roOIAF","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.358},{"b":"arc_ai2","dn":"GPT-J-6B","m":"gpt-j-6b","oid":"arc_ai2-113a6f8819430d51","rd":"2021-08-05","sid":"recPDicLmVESGltVO","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.363},{"b":"arc_ai2","dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"arc_ai2-8ba7d8b2c3c74ed7","rd":"2023-04-11","sid":"recpM34slUJwZbjqj","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.396},{"b":"arc_ai2","dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"arc_ai2-71c69dc0f4b5a767","rd":"2023-03-20","sid":"recuKcLt2nDob6ZLa","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.324},{"b":"arc_ai2","dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"arc_ai2-bd81efdb35574b9a","rd":"2023-04-19","sid":"recHdiAAANd9V5aXp","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.27},{"b":"arc_ai2","dn":"INTELLECT-1-Instruct","m":"intellect-1-instruct","oid":"arc_ai2-2e3a444e2430b699","rd":"2024-11-29","sid":"recPfTdB5uSeoqZv6","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.5452},{"b":"balrog","cfg":{"effort":"max"},"e":"max","m":"gpt-6-astra","oid":"balrog-3321aa2a862a7460","rd":"2026-09-03","se":0.02,"sid":"reck3TISQryfKvBGC","sm":"gpt-6-astra_max","src":"https://balrogai.com/","v":0.6829999999999999},{"b":"balrog","cfg":{"effort":"max"},"e":"max","m":"claude-opus-5","oid":"balrog-7f932cefc23554c8","rd":"2026-07-24","se":0.018000000000000002,"sid":"recMrVUQA94E5HlOL","sm":"claude-opus-5_max","src":"https://balrogai.com/","v":0.634},{"b":"balrog","cfg":{"effort":"max"},"e":"max","m":"gpt-5-6-sol","oid":"balrog-b6a78a37a4a17534","rd":"2026-07-09","se":0.02,"sid":"recaKkZ8e2yjmmGzb","sm":"gpt-5.6-sol_max","src":"https://balrogai.com/","v":0.6},{"b":"balrog","m":"gemini-3-pro-preview","notes":"https://balrogai.com/","oid":"balrog-8942acafd416a26b","rd":"2025-11-18","se":0.021,"sid":"recu64HG3CcO8EhbZ","sm":"gemini-3-pro-preview","src":"https://balrogai.com/","v":0.5810000000000001},{"b":"balrog","m":"gemini-3-1-pro-preview","notes":"https://balrogai.com/","oid":"balrog-cd43edc958280d87","rd":"2026-02-19","se":0.02,"sid":"recagT5kg4LOzfJRU","sm":"gemini-3.1-pro-preview","src":"https://balrogai.com/","v":0.57},{"b":"balrog","m":"gemini-3-1-pro-preview","notes":"https://balrogai.com/","oid":"balrog-1fd81c4c4fbbda82","rd":"2026-02-19","se":0.02,"sid":"recrPB4QFDwqkL1vd","sm":"gemini-3.1-pro-preview","src":"https://balrogai.com/","v":0.569},{"b":"balrog","cfg":{"effort":"max"},"e":"max","m":"gpt-5-6-terra","oid":"balrog-a86ff6c89d55bebf","rd":"2026-07-09","se":0.02,"sid":"recMKrSoVmhmeeJia","sm":"gpt-5.6-terra_max","src":"https://balrogai.com/","v":0.532},{"b":"balrog","m":"gemini-3-flash-preview","oid":"balrog-3b813aed962ab083","rd":"2025-12-17","se":0.024,"sid":"recoMyTSHduCvqkGN","sm":"gemini-3-flash-preview","src":"https://balrogai.com/","v":0.48100000000000004},{"b":"balrog","cfg":{"effort":"max"},"e":"max","m":"gpt-5-6-luna","oid":"balrog-ff2e86c5e50c770a","rd":"2026-07-09","se":0.017,"sid":"rec1Pr58yODRUF3Xo","sm":"gpt-5.6-luna_max","src":"https://balrogai.com/","v":0.456},{"b":"balrog","m":"grok-4-0709","oid":"balrog-403f38d69e071e5a","rd":"2025-07-09","se":0.022000000000000002,"sid":"recj1cynwUw30DZ24","sm":"grok-4-0709","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250713_naive_grok-4","v":0.436},{"b":"balrog","m":"claude-opus-4-5-2025-11-01","notes":"https://balrogai.com/","oid":"balrog-d71eee67e5356b79","rd":"2025-11-24","se":0.023,"sid":"rechOsDF5zTo7ToVP","sm":"claude-opus-4-5-20251101","src":"https://balrogai.com/","v":0.435},{"b":"balrog","m":"gemini-2-5-pro-exp-03-25","oid":"balrog-ecf1488585957100","rd":"2025-03-25","se":0.023,"sid":"recSOJ5L6wiyC21mt","sm":"gemini-2.5-pro-exp-03-25","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250425_naive_Gemini-2.5-Pro-Exp-03-25","v":0.433},{"b":"balrog","m":"claude-opus-4-5-20251101-64k","notes":"https://balrogai.com/","oid":"balrog-bff0d57301d44d90","rd":"2025-11-24","se":0.023,"sid":"recHOhXq4T9nx6ffH","sm":"claude-opus-4-5-20251101_64K","src":"https://balrogai.com/","v":0.43},{"b":"balrog","m":"claude-opus-4-5-2025-11-01","oid":"balrog-cd855a6a56460266","rd":"2025-11-24","se":0.023,"sid":"recu7vRmCRnKAPbOn","sm":"claude-opus-4-5-20251101_unknown","src":"https://balrogai.com/","v":0.43},{"b":"balrog","m":"deepseek-r1","oid":"balrog-9eebdae6cc490757","rd":"2025-01-20","se":0.021,"sid":"recHSLh1tcRzi8OcW","sm":"DeepSeek-R1","src":"https://balrogai.com/","v":0.349},{"b":"balrog","m":"gemini-2-5-flash","oid":"balrog-32a602bf93fb5d80","rd":"2025-06-17","se":0.022000000000000002,"sid":"recUkPodDLc0rmUT6","sm":"gemini-2.5-flash","src":"https://balrogai.com/","v":0.335},{"b":"balrog","cfg":{"effort":"minimal"},"e":"minimal","m":"gpt-5-2025-08-07","oid":"balrog-741573ab2ba42b3b","rd":"2025-08-07","se":0.022000000000000002,"sid":"recWQdFmqwZIHfPD9","sm":"gpt-5-2025-08-07_minimal","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250808_naive_gpt-5-minimal","v":0.32799999999999996},{"b":"balrog","m":"claude-3-5-sonnet-2024-10-22","oid":"balrog-923d0894a534d1e9","rd":"2024-10-22","se":0.019,"sid":"recLWfLdjfhmgI3Ap","sm":"claude-3-5-sonnet-20241022","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241103_Claude-3.5-Sonnet","v":0.326},{"b":"balrog","m":"gpt-4o-2024-05-13","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-a58b6626cc842ac3","rd":"2024-05-13","se":0.015,"sid":"recGsilOFpcHF89qF","sm":"gpt-4o-2024-05-13","src":"https://balrogai.com/","v":0.32299999999999995},{"b":"balrog","m":"claude-haiku-4-5-2025-10-01","notes":"https://balrogai.com/","oid":"balrog-522ab0ad23cba4bf","rd":"2025-10-15","se":0.021,"sid":"recOx8y3hn2Y1Jbml","sm":"claude-haiku-4-5-20251001","src":"https://balrogai.com/","v":0.312},{"b":"balrog","m":"claude-haiku-4-5-20251001-1k","oid":"balrog-41d4dabedc461f8e","rd":"2025-10-15","se":0.021,"sid":"recFZEHGMrDgjjNjD","sm":"claude-haiku-4-5-20251001_1K","src":"https://balrogai.com/","v":0.312},{"b":"balrog","m":"grok-3-beta","oid":"balrog-83fe0ce7a203e667","rd":"2025-04-09","se":0.022000000000000002,"sid":"recOo6W8q0IzNh3Fz","sm":"grok-3-beta","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250425_naive_grok-3","v":0.295},{"b":"balrog","m":"reka-flash-3","oid":"balrog-f264c4c77da914d1","rd":"2025-03-10","se":0.018000000000000002,"sid":"recNhhny2hCRlcZ6O","sm":"reka-flash-3","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250313_robust_cot_Reka-Flash-3","v":0.292},{"b":"balrog","m":"llama-3-1-70b-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-9fd109c3abd876cc","rd":"2024-07-23","se":0.013999999999999999,"sid":"recteB2cIvITYSNNg","sm":"Llama-3.1-70B-Instruct","src":"https://balrogai.com/","v":0.27899999999999997},{"b":"balrog","m":"llama-3-2-90b-vision-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-22f968c37e81ba7d","rd":"2024-09-24","se":0.013999999999999999,"sid":"recHmm4u7kk4i2kTV","sm":"Llama-3.2-90B-Vision-Instruct","src":"https://balrogai.com/","v":0.273},{"b":"balrog","m":"llama-3-3-70b-instruct","oid":"balrog-ffb216a0e4f16ebe","rd":"2024-12-06","se":0.017,"sid":"recY4fmN8bgBjHj5R","sm":"Llama-3.3-70B-Instruct","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241209_naive_Llama-3.3-70B-Instruct","v":0.23},{"b":"balrog","m":"gemini-1-5-pro-002","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-e81285733e88a7d8","rd":"2024-09-24","se":0.012,"sid":"recWRZ7DAviIRftFN","sm":"gemini-1.5-pro-002","src":"https://balrogai.com/","v":0.21},{"b":"balrog","m":"deepseek-r1-distill-qwen-32b","oid":"balrog-04525896e58375d5","rd":"2025-01-20","se":0.016,"sid":"recEyZ7hXqvBVFI5X","sm":"DeepSeek-R1-Distill-Qwen-32B","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250126_robust_cot_deepseek_R1_distill_qwen32B","v":0.195},{"b":"balrog","m":"claude-3-5-haiku-2024-10-22","oid":"balrog-390aa6b13d5daf25","rd":"2024-10-22","se":0.018000000000000002,"sid":"recCcD32JXmEOmxbh","sm":"claude-3-5-haiku-20241022","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241209_naive-claude-3-5-haiku","v":0.193},{"b":"balrog","m":"mistral-nemo-instruct-2407","oid":"balrog-9a6b19ff0dbbf30a","rd":"2024-07-18","se":0.015,"sid":"rec49SaiDl0h4hTar","sm":"Mistral-Nemo-Instruct-2407","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241209_naive-mistral-nemo-instruct","v":0.17600000000000002},{"b":"balrog","m":"gpt-4o-mini-2024-07-18","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-4663c8ee41066888","rd":"2024-07-18","se":0.013999999999999999,"sid":"reccmCPuiookWJZbw","sm":"gpt-4o-mini-2024-07-18","src":"https://balrogai.com/","v":0.174},{"b":"balrog","m":"llama-3-2-11b-vision-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-4cc7a5008a61ebae","rd":"2024-09-24","se":0.015,"sid":"recPgyvQTfFeS5y6w","sm":"Llama-3.2-11B-Vision-Instruct","src":"https://balrogai.com/","v":0.168},{"b":"balrog","m":"qwen2-5-72b-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-12146c973332849c","rd":"2024-09-19","se":0.016,"sid":"recpSxQQcNcrGbyaf","sm":"qwen2.5-72b-instruct","src":"https://balrogai.com/","v":0.162},{"b":"balrog","m":"llama-3-1-8b-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-a9b49174b4a431b9","rd":"2024-07-23","se":0.016,"sid":"recnsJM2l3foJ2IOJ","sm":"Llama-3.1-8B-Instruct","src":"https://balrogai.com/","v":0.151},{"b":"balrog","m":"gemini-1-5-flash-002","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-cb9697e44b63135e","rd":"2024-09-24","se":0.013999999999999999,"sid":"recQrinDnBT5cObDo","sm":"gemini-1.5-flash-002","src":"https://balrogai.com/","v":0.146},{"b":"balrog","m":"qwen2-vl-72b-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-99a626ea5cda3fb0","rd":"2024-08-29","se":0.016,"sid":"recArad880ZgrbLfj","sm":"Qwen2-VL-72B-Instruct","src":"https://balrogai.com/","v":0.128},{"b":"balrog","m":"phi-4","oid":"balrog-956dcdc897bba576","rd":"2024-12-12","se":0.013999999999999999,"sid":"recsjSXTW4U8imj8U","sm":"phi-4","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20250113_robust_naive_microsoft_phi-4","v":0.11599999999999999},{"b":"balrog","m":"llama-3-2-3b-instruct","oid":"balrog-75f1a1a4f5090899","rd":"2024-09-24","se":0.013000000000000001,"sid":"recwneInyMuZawoFC","sm":"Llama-3.2-3B-Instruct","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241030_naive-Llama-3.2-3B-Instruct","v":0.10099999999999999},{"b":"balrog","m":"qwen2-5-7b-instruct","oid":"balrog-09c6f933d98c6487","rd":"2024-09-19","se":0.011000000000000001,"sid":"recEMm4aiOXvIdY1V","sm":"qwen2.5-7b-instruct","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241115_naive-Qwen2.5-7B-it","v":0.078},{"b":"balrog","m":"llama-3-2-1b-instruct","oid":"balrog-ef1f41809e0633a6","rd":"2024-09-24","se":0.01,"sid":"recX2FEa3w9GQpsGc","sm":"Llama-3.2-1B-Instruct","src":"https://balrogai.com/","trace":"https://github.com/balrog-ai/experiments/tree/main/submissions/LLM/20241030_naive-Llama-3.2-1B-Instruct","v":0.066},{"b":"balrog","m":"qwen2-vl-7b-instruct","notes":"No trajectory available as of March 31, 2025.","oid":"balrog-c7902271f0cb641a","rd":"2024-08-29","se":0.008,"sid":"recBFyLHvaPPkNhv7","sm":"Qwen2-VL-7B-Instruct","src":"https://balrogai.com/","v":0.037000000000000005},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen 1.8B","m":"qwen-1-8b","oid":"bbh-3dc0d5232e484fb4","rd":"2023-11-30","sid":"recwEiyaFKdx0PezD","sm":"Qwen-1_8B","src":"https://arxiv.org/pdf/2309.16609","v":0.282},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen 7B","m":"qwen-7b","oid":"bbh-33fb967fa0ef2705","rd":"2023-09-28","sid":"rec6aaTYSwDqfD2sO","sm":"Qwen-7B","src":"https://arxiv.org/pdf/2309.16609","v":0.45},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen 14B","m":"qwen-14b","oid":"bbh-07912efbf69bbbf9","rd":"2023-09-28","sid":"recv5qI392enMRudT","sm":"Qwen-14B","src":"https://arxiv.org/pdf/2309.16609","v":0.534},{"b":"bbh","dn":"Gemma 2B","m":"gemma-2b","notes":"Couldn't find shot count","oid":"bbh-faf7304bb683d1b2","rd":"2024-02-21","sid":"rec6WR9YbYjnhvHbQ","sm":"gemma-2b","src":"https://arxiv.org/pdf/2403.08295","v":0.352},{"b":"bbh","dn":"Gemma 7B","m":"gemma-7b","notes":"Couldn't find shot count","oid":"bbh-748b9a6a0afe210b","rd":"2024-02-21","sid":"recQewDuedNzOm3m5","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.551},{"b":"bbh","dn":"Mistral 7B","m":"mistral-7b-v0-1","notes":"Couldn't find shot count","oid":"bbh-57ed73bfc9c64a3d","rd":"2023-09-27","sid":"rec8po7QMvux1PqYW","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2403.08295","v":0.561},{"b":"bbh","dn":"LLAMA-2 7B","m":"llama-2-7b","oid":"bbh-294e390d64eba3d4","rd":"2023-07-18","sid":"recorz7iAu0wehUWu","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.326},{"b":"bbh","dn":"LLAMA-2 13B","m":"llama-2-13b","oid":"bbh-3aa0bfb568b20316","rd":"2023-07-18","sid":"rec1PIPJMsc2vzPQA","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2403.08295","v":0.394},{"b":"bbh","cfg":{"shots":"3"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"bbh-d9b78163d0400603","rd":"2024-05-07","sid":"rec0HIGEri0gb87de","sm":"DeepSeek-V2","src":"https://arxiv.org/pdf/2412.19437","v":0.788},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"bbh-47eb8eef37cb47e2","rd":"2024-09-19","sid":"rec6UJmvJKZ0mSkf7","sm":"Qwen2.5-72B","src":"https://arxiv.org/pdf/2412.19437","v":0.798},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"bbh-955473fe06300b8f","rd":"2024-07-23","sid":"rechp4lRfnnkUlMX9","sm":"Llama-3.1-405B","src":"https://arxiv.org/pdf/2412.19437","v":0.829},{"b":"bbh","cfg":{"shots":"3"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"bbh-841ff967afdb7ff4","rd":"2024-12-26","sid":"recyTTQiA1OzvdQei","sm":"DeepSeek-V3","src":"https://arxiv.org/pdf/2412.19437","v":0.829},{"b":"bbh","cfg":{"shots":"3"},"dn":"Yi 6B","m":"yi-6b","oid":"bbh-1124cf11c530d520","rd":"2023-11-02","sid":"recrxjTojfWuyFdm1","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.428},{"b":"bbh","cfg":{"shots":"3"},"dn":"Yi 34B","m":"yi-34b","oid":"bbh-0e964e49f34421d1","rd":"2023-11-02","sid":"recCgyVX0fWUPFOkO","sm":"Yi-34B","src":"http://arxiv.org/abs/2403.04652","v":0.543},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"bbh-f2e7c77479cdc205","rd":"2023-07-18","sid":"rec7clZdHcyLSzBWa","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.394},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"bbh-5b5e0dab524313d6","rd":"2023-07-18","sid":"reckPGiU6K2bnS6LD","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.441},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan-2 13B","m":"baichuan-2-13b-base","oid":"bbh-54cac4403e15a4ee","rd":"2023-09-06","sid":"recBTqkLV3YTdBEKl","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2402.16819","v":0.488},{"b":"bbh","cfg":{"shots":"3"},"dn":"QWEN 14B","m":"qwen-14b","oid":"bbh-d1d32f585b19c5ef","rd":"2023-09-28","sid":"recKyeOiSeuHe7mjt","sm":"Qwen-14B","src":"http://arxiv.org/abs/2402.16819","v":0.534},{"b":"bbh","cfg":{"shots":"3"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"bbh-1d3ff15b1ccfd746","rd":"2023-09-27","sid":"recAbnFHkWY8M8u4C","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.395},{"b":"bbh","cfg":{"shots":"3"},"dn":"Gemma 7B","m":"gemma-7b","oid":"bbh-5a8b1982d63459ca","rd":"2024-02-21","sid":"recAL9kUMxLJLFs1G","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.551},{"b":"bbh","cfg":{"shots":"3"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"bbh-7b6c32cb0b6b1ec2","rd":"2024-02-26","sid":"recppgx1TV7PJqqoR","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.587},{"b":"bbh","cfg":{"shots":"3"},"dn":"GPT-4","m":"gpt-4-0613","oid":"bbh-ce1a8a45815e7eb9","rd":"2023-06-13","sid":"recw72u6BTbC1nVaC","sm":"gpt-4-0613","src":"http://arxiv.org/abs/2309.10305","v":0.7512},{"b":"bbh","cfg":{"shots":"3"},"dn":"GPT-3.5 Turbo","m":"gpt-3-5-turbo-0613","oid":"bbh-f83a6bce6868aa6e","rd":"2023-06-13","sid":"recUwVGAl716WVpBU","sm":"gpt-3.5-turbo-0613","src":"http://arxiv.org/abs/2309.10305","v":0.6159},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-7B","m":"llama-7b","oid":"bbh-fa6c97831c572374","rd":"2023-02-24","sid":"recbBaZ80LOgi2XOG","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.10305","v":0.3238},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2-7B","m":"llama-2-7b","oid":"bbh-da48f349bd13f8a3","rd":"2023-07-18","sid":"reclRzjJVPPiU3zg8","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.10305","v":0.3916},{"b":"bbh","cfg":{"shots":"3"},"dn":"MPT-7B","m":"mpt-7b","oid":"bbh-3399902cd41978fa","rd":"2023-05-05","sid":"rechXIS7X6UznLYiO","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.10305","v":0.352},{"b":"bbh","cfg":{"shots":"3"},"dn":"Falcon-7B","m":"falcon-7b","oid":"bbh-6587a65a9813d865","rd":"2023-04-24","sid":"recAQgXvN1LyZOMbb","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.10305","v":0.2877},{"b":"bbh","cfg":{"shots":"3"},"dn":"ChatGLM 2-6B (base)*","m":"chatglm2-6b","oid":"bbh-b5241de4782e9124","rd":"2023-06-24","sid":"recSk2YMS6sP9ybZP","sm":"chatglm2-6b","src":"http://arxiv.org/abs/2309.10305","v":0.3368},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan 1-7B","m":"baichuan-7b","oid":"bbh-ab92e6febe4ef3bc","rd":"2023-06-01","sid":"recMowz1c7WAeQP0r","sm":"Baichuan-7B","src":"http://arxiv.org/abs/2309.10305","v":0.3248},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan 2-7B-Base","m":"baichuan-2-7b-base","oid":"bbh-5cefc586e105f28c","rd":"2023-09-20","sid":"recAPbLpHdl6vjC16","sm":"Baichuan-2-7B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.4156},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-13B","m":"llama-13b","oid":"bbh-e53e71e8fd74a8ff","rd":"2023-02-24","sid":"recDxU7yqbO95fPBY","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2309.10305","v":0.3789},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2-13B","m":"llama-2-13b","oid":"bbh-2d0e2b78868fbd4b","rd":"2023-07-18","sid":"rec5TUJ5B5b5jcApF","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2309.10305","v":0.4698},{"b":"bbh","cfg":{"shots":"3"},"dn":"Vicuna-13B","m":"vicuna-13b-v1-1","oid":"bbh-822e931eaafe0d4e","rd":"2023-04-12","sid":"recX0LPusfvFRXmeh","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.10305","v":0.4304},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan 1-13B-Base","m":"baichuan-13b-base","oid":"bbh-5777f07f8ef99e1e","rd":"2023-07-11","sid":"recR9iM0H794LUNfw","sm":"Baichuan-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.4301},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan 2-13B-Base","m":"baichuan-2-13b-base","oid":"bbh-a6d0daa033103471","rd":"2023-09-06","sid":"recavR8MmJuWQQ4rY","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.4878},{"b":"bbh","cfg":{"shots":"3"},"dn":"Phi-3-mini","m":"phi-3-mini-4k-instruct","notes":"3-shot; CoT","oid":"bbh-36b091bbb9ca154a","rd":"2024-04-23","sid":"recUKkMoByKnWpuUZ","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.717},{"b":"bbh","cfg":{"shots":"3"},"dn":"Phi-3-small","m":"phi-3-small-8k-instruct","notes":"3-shot; CoT","oid":"bbh-43070013c8714e3f","rd":"2024-04-23","sid":"recKjdHfv3VjEMtsb","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.791},{"b":"bbh","cfg":{"shots":"3"},"dn":"Phi-3-medium","m":"phi-3-medium-128k-instruct","notes":"3-shot; CoT","oid":"bbh-d4238cbbccf5c481","rd":"2024-04-23","sid":"rec9iYkPYoTR9pzyx","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.814},{"b":"bbh","cfg":{"shots":"3"},"dn":"Phi-2","m":"phi-2","notes":"3-shot; CoT","oid":"bbh-bcffc145b910ffec","rd":"2023-12-12","sid":"recQSNSyrXQd8UgOd","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.594},{"b":"bbh","cfg":{"shots":"3"},"dn":"MPT 7B","m":"mpt-7b","oid":"bbh-1728a3e6be02885e","rd":"2023-05-05","sid":"recnDuyY5Dzi2cIHx","sm":"mpt-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.356},{"b":"bbh","cfg":{"shots":"3"},"dn":"MPT 30B","m":"mpt-30b","oid":"bbh-b202687cc9765176","rd":"2023-06-22","sid":"recNa6pWGs4Idzlug","sm":"mpt-30b","src":"https://arxiv.org/pdf/2309.16609","v":0.38},{"b":"bbh","cfg":{"shots":"3"},"dn":"Falcon 7B","m":"falcon-7b","oid":"bbh-d5c6ba0cc5a0bf38","rd":"2023-04-24","sid":"recY8LrwUj1lJj3Zi","sm":"falcon-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.28},{"b":"bbh","cfg":{"shots":"3"},"dn":"Falcon 40B","m":"falcon-40b","oid":"bbh-b1d9df332f151eb1","rd":"2023-03-15","sid":"recCuhggo9409cVhu","sm":"falcon-40b","src":"https://arxiv.org/pdf/2309.16609","v":0.371},{"b":"bbh","cfg":{"shots":"3"},"dn":"ChatGLM2","m":"chatglm2-6b","oid":"bbh-3177542b97ee7a81","rd":"2023-06-24","sid":"recSt1c1mDxWjT2kw","sm":"chatglm2-6b","src":"https://arxiv.org/pdf/2309.16609","v":0.337},{"b":"bbh","cfg":{"shots":"3"},"dn":"InternLM 7B","m":"internlm-7b","oid":"bbh-13fb9b688a0e4c7e","rd":"2023-07-05","sid":"recGLYH2QJ3KDou88","sm":"internlm-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.37},{"b":"bbh","cfg":{"shots":"3"},"dn":"InternLM 20B","m":"internlm-20b","oid":"bbh-72588fbac38c49de","rd":"2023-09-18","sid":"recnmQ4nbxuvotbeb","sm":"internlm-20b","src":"https://arxiv.org/pdf/2309.16609","v":0.525},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan2 7B","m":"baichuan-2-7b-base","oid":"bbh-2faf7db8378d3b9a","rd":"2023-09-20","sid":"recbMXxc5kgkoMubB","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/pdf/2309.16609","v":0.416},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"bbh-750cac20d7808ef0","rd":"2023-09-06","sid":"recmbmOOCOgqxb1g3","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/pdf/2309.16609","v":0.49},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 7B","m":"llama-7b","oid":"bbh-d10b6c1e1b7cc7fc","rd":"2023-02-24","sid":"recigmBKoJgg9WY2a","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2309.16609","v":0.335},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 13B","m":"llama-13b","oid":"bbh-636b324a10e824de","rd":"2023-02-24","sid":"recn0VFcI3UQDghGP","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2309.16609","v":0.379},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 33B","m":"llama-33b","oid":"bbh-4418c90871f5b3c0","rd":"2023-02-24","sid":"rec43eD7SvXQ5Dzq4","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2309.16609","v":0.5},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 65B","m":"llama-65b","oid":"bbh-71e5d37b06f48b5b","rd":"2023-02-24","sid":"recSRK6t1MK8dM2dx","sm":"LLaMA-65B","src":"https://arxiv.org/pdf/2309.16609","v":0.584},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 34B","m":"llama-2-34b","oid":"bbh-82fbd09fc9f088c5","rd":"2023-07-18","sid":"rec1c1obGEgm8UJON","sm":"Llama-2-34b","src":"https://arxiv.org/pdf/2309.16609","v":0.441},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"bbh-bda580b7e8e3552a","rd":"2023-07-18","sid":"recSGabK4Sqv5tGq4","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2309.16609","v":0.649},{"b":"bbh","cfg":{"shots":"3"},"dn":"StableBeluga2","m":"stablebeluga2","oid":"bbh-d826006ce9f37546","rd":"2023-07-20","sid":"recQpPjUchLLKHnr0","sm":"StableBeluga2","src":"https://arxiv.org/pdf/2309.16609","v":0.693},{"b":"bbh","cfg":{"shots":"3"},"dn":"QWEN 1.8B","m":"qwen-1-8b","oid":"bbh-4cf30fa0c69a149a","rd":"2023-11-30","sid":"recVIooB7cvf68T0j","sm":"Qwen-1_8B","src":"https://arxiv.org/pdf/2309.16609","v":0.282},{"b":"bbh","cfg":{"shots":"3"},"dn":"QWEN 7B","m":"qwen-7b","oid":"bbh-ff26e9ff1620e00a","rd":"2023-09-28","sid":"reckyn9rrrmjmrV6z","sm":"Qwen-7B","src":"https://arxiv.org/pdf/2309.16609","v":0.45},{"b":"bbh","cfg":{"shots":"3"},"dn":"QWEN 14B","m":"qwen-14b","oid":"bbh-fc0e158c8941b7a0","rd":"2023-09-28","sid":"rec8lvLyj6LfxLcTT","sm":"Qwen-14B","src":"https://arxiv.org/pdf/2309.16609","v":0.534},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 7B","m":"llama-7b","oid":"bbh-2bf20eb906ab688e","rd":"2023-02-24","sid":"rec9x2hh5bO75waHt","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.303},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 13B","m":"llama-13b","oid":"bbh-2e2d5bc3ccef8fcb","rd":"2023-02-24","sid":"recVjjfDkF3dxSUAL","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.37},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 33B","m":"llama-33b","oid":"bbh-a1c9917e6185bb39","rd":"2023-02-24","sid":"recsGL3NirdzuPYxz","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.398},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 65B","m":"llama-65b","oid":"bbh-7443b698256049c2","rd":"2023-02-24","sid":"recEPEBlWR8m0MZrr","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.435},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"bbh-0e1af5f06c2d7c43","rd":"2023-07-18","sid":"recjadyR0KerqJKuo","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.326},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"bbh-e46996e5d6a845f9","rd":"2023-07-18","sid":"recLcZqVyAA4OEgfQ","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.394},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 34B","m":"llama-2-34b","oid":"bbh-ba8efac6f72f7c62","rd":"2023-07-18","sid":"recatk8RloeAtyCmo","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.441},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"bbh-4287ff31caf2946b","rd":"2023-07-18","sid":"recubgr9UF2uUXY3E","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.512},{"b":"bbh","cfg":{"shots":"3"},"dn":"MPT 7B","m":"mpt-7b","oid":"bbh-1f70a27974a83f90","rd":"2023-05-05","sid":"recVOjXMkiQCv2yPt","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.31},{"b":"bbh","cfg":{"shots":"3"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"bbh-65af92ceccd8de65","rd":"2024-05-07","sid":"recEi33ywW8PezgV3","sm":"DeepSeek-V2","src":"http://arxiv.org/abs/2412.19437","v":0.788},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"bbh-4158ce3a17f48d99","rd":"2024-09-19","sid":"recHEqDIkyl72Zpiw","sm":"Qwen2.5-72B","src":"http://arxiv.org/abs/2412.19437","v":0.798},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"bbh-f70d5e589d103893","rd":"2024-07-23","sid":"recBt36OsYvimRxTu","sm":"Llama-3.1-405B","src":"http://arxiv.org/abs/2412.19437","v":0.829},{"b":"bbh","cfg":{"shots":"3"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"bbh-1e7d47d0f933b6ae","rd":"2024-12-26","sid":"rechnyAjkN3CHnI5g","sm":"DeepSeek-V3","src":"http://arxiv.org/abs/2412.19437","v":0.875},{"b":"bbh","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"bbh-7ceec3eecbabd754","rd":"2023-07-18","sid":"recYo1J8Qmz0VOJEw","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.329},{"b":"bbh","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"bbh-49ac58a62b4008b9","rd":"2023-07-18","sid":"rece5fh6ERllQI4iK","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.424},{"b":"bbh","cfg":{"shots":"0"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"bbh-39cf95373730d42f","rd":"2023-09-06","sid":"recbtRKdpDgysDGSC","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.388},{"b":"bbh","cfg":{"shots":"0"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"bbh-b400e469517d0f99","rd":"2023-09-24","sid":"recMoa8pBH75GkNPC","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.497},{"b":"bbh","cfg":{"shots":"0"},"dn":"InternLM-Chat 20B","m":"internlm-chat-20b","oid":"bbh-62f214eaa19c3d2e","rd":"2023-09-17","sid":"recMU394COBhaIEAe","sm":"internlm-chat-20b","src":"http://arxiv.org/abs/2403.04652","v":0.424},{"b":"bbh","cfg":{"shots":"0"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"bbh-7484e132f44be6c3","rd":"2023-11-22","sid":"recwwBrVHafqCEK3w","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.397},{"b":"bbh","cfg":{"shots":"0"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"bbh-3144c5e768f926a8","rd":"2023-11-22","sid":"recL10qlbGM5RuDG7","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.514},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"bbh-957b0f9add96df7e","rd":"2023-07-18","sid":"recFvpL4cwIM1jkzD","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.582},{"b":"bbh","cfg":{"shots":"3"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"bbh-8f98c8fef1639f91","rd":"2023-07-18","sid":"recDy6LbjY6SvJbKC","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.585},{"b":"bbh","cfg":{"shots":"3"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"bbh-b1021f999570855a","rd":"2023-09-06","sid":"recTQ6AnVQI3HzzDR","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.472},{"b":"bbh","cfg":{"shots":"3"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"bbh-c9ef2f2f8c86740e","rd":"2023-09-24","sid":"recwg8hk8QR2EAEmW","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.55},{"b":"bbh","cfg":{"shots":"3"},"dn":"InternLM-Chat 20B","m":"internlm-chat-20b","oid":"bbh-30c5c3e7faa233fe","rd":"2023-09-17","sid":"recdeDl0iYyeqCdZf","sm":"internlm-chat-20b","src":"http://arxiv.org/abs/2403.04652","v":0.367},{"b":"bbh","cfg":{"shots":"3"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"bbh-874f042fbe045553","rd":"2023-11-22","sid":"rec7U0UkrJXKMFZGe","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.472},{"b":"bbh","cfg":{"shots":"3"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"bbh-60ef6731845b5ec7","rd":"2023-11-22","sid":"recRvnph65VrLPHGb","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.717},{"b":"bbh","dn":"INTELLECT-1-Instruct","m":"intellect-1-instruct","oid":"bbh-14a6a3fe70cd4a2f","rd":"2024-11-29","sid":"rec4Pqqt0Y9MQ1HfR","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.3485},{"b":"bbh","cfg":{"shots":"3"},"dn":"Gemini 1.5 Pro (Feb 2024)","m":"gemini-1-5-pro-001-feb24","oid":"bbh-f866770436a6d1df","rd":"2024-02-15","sid":"recIJaCyW2MtzGFlO","sm":"gemini-1.5-pro-001-feb24","src":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","v":0.84},{"b":"bbh","cfg":{"shots":"3"},"dn":"Gemini 1.5 Pro (May 2024)","m":"gemini-1-5-pro-001","oid":"bbh-8b6b3365c0b3cf49","rd":"2024-05-14","sid":"recftOkRLxL5JX8iB","sm":"gemini-1.5-pro-001","src":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","v":0.892},{"b":"blueprint_bench_2","dn":"Claude Fable 5","m":"claude-fable-5","oid":"blueprint_bench_2-f43f760cd9578a83","rd":"2026-06-09","se":0.008,"sm":"claude-fable-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.3861171366594359},{"b":"blueprint_bench_2","dn":"GPT-5.5","m":"gpt-5-5","oid":"blueprint_bench_2-e8a4a8697c0767ab","rd":"2026-04-23","se":0.008,"sm":"gpt-5.5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.36225596529284154},{"b":"blueprint_bench_2","dn":"Gemini 3.5 Flash","m":"gemini-3-5-flash","oid":"blueprint_bench_2-9a5abfca0a391102","rd":"2026-05-19","se":0.006,"sm":"gemini-3.5-flash_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.336225596529284},{"b":"blueprint_bench_2","dn":"GPT-5.6 Sol","m":"gpt-5-6-sol","oid":"blueprint_bench_2-7ba53c94145d26df","rd":"2026-07-09","se":0.006,"sm":"gpt-5.6-sol_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.336225596529284},{"b":"blueprint_bench_2","dn":"GPT-5.6 Terra","m":"gpt-5-6-terra","oid":"blueprint_bench_2-91ebf79153103a48","rd":"2026-07-09","se":0.006,"sm":"gpt-5.6-terra_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.30802603036876364},{"b":"blueprint_bench_2","dn":"Grok 4.5","m":"grok-4-5","oid":"blueprint_bench_2-710829ddb0f2e01d","rd":"2026-07-08","se":0.007000000000000001,"sm":"grok-4.5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.2733188720173536},{"b":"blueprint_bench_2","dn":"GPT-5.4","m":"gpt-5-4-2026-03-05","oid":"blueprint_bench_2-3e51a2999f2f55b3","rd":"2026-03-05","se":0.018,"sm":"gpt-5.4-2026-03-05_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.27114967462039047},{"b":"blueprint_bench_2","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"blueprint_bench_2-1e0ec05de361120b","rd":"2026-02-19","se":0.011,"sm":"gemini-3.1-pro-preview","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.2646420824295011},{"b":"blueprint_bench_2","dn":"Claude Opus 4.7","m":"claude-opus-4-7","oid":"blueprint_bench_2-073a9d3e214b0f6c","rd":"2026-04-16","se":0.009,"sm":"claude-opus-4-7_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.245119305856833},{"b":"blueprint_bench_2","dn":"GPT-5.6 Luna","m":"gpt-5-6-luna","oid":"blueprint_bench_2-b2243aebb1f0a8bb","rd":"2026-07-09","se":0.004,"sm":"gpt-5.6-luna_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.22559652928416485},{"b":"blueprint_bench_2","dn":"Claude Opus 4.8","m":"claude-opus-4-8","oid":"blueprint_bench_2-9ebf3796bfb518a2","rd":"2026-05-28","se":0.01,"sm":"claude-opus-4-8_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.14533622559652917},{"b":"blueprint_bench_2","dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","oid":"blueprint_bench_2-83c2c00053d9cf56","rd":"2026-02-17","se":0.011,"sm":"claude-sonnet-4-6_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.06724511930585665},{"b":"blueprint_bench_2","dn":"Kimi K2.6","m":"kimi-k2-6","oid":"blueprint_bench_2-c0abce3cbd70f17a","rd":"2026-04-20","se":0.015,"sm":"kimi-k2.6","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.03904555314533626},{"b":"blueprint_bench_2","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"blueprint_bench_2-e42d082fc04db136","rd":"2025-10-15","se":0.017,"sm":"claude-haiku-4-5-20251001_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0},{"b":"blueprint_bench_2","dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"blueprint_bench_2-739d71dbe9b6fbf7","rd":"2025-12-17","se":0.019,"sm":"gemini-3-flash-preview","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0},{"b":"blueprint_bench_2","dn":"Grok 4.20 Reasoning","m":"grok-4-20","oid":"blueprint_bench_2-c6ba47767965e743","rd":"2026-02-17","se":0.011,"sm":"grok-4-20","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0},{"b":"blueprint_bench_2","dn":"Grok 4.3","m":"grok-4-3","oid":"blueprint_bench_2-e8c46240d35c69dc","rd":"2026-04-17","se":0.024,"sm":"grok-4.3_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0},{"b":"blueprint_bench_2","dn":"Gemini 3.6 Flash","m":"gemini-3-6-flash","oid":"blueprint_bench_2-c0ddef9485df4b72","rd":"2026-07-21","se":0.01,"sm":"gemini-3.6-flash_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.31236442516268986},{"b":"blueprint_bench_2","dn":"Claude Opus 5","m":"claude-opus-5","oid":"blueprint_bench_2-fab9697e33ca5320","rd":"2026-07-24","se":0.005,"sm":"claude-opus-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.30368763557483736},{"b":"blueprint_bench_2","dn":"Kimi K3","m":"kimi-k3","oid":"blueprint_bench_2-7eeeb2073bedf7cf","rd":"2026-07-16","se":0.005,"sm":"kimi-k3_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.29501084598698485},{"b":"blueprint_bench_2","dn":"Gemini Robotics-ER 1.6","m":"gemini-robotics-er-1-6-preview","oid":"blueprint_bench_2-6ab63c5730d3bd45","rd":"2026-04-14","se":0.021,"sm":"gemini-robotics-er-1.6-preview","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0},{"b":"blueprint_bench_2","dn":"Grok 4.6","m":"grok-4-6","oid":"blueprint_bench_2-f7dcf13a08cc4d41","rd":"2026-08-12","se":0.009,"sm":"grok-4.6_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.33188720173535774},{"b":"blueprint_bench_2","dn":"Claude Sonnet 5","m":"claude-sonnet-5","oid":"blueprint_bench_2-77536abdfe9d56d5","rd":"2026-06-30","se":0.007000000000000001,"sm":"claude-sonnet-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.24945770065075923},{"b":"blueprint_bench_2","dn":"Claude Fable 5.1","m":"claude-fable-5-1","oid":"blueprint_bench_2-aff33c89a274ae32","rd":"2026-09-01","se":0.008,"sm":"claude-fable-5-1_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.4186550976138828},{"b":"blueprint_bench_2","dn":"GPT-6 Astra","m":"gpt-6-astra","oid":"blueprint_bench_2-655106020513e829","rd":"2026-09-03","se":0.004,"sm":"gpt-6-astra_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.4967462039045553},{"b":"blueprint_bench_2","dn":"Gemini 3.8 Flash","m":"gemini-3-8-flash","oid":"blueprint_bench_2-202f3ca3ede6c092","rd":"2026-09-02","se":0.005,"sm":"gemini-3.8-flash_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.3861171366594359},{"b":"blueprint_bench_2","dn":"GPT-6 Sol","m":"gpt-6-sol","oid":"blueprint_bench_2-996323d9c904c72a","rd":"2026-09-22","se":0.007000000000000001,"sm":"gpt-6-sol_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.36876355748373096},{"b":"blueprint_bench_2","dn":"GPT-6 Luna","m":"gpt-6-luna","oid":"blueprint_bench_2-e1c684e2e57623d5","rd":"2026-09-22","se":0.009,"sm":"gpt-6-luna_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.31236442516268986},{"b":"blueprint_bench_2","dn":"Claude Opus 5.5","m":"claude-opus-5-5","oid":"blueprint_bench_2-980881590663466b","rd":"2026-09-22","se":0.01,"sm":"claude-opus-5-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.5119305856832972},{"b":"blueprint_bench_2","dn":"Gemini 4 Argon","m":"gemini-4-argon","oid":"blueprint_bench_2-4f5245b5f98a284d","rd":"2026-09-30","se":0.006,"sm":"gemini-4-argon_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.5444685466377441},{"b":"blueprint_bench_2","dn":"Grok 4.7","m":"grok-4-7","oid":"blueprint_bench_2-ec3a579ee85939f9","rd":"2026-09-21","se":0.009,"sm":"grok-4.7_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.3253796095444684},{"b":"blueprint_bench_2","dn":"Claude Sonnet 5.5","m":"claude-sonnet-5-5","oid":"blueprint_bench_2-5f85022a4692d910","rd":"2026-09-28","se":0.006,"sm":"claude-sonnet-5-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.3926247288503253},{"b":"blueprint_bench_2","dn":"Claude Haiku 5.5","m":"claude-haiku-5-5","oid":"blueprint_bench_2-910562e34334b706","rd":"2026-10-07","se":0.006,"sm":"claude-haiku-5-5_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.32971800433839465},{"b":"blueprint_bench_2","dn":"Gemini 4 Carbon","m":"gemini-4-carbon","oid":"blueprint_bench_2-5479f40050e72409","rd":"2026-10-28","se":0.007000000000000001,"sm":"gemini-4-carbon_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.5856832971800434},{"b":"blueprint_bench_2","dn":"GPT-6.1 Terra","m":"gpt-6-1-terra","oid":"blueprint_bench_2-8f62cb6c25079b25","rd":"2026-10-26","se":0.004,"sm":"gpt-6.1-terra_unknown","src":"https://andonlabs.com/evals/blueprint-bench-2","v":0.4208242950108459},{"b":"bool_q","m":"mpt-7b","oid":"bool_q-c2d4156d20c4c90c","rd":"2023-05-05","sid":"recHVOcpl5EdpYNHo","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.75},{"b":"bool_q","m":"falcon-7b","oid":"bool_q-06dafc713d721f6a","rd":"2023-04-24","sid":"recZ0BDsmFyBaGQ3q","sm":"falcon-7b","src":"https://arxiv.org/abs/2309.16609","v":0.675},{"b":"bool_q","dn":"ChatGLM2","m":"chatglm2-6b","oid":"bool_q-49d13b22f00e8749","rd":"2023-06-24","sid":"rect9TCmUKTJmVpeh","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.79},{"b":"bool_q","m":"internlm-7b","oid":"bool_q-e363c9093a68e4cd","rd":"2023-07-05","sid":"recHXC9HRaDYhPd1W","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.641},{"b":"bool_q","m":"internlm-20b","oid":"bool_q-914f864fd2c1deab","rd":"2023-09-18","sid":"reczVJsR2mliD98Nl","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.875},{"b":"bool_q","m":"baichuan-2-7b-base","oid":"bool_q-8104d2d86bef4b10","rd":"2023-09-20","sid":"recXyZbdJhwQiIzZO","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.632},{"b":"bool_q","m":"baichuan-2-13b-base","oid":"bool_q-ede2736932f5a4d4","rd":"2023-09-06","sid":"recyIRzjbpg4lwyF3","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.67},{"b":"bool_q","m":"llama-7b","oid":"bool_q-4fa249bae1c72a74","rd":"2023-02-24","sid":"recpwKBXvQCelNvqc","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.765},{"b":"bool_q","m":"llama-13b","oid":"bool_q-b4e782fe0b360ede","rd":"2023-02-24","sid":"recE0zMVHow2T8FrJ","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.787},{"b":"bool_q","m":"llama-33b","oid":"bool_q-029830195ffd7ad5","rd":"2023-02-24","sid":"rec82RLAgLSGjWYqo","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.844},{"b":"bool_q","m":"llama-65b","oid":"bool_q-c65eca73ceac0195","rd":"2023-02-24","sid":"recmtoGmjxfWqgb9P","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.866},{"b":"bool_q","m":"llama-2-7b","oid":"bool_q-638404ceaf011527","rd":"2023-07-18","sid":"recqrWI0B1Ddw0iFT","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.774},{"b":"bool_q","m":"llama-2-13b","oid":"bool_q-95d61f35d636d655","rd":"2023-07-18","sid":"recRE7Bx5SvbT0CeF","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.824},{"b":"bool_q","m":"llama-2-70b-hf","oid":"bool_q-12a033dbe332a3aa","rd":"2023-07-18","sid":"rec0UwLPLa3ZNqSaH","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.877},{"b":"bool_q","m":"stablebeluga2","oid":"bool_q-30a63cb402663ec6","rd":"2023-07-20","sid":"recaU8WSEEzPkxd7t","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.894},{"b":"bool_q","m":"qwen-1-8b","oid":"bool_q-7374f465ff63db39","rd":"2023-11-30","sid":"recdNVDZXTh5DKble","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.68},{"b":"bool_q","m":"qwen-7b","oid":"bool_q-43b3a7835d5f8b36","rd":"2023-09-28","sid":"recoOIXstoGV17NnV","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.764},{"b":"bool_q","m":"qwen-14b","oid":"bool_q-6c081c56cbe4899c","rd":"2023-09-28","sid":"recMLnbsklYmrgR72","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.862},{"b":"bool_q","cfg":{"shots":"0"},"m":"llama-2-7b","oid":"bool_q-5a65deee72f89c82","rd":"2023-07-18","sid":"recjT2p6s5emF8ljV","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2403.08295","v":0.774},{"b":"bool_q","cfg":{"shots":"0"},"m":"llama-2-13b","oid":"bool_q-86bffb128fe4a9bf","rd":"2023-07-18","sid":"recwePxenE1s2MEDc","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2403.08295","v":0.817},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Mistral 7b","m":"mistral-7b-instruct-v0-2","oid":"bool_q-611061c8f7cc00f6","rd":"2023-12-11","sid":"rechrxxd2cyc5qvCK","sm":"Mistral-7B-Instruct-v0.2","src":"http://arxiv.org/abs/2403.08295","v":0.832},{"b":"bool_q","cfg":{"shots":"0"},"m":"gemma-2b","oid":"bool_q-69cc7ef3fd400cba","rd":"2024-02-21","sid":"recL1IR83icoHlfOo","sm":"gemma-2b","src":"http://arxiv.org/abs/2403.08295","v":0.694},{"b":"bool_q","cfg":{"shots":"0"},"m":"gemma-7b","oid":"bool_q-1890fae3598b0cd8","rd":"2024-02-21","sid":"reczCGCIi9ogRGKSQ","sm":"gemma-7b","src":"http://arxiv.org/abs/2403.08295","v":0.832},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Llama 2 (70B)","m":"llama-2-70b-hf","oid":"bool_q-4883aa1f5fb1be12","rd":"2023-07-18","sid":"recBQdP8b3eIHdgG6","sm":"Llama-2-70b-hf","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.886},{"b":"bool_q","cfg":{"shots":"5"},"dn":"text-davinci-003","m":"text-davinci-003","oid":"bool_q-3a90ca3f8c62770a","rd":"2022-11-28","sid":"recH6d8ExyapqWoaZ","sm":"text-davinci-003","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.881},{"b":"bool_q","cfg":{"shots":"5"},"dn":"text-davinci-002","m":"text-davinci-002","oid":"bool_q-4972caa37ad768f1","rd":"2022-03-15","sid":"rec21epodjC4RKZOx","sm":"text-davinci-002","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.877},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Mistral v0.1 (7B)","m":"mistral-7b-v0-1","oid":"bool_q-e9e5f61f74bf14f8","rd":"2023-09-27","sid":"recfRB8rclD0fcc5y","sm":"Mistral-7B-v0.1","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.874},{"b":"bool_q","cfg":{"shots":"5"},"dn":"LLaMA (65B)","m":"llama-65b","oid":"bool_q-3c775ebcdfccb205","rd":"2023-02-24","sid":"reczBOxXmMvGByV42","sm":"LLaMA-65B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.871},{"b":"bool_q","cfg":{"shots":"5"},"dn":"gpt-3.5-turbo-0613","m":"gpt-3-5-turbo-0613","oid":"bool_q-e1267bfcd1487719","rd":"2023-06-13","sid":"recyVcZ23aeVIuZ1t","sm":"gpt-3.5-turbo-0613","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.87},{"b":"bool_q","cfg":{"shots":"5"},"dn":"LLaMA (30B)","m":"llama-33b","oid":"bool_q-483c241eba3219cf","rd":"2023-02-24","sid":"recwJIq7xykbFffHt","sm":"LLaMA-33B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.861},{"b":"bool_q","cfg":{"shots":"5"},"dn":"MPT-Instruct (30B)","m":"mpt-30b-instruct","oid":"bool_q-bc7351cdc1f204e7","rd":"2023-06-22","sid":"recn1pl2GKarqRqur","sm":"mpt-30b-instruct","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.85},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Falcon (40B)","m":"falcon-40b","oid":"bool_q-b66f3d5c648c3ff5","rd":"2023-03-15","sid":"recuQ5Yr1laHCD3E1","sm":"falcon-40b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.819},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Llama 2 (13B)","m":"llama-2-13b","oid":"bool_q-847fa296f1035573","rd":"2023-07-18","sid":"recTu0x07WmvoVj5H","sm":"Llama-2-13b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.811},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Vicuna v1.3 (13B)","m":"vicuna-13b-v1-3","oid":"bool_q-34f4d316d948f230","rd":"2023-06-18","sid":"recPPbDL9r2HfV1gY","sm":"vicuna-13b-v1.3","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.808},{"b":"bool_q","cfg":{"shots":"5"},"dn":"OPT (175B)","m":"opt-175b","oid":"bool_q-7970babe07583807","rd":"2022-05-02","sid":"recdUJtVpunC6bget","sm":"opt-175b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.793},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Llama 2 (7B)","m":"llama-2-7b","oid":"bool_q-58ff111c2eff2539","rd":"2023-07-18","sid":"reczARKbf0v4uqrJ7","sm":"Llama-2-7b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.762},{"b":"bool_q","dn":"T5 (11B)","m":"t5-11b","oid":"bool_q-76a26aa5baf2511d","sid":"rec4qGZsK5ZSaigQY","sm":"T5-11B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.761},{"b":"bool_q","cfg":{"shots":"5"},"dn":"OPT (66B)","m":"opt-66b","oid":"bool_q-c44c76e082b95aea","rd":"2022-05-03","sid":"recoTFyMtUPbNMDlF","sm":"opt-66b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.76},{"b":"bool_q","cfg":{"shots":"5"},"dn":"LLaMA (7B)","m":"llama-7b","oid":"bool_q-7039ebf559cebd0e","rd":"2023-02-24","sid":"recb9Gg4qx25jTl8T","sm":"LLaMA-7B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.756},{"b":"bool_q","cfg":{"shots":"5"},"dn":"Falcon (7B)","m":"falcon-7b","oid":"bool_q-a9ae2902613e9b71","rd":"2023-04-24","sid":"recqpN4PPkUHOfuzM","sm":"falcon-7b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.753},{"b":"bool_q","cfg":{"shots":"5"},"dn":"davinci (175B)","m":"davinci","oid":"bool_q-47abf59bb934865c","sid":"recfjfOx271bgYCFm","sm":"davinci","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.722},{"b":"bool_q","cfg":{"shots":"5"},"dn":"LLaMA (13B)","m":"llama-13b","oid":"bool_q-6493d567a8e6c9d6","rd":"2023-02-24","sid":"rec8uZxM89GP1el28","sm":"LLaMA-13B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.714},{"b":"bool_q","cfg":{"shots":"5"},"dn":"BLOOM (176B)","m":"bloom","oid":"bool_q-1a58faf501d723cd","rd":"2022-07-06","sid":"recAvnz7EM41lTjc4","sm":"bloom","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.704},{"b":"bool_q","cfg":{"shots":"5"},"dn":"MPT (30B)","m":"mpt-30b","oid":"bool_q-16038133437af448","rd":"2023-06-22","sid":"rectEiEz1royh5rn1","sm":"mpt-30b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.704},{"b":"bool_q","cfg":{"shots":"5"},"dn":"curie (6.7B)","m":"curie","oid":"bool_q-4ce5c9847827ba1b","sid":"recopuZrvouRmaJuY","sm":"curie","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.656},{"b":"bool_q","cfg":{"shots":"5"},"dn":"text-curie-001","m":"text-curie-001","oid":"bool_q-3320fa4a3279e3b9","sid":"recZv2dfvg2B38Q8n","sm":"text-curie-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.62},{"b":"bool_q","cfg":{"shots":"5"},"dn":"ada (350M)","m":"ada","oid":"bool_q-cb5fcd4a50aaba14","sid":"recuxYuqda0BgQhJY","sm":"ada","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.581},{"b":"bool_q","cfg":{"shots":"5"},"dn":"babbage (1.3B)","m":"babbage","oid":"bool_q-5f8ce7f1cd1c6819","rd":"2020-06-22","sid":"rec3r9ODWHvb0JhHT","sm":"babbage","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.574},{"b":"bool_q","cfg":{"shots":"5"},"dn":"text-ada-001","m":"text-ada-001","oid":"bool_q-a845296b23e520e2","sid":"recMowoIcCL0xyNCc","sm":"text-ada-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.464},{"b":"bool_q","cfg":{"shots":"5"},"dn":"text-babbage-001","m":"text-babbage-001","oid":"bool_q-16498741daf7f63a","sid":"recwFyj4k7N8CWhsj","sm":"text-babbage-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/boolq","v":0.451},{"b":"bool_q","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"bool_q-3be7c9ae4c5d31fd","rd":"2022-01-27","sid":"rechcklkYf759OmcE","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.767},{"b":"bool_q","cfg":{"shots":"few"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"bool_q-08086d67eeb48265","rd":"2022-01-27","sid":"rec4h1FDNDFDmV32Z","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.775},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Phi-3.5-mini","m":"phi-3-5-mini-instruct","oid":"bool_q-a15bbb307cc5b242","rd":"2024-08-16","sid":"recZhel5rsnPzr7yb","sm":"Phi-3.5-mini-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.78},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Phi-3.5-MoE","m":"phi-3-5-moe-instruct","oid":"bool_q-d32269e002340a4d","rd":"2024-08-17","sid":"rec83R4zQ59VpXJGd","sm":"Phi-3.5-MoE-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.846},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"bool_q-ec966ed1e1fad1b6","rd":"2023-09-27","sid":"rece8qjazdWTKnvG1","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.805},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Mistral-Nemo 12B","m":"mistral-nemo-base-2407","oid":"bool_q-ec6bc5a4c7d151cc","rd":"2024-07-18","sid":"recJXBsbsGjM6r8IP","sm":"Mistral-Nemo-Base-2407","src":"http://arxiv.org/abs/2404.14219","v":0.825},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Llama-3.1-In 8B","m":"llama-3-1-8b-instruct","oid":"bool_q-80b4586de112537e","rd":"2024-07-23","sid":"recauId7IvSgbcDLC","sm":"Llama-3.1-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.828},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Gemma-2 9B","m":"gemma-2-9b","oid":"bool_q-44f42a278bcfbc9f","sid":"recht7f4a80DWbgMN","sm":"gemma-2-9b","src":"http://arxiv.org/abs/2404.14219","v":0.857},{"b":"bool_q","cfg":{"shots":"2"},"dn":"Gemini-1.5 Flash","m":"gemini-1-5-flash-001","oid":"bool_q-980a4ff28056f4e9","rd":"2024-05-23","sid":"recLLijFlIEJQiniu","sm":"gemini-1.5-flash-001","src":"http://arxiv.org/abs/2404.14219","v":0.858},{"b":"bool_q","cfg":{"shots":"2"},"dn":"GPT-4o-mini","m":"gpt-4o-mini-2024-07-18","oid":"bool_q-2a475d0407937f28","rd":"2024-07-18","sid":"recizVJrP3lam4Dj6","sm":"gpt-4o-mini-2024-07-18","src":"http://arxiv.org/abs/2404.14219","v":0.887},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-4ff0dacf4f0fbe01","rd":"2022-01-27","sid":"recOueG0773kjBUnT","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.605},{"b":"bool_q","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-4c3b737e6b380df7","rd":"2022-01-27","sid":"reczpDrT20I92v7cm","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.767},{"b":"bool_q","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-3c27fb88eed21285","rd":"2022-01-27","sid":"recRdRHAlrhly8hu3","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.775},{"b":"bool_q","cfg":{"shots":"0"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"bool_q-f86ebbacec2dc9b7","rd":"2021-10-11","sid":"recSrLGObTDXVTGLX","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.782},{"b":"bool_q","cfg":{"shots":"1"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"bool_q-4b1fc793cc52a40e","rd":"2021-10-11","sid":"recvGG1jljsdpWBxK","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8251},{"b":"bool_q","cfg":{"shots":"few"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"bool_q-1e5f0f62cde1b66e","rd":"2021-10-11","sid":"recIZKFQQyhw23W0n","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8483},{"b":"bool_q","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-02e8e54df8906442","rd":"2022-01-27","sid":"recm7HR2MIA1lJTvd","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.764},{"b":"bool_q","dn":"MPT 7B","m":"mpt-7b","oid":"bool_q-0970dcf799c8ac10","rd":"2023-05-05","sid":"recfTRiyofC2FAKWk","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.75},{"b":"bool_q","dn":"MPT 30B","m":"mpt-30b","oid":"bool_q-e3461572c5ff33f5","rd":"2023-06-22","sid":"recy4wDNfg4zU5Kfm","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.79},{"b":"bool_q","dn":"Falcon 7B","m":"falcon-7b","oid":"bool_q-f3c5d19ee5564316","rd":"2023-04-24","sid":"recbAlTSkblYogQHw","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.675},{"b":"bool_q","dn":"Falcon 40B","m":"falcon-40b","oid":"bool_q-b08415a31ad3345b","rd":"2023-03-15","sid":"recE8phjP9jiZWBfT","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.831},{"b":"bool_q","dn":"LLAMA 1 7B","m":"llama-7b","oid":"bool_q-39131c2582f4d9f7","rd":"2023-02-24","sid":"reczhmMSNPZ9EJiyg","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.765},{"b":"bool_q","dn":"LLAMA 1 13B","m":"llama-13b","oid":"bool_q-b4baf54b7b3da6c0","rd":"2023-02-24","sid":"recgnN499a9fPoAEL","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.781},{"b":"bool_q","dn":"LLAMA 1 33B","m":"llama-33b","oid":"bool_q-cd88d5e93b249b28","rd":"2023-02-24","sid":"recFao7zOzUGLtIjN","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.831},{"b":"bool_q","dn":"LLAMA 1 65B","m":"llama-65b","oid":"bool_q-e2fd07256e8c1e9d","rd":"2023-02-24","sid":"recvXX52834VbbFW3","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.853},{"b":"bool_q","dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"bool_q-699727992447b0dd","rd":"2023-07-18","sid":"recG4WRS5BghAvk6Y","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.774},{"b":"bool_q","dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"bool_q-f7d2bd4785c12456","rd":"2023-07-18","sid":"rech3JdcTT3kv3qu5","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.817},{"b":"bool_q","dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"bool_q-19be63d8ec536a2c","rd":"2023-07-18","sid":"recfEGMnrNkXNwPPi","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.837},{"b":"bool_q","dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"bool_q-c70d6ab52b6b60c9","rd":"2023-07-18","sid":"reciadFY9H9wkmY3z","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.85},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-3 175B","m":"text-davinci-001","oid":"bool_q-11fc2c18a6074690","rd":"2022-01-27","sid":"recn2QgU7kt60wQMf","sm":"text-davinci-001","src":"http://arxiv.org/abs/2302.13971","v":0.605},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Gopher 280B","m":"gopher-280b","oid":"bool_q-e5639d358df50a63","rd":"2021-12-08","sid":"recuZQPvxb8IaEpqF","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.793},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"bool_q-cda7c9ae25a3b1a1","rd":"2022-03-29","sid":"recMWwNiBWKAOg6kJ","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.837},{"b":"bool_q","cfg":{"shots":"0"},"dn":"PaLM 62B","m":"palm-62b","oid":"bool_q-d093b018f0197885","rd":"2022-04-04","sid":"reck42cJ26yoLKNtU","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.848},{"b":"bool_q","cfg":{"shots":"0"},"dn":"PaLM 540B","m":"palm-540b","oid":"bool_q-f6d273ddd4532e03","rd":"2022-04-04","sid":"rectUVBLNget8omGA","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.88},{"b":"bool_q","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"bool_q-b405145343eddd16","rd":"2023-02-24","sid":"recWJYo2XN4WQclvW","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.765},{"b":"bool_q","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"bool_q-8b830779e2499103","rd":"2023-02-24","sid":"recD09aNkSfpjpNGd","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.781},{"b":"bool_q","cfg":{"shots":"0"},"dn":"LLaMA 33B","m":"llama-33b","oid":"bool_q-98e4595da9958e49","rd":"2023-02-24","sid":"recTV8p9ldsizwhxf","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.831},{"b":"bool_q","cfg":{"shots":"0"},"dn":"LLaMA 65B","m":"llama-65b","oid":"bool_q-046725ad07eb325d","rd":"2023-02-24","sid":"recvFXQQ05stZUOqm","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.853},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Vicuna-13B (v1.1)","m":"vicuna-13b-v1-1","oid":"bool_q-3ecf9e1946186749","rd":"2023-04-12","sid":"recL6VYFLDmi24iX1","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.835},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Llama2-7B","m":"llama-2-7b","oid":"bool_q-f81d6a36eafbe032","rd":"2023-07-18","sid":"rec0J7W0GcCwspbI5","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.779},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Llama-7B","m":"llama-7b","oid":"bool_q-a2c042d1b694bd2b","rd":"2023-02-24","sid":"recwSmbQFWdIq6oNj","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.732},{"b":"bool_q","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"bool_q-893bb666230a482d","rd":"2023-05-05","sid":"recAepoQ1X3IV1zKD","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.739},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"bool_q-632b6e5fbc4d513e","rd":"2023-04-24","sid":"rec7MXZRUX9RKaxzq","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.685},{"b":"bool_q","cfg":{"shots":"0"},"dn":"OPT-1.3B","m":"opt-1-3b","oid":"bool_q-234d465ae61b9add","rd":"2022-05-11","sid":"recjiJINwIbUczh5B","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.596},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-Neo-2.7B","m":"gpt-neo-2-7b","oid":"bool_q-7523273955fdd093","rd":"2021-03-30","sid":"rec2lQCso9gS2nRx3","sm":"gpt-neo-2.7B","src":"http://arxiv.org/abs/2309.05463","v":0.618},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"bool_q-24a0a162edfd4588","rd":"2019-11-05","sid":"recLUbtWDOXZleRu3","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.618},{"b":"bool_q","cfg":{"shots":"0"},"dn":"phi-1.5 (1.3B)","m":"phi-1-5","oid":"bool_q-0245c93a4d07b0fd","rd":"2023-09-11","sid":"recL7N3v3HNGYrMYP","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.758},{"b":"bool_q","cfg":{"shots":"1"},"dn":"PalM","m":"palm-540b","oid":"bool_q-fa10fd9de7fbd4df","rd":"2022-04-04","sid":"recRhDl4Z6NbPmtxb","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.887},{"b":"bool_q","cfg":{"shots":"1"},"dn":"PaLM-2 S","m":"palm-2-s","oid":"bool_q-c340c9414e104d9a","rd":"2023-05-17","sid":"recxf15f7V0ann295","sm":"PaLM 2-S","src":"http://arxiv.org/abs/2311.16867","v":0.881},{"b":"bool_q","cfg":{"shots":"1"},"dn":"PaLM-2 M","m":"palm-2-m","oid":"bool_q-f43c7dfbf2c9b8b5","rd":"2023-05-17","sid":"rec8eL2GH4EnQBBS6","sm":"PaLM 2-M","src":"http://arxiv.org/abs/2311.16867","v":0.886},{"b":"bool_q","cfg":{"shots":"1"},"dn":"PaLM-2 L","m":"palm-2-l","oid":"bool_q-20eff085df58f255","rd":"2023-05-17","sid":"recEIFrHcE6dr8nui","sm":"PaLM 2-L","src":"http://arxiv.org/abs/2311.16867","v":0.909},{"b":"bool_q","cfg":{"shots":"1"},"dn":"Falcon 180B","m":"falcon-180b","oid":"bool_q-554eccae6ad994c4","rd":"2023-09-06","sid":"rec0s5hZESCO9CSIm","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.89},{"b":"bool_q","dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-accc25ec5be1c905","rd":"2022-01-27","sid":"recwKMMtCzOuwW4c4","sm":"text-davinci-001","src":"http://arxiv.org/abs/2311.16867","v":0.605},{"b":"bool_q","dn":"Gopher","m":"gopher-280b","oid":"bool_q-0673793845120c22","rd":"2021-12-08","sid":"rech2f4n7o3GTNuiE","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2311.16867","v":0.794},{"b":"bool_q","dn":"Chinchilla","m":"chinchilla-70b","oid":"bool_q-7b80eeca2c5f7a2a","rd":"2022-03-29","sid":"recd1SuMlPypqXymS","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2311.16867","v":0.837},{"b":"bool_q","dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"bool_q-008d4cb8ae7a8d3b","rd":"2021-10-11","sid":"recTl1Vbdzvo45D7K","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2311.16867","v":0.782},{"b":"bool_q","dn":"PaLM","m":"palm-540b","oid":"bool_q-e5e5bfdd03c5a6ce","rd":"2022-04-04","sid":"recmmwz1UjW3jHsyr","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.88},{"b":"bool_q","dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"bool_q-ad8e8dd66a03e8f6","rd":"2023-07-18","sid":"recqU7IALE6TynEe2","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2311.16867","v":0.774},{"b":"bool_q","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"bool_q-ac24d79ec404f5ec","rd":"2023-07-18","sid":"recPkoBqZap7jXzgB","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2311.16867","v":0.817},{"b":"bool_q","dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"bool_q-2580b251d11d3dd9","rd":"2023-07-18","sid":"rechAJ1zWpe1ZCwMR","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2311.16867","v":0.837},{"b":"bool_q","dn":"LLaMA-2 70B","m":"llama-2-70b-hf","oid":"bool_q-6370c83997894043","rd":"2023-07-18","sid":"recbiZ4NS4ck3z8RR","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2311.16867","v":0.85},{"b":"bool_q","dn":"Inflection-1","m":"inflection-1","oid":"bool_q-413dd986005edf9a","rd":"2023-06-22","sid":"recjqB6KY2VD9lRq4","sm":"Inflection-1","src":"http://arxiv.org/abs/2311.16867","v":0.897},{"b":"bool_q","dn":"Falcon 7B","m":"falcon-7b","oid":"bool_q-6c16cec1be5d3832","rd":"2023-04-24","sid":"recPpHF47MVPuGYHh","sm":"falcon-7b","src":"http://arxiv.org/abs/2311.16867","v":0.738},{"b":"bool_q","dn":"Falcon 40B","m":"falcon-40b","oid":"bool_q-1318168edaa9efe8","rd":"2023-03-15","sid":"recuoAdmvVf1gxS0Y","sm":"falcon-40b","src":"http://arxiv.org/abs/2311.16867","v":0.819},{"b":"bool_q","dn":"Falcon 180B","m":"falcon-180b","oid":"bool_q-a7ff043d874252c8","rd":"2023-09-06","sid":"recAGePlElM7Iyr35","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.878},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Chinchilla","m":"chinchilla-70b","oid":"bool_q-093b8cd0cde71894","rd":"2022-03-29","sid":"recEThKkmruDgpAbW","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2203.15556","v":0.837},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"bool_q-d0801593a0987007","rd":"2021-12-08","sid":"recqIqa15nYxZ9gnF","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2203.15556","v":0.793},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"bool_q-df4d44d4be0e5097","rd":"2022-01-27","sid":"rec4Pxnfze9hfW9Rv","sm":"text-davinci-001","src":"http://arxiv.org/abs/2203.15556","v":0.605},{"b":"bool_q","cfg":{"shots":"0"},"dn":"MT-NLG 530B","m":"megatron-turing-nlg-530b","oid":"bool_q-dee7e01214d276b5","rd":"2021-10-11","sid":"recdV90DkEOlAJrx3","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2203.15556","v":0.782},{"b":"bool_q","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"bool_q-24b114e8a5380d04","rd":"2023-06-27","sid":"recyTyx10NhAUx6Tx","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.743},{"b":"bool_q","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"bool_q-622d7207e26b1666","rd":"2023-02-24","sid":"recge42MZtxHObbhV","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.749},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"bool_q-4e437eb817647da3","rd":"2023-04-24","sid":"recl6xPFTAG6JhJz6","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.738},{"b":"bool_q","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"bool_q-58a4ad6b5f2b3f30","rd":"2023-05-05","sid":"recaiOmdDGH74rhYr","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.741},{"b":"bool_q","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"bool_q-63ce53713fe4dc12","rd":"2023-06-07","sid":"recWFedHzxqXyOy0H","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.706},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"bool_q-c7a38d1a30780356","rd":"2023-05-04","sid":"reckzjCj8MFWX4ciG","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.693},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"bool_q-42febdc745646e09","rd":"2022-04-07","sid":"recxDRq4b1ufMQvK9","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.649},{"b":"bool_q","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"bool_q-37a9ea08745b2596","rd":"2022-05-11","sid":"recy780ljZMfx4lsm","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.65},{"b":"bool_q","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"bool_q-6741795f28590a27","rd":"2021-08-05","sid":"recZKmrERvEZF19dQ","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.654},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"bool_q-cacb838926ece51c","rd":"2023-04-11","sid":"rec2kgz52vP9geaQT","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.563},{"b":"bool_q","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"bool_q-60ed7a5d6216e4e4","rd":"2023-03-20","sid":"rec16ATP420iWWI5Q","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.611},{"b":"bool_q","cfg":{"shots":"0"},"dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"bool_q-97dee6b1ac9620a5","rd":"2023-04-19","sid":"recDlV2VRTJZln7b2","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.59},{"b":"bool_q","dn":"T5-Small","m":"t5-small","oid":"bool_q-9f95505a8251a7e0","sid":"rec48lEgo5a0lIcCQ","sm":"T5-Small","src":"https://arxiv.org/pdf/1910.10683","v":0.764},{"b":"bool_q","dn":"T5-Base","m":"t5-base","oid":"bool_q-f82b7a414bd16c35","sid":"recOwAD9Jhe7hSl9k","sm":"T5-Base","src":"https://arxiv.org/pdf/1910.10683","v":0.814},{"b":"bool_q","dn":"T5-Large","m":"t5-large","oid":"bool_q-89074111b4337a39","sid":"recvh3VHfWBGU2p2P","sm":"T5-Large","src":"https://arxiv.org/pdf/1910.10683","v":0.854},{"b":"bool_q","dn":"T5-3B","m":"t5-3b","oid":"bool_q-d891f97cbcaf1f90","sid":"recwuCAISo7vgsHoh","sm":"T5-3B","src":"https://arxiv.org/pdf/1910.10683","v":0.899},{"b":"bool_q","dn":"T5-11B","m":"t5-11b","oid":"bool_q-701501fc2eb2a47d","sid":"recZoSfzBQ8qwHGbj","sm":"T5-11B","src":"https://arxiv.org/pdf/1910.10683","v":0.912},{"b":"btf3","cfg":{"effort":"xhigh","harness":"FutureSearch agent"},"dn":"Claude Opus 4.8 (xhigh)","e":"xhigh","hi":0.139,"lo":0.125,"m":"claude-opus-4-8","oid":"btf3-2bc0696ca3d46cee","rd":"2026-05-28","sm":"claude-opus-4-8_xhigh","src":"https://evals.futuresearch.ai/#btf3","v":0.132},{"b":"btf3","cfg":{"effort":"high","harness":"FutureSearch agent"},"dn":"Claude Fable 5 (high)","e":"high","hi":0.138,"lo":0.123,"m":"claude-fable-5","oid":"btf3-c4cbbb79bd16d66d","rd":"2026-06-09","sm":"claude-fable-5_high","src":"https://evals.futuresearch.ai/#btf3","v":0.13},{"b":"btf3","cfg":{"effort":"high","harness":"Vendor agent SDK"},"dn":"GPT-5.5 (high, agent SDK)","e":"high","hi":0.14,"lo":0.128,"m":"gpt-5-5","oid":"btf3-a06aeff0ddb655f5","rd":"2026-04-23","sm":"gpt-5.5_high","src":"https://evals.futuresearch.ai/#btf3","v":0.134},{"b":"btf3","cfg":{"effort":"high","harness":"FutureSearch agent"},"dn":"GPT-5.6 Sol (high)","e":"high","hi":0.143,"lo":0.128,"m":"gpt-5-6-sol","oid":"btf3-4676e8e1decc906b","rd":"2026-07-09","sm":"gpt-5.6-sol_high","src":"https://evals.futuresearch.ai/#btf3","v":0.135},{"b":"btf3","cfg":{"effort":"high","harness":"Vendor agent SDK"},"dn":"Claude Opus 4.8 (high, agent SDK)","e":"high","hi":0.145,"lo":0.129,"m":"claude-opus-4-8","oid":"btf3-d421666518310794","rd":"2026-05-28","sm":"claude-opus-4-8_high","src":"https://evals.futuresearch.ai/#btf3","v":0.137},{"b":"btf3","cfg":{"effort":"high","harness":"FutureSearch agent"},"dn":"Claude Opus 4.8 (high)","e":"high","hi":0.147,"lo":0.132,"m":"claude-opus-4-8","oid":"btf3-b3891d27c50e45dd","rd":"2026-05-28","sm":"claude-opus-4-8_high","src":"https://evals.futuresearch.ai/#btf3","v":0.14},{"b":"btf3","cfg":{"effort":"high","harness":"FutureSearch agent"},"dn":"GPT-5.5 (high)","e":"high","hi":0.149,"lo":0.136,"m":"gpt-5-5","oid":"btf3-4ad1e26335363147","rd":"2026-04-23","sm":"gpt-5.5_high","src":"https://evals.futuresearch.ai/#btf3","v":0.143},{"b":"btf3","cfg":{"effort":"xhigh","harness":"FutureSearch agent"},"dn":"Claude Sonnet 5 (xhigh)","e":"xhigh","hi":0.149,"lo":0.135,"m":"claude-sonnet-5","oid":"btf3-b262ada2fe1d4642","rd":"2026-06-30","sm":"claude-sonnet-5_xhigh","src":"https://evals.futuresearch.ai/#btf3","v":0.142},{"b":"btf3","cfg":{"effort":"xhigh","harness":"FutureSearch agent"},"dn":"Claude Opus 5 (xhigh) §","e":"xhigh","hi":0.128,"lo":0.113,"m":"claude-opus-5","oid":"btf3-4392879646aa2437","rd":"2026-07-24","sm":"claude-opus-5_xhigh","src":"https://evals.futuresearch.ai/#btf3","v":0.12},{"b":"btf3","cfg":{"effort":"max","harness":"FutureSearch agent"},"dn":"GPT-5.6 Sol (max)","e":"max","hi":0.145,"lo":0.13,"m":"gpt-5-6-sol","oid":"btf3-d94fcdd9201357e2","rd":"2026-07-09","sm":"gpt-5.6-sol_max","src":"https://evals.futuresearch.ai/#btf3","v":0.137},{"b":"btf3","cfg":{"effort":"low","harness":"FutureSearch agent"},"dn":"GPT-6 Astra (low)","e":"low","hi":0.141,"lo":0.129,"m":"gpt-6-astra","oid":"btf3-ea7f768b66f9388f","rd":"2026-09-03","sm":"gpt-6-astra_low","src":"https://evals.futuresearch.ai/#btf3","v":0.135},{"b":"btf3","cfg":{"effort":"medium","harness":"FutureSearch agent"},"dn":"GPT-5.6 Sol (medium)","e":"medium","hi":0.144,"lo":0.131,"m":"gpt-5-6-sol","oid":"btf3-dbdf1358c841d2a5","rd":"2026-07-09","sm":"gpt-5.6-sol_medium","src":"https://evals.futuresearch.ai/#btf3","v":0.137},{"b":"btf3","cfg":{"effort":"xhigh","harness":"FutureSearch agent"},"dn":"Muse Spark 1.3 (xhigh)","e":"xhigh","hi":0.148,"lo":0.134,"m":"muse-spark-1-3","oid":"btf3-b43a8420a249c196","rd":"2026-09-02","sm":"muse-spark-1.3_xhigh","src":"https://evals.futuresearch.ai/#btf3","v":0.141},{"b":"btf3","cfg":{"harness":"FutureSearch agent"},"dn":"GLM-5.3-Flash","hi":0.156,"lo":0.142,"m":"glm-5-3-flash","oid":"btf3-39cf28a325d19d47","rd":"2026-08-20","sm":"glm-5.3-flash_unknown","src":"https://evals.futuresearch.ai/#btf3","v":0.149},{"b":"btf3","cfg":{"harness":"FutureSearch agent"},"dn":"GLM-5.3","hi":0.155,"lo":0.143,"m":"glm-5-3","oid":"btf3-a738e746fe47d9b0","rd":"2026-08-14","sm":"glm-5.3_unknown","src":"https://evals.futuresearch.ai/#btf3","v":0.149},{"b":"cad_eval","dn":"claude-3-5-haiku-20241022","m":"claude-3-5-haiku-2024-10-22","oid":"cad_eval-23fe5a559fa8101d","rd":"2024-10-22","sid":"recd89YB5ghMG5qFq","sm":"claude-3-5-haiku-20241022","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.32},{"b":"cad_eval","dn":"claude-3-5-sonnet-20241022","m":"claude-3-5-sonnet-2024-10-22","oid":"cad_eval-3fc09216dc49618a","rd":"2024-10-22","sid":"recVkIinzo3EVhob6","sm":"claude-3-5-sonnet-20241022","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.48},{"b":"cad_eval","dn":"claude-3-7-sonnet-20250219","m":"claude-3-7-sonnet-2025-02-19","oid":"cad_eval-493d1f96e9f08d4d","rd":"2025-02-24","sid":"recfhRjXU8DqyvaLJ","sm":"claude-3-7-sonnet-20250219","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.54},{"b":"cad_eval","dn":"claude-3-haiku-20240307","m":"claude-3-haiku-2024-03-07","oid":"cad_eval-d8fed7394155ea08","rd":"2024-03-07","sid":"recXk8aKqbCUZ9FSa","sm":"claude-3-haiku-20240307","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.12},{"b":"cad_eval","dn":"gemini-1.5-pro","m":"gemini-1-5-pro-002","notes":"Assuming latest version of Gemini 1.5 Pro","oid":"cad_eval-85c2f72a0d1db261","rd":"2024-09-24","sid":"recHocZbBBZGwraMJ","sm":"gemini-1.5-pro-002","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.34},{"b":"cad_eval","dn":"gemini-2.0-flash","m":"gemini-2-0-flash-001","oid":"cad_eval-3177f0bb11511353","rd":"2025-02-05","sid":"rec4KyH2HNvrzfxse","sm":"gemini-2.0-flash-001","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.3},{"b":"cad_eval","dn":"gemini-2.5-pro-preview-03-25","m":"gemini-2-5-pro-preview-03-25","oid":"cad_eval-f60f9f5da02b3fbe","rd":"2025-03-31","sid":"recIwzxtVNgzTR2iZ","sm":"gemini-2.5-pro-preview-03-25","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.64},{"b":"cad_eval","dn":"gpt-4.1-2025-04-14","m":"gpt-4-1-2025-04-14","oid":"cad_eval-d3c9786b02fe1984","rd":"2025-04-14","sid":"recvDzUaVY5lEc1YR","sm":"gpt-4.1-2025-04-14","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.42},{"b":"cad_eval","dn":"gpt-4.1-mini-2025-04-14","m":"gpt-4-1-mini-2025-04-14","oid":"cad_eval-f6ee47235a8f75f4","rd":"2025-04-14","sid":"recZA4b2nHwHQAFhL","sm":"gpt-4.1-mini-2025-04-14","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.16},{"b":"cad_eval","dn":"gpt-4o-2024-08-06","m":"gpt-4o-2024-08-06","oid":"cad_eval-b4a38fbc86428604","rd":"2024-08-06","sid":"recMLAVAe2s9Zys4V","sm":"gpt-4o-2024-08-06","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.26},{"b":"cad_eval","cfg":{"effort":"medium"},"dn":"o1-2024-12-17","e":"medium","m":"o1-2024-12-17","oid":"cad_eval-7891e83185f8cbd1","rd":"2024-12-17","sid":"recalCdqLvbTfE1Bj","sm":"o1-2024-12-17_medium","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.56},{"b":"cad_eval","cfg":{"effort":"medium"},"dn":"o3-2025-04-16","e":"medium","m":"o3-2025-04-16","notes":"Assuming medium reasoning effort.","oid":"cad_eval-a4f54e39edf52c48","rd":"2025-04-16","sid":"receLc2LaThHq7BXT","sm":"o3-2025-04-16_medium","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.74},{"b":"cad_eval","cfg":{"effort":"medium"},"dn":"o3-mini-2025-01-31","e":"medium","m":"o3-mini-2025-01-31","notes":"Assuming medium reasoning effort.","oid":"cad_eval-0038b3a4ff4adb79","rd":"2025-01-31","sid":"recFt9Ufy79jEUKw6","sm":"o3-mini-2025-01-31_medium","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.54},{"b":"cad_eval","cfg":{"effort":"medium"},"dn":"o4-mini-2025-04-16","e":"medium","m":"o4-mini-2025-04-16","oid":"cad_eval-1b4a9286f0bc63e0","rd":"2025-04-16","sid":"recK4177uAx9FjJAz","sm":"o4-mini-2025-04-16_medium","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.62},{"b":"cad_eval","dn":"zoo-ml-text-to-cad","m":"ml-elephant","notes":"text-to-cad appears to be a tool that calls an underlying model which is otherwise exposed via API as the service 'ml-ephant'","oid":"cad_eval-d2e9211323ff5096","sid":"recHen3pRfSp2ljo7","sm":"ml-elephant","src":"https://willpatrick.xyz/cadevalresults_20250422_095709/","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-11-06","e":"max","evaluated_at":"2026-11-06T00:59:23.000Z","m":"kimi-k3-1-pro-together","oid":"chess_puzzles-393429a65ad21cad","rd":"2026-10-23","se":0.05,"sid":"8eSYUWwajdBjKiiM9CwnKu","sm":"kimi-k3.1-pro-together_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8eSYUWwajdBjKiiM9CwnKu.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T20:57:35.000Z","m":"doubao-seed-evolving","oid":"chess_puzzles-8ee68b1024e07fec","rd":"2026-11-02","se":0.0492365963917331,"sid":"DDxB37H4uXfuBeSeD3UJWJ","sm":"doubao-seed-evolving_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDDxB37H4uXfuBeSeD3UJWJ.eval","v":0.4},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T13:47:17.000Z","m":"mistral-large-4-0-oss","oid":"chess_puzzles-c14b81f48c4b3f93","rd":"2026-10-29","se":0.045604802157206865,"sid":"PuHzUbC9C2fJAq7iA5wQpv","sm":"mistral-large-4-0-oss_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPuHzUbC9C2fJAq7iA5wQpv.eval","v":0.29},{"b":"chess_puzzles","d":"2026-10-30","evaluated_at":"2026-10-30T18:28:17.000Z","m":"seed-1-6","oid":"chess_puzzles-c3cc453927af4e70","rd":"2025-06-11","se":0.021904291355759033,"sid":"B2LLDDBek4ULJdZBptXWSV","sm":"seed-1.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FB2LLDDBek4ULJdZBptXWSV.eval","v":0.05},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T12:30:09.000Z","m":"doubao-seed-2-1-turbo-260628","oid":"chess_puzzles-86cc66b6ff3a4990","rd":"2026-06-23","se":0.0440844002276808,"sid":"8EDZwAT98nwY8hB5DD75Br","sm":"doubao-seed-2.1-turbo-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8EDZwAT98nwY8hB5DD75Br.eval","v":0.26},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:56.000Z","m":"doubao-seed-2-0-lite-260428","oid":"chess_puzzles-baafb36b49661a55","rd":"2026-04-28","se":0.04020151261036849,"sid":"eqmrbJKgZ9iXXB9MvYThrJ","sm":"doubao-seed-2.0-lite-260428_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeqmrbJKgZ9iXXB9MvYThrJ.eval","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:39.000Z","m":"doubao-seed-2-0-pro-260215","oid":"chess_puzzles-67b048b9885bc4a7","rd":"2026-02-14","se":0.04020151261036849,"sid":"Bdoepct3UJMwAXZEGdYqxP","sm":"doubao-seed-2.0-pro-260215_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBdoepct3UJMwAXZEGdYqxP.eval","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:29.000Z","m":"doubao-seed-2-1-pro-260628","oid":"chess_puzzles-248e6eb3ea1e577c","rd":"2026-06-23","se":0.040936018074033236,"sid":"hF9zXdtPjSgoDRQuPyfncZ","sm":"doubao-seed-2.1-pro-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhF9zXdtPjSgoDRQuPyfncZ.eval","v":0.21},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-19","e":"max","evaluated_at":"2026-10-19T22:39:00.000Z","m":"deepseek-v4-1-flash-1017","oid":"chess_puzzles-98022b7ea1c5acd4","rd":"2026-10-17","se":0.05009082659620331,"sid":"4sRo7TxavQAsjaEALPZdDy","sm":"deepseek-v4.1-flash-1017_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4sRo7TxavQAsjaEALPZdDy.eval","v":0.46},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-15","e":"high","evaluated_at":"2026-10-15T22:52:59.000Z","m":"mistral-large-4-0","oid":"chess_puzzles-d0bf91b6ac98cd7e","rd":"2026-10-06","se":0.04725815626252608,"sid":"eNdDCt2u6A8mPisK56SApU","sm":"mistral-large-4-0_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeNdDCt2u6A8mPisK56SApU.eval","v":0.33},{"b":"chess_puzzles","cfg":{"effort":"promax"},"d":"2026-10-15","e":"promax","evaluated_at":"2026-10-15T16:14:15.000Z","m":"gpt-6-astra","oid":"chess_puzzles-7fd8a611e8b5413c","rd":"2026-09-03","se":0.04292346959909278,"sid":"76P4iR5BFFPS5jtR3VRWDE","sm":"gpt-6-astra_promax","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F76P4iR5BFFPS5jtR3VRWDE.eval","v":0.76},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-14","e":"xhigh","evaluated_at":"2026-10-14T18:22:26.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"chess_puzzles-14b66939b4f51dcb","rd":"2026-03-17","se":0.05,"sid":"MeFyTsrHFWVrQJ8QHRjnRw","sm":"gpt-5.4-nano-2026-03-17_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMeFyTsrHFWVrQJ8QHRjnRw.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-14","e":"xhigh","evaluated_at":"2026-10-14T16:16:30.000Z","m":"gpt-5-2-2025-12-11","oid":"chess_puzzles-a286bab045487c13","rd":"2025-12-11","se":0.05021167315686782,"sid":"YGX23vJ7GJBaYUVLvHnXn5","sm":"gpt-5.2-2025-12-11_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYGX23vJ7GJBaYUVLvHnXn5.eval","v":0.48},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-14","e":"high","evaluated_at":"2026-10-14T09:56:39.000Z","m":"gpt-5-pro-2025-10-06","oid":"chess_puzzles-fdfb0eed87fa75c1","rd":"2025-10-07","se":0.05021167315686782,"sid":"RwHkGk6Qhd4NVTtRLDwLmc","sm":"gpt-5-pro-2025-10-06_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRwHkGk6Qhd4NVTtRLDwLmc.eval","v":0.48},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-14","e":"xhigh","evaluated_at":"2026-10-14T09:56:34.000Z","m":"gpt-5-2-pro-2025-12-11","oid":"chess_puzzles-44edcd8d25425053","rd":"2025-12-11","se":0.0492365963917331,"sid":"LgRDo2HcNECAGiD8UnHYjx","sm":"gpt-5.2-pro-2025-12-11_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLgRDo2HcNECAGiD8UnHYjx.eval","v":0.6},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-14","e":"xhigh","evaluated_at":"2026-10-14T09:56:24.000Z","m":"gpt-5-4-pro-2026-03-05","oid":"chess_puzzles-a804cab51065f025","rd":"2026-03-05","se":0.049431107042371025,"sid":"5yc9mNtsSiFQ5PducfGZGT","sm":"gpt-5.4-pro-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5yc9mNtsSiFQ5PducfGZGT.eval","v":0.59},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-14","e":"xhigh","evaluated_at":"2026-10-14T09:56:16.000Z","m":"gpt-5-5-pro","oid":"chess_puzzles-756f495837ce278a","rd":"2026-04-23","se":0.04878317312145634,"sid":"5u4XrDnQSRnuXSK6oXKAth","sm":"gpt-5.5-pro_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5u4XrDnQSRnuXSK6oXKAth.eval","v":0.62},{"b":"chess_puzzles","cfg":{"effort":"promax"},"d":"2026-10-14","e":"promax","evaluated_at":"2026-10-14T09:56:09.000Z","m":"gpt-5-6-sol","oid":"chess_puzzles-75a98b16a9dd4d12","rd":"2026-07-09","se":0.04688261722621507,"sid":"WAavFdihYDHwjGsDNh9toF","sm":"gpt-5.6-sol_promax","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWAavFdihYDHwjGsDNh9toF.eval","v":0.68},{"b":"chess_puzzles","d":"2026-10-14","evaluated_at":"2026-10-14T09:36:09.000Z","m":"glm-4-7","oid":"chess_puzzles-ebf82915be09f61a","rd":"2025-12-22","se":0.040936018074033236,"sid":"9rDLegdh3zmBfFzTj3fkCc","sm":"glm-4.7","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9rDLegdh3zmBfFzTj3fkCc.eval","v":0.21},{"b":"chess_puzzles","d":"2026-10-14","evaluated_at":"2026-10-14T06:28:37.000Z","m":"glm-5-1","oid":"chess_puzzles-646ad853a02b00a3","rd":"2026-04-07","se":0.04020151261036849,"sid":"R7XgJMi2R9sv7tZdx5SZTv","sm":"glm-5.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR7XgJMi2R9sv7tZdx5SZTv.eval","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-14","e":"max","evaluated_at":"2026-10-14T05:10:27.000Z","m":"glm-5-2","oid":"chess_puzzles-96adcbb7ce6968e4","rd":"2026-06-16","se":0.041633319989322654,"sid":"Hui8iZgjwKPxz7DkRU8sJX","sm":"glm-5.2_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHui8iZgjwKPxz7DkRU8sJX.eval","v":0.22},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-14","e":"low","evaluated_at":"2026-10-14T04:18:44.000Z","m":"glm-5-2","oid":"chess_puzzles-4f64c5ae2a0727fb","rd":"2026-06-16","se":0.03861229196653691,"sid":"6gYpimjqJm3CL6mvKEN9qs","sm":"glm-5.2_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6gYpimjqJm3CL6mvKEN9qs.eval","v":0.18},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-14","e":"none","evaluated_at":"2026-10-14T03:42:13.000Z","m":"glm-5-2","oid":"chess_puzzles-de590f03489df836","rd":"2026-06-16","se":0.039427724440366255,"sid":"Xi45sLZGyn5sutf7nKRHTk","sm":"glm-5.2_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXi45sLZGyn5sutf7nKRHTk.eval","v":0.19},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T21:22:18.000Z","m":"glm-5-3","oid":"chess_puzzles-bb80435cf9cae471","rd":"2026-08-14","se":0.048523658709390974,"sid":"kG3hbd8qLEd6fLAJHyQTiN","sm":"glm-5.3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkG3hbd8qLEd6fLAJHyQTiN.eval","v":0.37},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T17:16:43.000Z","m":"kimi-k2-6","oid":"chess_puzzles-8eaa7a3adad2d44a","rd":"2026-04-20","se":0.04020151261036849,"sid":"ehqiuEDETZTqJ8zt6Mc7Ja","sm":"kimi-k2.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FehqiuEDETZTqJ8zt6Mc7Ja.eval","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:15:18.000Z","m":"gemini-4-argon","oid":"chess_puzzles-fb0d4016a20e3206","rd":"2026-09-30","se":0.04943110704237103,"sid":"GJtYeC7ZV4fYgN3wu4rRvq","sm":"gemini-4-argon_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGJtYeC7ZV4fYgN3wu4rRvq.eval","v":0.59},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T17:13:43.000Z","m":"kimi-k2-7-code","oid":"chess_puzzles-8449957ce6ce6577","rd":"2026-06-12","se":0.0348735088019777,"sid":"bSyTSKD8RiRNZCGNu6MnFp","sm":"kimi-k2.7-code","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbSyTSKD8RiRNZCGNu6MnFp.eval","v":0.14},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T17:13:35.000Z","m":"kimi-k3","oid":"chess_puzzles-b416c7f1070fd7eb","rd":"2026-07-16","se":0.048241815132442176,"sid":"nzQV9YvictnDF2EwxqTJQs","sm":"kimi-k3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnzQV9YvictnDF2EwxqTJQs.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:10:13.000Z","m":"kimi-k3","oid":"chess_puzzles-fc4839a50d7db3b2","rd":"2026-07-16","se":0.048523658709390974,"sid":"X2QNPHFXuWosA6AiuvVtDv","sm":"kimi-k3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX2QNPHFXuWosA6AiuvVtDv.eval","v":0.37},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T17:06:54.000Z","m":"kimi-k3","oid":"chess_puzzles-cab2a304c913168a","rd":"2026-07-16","se":0.03861229196653691,"sid":"VPSX7EnwWuBrCPnpYGMUEM","sm":"kimi-k3_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVPSX7EnwWuBrCPnpYGMUEM.eval","v":0.18},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T15:54:34.000Z","m":"glm-5-3-flash","oid":"chess_puzzles-fdaa5d8241065435","rd":"2026-08-20","se":0.040936018074033236,"sid":"fWNxmiSUC7BX8whqvzssY8","sm":"glm-5.3-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfWNxmiSUC7BX8whqvzssY8.eval","v":0.21},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T14:12:48.000Z","m":"gpt-4o-mini-2024-07-18","oid":"chess_puzzles-14178c502b9f2f2f","rd":"2024-07-18","se":0.01714466079977653,"sid":"QGrgPmo24qW4h9SAvELP3Y","sm":"gpt-4o-mini-2024-07-18","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQGrgPmo24qW4h9SAvELP3Y.eval","v":0.03},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:12:19.000Z","m":"o1-2024-12-17","oid":"chess_puzzles-dc37ccab1e305916","rd":"2024-12-17","se":0.03015113445777636,"sid":"UtRhB562A5f4SuFsg5MPrJ","sm":"o1-2024-12-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUtRhB562A5f4SuFsg5MPrJ.eval","v":0.1},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:12:16.000Z","m":"o1-2024-12-17","oid":"chess_puzzles-eb37951705ac75a4","rd":"2024-12-17","se":0.027265992434429075,"sid":"8hGkBXjvgrhMoviNRydRTy","sm":"o1-2024-12-17_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8hGkBXjvgrhMoviNRydRTy.eval","v":0.08},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T14:12:01.000Z","m":"gpt-4-1-2025-04-14","oid":"chess_puzzles-caedcf345605074a","rd":"2025-04-14","se":0.031446603773522014,"sid":"TzdbTbqAFB73Tfet8GFzQQ","sm":"gpt-4.1-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTzdbTbqAFB73Tfet8GFzQQ.eval","v":0.11},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T14:12:01.000Z","m":"gpt-4o-2024-08-06","oid":"chess_puzzles-7f0eb19d79a61a50","rd":"2024-08-06","se":0.028762349126466143,"sid":"Cjt9Xa7iPVBf9dN5cfJ88q","sm":"gpt-4o-2024-08-06","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCjt9Xa7iPVBf9dN5cfJ88q.eval","v":0.09},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:12:00.000Z","m":"o1-2024-12-17","oid":"chess_puzzles-7ecb74aa7c82f0e8","rd":"2024-12-17","se":0.03144660377352202,"sid":"GX3T3cWkC6n2Dc6ufeQN6y","sm":"o1-2024-12-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGX3T3cWkC6n2Dc6ufeQN6y.eval","v":0.11},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:11:33.000Z","m":"o3-mini-2025-01-31","oid":"chess_puzzles-d7a145d008d6c07e","rd":"2025-01-31","se":0.031446603773522014,"sid":"KX6LwuDrXwKvJ6RXod52j4","sm":"o3-mini-2025-01-31_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKX6LwuDrXwKvJ6RXod52j4.eval","v":0.11},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T14:11:28.000Z","m":"gpt-4-1-nano-2025-04-14","oid":"chess_puzzles-4533de6a4ace3910","rd":"2025-04-14","se":0,"sid":"FLJ6bNrtaJQGCU2dP3CvWH","sm":"gpt-4.1-nano-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFLJ6bNrtaJQGCU2dP3CvWH.eval","v":0},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:11:18.000Z","m":"o3-mini-2025-01-31","oid":"chess_puzzles-d2aca7997bb9ac9f","rd":"2025-01-31","se":0.03588702812826367,"sid":"65oGRSHSEY6edS9mqbsCfE","sm":"o3-mini-2025-01-31_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F65oGRSHSEY6edS9mqbsCfE.eval","v":0.15},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:11:15.000Z","m":"o3-mini-2025-01-31","oid":"chess_puzzles-98947b3d585263c2","rd":"2025-01-31","se":0.023868325657594204,"sid":"gE8uyjscAzcjpuZZNow3d8","sm":"o3-mini-2025-01-31_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgE8uyjscAzcjpuZZNow3d8.eval","v":0.06},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:10:35.000Z","m":"o4-mini-2025-04-16","oid":"chess_puzzles-a7caa77e89e4654c","rd":"2025-04-16","se":0.03379976689896311,"sid":"4MyKFuqZDpWQCFpKuHbMyD","sm":"o4-mini-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4MyKFuqZDpWQCFpKuHbMyD.eval","v":0.13},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T14:10:21.000Z","m":"gpt-4-1-mini-2025-04-14","oid":"chess_puzzles-757d644f22aa4306","rd":"2025-04-14","se":0.009999999999999998,"sid":"ZyRri9YZ8CNz6CrUWXFjBA","sm":"gpt-4.1-mini-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZyRri9YZ8CNz6CrUWXFjBA.eval","v":0.01},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:10:20.000Z","m":"o4-mini-2025-04-16","oid":"chess_puzzles-ec7b35740cad328c","rd":"2025-04-16","se":0.041633319989322654,"sid":"dfonReEmxqpRWn5kxGSSES","sm":"o4-mini-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdfonReEmxqpRWn5kxGSSES.eval","v":0.22},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:10:10.000Z","m":"o4-mini-2025-04-16","oid":"chess_puzzles-a32ab7ba16b0ae5f","rd":"2025-04-16","se":0.03588702812826367,"sid":"EjErEBJsumLKEs2LByVSst","sm":"o4-mini-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEjErEBJsumLKEs2LByVSst.eval","v":0.15},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:09:58.000Z","m":"gpt-5-mini-2025-08-07","oid":"chess_puzzles-f482dfcd9756d3fc","rd":"2025-08-07","se":0.036845294917747066,"sid":"Wk25aTN6ZVjg5BwRUdcYKo","sm":"gpt-5-mini-2025-08-07_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWk25aTN6ZVjg5BwRUdcYKo.eval","v":0.16},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:09:51.000Z","m":"o3-2025-04-16","oid":"chess_puzzles-7ad57586e5509c32","rd":"2025-04-16","se":0.048241815132442176,"sid":"mjrcPbSQbMApmXPnqhoawd","sm":"o3-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmjrcPbSQbMApmXPnqhoawd.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:09:48.000Z","m":"gpt-5-mini-2025-08-07","oid":"chess_puzzles-b07482abd739f0e0","rd":"2025-08-07","se":0.04292346959909278,"sid":"5tdGvpe8Ahh8xsUnSwhdAF","sm":"gpt-5-mini-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5tdGvpe8Ahh8xsUnSwhdAF.eval","v":0.24},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:09:35.000Z","m":"o3-2025-04-16","oid":"chess_puzzles-ba781ddb29405904","rd":"2025-04-16","se":0.048241815132442176,"sid":"RBFypLgCtFRyC4gsCewHfj","sm":"o3-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRBFypLgCtFRyC4gsCewHfj.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:09:29.000Z","m":"o3-2025-04-16","oid":"chess_puzzles-c18250a28dbf3c61","rd":"2025-04-16","se":0.048523658709390974,"sid":"MiKNm9AZG7MxP6hibnYN8a","sm":"o3-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMiKNm9AZG7MxP6hibnYN8a.eval","v":0.37},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:09:27.000Z","m":"gpt-5-nano-2025-08-07","oid":"chess_puzzles-5b36549a2e9a80b5","rd":"2025-08-07","se":0.041633319989322654,"sid":"gQHCk4faPpTNupNZ3xmr4c","sm":"gpt-5-nano-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgQHCk4faPpTNupNZ3xmr4c.eval","v":0.22},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:09:19.000Z","m":"gpt-5-nano-2025-08-07","oid":"chess_puzzles-4c2eec3a48928169","rd":"2025-08-07","se":0.03588702812826367,"sid":"kdPFdnbdEGaSw4krPzeJtY","sm":"gpt-5-nano-2025-08-07_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkdPFdnbdEGaSw4krPzeJtY.eval","v":0.15},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T14:08:37.000Z","m":"gpt-5-nano-2025-08-07","oid":"chess_puzzles-10617c967968eca0","rd":"2025-08-07","se":0.014070529413628966,"sid":"BdrAtxC8wKnKiaEgedzdYY","sm":"gpt-5-nano-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBdrAtxC8wKnKiaEgedzdYY.eval","v":0.02},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T14:08:10.000Z","m":"gpt-5-mini-2025-08-07","oid":"chess_puzzles-6768e548b6660bb9","rd":"2025-08-07","se":0,"sid":"ELe4PgXXEJQ8TDtUCQRKae","sm":"gpt-5-mini-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FELe4PgXXEJQ8TDtUCQRKae.eval","v":0},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:07:43.000Z","m":"gpt-5-2025-08-07","oid":"chess_puzzles-8f35435602c9c3e1","rd":"2025-08-07","se":0.04824181513244218,"sid":"CxCwksDLsyrgTJ5ievyAoq","sm":"gpt-5-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCxCwksDLsyrgTJ5ievyAoq.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:07:34.000Z","m":"gpt-5-2025-08-07","oid":"chess_puzzles-3ae3f41c495da707","rd":"2025-08-07","se":0.04688261722621506,"sid":"4QRM7D5HeCwCvWVZqhJKRS","sm":"gpt-5-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4QRM7D5HeCwCvWVZqhJKRS.eval","v":0.32},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:07:24.000Z","m":"gpt-5-2025-08-07","oid":"chess_puzzles-aa964192b7c2dcf9","rd":"2025-08-07","se":0.04461960433384737,"sid":"iygB8rn4Xogj5N5x9W7ifT","sm":"gpt-5-2025-08-07_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiygB8rn4Xogj5N5x9W7ifT.eval","v":0.27},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T14:06:12.000Z","m":"gpt-5-2025-08-07","oid":"chess_puzzles-74d08fa251513b48","rd":"2025-08-07","se":0.03144660377352202,"sid":"gz54krKiBUPt4BfJASJZqi","sm":"gpt-5-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fgz54krKiBUPt4BfJASJZqi.eval","v":0.11},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T14:05:48.000Z","m":"gpt-5-1-2025-11-13","oid":"chess_puzzles-d55adbff5832dd55","rd":"2025-11-13","se":0.0238683256575942,"sid":"5HoXvfpXgnMT3mZ2a8ezkC","sm":"gpt-5.1-2025-11-13_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5HoXvfpXgnMT3mZ2a8ezkC.eval","v":0.06},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:05:35.000Z","m":"gpt-5-2-2025-12-11","oid":"chess_puzzles-8e33bf86e3d0142b","rd":"2025-12-11","se":0.04824181513244218,"sid":"WpdE2FAG4gvAbvgemVXgPA","sm":"gpt-5.2-2025-12-11_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWpdE2FAG4gvAbvgemVXgPA.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:05:29.000Z","m":"gpt-5-1-2025-11-13","oid":"chess_puzzles-345e2e0f7e50d641","rd":"2025-11-13","se":0.04960449637488583,"sid":"HKQxqBGokn83tHr5wGJiZX","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHKQxqBGokn83tHr5wGJiZX.eval","v":0.42},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:05:25.000Z","m":"gpt-5-1-2025-11-13","oid":"chess_puzzles-bb00334c5db50a0b","rd":"2025-11-13","se":0.04229525846816507,"sid":"LfDm9pJ9Uttcjcacy9DGv3","sm":"gpt-5.1-2025-11-13_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLfDm9pJ9Uttcjcacy9DGv3.eval","v":0.23},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:05:16.000Z","m":"gpt-5-2-2025-12-11","oid":"chess_puzzles-0407b96c6973e08c","rd":"2025-12-11","se":0.05,"sid":"Y9hzf2ioQTW9ezDvCGzTCu","sm":"gpt-5.2-2025-12-11_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY9hzf2ioQTW9ezDvCGzTCu.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:04:39.000Z","m":"gpt-5-2-2025-12-11","oid":"chess_puzzles-34a677e3fc5b559e","rd":"2025-12-11","se":0.04408440022768081,"sid":"gajcfzsiUXo8AyTudT22bP","sm":"gpt-5.2-2025-12-11_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgajcfzsiUXo8AyTudT22bP.eval","v":0.26},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T14:04:13.000Z","m":"gpt-5-2-2025-12-11","oid":"chess_puzzles-0d38d9bfdbc20e0d","rd":"2025-12-11","se":0.014070529413628968,"sid":"MuBMyrNUmTRebT8AA8AY8g","sm":"gpt-5.2-2025-12-11_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMuBMyrNUmTRebT8AA8AY8g.eval","v":0.02},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T14:04:04.000Z","m":"gpt-5-4-2026-03-05","oid":"chess_puzzles-75988e604b1f0ec0","rd":"2026-03-05","se":0.05021167315686782,"sid":"92ZBraB6bB98s8raSiPgUw","sm":"gpt-5.4-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F92ZBraB6bB98s8raSiPgUw.eval","v":0.48},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:04:01.000Z","m":"gpt-5-4-2026-03-05","oid":"chess_puzzles-4c7ec0e45a8a1223","rd":"2026-03-05","se":0.04960449637488583,"sid":"QxrNbLxE4V9iHrUVmVPFHk","sm":"gpt-5.4-2026-03-05_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQxrNbLxE4V9iHrUVmVPFHk.eval","v":0.42},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T14:03:43.000Z","m":"gpt-5-4-2026-03-05","oid":"chess_puzzles-8bd004b03a47f616","rd":"2026-03-05","se":0.048523658709390974,"sid":"YA6uTqem8RwiNreRbaarAz","sm":"gpt-5.4-2026-03-05_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYA6uTqem8RwiNreRbaarAz.eval","v":0.37},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:03:36.000Z","m":"gpt-5-4-2026-03-05","oid":"chess_puzzles-819a108921b36539","rd":"2026-03-05","se":0.039427724440366255,"sid":"8fpUqaaFFUfWWNZskndSpp","sm":"gpt-5.4-2026-03-05_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8fpUqaaFFUfWWNZskndSpp.eval","v":0.19},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T14:03:23.000Z","m":"gpt-5-4-2026-03-05","oid":"chess_puzzles-2fbb793e3ca2ca4b","rd":"2026-03-05","se":0.017144660799776525,"sid":"TfUnmjxgDbgUvDAo6JrD9a","sm":"gpt-5.4-2026-03-05_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTfUnmjxgDbgUvDAo6JrD9a.eval","v":0.03},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T14:02:58.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"chess_puzzles-5d3645e0fec6ad9b","rd":"2026-03-17","se":0.048241815132442176,"sid":"N5YuEXtrXeXuhDN2qRssSK","sm":"gpt-5.4-nano-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FN5YuEXtrXeXuhDN2qRssSK.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T14:02:49.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"chess_puzzles-576704dd57990727","rd":"2026-03-17","se":0.04020151261036849,"sid":"isrAz3SUJUJHGUxsk3sG6V","sm":"gpt-5.4-nano-2026-03-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FisrAz3SUJUJHGUxsk3sG6V.eval","v":0.2},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T14:02:42.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"chess_puzzles-dc72d1abf1fe683e","rd":"2026-03-17","se":0.014070529413628968,"sid":"at7F8ntqX3M6rXA38mKkWr","sm":"gpt-5.4-nano-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fat7F8ntqX3M6rXA38mKkWr.eval","v":0.02},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T14:02:26.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"chess_puzzles-d7862ba53978b7a7","rd":"2026-03-17","se":0.03775251680686369,"sid":"8Wf3UWZzc4nxwzzPv9ckvF","sm":"gpt-5.4-mini-2026-03-17_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8Wf3UWZzc4nxwzzPv9ckvF.eval","v":0.17},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:56:43.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"chess_puzzles-1c3d74397b81613d","rd":"2026-03-17","se":0.04229525846816507,"sid":"fzgzPmgay5wxMGHjevRjrm","sm":"gpt-5.4-mini-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfzgzPmgay5wxMGHjevRjrm.eval","v":0.23},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:54:48.000Z","m":"gpt-5-5","oid":"chess_puzzles-9a20660196c060d3","rd":"2026-04-23","se":0.048523658709390974,"sid":"DFWQcrbMegFAcEWZKWoJxC","sm":"gpt-5.5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDFWQcrbMegFAcEWZKWoJxC.eval","v":0.63},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:54:47.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"chess_puzzles-971ba9305a7bce0a","rd":"2026-03-17","se":0.014070529413628966,"sid":"Jpv6S8xWEHQQTzNHD35dcw","sm":"gpt-5.4-mini-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJpv6S8xWEHQQTzNHD35dcw.eval","v":0.02},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:54:20.000Z","m":"gpt-5-5","oid":"chess_puzzles-3ae5b6d0b48c6d22","rd":"2026-04-23","se":0.04605661864718382,"sid":"4EayGByBV2xodNBkrZMcFb","sm":"gpt-5.5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4EayGByBV2xodNBkrZMcFb.eval","v":0.3},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:53:55.000Z","m":"gpt-5-5","oid":"chess_puzzles-7a9e32b9f2c68e62","rd":"2026-04-23","se":0.028762349126466143,"sid":"QfS2PrjUFMNnceRZmCmewP","sm":"gpt-5.5_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQfS2PrjUFMNnceRZmCmewP.eval","v":0.09},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:53:45.000Z","m":"gpt-5-6-luna","oid":"chess_puzzles-d1d1c14dc2b8b17b","rd":"2026-07-09","se":0.0497569851956243,"sid":"LUfGWqWQim7rdBPgSpbrXS","sm":"gpt-5.6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLUfGWqWQim7rdBPgSpbrXS.eval","v":0.43},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:53:37.000Z","m":"gpt-5-6-luna","oid":"chess_puzzles-b6e254b61a1b4ce7","rd":"2026-07-09","se":0.03775251680686369,"sid":"FagAUZ7m5PJHseDATvp2vf","sm":"gpt-5.6-luna_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFagAUZ7m5PJHseDATvp2vf.eval","v":0.17},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:53:29.000Z","m":"gpt-5-6-luna","oid":"chess_puzzles-7934f24812045fef","rd":"2026-07-09","se":0.01714466079977653,"sid":"5Q7zWYmoRik8mhABL4wpVR","sm":"gpt-5.6-luna_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5Q7zWYmoRik8mhABL4wpVR.eval","v":0.03},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:53:13.000Z","m":"gpt-5-6-terra","oid":"chess_puzzles-8c8d276125e7dd7e","rd":"2026-07-09","se":0.05016135580465918,"sid":"J2ZgtxyaQZfyDMPTPxS8nm","sm":"gpt-5.6-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ2ZgtxyaQZfyDMPTPxS8nm.eval","v":0.47},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:51:29.000Z","m":"gpt-5-6-terra","oid":"chess_puzzles-47e5d23aac1dd9f7","rd":"2026-07-09","se":0.044619604333847374,"sid":"bA5j9K2shzZVFGTMQ5g6EU","sm":"gpt-5.6-terra_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbA5j9K2shzZVFGTMQ5g6EU.eval","v":0.27},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:50:17.000Z","m":"gpt-5-6-terra","oid":"chess_puzzles-cbfcd592ad6c2513","rd":"2026-07-09","se":0.02190429135575903,"sid":"HzNnNaRmyNExSVaMjmFCHe","sm":"gpt-5.6-terra_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHzNnNaRmyNExSVaMjmFCHe.eval","v":0.05},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:50:12.000Z","m":"gpt-5-6-sol","oid":"chess_puzzles-84c9b645f580f167","rd":"2026-07-09","se":0.049431107042371025,"sid":"oWqwMmwEd7LU3eU5Ftak64","sm":"gpt-5.6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoWqwMmwEd7LU3eU5Ftak64.eval","v":0.59},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:49:48.000Z","m":"gpt-5-6-sol","oid":"chess_puzzles-60bc6639952e0db8","rd":"2026-07-09","se":0.04648231987117317,"sid":"V6aVGtn9gAmRep2oBGsqVN","sm":"gpt-5.6-sol_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV6aVGtn9gAmRep2oBGsqVN.eval","v":0.31},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:49:41.000Z","m":"gpt-5-6-sol","oid":"chess_puzzles-374f5838ebe0b51a","rd":"2026-07-09","se":0.02564323999762428,"sid":"WvV6QREDDXeDEGdrKztDsy","sm":"gpt-5.6-sol_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWvV6QREDDXeDEGdrKztDsy.eval","v":0.07},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:46:15.000Z","m":"claude-sonnet-4-5-20250929-59k","oid":"chess_puzzles-7ad39af5fce3e3e8","rd":"2025-09-29","se":0.03015113445777636,"sid":"RuQX7KA7owzqzf8Le9veeM","sm":"claude-sonnet-4-5-20250929_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRuQX7KA7owzqzf8Le9veeM.eval","v":0.1},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:45:40.000Z","m":"claude-3-opus-2024-02-29","oid":"chess_puzzles-1cb8d0dacb41d269","rd":"2024-02-29","se":0.01969463855669324,"sid":"WAbi8sZpyNAZoNN3Gmg22y","sm":"claude-3-opus-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWAbi8sZpyNAZoNN3Gmg22y.eval","v":0.04},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:45:23.000Z","m":"claude-sonnet-4-5-20250929-32k","oid":"chess_puzzles-280541b7682d0686","rd":"2025-09-29","se":0.03015113445777636,"sid":"gJizntrYwhnbpCviv9mNBN","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgJizntrYwhnbpCviv9mNBN.eval","v":0.1},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:44:41.000Z","m":"minimax-m3","oid":"chess_puzzles-a4f6c09309c07cd7","rd":"2026-06-01","se":0.027265992434429072,"sid":"Ws3g7xtto2fZ7AhuXJfecf","sm":"MiniMax-M3_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWs3g7xtto2fZ7AhuXJfecf.eval","v":0.08},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:44:15.000Z","m":"claude-sonnet-4-6","oid":"chess_puzzles-dd20ba0da1d8f8ab","rd":"2026-02-17","se":0.04229525846816507,"sid":"6P9AJQmBzaPmE7ossbqLjz","sm":"claude-sonnet-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6P9AJQmBzaPmE7ossbqLjz.eval","v":0.23},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:43:54.000Z","m":"claude-sonnet-4-5-2025-09-29","oid":"chess_puzzles-054a82566516dbc4","rd":"2025-09-29","se":0.023868325657594204,"sid":"VajBDnUCyX7uqAfMTbi63j","sm":"claude-sonnet-4-5-20250929","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVajBDnUCyX7uqAfMTbi63j.eval","v":0.06},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:43:41.000Z","m":"claude-haiku-4-5-20251001-59k","oid":"chess_puzzles-ad879557468d66e8","rd":"2025-10-15","se":0.027265992434429072,"sid":"6bmTzBjpMySeHn65gc5Hu5","sm":"claude-haiku-4-5-20251001_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6bmTzBjpMySeHn65gc5Hu5.eval","v":0.08},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:43:19.000Z","m":"claude-opus-4-6","oid":"chess_puzzles-bcea2053e7416a17","rd":"2026-02-05","se":0.040936018074033236,"sid":"aKc2rzsA6YPYwCkE7nhvvB","sm":"claude-opus-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaKc2rzsA6YPYwCkE7nhvvB.eval","v":0.21},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:43:11.000Z","m":"claude-haiku-4-5-20251001-32k","oid":"chess_puzzles-991dbe5dc6ac1f0b","rd":"2025-10-15","se":0.021904291355759033,"sid":"BLgwhLGGofDBzxoc34YFHK","sm":"claude-haiku-4-5-20251001_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBLgwhLGGofDBzxoc34YFHK.eval","v":0.05},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:42:53.000Z","m":"claude-opus-4-5-20251101-59k","oid":"chess_puzzles-d3ebe0f86342fdf1","rd":"2025-11-24","se":0.0348735088019777,"sid":"DPuTDKvupUqKgkdVj7Awsp","sm":"claude-opus-4-5-20251101_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDPuTDKvupUqKgkdVj7Awsp.eval","v":0.14},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:42:41.000Z","m":"claude-opus-4-5-20251101-32k","oid":"chess_puzzles-b69cf96737c70fb3","rd":"2025-11-24","se":0.02190429135575903,"sid":"V7seJ9yeHu4LYjrkomhEQm","sm":"claude-opus-4-5-20251101_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV7seJ9yeHu4LYjrkomhEQm.eval","v":0.05},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:42:38.000Z","m":"gpt-6-astra","oid":"chess_puzzles-a168e97c8877ca5a","rd":"2026-09-03","se":0.0440844002276808,"sid":"KaEir82ukWPm2Fq4FanY2M","sm":"gpt-6-astra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKaEir82ukWPm2Fq4FanY2M.eval","v":0.74},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:42:24.000Z","m":"claude-opus-4-5-2025-11-01","oid":"chess_puzzles-43f6a858d8c7def7","rd":"2025-11-24","se":0.021904291355759033,"sid":"4afZ2hjLiJnPy5pD7TfE3K","sm":"claude-opus-4-5-20251101","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4afZ2hjLiJnPy5pD7TfE3K.eval","v":0.05},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:42:10.000Z","m":"gemini-3-flash-preview","oid":"chess_puzzles-c42f66af74a11a87","rd":"2025-12-17","se":0.0492365963917331,"sid":"UUNZJtgib9XbCjLEdhtGaT","sm":"gemini-3-flash-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUUNZJtgib9XbCjLEdhtGaT.eval","v":0.4},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:41:42.000Z","m":"gpt-6-sol","oid":"chess_puzzles-732c48487ade2b42","rd":"2026-09-22","se":0.049888765156985884,"sid":"N9uFn7EZFsvHNcHEwEy3J6","sm":"gpt-6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FN9uFn7EZFsvHNcHEwEy3J6.eval","v":0.56},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:39:53.000Z","m":"qwen3-5-plus","oid":"chess_puzzles-5082af56a850538f","rd":"2026-02-16","se":0.028762349126466143,"sid":"XEaDaqM3oaPcrtQ6XH5fGS","sm":"qwen3.5-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXEaDaqM3oaPcrtQ6XH5fGS.eval","v":0.09},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:39:44.000Z","m":"qwen3-5-flash","oid":"chess_puzzles-9a69503e01ce1961","rd":"2026-02-25","se":0.034873508801977704,"sid":"fEgRyzyFYZraNnkkbRVSSg","sm":"qwen3.5-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfEgRyzyFYZraNnkkbRVSSg.eval","v":0.14},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:38:57.000Z","m":"qwen3-6-plus","oid":"chess_puzzles-50453417bedd4dcf","rd":"2026-03-31","se":0.036845294917747066,"sid":"SP9UFPzxVnsbzs2kExP2bh","sm":"qwen3.6-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSP9UFPzxVnsbzs2kExP2bh.eval","v":0.16},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:38:19.000Z","m":"deepseek-v4-pro","oid":"chess_puzzles-d4c1c71c7c8debe7","rd":"2026-04-24","se":0.039427724440366255,"sid":"DSyqeLiA9LLZF4iaxsTVHL","sm":"deepseek-v4-pro_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDSyqeLiA9LLZF4iaxsTVHL.eval","v":0.19},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:38:10.000Z","m":"deepseek-v4-pro","oid":"chess_puzzles-af44aa539b88c680","rd":"2026-04-24","se":0.036845294917747066,"sid":"duoDEpEXouGJfYrjd5DbvN","sm":"deepseek-v4-pro_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FduoDEpEXouGJfYrjd5DbvN.eval","v":0.16},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:37:06.000Z","m":"gpt-6-luna","oid":"chess_puzzles-ce9606192f257933","rd":"2026-09-22","se":0.04824181513244218,"sid":"Jevnk9hi9oRBbGcFu2JxBC","sm":"gpt-6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJevnk9hi9oRBbGcFu2JxBC.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:36:50.000Z","m":"claude-sonnet-4-6","oid":"chess_puzzles-35a85a7294e69507","rd":"2026-02-17","se":0.039427724440366255,"sid":"Atc4F5iuMZu3vNBWVGCBnv","sm":"claude-sonnet-4-6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAtc4F5iuMZu3vNBWVGCBnv.eval","v":0.19},{"b":"chess_puzzles","cfg":{"effort":"medium"},"d":"2026-10-13","e":"medium","evaluated_at":"2026-10-13T13:36:47.000Z","m":"claude-sonnet-4-6","oid":"chess_puzzles-9436917ea3429e94","rd":"2026-02-17","se":0.03684529491774706,"sid":"htvM9QKQJ8TfBxTzDxfZfp","sm":"claude-sonnet-4-6_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhtvM9QKQJ8TfBxTzDxfZfp.eval","v":0.16},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:36:23.000Z","m":"claude-opus-4-7","oid":"chess_puzzles-c63c4933a96e6e5c","rd":"2026-04-16","se":0.04943110704237103,"sid":"XXPqVNR7WCFFA2Rnzjcbk6","sm":"claude-opus-4-7_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXXPqVNR7WCFFA2Rnzjcbk6.eval","v":0.41},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:35:58.000Z","m":"claude-opus-4-7","oid":"chess_puzzles-6f06ac012bd98634","rd":"2026-04-16","se":0.04824181513244218,"sid":"RHbNnWqK6x8f6pR2AZ8VAq","sm":"claude-opus-4-7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRHbNnWqK6x8f6pR2AZ8VAq.eval","v":0.36},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:35:50.000Z","m":"claude-opus-4-7","oid":"chess_puzzles-402eb454bfe51714","rd":"2026-04-16","se":0.0440844002276808,"sid":"NyAvSf73nUe84HuY6Vpjkh","sm":"claude-opus-4-7_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNyAvSf73nUe84HuY6Vpjkh.eval","v":0.26},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:35:27.000Z","m":"claude-opus-4-8","oid":"chess_puzzles-079a3b58f07b4c92","rd":"2026-05-28","se":0.05021167315686782,"sid":"PEZpuEuU65bu3ZtvjLytHq","sm":"claude-opus-4-8_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPEZpuEuU65bu3ZtvjLytHq.eval","v":0.52},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:35:16.000Z","m":"claude-opus-4-8","oid":"chess_puzzles-d6c8ffd3ff63d038","rd":"2026-05-28","se":0.045604802157206865,"sid":"icTHLT2S2egxYtgbjQHUng","sm":"claude-opus-4-8_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FicTHLT2S2egxYtgbjQHUng.eval","v":0.29},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:34:59.000Z","m":"claude-opus-4-8","oid":"chess_puzzles-e270d2be37658239","rd":"2026-05-28","se":0.039427724440366255,"sid":"jDWEviMweCdzGxLfz8sdBm","sm":"claude-opus-4-8_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjDWEviMweCdzGxLfz8sdBm.eval","v":0.19},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:34:38.000Z","m":"grok-4-20-0309-reasoning","oid":"chess_puzzles-2ebd5b4ba52cd830","rd":"2026-02-17","se":0.045604802157206865,"sid":"ETNzxNbPKwj4qxkdo3VVgT","sm":"grok-4.20-0309-reasoning","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FETNzxNbPKwj4qxkdo3VVgT.eval","v":0.29},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:34:37.000Z","m":"claude-fable-5","oid":"chess_puzzles-5de402e2eee7537a","rd":"2026-06-09","se":0.05016135580465918,"sid":"bxS76CJuVws8iwWU2Wr2PM","sm":"claude-fable-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbxS76CJuVws8iwWU2Wr2PM.eval","v":0.47},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:34:31.000Z","m":"grok-4-3","oid":"chess_puzzles-ca30e2eb47647ce4","rd":"2026-04-17","se":0.03861229196653691,"sid":"m4WQuAN88hKsqbT9zXbg7z","sm":"grok-4.3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fm4WQuAN88hKsqbT9zXbg7z.eval","v":0.18},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:34:13.000Z","m":"gemini-3-1-pro-preview","oid":"chess_puzzles-80e7e0d74adf34ce","rd":"2026-02-19","se":0.05024183937956913,"sid":"FTDFCpXzDTgfgWBNdHLHGc","sm":"gemini-3.1-pro-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFTDFCpXzDTgfgWBNdHLHGc.eval","v":0.51},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:33:53.000Z","m":"gemini-3-5-flash","oid":"chess_puzzles-90e0c411005ea784","rd":"2026-05-19","se":0.05,"sid":"XUDdWKUZBhdWPPQBRgx8Pw","sm":"gemini-3.5-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXUDdWKUZBhdWPPQBRgx8Pw.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:33:40.000Z","m":"gemini-3-5-flash","oid":"chess_puzzles-5fa202acb163cf91","rd":"2026-05-19","se":0.04960449637488583,"sid":"68KxyCJbfMSq8JBL3LRJfr","sm":"gemini-3.5-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F68KxyCJbfMSq8JBL3LRJfr.eval","v":0.42},{"b":"chess_puzzles","cfg":{"effort":"none"},"d":"2026-10-13","e":"none","evaluated_at":"2026-10-13T13:33:35.000Z","m":"deepseek-v4-pro","oid":"chess_puzzles-db05e14e1091a060","rd":"2026-04-24","se":0.019694638556693237,"sid":"iKiaFJXjdHdUnbMXdKnq4Q","sm":"deepseek-v4-pro_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiKiaFJXjdHdUnbMXdKnq4Q.eval","v":0.04},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T13:33:29.000Z","m":"gemini-3-5-flash","oid":"chess_puzzles-04bbbb9cc0cce7ff","rd":"2026-05-19","se":0.04461960433384737,"sid":"H8CebpQyV2qEE8SQzjBrWe","sm":"gemini-3.5-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH8CebpQyV2qEE8SQzjBrWe.eval","v":0.27},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:33:28.000Z","m":"qwen3-6-flash","oid":"chess_puzzles-13cd6396db59008f","rd":"2026-04-27","se":0.032659863237109045,"sid":"j99TDv5LAYUSvTUcR47T75","sm":"qwen3.6-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fj99TDv5LAYUSvTUcR47T75.eval","v":0.12},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:33:00.000Z","m":"qwen3-7-max","oid":"chess_puzzles-5aff4c8d7f2d597b","rd":"2026-05-19","se":0.044619604333847374,"sid":"cbQnfsMSA3aQaBtjm3aFx2","sm":"qwen3.7-max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcbQnfsMSA3aQaBtjm3aFx2.eval","v":0.27},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:32:51.000Z","m":"deepseek-v4-flash-0731","oid":"chess_puzzles-0d5a33d7eb7b754a","rd":"2026-07-31","se":0.04688261722621506,"sid":"Azz52eFkXwVJiRCGpaUBwB","sm":"deepseek-v4-flash-0731_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAzz52eFkXwVJiRCGpaUBwB.eval","v":0.32},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:26:52.000Z","m":"gemini-3-5-flash-lite","oid":"chess_puzzles-d6a36c35dcbd2f00","rd":"2026-07-21","se":0.036845294917747066,"sid":"JJyzHWKJUA3iTZDU8iYefz","sm":"gemini-3.5-flash-lite_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJJyzHWKJUA3iTZDU8iYefz.eval","v":0.16},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:25:11.000Z","m":"gemini-3-5-flash-lite","oid":"chess_puzzles-1c2afbc7b276d867","rd":"2026-07-21","se":0.040936018074033236,"sid":"d5QCJNq9uLtxjhiMhqXLmg","sm":"gemini-3.5-flash-lite_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd5QCJNq9uLtxjhiMhqXLmg.eval","v":0.21},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:24:08.000Z","m":"claude-fable-5","oid":"chess_puzzles-003399ad3b9c1e13","rd":"2026-06-09","se":0.05,"sid":"9ZFhnyMNCKZ8XdR58WGD3D","sm":"claude-fable-5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9ZFhnyMNCKZ8XdR58WGD3D.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:23:41.000Z","m":"claude-fable-5","oid":"chess_puzzles-2a6c466b9ba77237","rd":"2026-06-09","se":0.04648231987117317,"sid":"2jKf4iEfVQnHkgAwEtD9Nn","sm":"claude-fable-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2jKf4iEfVQnHkgAwEtD9Nn.eval","v":0.31},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:23:26.000Z","m":"claude-sonnet-5","oid":"chess_puzzles-d8db1725b12c00a3","rd":"2026-06-30","se":0.05,"sid":"PN9exWpeJGXJB8tZGX32e4","sm":"claude-sonnet-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPN9exWpeJGXJB8tZGX32e4.eval","v":0.45},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:23:17.000Z","m":"claude-sonnet-5","oid":"chess_puzzles-28f3b8535b55a1bf","rd":"2026-06-30","se":0.04760952285695234,"sid":"DFzqUCfxnQPUdK8b8D7aYu","sm":"claude-sonnet-5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDFzqUCfxnQPUdK8b8D7aYu.eval","v":0.34},{"b":"chess_puzzles","d":"2026-10-13","evaluated_at":"2026-10-13T13:23:06.000Z","m":"minimax-m3","oid":"chess_puzzles-31956db1d5117b89","rd":"2026-06-01","se":0.03775251680686369,"sid":"CtmxkTPbFRofQWTkEw5aSe","sm":"MiniMax-M3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCtmxkTPbFRofQWTkEw5aSe.eval","v":0.17},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T13:19:48.000Z","m":"gemini-3-5-flash-lite","oid":"chess_puzzles-8ac0c43aa3bc60f0","rd":"2026-07-21","se":0.02564323999762428,"sid":"gzQH7StUXaxAtXgxkTgyxJ","sm":"gemini-3.5-flash-lite_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgzQH7StUXaxAtXgxkTgyxJ.eval","v":0.07},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:19:00.000Z","m":"claude-opus-5","oid":"chess_puzzles-4da4372d4ca68821","rd":"2026-07-24","se":0.0497569851956243,"sid":"RLN98bmQSMnc5Zv39BF4M2","sm":"claude-opus-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRLN98bmQSMnc5Zv39BF4M2.eval","v":0.43},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:14:52.000Z","m":"gemini-3-6-flash","oid":"chess_puzzles-0dca5d436e3c3aef","rd":"2026-07-21","se":0.0492365963917331,"sid":"ZDeGG5QYkdgVqzGPmmnbfx","sm":"gemini-3.6-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZDeGG5QYkdgVqzGPmmnbfx.eval","v":0.4},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:14:23.000Z","m":"grok-4-5","oid":"chess_puzzles-ddd3e1bbc4452eb4","rd":"2026-07-08","se":0.047937248544110224,"sid":"jTuYE8VJTos73Kw77UFevq","sm":"grok-4.5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjTuYE8VJTos73Kw77UFevq.eval","v":0.35},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:14:03.000Z","m":"grok-4-6","oid":"chess_puzzles-87a64e8b047c22ff","rd":"2026-08-12","se":0.04725815626252608,"sid":"NiuExVccWkWbWqi6ex6vvf","sm":"grok-4.6_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNiuExVccWkWbWqi6ex6vvf.eval","v":0.33},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:13:49.000Z","m":"qwen3-8-max","oid":"chess_puzzles-35bb27dd547ea616","rd":"2026-08-02","se":0.05009082659620331,"sid":"bq5y6Tp83KLtxC58UDfpyN","sm":"qwen3.8-max_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fbq5y6Tp83KLtxC58UDfpyN.eval","v":0.46},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:13:34.000Z","m":"deepseek-v4-pro-0813","oid":"chess_puzzles-a100be535d4c1224","rd":"2026-08-13","se":0.05009082659620331,"sid":"C56VHYLMeww2wXebeJh3gg","sm":"deepseek-v4-pro-0813_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FC56VHYLMeww2wXebeJh3gg.eval","v":0.46},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:13:33.000Z","m":"muse-spark-1-1","oid":"chess_puzzles-f8d36425e6166eb5","rd":"2026-07-09","se":0.04760952285695234,"sid":"4pbEvTxsbzNRCh9kU9Xhun","sm":"muse-spark-1.1_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4pbEvTxsbzNRCh9kU9Xhun.eval","v":0.34},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:13:14.000Z","m":"claude-opus-5","oid":"chess_puzzles-e7b5aafcb1d518c9","rd":"2026-07-24","se":0.044619604333847374,"sid":"XTW92PMsypptnTGBFuLCxe","sm":"claude-opus-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXTW92PMsypptnTGBFuLCxe.eval","v":0.27},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:13:06.000Z","m":"claude-fable-5-1","oid":"chess_puzzles-f44752cc2ac0980f","rd":"2026-09-01","se":0.05021167315686782,"sid":"aLrHd8RCiNo3Yb4GqSawM7","sm":"claude-fable-5-1_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaLrHd8RCiNo3Yb4GqSawM7.eval","v":0.48},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T13:12:52.000Z","m":"gemini-3-6-flash","oid":"chess_puzzles-b5fb30d6076b7053","rd":"2026-07-21","se":0.04960449637488583,"sid":"hJ6XXLBm9yfXtSbbhWuwpw","sm":"gemini-3.6-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhJ6XXLBm9yfXtSbbhWuwpw.eval","v":0.42},{"b":"chess_puzzles","cfg":{"effort":"minimal"},"d":"2026-10-13","e":"minimal","evaluated_at":"2026-10-13T13:12:45.000Z","m":"gemini-3-6-flash","oid":"chess_puzzles-765e6a9c52616407","rd":"2026-07-21","se":0.04560480215720686,"sid":"beSbPHiiWcwJBSzYRh7Kh4","sm":"gemini-3.6-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbeSbPHiiWcwJBSzYRh7Kh4.eval","v":0.29},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:05:03.000Z","m":"muse-spark-1-2","oid":"chess_puzzles-d6e021abcb3f11bb","rd":"2026-08-05","se":0.044619604333847374,"sid":"FueK5bFAzGbkUao9JAibms","sm":"muse-spark-1.2_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFueK5bFAzGbkUao9JAibms.eval","v":0.27},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:01:31.000Z","m":"claude-opus-5-5","oid":"chess_puzzles-d68ff2c421a2f244","rd":"2026-09-22","se":0.04725815626252609,"sid":"F2tuCm5VmNDAaYYh6qYdRy","sm":"claude-opus-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF2tuCm5VmNDAaYYh6qYdRy.eval","v":0.67},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:00:38.000Z","m":"claude-sonnet-5-5","oid":"chess_puzzles-53edbe5d45a1ff56","rd":"2026-09-28","se":0.04943110704237103,"sid":"kXU32N6tFkQLx5ei6nRJFS","sm":"claude-sonnet-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkXU32N6tFkQLx5ei6nRJFS.eval","v":0.59},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T13:00:28.000Z","m":"muse-spark-1-3","oid":"chess_puzzles-7e9195dfa3a15fdc","rd":"2026-09-02","se":0.048523658709390974,"sid":"XkHyFXCWYDv2nAB5vmmWrU","sm":"muse-spark-1.3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXkHyFXCWYDv2nAB5vmmWrU.eval","v":0.37},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:00:23.000Z","m":"gemini-3-7-flash","oid":"chess_puzzles-d25889e155d46922","rd":"2026-08-13","se":0.04760952285695234,"sid":"Xu34dKTAdpBWL6advf6DMg","sm":"gemini-3.7-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXu34dKTAdpBWL6advf6DMg.eval","v":0.66},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T13:00:11.000Z","m":"qwen3-8-max-0902","oid":"chess_puzzles-0eab924a01a29b61","rd":"2026-09-01","se":0.048523658709390974,"sid":"Ut4d4rvkUygdLTRomFb5rh","sm":"qwen3.8-max-0902_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUt4d4rvkUygdLTRomFb5rh.eval","v":0.37},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T13:00:07.000Z","m":"grok-4-6","oid":"chess_puzzles-0bcce5bb3d576db2","rd":"2026-08-12","se":0.04725815626252608,"sid":"YoAJyC3j8dZ6ZpDZ5rSDLw","sm":"grok-4.6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYoAJyC3j8dZ6ZpDZ5rSDLw.eval","v":0.33},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T12:49:23.000Z","m":"inkling","oid":"chess_puzzles-05960b3c4ab243cf","rd":"2026-07-15","se":0.040936018074033236,"sid":"UZ9LtVBCjHPMN9SKFUo7Wa","sm":"Inkling_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUZ9LtVBCjHPMN9SKFUo7Wa.eval","v":0.21},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T12:40:12.000Z","m":"gpt-6-1-sol","oid":"chess_puzzles-490968c7913967ef","rd":"2026-09-29","se":0.04688261722621507,"sid":"KkGEgrjYDnofMoUJBCykcH","sm":"gpt-6.1-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKkGEgrjYDnofMoUJBCykcH.eval","v":0.68},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T12:40:01.000Z","m":"claude-haiku-5-5","oid":"chess_puzzles-d7a5489ef557d35b","rd":"2026-10-07","se":0.049999999999999996,"sid":"KEyuvehfRGtcdgNfNavy62","sm":"claude-haiku-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKEyuvehfRGtcdgNfNavy62.eval","v":0.55},{"b":"chess_puzzles","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T12:39:44.000Z","m":"deepseek-v4-1-flash","oid":"chess_puzzles-15106011fdb794b0","rd":"2026-09-09","se":0.05021167315686782,"sid":"ACpDTbhRPwce2BWQTWD8iC","sm":"deepseek-v4.1-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FACpDTbhRPwce2BWQTWD8iC.eval","v":0.48},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T12:39:26.000Z","m":"muse-spark-1-3","oid":"chess_puzzles-024a90977fafa413","rd":"2026-09-02","se":0.04760952285695234,"sid":"Hi6xJczgqR4zfCXCXSF6rz","sm":"muse-spark-1.3_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHi6xJczgqR4zfCXCXSF6rz.eval","v":0.34},{"b":"chess_puzzles","cfg":{"effort":"xhigh"},"d":"2026-10-13","e":"xhigh","evaluated_at":"2026-10-13T12:39:21.000Z","m":"grok-4-7","oid":"chess_puzzles-f87533ceabae85a4","rd":"2026-09-21","se":0.04943110704237103,"sid":"L5Ck7pMy8bVmVMMLwxJN7f","sm":"grok-4.7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FL5Ck7pMy8bVmVMMLwxJN7f.eval","v":0.41},{"b":"chess_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T12:39:05.000Z","m":"gemini-3-8-flash","oid":"chess_puzzles-33d1224ea9830a80","rd":"2026-09-02","se":0.04960449637488583,"sid":"Hpyy6CpmXmd5q7Zbdq63CR","sm":"gemini-3.8-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHpyy6CpmXmd5q7Zbdq63CR.eval","v":0.58},{"b":"chess_puzzles","cfg":{"effort":"low"},"d":"2026-10-13","e":"low","evaluated_at":"2026-10-13T07:50:03.000Z","m":"claude-haiku-5-5","oid":"chess_puzzles-f44b2d17d69fc877","rd":"2026-10-07","se":0.04292346959909278,"sid":"KSXLphjiiXtuGdqv9f7ENL","sm":"claude-haiku-5-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKSXLphjiiXtuGdqv9f7ENL.eval","v":0.24},{"b":"cl_bench","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (xhigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"cl_bench-c8aeef3000d5b76c","rd":"2026-03-05","sd":0,"sid":"recF5Go9AZ5xlvAkF","sm":"gpt-5.4-2026-03-05_xhigh","v":0.279},{"b":"cl_bench","cfg":{"effort":"high"},"dn":"GPT 5.1 (High)","e":"high","m":"gpt-5-1-2025-11-13","oid":"cl_bench-033f6956ddafdca7","rd":"2025-11-13","sd":0.005,"sid":"recECMAlNLLCncgpy","sm":"gpt-5.1-2025-11-13_high","v":0.237},{"b":"cl_bench","dn":"Grok 4.20 (reasoning)","m":"grok-4-20","oid":"cl_bench-f6daa2039bf7de5c","rd":"2026-02-17","sd":0,"sid":"recGOUZrtjVucfr50","sm":"grok-4-20","v":0.222},{"b":"cl_bench","dn":"GPT 5.1","m":"gpt-5-1-2025-11-13","oid":"cl_bench-06f5299f7cb6ae75","rd":"2025-11-13","sd":0.002,"sid":"recM0YyEtnfawuPRG","sm":"gpt-5.1-2025-11-13_unknown","v":0.211},{"b":"cl_bench","dn":"Claude Opus 4.5 Thinking","m":"claude-opus-4-5-2025-11-01","oid":"cl_bench-95750a63b01a45ec","rd":"2025-11-24","sd":0.013999999999999999,"sid":"recoZ0jztT3GiULHi","sm":"claude-opus-4-5-20251101_unknown","v":0.211},{"b":"cl_bench","cfg":{"effort":"high"},"dn":"Gemini 3.1 Pro (high)","e":"high","m":"gemini-3-1-pro-preview","oid":"cl_bench-3164305bb1b49218","rd":"2026-02-19","sd":0,"sid":"reckpokJDAOxCx2S3","sm":"gemini-3.1-pro-preview","v":0.20800000000000002},{"b":"cl_bench","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"cl_bench-ef0c6878407ff966","rd":"2026-02-05","sd":0,"sid":"reckTL3wR1Z9aGDlt","sm":"claude-opus-4-6_unknown","v":0.207},{"b":"cl_bench","dn":"Qwen 3.6 Plus","m":"qwen3-6-plus","oid":"cl_bench-90b072c4cbb6ed9c","rd":"2026-03-31","sd":0,"sid":"recURyoAkx1koU7Mk","sm":"qwen3.6-plus","v":0.203},{"b":"cl_bench","cfg":{"effort":"thinking"},"dn":"Qwen 3.5 Plus (thinking)","e":"thinking","m":"qwen3-5-plus","oid":"cl_bench-339647d3e2f7ca1b","rd":"2026-02-16","sd":0,"sid":"rect5z4YBbZG9s8cs","sm":"qwen3.5-plus","v":0.198},{"b":"cl_bench","dn":"Kimi K2.5","m":"kimi-k2-5","oid":"cl_bench-22795e4d781e3cf3","rd":"2026-01-27","sd":0,"sid":"recWjy9h3wfHdIktv","sm":"kimi-k2.5","v":0.193},{"b":"cl_bench","dn":"GLM 5.0","m":"glm-5","oid":"cl_bench-ff44bfb1b3bca82d","rd":"2026-02-11","sd":0,"sid":"recRqKIsEVV6JXeQp","sm":"glm-5","v":0.187},{"b":"cl_bench","dn":"GPT 5.2","m":"gpt-5-2-2025-12-11","oid":"cl_bench-d47875db53551f13","rd":"2025-12-11","sd":0.005,"sid":"recey2jawoJyDioup","sm":"gpt-5.2-2025-12-11_unknown","v":0.182},{"b":"cl_bench","cfg":{"effort":"high"},"dn":"GPT 5.2 (High)","e":"high","m":"gpt-5-2-2025-12-11","oid":"cl_bench-e2cbd360e485f089","rd":"2025-12-11","sd":0.008,"sid":"recLTWNGyZ0seKudZ","sm":"gpt-5.2-2025-12-11_high","v":0.181},{"b":"cl_bench","cfg":{"effort":"high"},"dn":"o3 (High)","e":"high","m":"o3-2025-04-16","oid":"cl_bench-6f2f8219b58c559e","rd":"2025-04-16","sd":0.002,"sid":"reccVub2zmLp7euhf","sm":"o3-2025-04-16_high","v":0.17800000000000002},{"b":"cl_bench","cfg":{"effort":"thinking"},"dn":"Kimi K2 Thinking","e":"thinking","m":"moonshotaikimi-k2-thinking","oid":"cl_bench-3d33934575d17c7a","rd":"2025-11-06","sd":0.006,"sid":"recTf4BEBqZlUcV8G","sm":"moonshotai/Kimi-K2-Thinking","v":0.176},{"b":"cl_bench","dn":"GLM 4.7","m":"zai-orgglm-4-7","oid":"cl_bench-d68f3b24146ea73f","rd":"2025-12-22","sd":0,"sid":"reczKXbGAHjIRQ6cI","sm":"zai-org/glm-4.7","v":0.159},{"b":"cl_bench","cfg":{"effort":"high"},"dn":"Gemini 3 Pro (High)","e":"high","m":"gemini-3-pro-preview","oid":"cl_bench-05ed49deadba228d","rd":"2025-11-18","sd":0.003,"sid":"recWhJzlCOU367ohN","sm":"gemini-3-pro-preview","v":0.158},{"b":"cl_bench","dn":"MiMo V2 Pro","m":"mimo-v2-pro","oid":"cl_bench-02530ddcaa8beb3b","sd":0,"sid":"recxv85zQUCC6fGH8","sm":"mimo-v2-pro","v":0.157},{"b":"cl_bench","cfg":{"effort":"max"},"dn":"Qwen 3 Max","e":"max","m":"qwen3-max-2025-09-23","oid":"cl_bench-af9adc55056a989f","rd":"2025-09-24","sd":0.006,"sid":"recAjmVPbaPboFz0Y","sm":"qwen3-max-2025-09-23","v":0.145},{"b":"cl_bench","cfg":{"effort":"thinking"},"dn":"DeepSeek V3.2 Thinking","e":"thinking","m":"deepseek-v3-2-exp","oid":"cl_bench-56aa2df989040e08","rd":"2025-09-29","sd":0.004,"sid":"recFRchEtTpijiHzY","sm":"DeepSeek-V3.2-Exp_thinking","v":0.132},{"b":"cl_bench","dn":"DeepSeek V3.2","m":"deepseekdeepseek-v3-2","oid":"cl_bench-b552826a3413f2d3","rd":"2025-12-01","sd":0.005,"sid":"reciJ4L8W7oaNa5rF","sm":"deepseek/deepseek-v3.2","v":0.12400000000000001},{"b":"cl_bench","dn":"Kimi K2","m":"kimi-k2-thinking","oid":"cl_bench-9498f75c7ed50ef9","rd":"2025-11-06","sd":0.005,"sid":"recqE13ghQXf6Zvmm","sm":"kimi-k2-thinking","v":0.119},{"b":"cl_bench","dn":"MiniMax M2.5","m":"minimax-m2-5","oid":"cl_bench-d819c517508b54bb","rd":"2026-02-12","sd":0,"sid":"recDfs2bwO8BnfpYc","sm":"MiniMax-M2.5","v":0.114},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"GPT 5.5 (High)","e":"high","m":"gpt-5-5","oid":"cl_bench_life-f424a7f745454179","rd":"2026-04-23","sd":0.005,"sid":"recg5QiqL5ur0TMP6","sm":"gpt-5.5_high","v":0.222},{"b":"cl_bench_life","cfg":{"effort":"xhigh"},"dn":"GPT 5.4 (xHigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"cl_bench_life-9781e33787b3f8c1","rd":"2026-03-05","sd":0,"sid":"recNrp7ZrTpzx4Mgj","sm":"gpt-5.4-2026-03-05_xhigh","v":0.217},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"GPT 5.4 (High)","e":"high","m":"gpt-5-4-2026-03-05","oid":"cl_bench_life-e4093a86a376ea1b","rd":"2026-03-05","sd":0.005,"sid":"recMIYYtJTzv6SCNJ","sm":"gpt-5.4-2026-03-05_high","v":0.193},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"GPT 5.1 (High)","e":"high","m":"gpt-5-1-2025-11-13","oid":"cl_bench_life-f87c351e3a347ab2","rd":"2025-11-13","sd":0,"sid":"recCojnbwwVIS85Fs","sm":"gpt-5.1-2025-11-13_high","v":0.173},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"Claude Opus 4.6 (High)","e":"high","m":"claude-opus-4-6","oid":"cl_bench_life-cb787ff3619cc0b6","rd":"2026-02-05","sd":0.013000000000000001,"sid":"recteQfW5CfQPwtaY","sm":"claude-opus-4-6_high","v":0.17},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"Gemini 3.1 Pro (High)","e":"high","m":"gemini-3-1-pro-preview","oid":"cl_bench_life-20ee8aa6b14a6701","rd":"2026-02-19","sd":0.012,"sid":"recXnEvPXtnJGxu0B","sm":"gemini-3.1-pro-preview","v":0.169},{"b":"cl_bench_life","dn":"GPT 5.4","m":"gpt-5-4-2026-03-05","oid":"cl_bench_life-d18bd465065ae5d9","rd":"2026-03-05","sd":0,"sid":"rectS8ABFjpDFk5jA","sm":"gpt-5.4-2026-03-05_unknown","v":0.138},{"b":"cl_bench_life","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"cl_bench_life-8b65a61e56880e57","rd":"2026-02-05","sd":0,"sid":"recHnGNtXYiffKcWS","sm":"claude-opus-4-6_unknown","v":0.136},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"DeepSeek V4 Pro (High)","e":"high","m":"deepseek-v4-pro","oid":"cl_bench_life-bf4eec23c7388d3e","rd":"2026-04-24","sd":0.013999999999999999,"sid":"receUba4SjxSFW7n9","sm":"deepseek-v4-pro_high","v":0.135},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"Kimi K2.5 (High)","e":"high","m":"kimi-k2-5","oid":"cl_bench_life-46a04bb60d88c223","rd":"2026-01-27","sd":0.008,"sid":"rec9D98JysK1Sczay","sm":"kimi-k2.5","v":0.132},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"Qwen 3.5 Plus (High)","e":"high","m":"qwen3-5-plus","oid":"cl_bench_life-68a3e5f1c107c829","rd":"2026-02-16","sd":0.001,"sid":"rec634ax46X5sSTPm","sm":"qwen3.5-plus","v":0.12400000000000001},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"Grok 4.20 (High)","e":"high","m":"grok-4-20","oid":"cl_bench_life-3ec58eac2e3d21ff","rd":"2026-02-17","sd":0.004,"sid":"recb8nzTuuGW1x5wl","sm":"grok-4-20","v":0.119},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"GLM-4.7 (High)","e":"high","m":"zai-orgglm-4-7","oid":"cl_bench_life-0f2d5986ffae998e","rd":"2025-12-22","sd":0,"sid":"rec8ofxVXDYvOH6Mf","sm":"zai-org/glm-4.7","v":0.109},{"b":"cl_bench_life","cfg":{"effort":"thinking"},"dn":"DeepSeek V3.2 Thinking","e":"thinking","m":"deepseek-v3-2-exp","oid":"cl_bench_life-6d4c3a2eb330783f","rd":"2025-09-29","sd":0.004,"sid":"rec7DtKf0ASL4g5K9","sm":"DeepSeek-V3.2-Exp_thinking","v":0.095},{"b":"cl_bench_life","dn":"DeepSeek V3.2","m":"deepseekdeepseek-v3-2","oid":"cl_bench_life-5d5a55fe97d9916c","rd":"2025-12-01","sd":0,"sid":"recx2jDLimazkXNNc","sm":"deepseek/deepseek-v3.2","v":0.074},{"b":"cl_bench_life","cfg":{"effort":"high"},"dn":"MiMo v2 Pro (High)","e":"high","m":"mimo-v2-pro","oid":"cl_bench_life-e85bf6dae3c8f537","sd":0,"sid":"rec6PuXyRy8YfxMVo","sm":"mimo-v2-pro","v":0.069},{"b":"cl_bench_life","cfg":{"effort":"thinking"},"dn":"MiniMax M2.5 (Thinking)","e":"thinking","m":"minimax-m2-5","oid":"cl_bench_life-8cd1f56818308689","rd":"2026-02-12","sd":0.01,"sid":"recsyFLtORK2PIRNY","sm":"MiniMax-M2.5","v":0.063},{"b":"common_sense_qa_2","dn":"GPT-3","m":"text-davinci-001","oid":"common_sense_qa_2-dd87ca43baf96f51","rd":"2022-01-27","sid":"rec2N4ugHyIuFFHCg","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.05320","v":0.529},{"b":"common_sense_qa_2","dn":"T5-Large","m":"t5-large","oid":"common_sense_qa_2-7de394ae0807f660","sid":"recXRfZ560YNIjqvj","sm":"T5-Large","src":"https://arxiv.org/pdf/2201.05320","v":0.546},{"b":"common_sense_qa_2","dn":"T5-11B","m":"t5-11b","oid":"common_sense_qa_2-a8e067b47eac03c0","sid":"recjb7oltbflJgcnj","sm":"T5-11B","src":"https://arxiv.org/pdf/2201.05320","v":0.678},{"b":"common_sense_qa_2","dn":"T5-3B","m":"t5-3b","oid":"common_sense_qa_2-d61bb9f77b8d526c","sid":"recDaVPKR6YwbKmLZ","sm":"T5-3B","src":"https://arxiv.org/abs/2305.05921","v":0.602},{"b":"common_sense_qa_2","cfg":{"shots":"few"},"dn":"GPT-3.5 Turbo","m":"gpt-3-5-turbo-0613","notes":"Assuming original 3.5-turbo release","oid":"common_sense_qa_2-0a6871ddefb5dcb5","rd":"2023-06-13","sid":"recLJTHqISFdhqJwy","sm":"gpt-3.5-turbo-0613","src":"https://arxiv.org/pdf/2309.13165v1","v":0.57},{"b":"common_sense_qa_2","cfg":{"shots":"few"},"dn":"LLama-70B","m":"llama-2-70b-hf","notes":"Assuming LLAMA-70b was LLAMA 2","oid":"common_sense_qa_2-11bc6bc434cfe01f","rd":"2023-07-18","sid":"recdvV2jdEN8V6o76","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2309.13165v1","v":0.5},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 4 Carbon (High)","e":"high","m":"gemini-4-carbon","oid":"critpt-a9ad76959fd851e1","rd":"2026-10-28","sid":"rec6qQGC98m308hd4","sm":"gemini-4-carbon_high","v":0.36},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-5.6 Sol (Max)","e":"max","m":"gpt-5-6-sol","oid":"critpt-3b23186e8aec2d82","rd":"2026-07-09","sid":"recok7WFznQiAaI15","sm":"gpt-5.6-sol_max","v":0.322857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-6 Astra (Max)","e":"max","m":"gpt-6-astra","oid":"critpt-d2c2d9142542ad10","rd":"2026-09-03","sid":"recchwRSFo4NJR8y2","sm":"gpt-6-astra_max","v":0.317142857142857},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Opus 5.5 (Max, Default Fallback)","e":"max","m":"claude-opus-5-5","oid":"critpt-e8d66575ce8bd7e4","rd":"2026-09-22","sid":"recCMhoGESWuXr0FY","sm":"claude-opus-5-5_max","v":0.317142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Opus 5.5 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-opus-5-5","oid":"critpt-12e8c3d9b4065bf3","rd":"2026-09-22","sid":"rec3ntl7wmky8zgSD","sm":"claude-opus-5-5_xhigh","v":0.317142857142857},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-6.1 Sol (Max)","e":"max","m":"gpt-6-1-sol","oid":"critpt-a09d0b47916d167d","rd":"2026-09-29","sid":"rec2iiVrG9dijlo97","sm":"gpt-6.1-sol_max","v":0.317142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-6.1 Sol (Xhigh)","e":"xhigh","m":"gpt-6-1-sol","oid":"critpt-207e6e592c3e757f","rd":"2026-09-29","sid":"recuC1By9ytTjuaq8","sm":"gpt-6.1-sol_xhigh","v":0.317142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-6 Astra (Xhigh)","e":"xhigh","m":"gpt-6-astra","oid":"critpt-87bb5b1a5adf5288","rd":"2026-09-03","sid":"recONBqYxzUiCO8Iy","sm":"gpt-6-astra_xhigh","v":0.314285714285714},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Sonnet 5.5 (Max, Default Fallback)","e":"max","m":"claude-sonnet-5-5","oid":"critpt-b8f949b24767bd06","rd":"2026-09-28","sid":"recIMNF7yjEmzEOf3","sm":"claude-sonnet-5-5_max","v":0.314285714285714},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Fable 5.1 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-fable-5-1","oid":"critpt-f14a7d803adca7ac","rd":"2026-09-01","sid":"recDxO947Y2IBUxq1","sm":"claude-fable-5-1_xhigh","v":0.311428571428571},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Sonnet 5.5 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-sonnet-5-5","oid":"critpt-16722c089d9ced4c","rd":"2026-09-28","sid":"recYZxN3FPdlvh4L3","sm":"claude-sonnet-5-5_xhigh","v":0.311428571428571},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-6 Sol (Max)","e":"max","m":"gpt-6-sol","oid":"critpt-ca4edbcda9a779f1","rd":"2026-09-22","sid":"recryI0wi0k0mknzo","sm":"gpt-6-sol_max","v":0.308571428571429},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Opus 5.5 (High, Default Fallback)","e":"high","m":"claude-opus-5-5","oid":"critpt-d480abd84a4f2ed7","rd":"2026-09-22","sid":"recKCd0FGhGShnckc","sm":"claude-opus-5-5_high","v":0.308571428571429},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 Pro (Xhigh)","e":"xhigh","m":"gpt-5-5-pro","oid":"critpt-d4bf7c0895d7f187","rd":"2026-04-23","sid":"recB9otf2Iy1aL15O","sm":"gpt-5.5-pro_xhigh","v":0.305714285714286},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Fable 5.1 (High, Default Fallback)","e":"high","m":"claude-fable-5-1","oid":"critpt-551361ffb88f4366","rd":"2026-09-01","sid":"recdIJdHkHz5QHHB8","sm":"claude-fable-5-1_high","v":0.302857142857143},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 Pro (Xhigh)","e":"xhigh","m":"gpt-5-4-pro-2026-03-05","oid":"critpt-1582634c5e2d559e","rd":"2026-03-05","sid":"recpj4aTRQU6UUqUg","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.3},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-5.6 Terra (Max)","e":"max","m":"gpt-5-6-terra","oid":"critpt-c834050b70965f85","rd":"2026-07-09","sid":"rec5E8CdC6AxpPjGM","sm":"gpt-5.6-terra_max","v":0.3},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-6.1 Sol (High)","e":"high","m":"gpt-6-1-sol","oid":"critpt-eff7b6585b9a88a4","rd":"2026-09-29","sid":"recQzne2TYYeL4phc","sm":"gpt-6.1-sol_high","v":0.3},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Fable 5.1 (Max, Default Fallback)","e":"max","m":"claude-fable-5-1","oid":"critpt-d3bded0b47585214","rd":"2026-09-01","sid":"reclr0DLMwnSAaUs2","sm":"claude-fable-5-1_max","v":0.297142857142857},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-5","oid":"critpt-79b86f73034922f6","rd":"2026-07-24","sid":"rec5C4zdQpZvERrGC","sm":"claude-opus-5_max","v":0.291428571428571},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Fable 5.1 (Medium, Default Fallback)","e":"medium","m":"claude-fable-5-1","oid":"critpt-d91f17b726519857","rd":"2026-09-01","sid":"recZfmbm2weC0g1KH","sm":"claude-fable-5-1_medium","v":0.291428571428571},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-6 Astra (Medium)","e":"medium","m":"gpt-6-astra","oid":"critpt-0060df5e0314689b","rd":"2026-09-03","sid":"recnhCs1Rmx6NBPl6","sm":"gpt-6-astra_medium","v":0.291428571428571},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-6 Astra (High)","e":"high","m":"gpt-6-astra","oid":"critpt-e07a84f0f5f60fae","rd":"2026-09-03","sid":"rechVO3SVbWWqQQjo","sm":"gpt-6-astra_high","v":0.288571428571429},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol (Xhigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"critpt-fc0e8a7f9c614f79","rd":"2026-07-09","sid":"recCCEqC3wisl7qU4","sm":"gpt-5.6-sol_xhigh","v":0.285714285714286},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","e":"max","m":"claude-fable-5","oid":"critpt-143b8940893e8a94","rd":"2026-06-09","sid":"recVfxdxUtc8D2L0n","sm":"claude-fable-5_max","v":0.2857142857},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Opus 5 (Adaptive Reasoning, High Effort)","e":"high","m":"claude-opus-5","oid":"critpt-aadf0ed62f756829","rd":"2026-07-24","sid":"recQhcpUHUieJGpId","sm":"claude-opus-5_high","v":0.282857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"Muse Wonder (Max)","e":"max","m":"muse-wonder-1-0","oid":"critpt-e419929d27c06d69","rd":"2026-10-12","sid":"recq3eztuBfhsowDH","sm":"muse-wonder-1.0_max","v":0.282857142857143},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-6 Sol (Xhigh)","e":"xhigh","m":"gpt-6-sol","oid":"critpt-f79858a89a7406c1","rd":"2026-09-22","sid":"rec0MPvJkpAnRaw41","sm":"gpt-6-sol_xhigh","v":0.28},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-6.1 Terra (Max)","e":"max","m":"gpt-6-1-terra","oid":"critpt-92474914b6d37fb3","rd":"2026-10-26","sid":"rec9uQLMwE0DMdWIk","sm":"gpt-6.1-terra_max","v":0.28},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)","e":"xhigh","m":"claude-opus-5","oid":"critpt-3c475258e82ec963","rd":"2026-07-24","sid":"recqe0FCEPFJX8Adc","sm":"claude-opus-5_xhigh","v":0.277142857142857},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Fable 5.1 (Low, Default Fallback)","e":"low","m":"claude-fable-5-1","oid":"critpt-1bd41e03d2352960","rd":"2026-09-01","sid":"rec2RJ8EQ2qMVtCE8","sm":"claude-fable-5-1_low","v":0.277142857142857},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-6.1 Sol (Medium)","e":"medium","m":"gpt-6-1-sol","oid":"critpt-af604d16567dc864","rd":"2026-09-29","sid":"recAlv9eC6ITOjyIn","sm":"gpt-6.1-sol_medium","v":0.277142857142857},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Opus 5.5 (Medium, Default Fallback)","e":"medium","m":"claude-opus-5-5","oid":"critpt-883121bb2e9e1989","rd":"2026-09-22","sid":"recBDknvezsLl5kHO","sm":"claude-opus-5-5_medium","v":0.277142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.5 (Xhigh)","e":"xhigh","m":"gpt-5-5","oid":"critpt-0f4a3f2ee808c016","rd":"2026-04-23","sid":"rec7ob5La2ExbJeLM","sm":"gpt-5.5_xhigh","v":0.271428571428571},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra (Xhigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"critpt-2a9cee8a4c267413","rd":"2026-07-09","sid":"recVghFQgXkxpNwPc","sm":"gpt-5.6-terra_xhigh","v":0.271428571428571},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 4 Argon (High)","e":"high","m":"gemini-4-argon","oid":"critpt-ea4b3e6ab6c171da","rd":"2026-09-30","sid":"recpQOevVr4QIAPTK","sm":"gemini-4-argon_high","v":0.271428571428571},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)","e":"medium","m":"claude-opus-5","oid":"critpt-cf24d39d2fa9bef1","rd":"2026-07-24","sid":"recS9jY80Y6pwM6mV","sm":"claude-opus-5_medium","v":0.268571428571429},{"b":"critpt","dn":"MiMo-V2.6-Pro","m":"mimo-v2-6-pro","oid":"critpt-32569b2b40ffbb2a","rd":"2026-09-21","sid":"recU4tvMX9uI4D6rE","sm":"mimo-v2.6-pro","v":0.265714285714286},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-6.1 Terra (Xhigh)","e":"xhigh","m":"gpt-6-1-terra","oid":"critpt-402531a585a23622","rd":"2026-10-26","sid":"recLScFg6RmweUXw8","sm":"gpt-6.1-terra_xhigh","v":0.265714285714286},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-6 Astra (Low)","e":"low","m":"gpt-6-astra","oid":"critpt-a6861902eff32ed1","rd":"2026-09-03","sid":"recwsQ3t9s5hZ7bEI","sm":"gpt-6-astra_low","v":0.262857142857143},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Muse Spark 1.3 (Xhigh)","e":"xhigh","m":"muse-spark-1-3","oid":"critpt-24975a434ff4c1f6","rd":"2026-09-02","sid":"recnpxvLc3yRjYTCS","sm":"muse-spark-1.3_xhigh","v":0.26},{"b":"critpt","dn":"Gemini 3 Deep Think","m":"gemini-3-deep-think-preview","oid":"critpt-015564c2a5fcb022","rd":"2026-02-12","sid":"recq3pI3ZzF2fZiaI","sm":"gemini-3-deep-think-preview","v":0.257142857142857},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-5.6 Sol (High)","e":"high","m":"gpt-5-6-sol","oid":"critpt-36e66339c98c6828","rd":"2026-07-09","sid":"recrwnRfpJKrmslqN","sm":"gpt-5.6-sol_high","v":0.257142857142857},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-6.1 Terra (High)","e":"high","m":"gpt-6-1-terra","oid":"critpt-056edea9b7fbdb4c","rd":"2026-10-26","sid":"rechZa540NCHEYGdk","sm":"gpt-6.1-terra_high","v":0.257142857142857},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-5.5 (High)","e":"high","m":"gpt-5-5","oid":"critpt-59100169113c57a6","rd":"2026-04-23","sid":"recLoNQxQteSxrTSI","sm":"gpt-5.5_high","v":0.254285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-6 Sol (High)","e":"high","m":"gpt-6-sol","oid":"critpt-f03e92aea7a93196","rd":"2026-09-22","sid":"recwmk2GM9CClCJWE","sm":"gpt-6-sol_high","v":0.254285714285714},{"b":"critpt","cfg":{"effort":"max"},"dn":"Muse Spark 1.3 (Max)","e":"max","m":"muse-spark-1-3","oid":"critpt-25b881eeb9c6c149","rd":"2026-09-02","sid":"recWjWRh1an30zEd3","sm":"muse-spark-1.3_max","v":0.248571428571429},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-6.1 Sol (Low)","e":"low","m":"gpt-6-1-sol","oid":"critpt-fc639233e130177e","rd":"2026-09-29","sid":"reccnVv4LeJz0hsUh","sm":"gpt-6.1-sol_low","v":0.248571428571429},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Sonnet 5.5 (High, Default Fallback)","e":"high","m":"claude-sonnet-5-5","oid":"critpt-aa262215185ed65a","rd":"2026-09-28","sid":"recvQbjd5oXpVCYX7","sm":"claude-sonnet-5-5_high","v":0.245714285714286},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-6 Sol (Medium)","e":"medium","m":"gpt-6-sol","oid":"critpt-7c334a5316a90f7b","rd":"2026-09-22","sid":"recTlQ7nNgZFAwB8s","sm":"gpt-6-sol_medium","v":0.245714285714286},{"b":"critpt","dn":"Kimi K3.1 Pro","m":"kimi-k3-1-pro","oid":"critpt-8c0809e7ec4abc80","rd":"2026-10-23","sid":"recoLyC8auNcojeiT","sm":"kimi-k3.1-pro_unknown","v":0.245714285714286},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash 1017 (Max)","e":"max","m":"deepseek-v4-1-flash-1017","oid":"critpt-f1331180edc2793a","rd":"2026-10-17","sid":"rec1k4psulmy4GUPu","sm":"deepseek-v4.1-flash-1017_max","v":0.24},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 (xhigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"critpt-79cc620e310f355d","rd":"2026-03-05","sid":"recRZ2byfLFxfUg3P","sm":"gpt-5.4-2026-03-05_xhigh","v":0.23428571430000003},{"b":"critpt","cfg":{"effort":"max"},"dn":"Kimi K3","e":"max","m":"kimi-k3","oid":"critpt-7b7d4c09648b845b","rd":"2026-07-16","sid":"recQkvzVLQfVkOHT4","sm":"kimi-k3_max","v":0.234},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Opus 5 (Adaptive Reasoning, Low Effort)","e":"low","m":"claude-opus-5","oid":"critpt-313334197dc3fb71","rd":"2026-07-24","sid":"reckQnoYBUwhFIUoh","sm":"claude-opus-5_low","v":0.231428571428571},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-5.6 Terra (High)","e":"high","m":"gpt-5-6-terra","oid":"critpt-b4699d33cfcf4671","rd":"2026-07-09","sid":"rec5NRGXYI5SJra1r","sm":"gpt-5.6-terra_high","v":0.228571428571429},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-5.6 Sol (Medium)","e":"medium","m":"gpt-5-6-sol","oid":"critpt-440fd67361245d56","rd":"2026-07-09","sid":"recBeh9GpOB82lJDU","sm":"gpt-5.6-sol_medium","v":0.228571428571429},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Haiku 5.5 (Xhigh)","e":"xhigh","m":"claude-haiku-5-5","oid":"critpt-a33a8ad79d7bd5d3","rd":"2026-10-07","sid":"recBvV1GipGKHNyKH","sm":"claude-haiku-5-5_xhigh","v":0.225714285714286},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-6.1 Terra (Medium)","e":"medium","m":"gpt-6-1-terra","oid":"critpt-6a95fb12ed2755da","rd":"2026-10-26","sid":"recYqnWLNUmJX7xAf","sm":"gpt-6.1-terra_medium","v":0.214285714285714},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Opus 4.8 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-4-8","oid":"critpt-2aac4713d4280ed8","rd":"2026-05-28","sid":"recXxyRL6OkDzcqI8","sm":"claude-opus-4-8_max","v":0.20857142857142902},{"b":"critpt","cfg":{"effort":"max"},"dn":"GLM-5.2 (Max)","e":"max","m":"glm-5-2","oid":"critpt-8560421ae9224ee2","rd":"2026-06-16","sid":"reccz0GyjqLGPM3cH","sm":"glm-5.2_max","v":0.20857142857142902},{"b":"critpt","dn":"Step 5 Preview","m":"step-5-preview","oid":"critpt-c87f64819bb0dda5","rd":"2026-09-18","sid":"recBlAKrqitdTBlYA","sm":"step-5-preview_unknown","v":0.20857142857142902},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-5.6 Luna (Max)","e":"max","m":"gpt-5-6-luna","oid":"critpt-50118f03db758537","rd":"2026-07-09","sid":"recOr7SfqxNxcZELE","sm":"gpt-5.6-luna_max","v":0.205714285714286},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna (Xhigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"critpt-9d840bd99055290f","rd":"2026-07-09","sid":"recaayE1vDGtpM1wz","sm":"gpt-5.6-luna_xhigh","v":0.205714285714286},{"b":"critpt","dn":"Qwen3.8 Max","m":"qwen3-8-max","oid":"critpt-4ce32c68e6cefeb2","rd":"2026-08-02","sid":"recfs3SLfMeRAbppt","sm":"qwen3.8-max_unknown","v":0.2},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Grok 4.6 (Xhigh)","e":"xhigh","m":"grok-4-6","oid":"critpt-57b4f8a46931b139","rd":"2026-08-12","sid":"rec6XIxvRHFvIHFn1","sm":"grok-4.6_xhigh","v":0.197142857142857},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-6 Astra (Non-reasoning)","e":"none","m":"gpt-6-astra","oid":"critpt-eb3f93116f1e8e32","rd":"2026-09-03","sid":"recFv6W9bqROZOfW9","sm":"gpt-6-astra_none","v":0.197142857142857},{"b":"critpt","cfg":{"effort":"max"},"dn":"GPT-6 Luna (Max)","e":"max","m":"gpt-6-luna","oid":"critpt-6727e9f3b0b0adca","rd":"2026-09-22","sid":"recKcBHXINlji9VSJ","sm":"gpt-6-luna_max","v":0.19428571428571403},{"b":"critpt","cfg":{"effort":"max"},"dn":"GLM-5.3 (Max)","e":"max","m":"glm-5-3","oid":"critpt-6cf9ead868a7e120","rd":"2026-08-14","sid":"recLWpumgBpzZU4y1","sm":"glm-5.3_max","v":0.191428571428571},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Haiku 5.5 (Max)","e":"max","m":"claude-haiku-5-5","oid":"critpt-aa9665674fd4ded3","rd":"2026-10-07","sid":"recW9LZskaNg7hjRz","sm":"claude-haiku-5-5_max","v":0.188571428571429},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-5.5 (Medium)","e":"medium","m":"gpt-5-5","oid":"critpt-b09af6f94f635a0b","rd":"2026-04-23","sid":"rec9tobYSj0aIDyTz","sm":"gpt-5.5_medium","v":0.18571428571428603},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Haiku 5.5 (High)","e":"high","m":"claude-haiku-5-5","oid":"critpt-2701f05f447c823e","rd":"2026-10-07","sid":"rec0BoOLNB0KlFYZ7","sm":"claude-haiku-5-5_high","v":0.18571428571428603},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 3.8 Flash (High)","e":"high","m":"gemini-3-8-flash","oid":"critpt-8280b8a54f33c888","rd":"2026-09-02","sid":"recKtVPT8wSqnhvJT","sm":"gemini-3.8-flash_high","v":0.182857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4 Pro 0813 (Max)","e":"max","m":"deepseek-v4-pro-0813","oid":"critpt-42f7a4b1f7a94600","rd":"2026-08-13","sid":"recAmxzNixBWcVrYv","sm":"deepseek-v4-pro-0813_max","v":0.18},{"b":"critpt","cfg":{"effort":"high"},"dn":"Grok 4.7 (High)","e":"high","m":"grok-4-7","oid":"critpt-37d9922c0cb7f94e","rd":"2026-09-21","sid":"rec308gC8B3oEudLd","sm":"grok-4.7_high","v":0.18},{"b":"critpt","dn":"Gemini 3.1 Pro Preview","m":"gemini-3-1-pro-preview","oid":"critpt-9636cff1e5f19463","rd":"2026-02-19","sid":"recjxJ5hAbmtjQIIn","sm":"gemini-3.1-pro-preview","v":0.177142857142857},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Grok 4.6 (Medium)","e":"medium","m":"grok-4-6","oid":"critpt-08ec153492dc649c","rd":"2026-08-12","sid":"recLIPfWFb1YpaMHA","sm":"grok-4.6_medium","v":0.177142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Muse Spark 1.2 (Xhigh)","e":"xhigh","m":"muse-spark-1-2","oid":"critpt-e01ddb8f8bfe5ff8","rd":"2026-08-05","sid":"recZlkudytfNTJjHh","sm":"muse-spark-1.2_xhigh","v":0.177142857142857},{"b":"critpt","dn":"Qwen3.8 Max (0902)","m":"qwen3-8-max-0902","oid":"critpt-e51d8b9004f020cf","rd":"2026-09-01","sid":"recFmHGKzOkLYr23L","sm":"qwen3.8-max-0902_unknown","v":0.177142857142857},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Grok 4.7 (Xhigh)","e":"xhigh","m":"grok-4-7","oid":"critpt-f0e32d1de4bb39b2","rd":"2026-09-21","sid":"recOgLdIJweeu6Ha0","sm":"grok-4.7_xhigh","v":0.177142857142857},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Opus 5.5 (Low, Default Fallback)","e":"low","m":"claude-opus-5-5","oid":"critpt-59db38874fc0db14","rd":"2026-09-22","sid":"recVdUMQOOxAmGoKj","sm":"claude-opus-5-5_low","v":0.177142857142857},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-5.6 Terra (Medium)","e":"medium","m":"gpt-5-6-terra","oid":"critpt-914f663db2a23028","rd":"2026-07-09","sid":"recd5XH0K1FBEWW6T","sm":"gpt-5.6-terra_medium","v":0.174285714285714},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-6 Luna (Xhigh)","e":"xhigh","m":"gpt-6-luna","oid":"critpt-366e0a9b47ed8052","rd":"2026-09-22","sid":"recbxqWS7BcKj9AEN","sm":"gpt-6-luna_xhigh","v":0.174285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"Grok 4.6 (High)","e":"high","m":"grok-4-6","oid":"critpt-2db7ce4a4e4a58f7","rd":"2026-08-12","sid":"rec7IzR3Isff3P54Q","sm":"grok-4.6_high","v":0.171428571428571},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-sonnet-5","oid":"critpt-13c9019ac9a7cf84","rd":"2026-06-30","sid":"reczNcIVusSshoSMi","sm":"claude-sonnet-5_max","v":0.168571428571429},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Sonnet 5.5 (Medium, Default Fallback)","e":"medium","m":"claude-sonnet-5-5","oid":"critpt-05815dbb53a6b6fc","rd":"2026-09-28","sid":"recw20wmrUkfYehBO","sm":"claude-sonnet-5-5_medium","v":0.168571428571429},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.3 Codex (Xhigh)","e":"xhigh","m":"gpt-5-3-codex","oid":"critpt-6bb3484928c5ef0a","rd":"2026-02-05","sid":"recXG78uM89aZKW5U","sm":"gpt-5.3-codex_xhigh","v":0.168571428571429},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-5.6 Luna (High)","e":"high","m":"gpt-5-6-luna","oid":"critpt-d3aeee31ef3463b7","rd":"2026-07-09","sid":"recAl58eDFCqhYed8","sm":"gpt-5.6-luna_high","v":0.165714285714286},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-flash-0731","oid":"critpt-c2f045773aabb49d","rd":"2026-07-31","sid":"rec9qGfVvzMRr1hcF","sm":"deepseek-v4-flash-0731_max","v":0.165714285714286},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-6 Sol (Low)","e":"low","m":"gpt-6-sol","oid":"critpt-bc3c668cca6c8905","rd":"2026-09-22","sid":"recP8qQjotNOrlO7W","sm":"gpt-6-sol_low","v":0.162857142857143},{"b":"critpt","cfg":{"effort":"high"},"dn":"Grok 4.5 (High)","e":"high","m":"grok-4-5","oid":"critpt-3ec00b794c28eda3","rd":"2026-07-08","sid":"rec9SPFHtXmSqfBHy","sm":"grok-4.5_high","v":0.154285714285714},{"b":"critpt","dn":"GLM-5.3-Flash","m":"glm-5-3-flash","oid":"critpt-3a29afb81a0e86d1","rd":"2026-08-20","sid":"recTSPwVVUhGNlW9X","sm":"glm-5.3-flash_unknown","v":0.154285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-6 Luna (High)","e":"high","m":"gpt-6-luna","oid":"critpt-5071c4ef62db0859","rd":"2026-09-22","sid":"recWQBMbwTm6vLW0h","sm":"gpt-6-luna_high","v":0.154285714285714},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)","e":"xhigh","m":"claude-sonnet-5","oid":"critpt-be254a1b7cc8ac3e","rd":"2026-06-30","sid":"recj6KALtFlQIwjyc","sm":"claude-sonnet-5_xhigh","v":0.154285714285714},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Muse Spark 1.1 (Xhigh)","e":"xhigh","m":"muse-spark-1-1","oid":"critpt-e7c455f40ec930ab","rd":"2026-07-09","sid":"rec6dA1Ulc6YPFHdT","sm":"muse-spark-1.1_xhigh","v":0.151428571428571},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-6.1 Terra (Low)","e":"low","m":"gpt-6-1-terra","oid":"critpt-ec2846ab6b3d7aca","rd":"2026-10-26","sid":"recYSoziadIBCFPMA","sm":"gpt-6.1-terra_low","v":0.151428571428571},{"b":"critpt","dn":"Muse Spark 1.1","m":"muse-spark-1-1","oid":"critpt-7e022c1b28f8a9b0","rd":"2026-07-09","sid":"rec6l66FR3fDrLFJR","sm":"muse-spark-1.1","v":0.151},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-5.6 Sol (Low)","e":"low","m":"gpt-5-6-sol","oid":"critpt-e8d9228d6fd7a6f8","rd":"2026-07-09","sid":"reciJ89y9SSL8MjqP","sm":"gpt-5.6-sol_low","v":0.148571428571429},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Muse Spark Open (Xhigh)","e":"xhigh","m":"muse-spark-open-816b-a42b","oid":"critpt-2e79c0f38441b10c","rd":"2026-10-15","sid":"reco5BBRVh218jtrZ","sm":"Muse-Spark-Open-816B-A42B_xhigh","v":0.148571428571429},{"b":"critpt","cfg":{"effort":"low"},"dn":"GLM-5.3 (Low)","e":"low","m":"glm-5-3","oid":"critpt-c2778517465e6ec0","rd":"2026-08-14","sid":"recL8ApuHU4h00JJU","sm":"glm-5.3_low","v":0.145714285714286},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 3.7 Flash (High)","e":"high","m":"gemini-3-7-flash","oid":"critpt-07e7e77d82355e87","rd":"2026-08-13","sid":"recUWOMKzoTyIZBwR","sm":"gemini-3.7-flash_high","v":0.142857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-1-flash","oid":"critpt-9bce214555d0dd1b","rd":"2026-09-09","sid":"recWbpaQ6RRJBEsbG","sm":"deepseek-v4.1-flash_max","v":0.142857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"Qwen3.7 Max","e":"max","m":"qwen3-7-max","oid":"critpt-b563f418e90e1510","rd":"2026-05-19","sid":"recrFEli7KIbnH2Id","sm":"qwen3.7-max","v":0.134285714285714},{"b":"critpt","cfg":{"effort":"low"},"dn":"Grok 4.7 (Low)","e":"low","m":"grok-4-7","oid":"critpt-b3c79c063b8fc9f9","rd":"2026-09-21","sid":"recUDw9uLW6OUC9te","sm":"grok-4.7_low","v":0.134285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 3.5 Flash (High)","e":"high","m":"gemini-3-5-flash","oid":"critpt-9ed34f55a92d32b0","rd":"2026-05-19","sid":"recCV91VfXHxJp3bC","sm":"gemini-3.5-flash_high","v":0.131428571428571},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4 Pro (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-pro","oid":"critpt-a8bc226e1882d539","rd":"2026-04-24","sid":"rec1L46OeAGTWKlTb","sm":"deepseek-v4-pro_max","v":0.128571428571429},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Haiku 5.5 (Medium)","e":"medium","m":"claude-haiku-5-5","oid":"critpt-53781fade4c77685","rd":"2026-10-07","sid":"recHXjcWMBz3J0e4Y","sm":"claude-haiku-5-5_medium","v":0.128571428571429},{"b":"critpt","c":202.3,"cfg":{"effort":"high"},"dn":"GPT-5 (high, code \u0026 web)","e":"high","m":"gpt-5-2025-08-07","oid":"critpt-9136cbab08b5628f","rd":"2025-08-07","sid":"recdr3BrRL90Bz4BD","sm":"gpt-5-2025-08-07_high","v":0.126},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Gemini 3.8 Flash (Medium)","e":"medium","m":"gemini-3-8-flash","oid":"critpt-8bb78b87a92c255d","rd":"2026-09-02","sid":"rec4GH6UtiducrcoH","sm":"gemini-3.8-flash_medium","v":0.122857142857143},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Opus 4.7 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-4-7","oid":"critpt-6e55b59a52ecc9f0","rd":"2026-04-16","sid":"recet99FlOwSOYz79","sm":"claude-opus-4-7_max","v":0.12},{"b":"critpt","dn":"MiMo-V2.6-Flash","m":"mimo-v2-6-flash","oid":"critpt-957825e8d610120a","rd":"2026-09-21","sid":"rec4ROM8oWxvqp8kz","sm":"mimo-v2.6-flash","v":0.12},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Sonnet 5.5 (Low, Default Fallback)","e":"low","m":"claude-sonnet-5-5","oid":"critpt-ff5e621f9e63b4d6","rd":"2026-09-28","sid":"recpvVdkoBS6FSh1L","sm":"claude-sonnet-5-5_low","v":0.114285714285714},{"b":"critpt","dn":"Muse Spark","m":"muse-spark","oid":"critpt-4620f1deea8f779f","rd":"2026-04-08","sid":"recb3XQxDm31GDzUB","sm":"muse-spark","v":0.11330612244898002},{"b":"critpt","cfg":{"effort":"high"},"dn":"Gemini 3.6 Flash (High)","e":"high","m":"gemini-3-6-flash","oid":"critpt-9d4e6982f5ba1651","rd":"2026-07-21","sid":"recqwxOipLyu5aMnZ","sm":"gemini-3.6-flash_high","v":0.105714285714286},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-6 Luna (Medium)","e":"medium","m":"gpt-6-luna","oid":"critpt-05492e4313b7918b","rd":"2026-09-22","sid":"recbhkUf2Yx4RkpD7","sm":"gpt-6-luna_medium","v":0.105714285714286},{"b":"critpt","dn":"Mistral Large 4 Preview","m":"mistral-large-4-0","oid":"critpt-38a2793a7a96d2b9","rd":"2026-10-06","sid":"recq4PLydfJLH3soL","sm":"mistral-large-4-0_unknown","v":0.105714285714286},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 mini (Xhigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"critpt-373ad693ac375711","rd":"2026-03-17","sid":"recLA3xembg5N2dya","sm":"gpt-5.4-mini-2026-03-17_xhigh","v":0.1},{"b":"critpt","dn":"Kimi K2.7 Code","m":"kimi-k2-7-code","oid":"critpt-1dcd86308e55aa39","rd":"2026-06-12","sid":"recPed7pBpLPw6WNH","sm":"kimi-k2.7-code","v":0.1},{"b":"critpt","cfg":{"effort":"high"},"dn":"DeepSeek V4 Pro (Reasoning, High Effort)","e":"high","m":"deepseek-v4-pro","oid":"critpt-cec989ced1d7e717","rd":"2026-04-24","sid":"rect1awaZAAJmULnY","sm":"deepseek-v4-pro_high","v":0.1},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-5.6 Terra (Low)","e":"low","m":"gpt-5-6-terra","oid":"critpt-97af170a1ee69299","rd":"2026-07-09","sid":"recnChaP4Db7ldV91","sm":"gpt-5.6-terra_low","v":0.0942857142857143},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Gemini 3.7 Flash (medium)","e":"medium","m":"gemini-3-7-flash","oid":"critpt-0815a938914b6a88","rd":"2026-08-13","sid":"recoM4eoauNDO4OTU","sm":"gemini-3.7-flash_medium","v":0.0942857142857143},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"GPT-5.4 nano (Xhigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"critpt-52e7b1dc94f6d8be","rd":"2026-03-17","sid":"recEIYtCJHQeBZ99Q","sm":"gpt-5.4-nano-2026-03-17_xhigh","v":0.0925306122448979},{"b":"critpt","dn":"Qwen3.7 Plus","m":"qwen3-7-plus","oid":"critpt-4fac9a78b8040d51","rd":"2026-06-02","sid":"recno6rUOu5XpERYI","sm":"qwen3.7-plus","v":0.0914285714285714},{"b":"critpt","dn":"Grok Build 0.1 0616","m":"grok-build-0-1","oid":"critpt-602533f641b08e05","rd":"2026-05-29","sid":"recxDGS23zQpX5BmU","sm":"grok-build-0.1","v":0.0914285714285714},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Haiku 5.5 (Low)","e":"low","m":"claude-haiku-5-5","oid":"critpt-4c5def170e2e14f0","rd":"2026-10-07","sid":"recP0jtGizZ0j0yJj","sm":"claude-haiku-5-5_low","v":0.0914285714285714},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)","e":"medium","m":"claude-sonnet-5","oid":"critpt-e1cc1585fb4e531d","rd":"2026-06-30","sid":"recshOK4pTnH31VBX","sm":"claude-sonnet-5_medium","v":0.0857142857142857},{"b":"critpt","dn":"Inkling Small","m":"inkling-small","oid":"critpt-a54cbb9137d02ed9","rd":"2026-07-15","sid":"recpqBnouqWiNiwo4","sm":"inkling-small","v":0.08285714285714291},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-5.5 (low)","e":"low","m":"gpt-5-5","oid":"critpt-f2e7c195bd71dea9","rd":"2026-04-23","sid":"recgwhnZGia2230nD","sm":"gpt-5.5_low","v":0.08},{"b":"critpt","dn":"Kimi K2.6","m":"kimi-k2-6","oid":"critpt-d140b23203485816","rd":"2026-04-20","sid":"recMFITGa7bQcUxFW","sm":"kimi-k2.6","v":0.08},{"b":"critpt","cfg":{"effort":"high"},"dn":"Grok 4.3 (High)","e":"high","m":"grok-4-3","oid":"critpt-3f93d446a740a92d","rd":"2026-04-17","sid":"recX7QxI0juHRS4eW","sm":"grok-4.3_high","v":0.08},{"b":"critpt","cfg":{"effort":"max"},"dn":"DeepSeek V4 Flash (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-flash","oid":"critpt-cf15b12c8a8994c2","rd":"2026-04-24","sid":"recTQseYAWSYqUQWV","sm":"deepseek-v4-flash_max","v":0.0714285714285714},{"b":"critpt","c":95.3,"dn":"Gemini-3 Pro","m":"gemini-3-pro-preview","oid":"critpt-57c06d4deee4ab53","rd":"2025-11-18","sid":"recyLEBt2xmVn7fKW","sm":"gemini-3-pro-preview","v":0.069},{"b":"critpt","cfg":{"effort":"low"},"dn":"Gemini 3.7 Flash (low)","e":"low","m":"gemini-3-7-flash","oid":"critpt-03949b2e2e8311d7","rd":"2026-08-13","sid":"recNIzLCFdn6UO4EM","sm":"gemini-3.7-flash_low","v":0.0571428571428571},{"b":"critpt","cfg":{"effort":"low"},"dn":"Grok 4.6 (Low)","e":"low","m":"grok-4-6","oid":"critpt-94244543aab968a8","rd":"2026-08-12","sid":"recNzlblHrjDoaZ6g","sm":"grok-4.6_low","v":0.0571428571428571},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Inkling (Xhigh)","e":"xhigh","m":"inkling","oid":"critpt-b6064ca22081d970","rd":"2026-07-15","sid":"recGBFX27sL04thDg","sm":"Inkling_xhigh","v":0.0542857142857143},{"b":"critpt","cfg":{"effort":"xhigh"},"dn":"Qwen3.8 27B (Xhigh)","e":"xhigh","m":"qwen3-8-27b","oid":"critpt-c3e2607b1d11e5c7","rd":"2026-08-14","sid":"recqtkPIAwJWQVvzb","sm":"qwen3.8-27b_xhigh","v":0.0542857142857143},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-5.6 Sol (Non-reasoning)","e":"none","m":"gpt-5-6-sol","oid":"critpt-d6b118a50c1f447a","rd":"2026-07-09","sid":"recYXEB0E6azp3wh8","sm":"gpt-5.6-sol_none","v":0.0514285714285714},{"b":"critpt","dn":"GPT-5.1 (high)","m":"gpt-5-1-2025-11-13","oid":"critpt-57f0cae005baa18b","rd":"2025-11-13","sid":"rec8GXNpcmctYJZAg","sm":"gpt-5.1-2025-11-13_unknown","v":0.0485714285714286},{"b":"critpt","cfg":{"effort":"medium"},"dn":"GPT-5.6 Luna (Medium)","e":"medium","m":"gpt-5-6-luna","oid":"critpt-81e1763ccd347715","rd":"2026-07-09","sid":"recv0na0ZTNy7xqng","sm":"gpt-5.6-luna_medium","v":0.0485714285714286},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-6.1 Terra (Non-reasoning)","e":"none","m":"gpt-6-1-terra","oid":"critpt-677e8231dda0786b","rd":"2026-10-26","sid":"recFxC4rih5HgvTod","sm":"gpt-6.1-terra_none","v":0.0485714285714286},{"b":"critpt","dn":"GLM-5.1 (Reasoning)","m":"glm-5-1","oid":"critpt-88fcb7f66761eed4","rd":"2026-04-07","sid":"rec1pBzZ6UB4QOi5L","sm":"glm-5.1","v":0.0457142857142857},{"b":"critpt","cfg":{"effort":"low"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)","e":"low","m":"claude-sonnet-5","oid":"critpt-56f499a0c79a0030","rd":"2026-06-30","sid":"recnrPRrF9HlsmJIB","sm":"claude-sonnet-5_low","v":0.0457142857142857},{"b":"critpt","dn":"MiMo-V2.5-Pro","m":"mimo-v2-5-pro","oid":"critpt-b47afbacef7a1571","sid":"recnK6cBmJSuoZHMx","sm":"mimo-v2.5-pro","v":0.04},{"b":"critpt","cfg":{"effort":"low"},"dn":"Gemini 3.8 Flash (Low)","e":"low","m":"gemini-3-8-flash","oid":"critpt-98e2e69be3cd51f1","rd":"2026-09-02","sid":"recybP63W1fQVK2Gh","sm":"gemini-3.8-flash_low","v":0.04},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-6 Sol (Non-reasoning)","e":"none","m":"gpt-6-sol","oid":"critpt-b8fe297f95447ede","rd":"2026-09-22","sid":"reclrdB6fUm8KdvHz","sm":"gpt-6-sol_none","v":0.04},{"b":"critpt","dn":"MiniMax-M3","m":"minimax-m3","oid":"critpt-b84ccf0603724a90","rd":"2026-06-01","sid":"recW7eWCnlhJL30vW","sm":"MiniMax-M3","v":0.0371428571428571},{"b":"critpt","dn":"Ring-2.6-1T","m":"ring-2-6-1t","oid":"critpt-68ffb56538b1172b","rd":"2026-05-14","sid":"recCjySOHKPVKzoIR","sm":"Ring-2.6-1T","v":0.0371428571428571},{"b":"critpt","dn":"MiMo-V2.5","m":"mimo-v2-5","oid":"critpt-4eda8c7395776426","sid":"recDjobS7nuxZ088N","sm":"mimo-v2.5","v":0.0371428571428571},{"b":"critpt","cfg":{"effort":"high"},"dn":"DeepSeek V4 Flash (Reasoning, High Effort)","e":"high","m":"deepseek-v4-flash","oid":"critpt-8c70deef751ba501","rd":"2026-04-24","sid":"recUcNlOyTisHxSMw","sm":"deepseek-v4-flash_high","v":0.0342857142857143},{"b":"critpt","dn":"Nemotron 3 Ultra 550B A55B (Reasoning)","m":"nemotron-3-ultra","oid":"critpt-a8887e611e1a851f","rd":"2026-06-04","sid":"recpD8mZ2K1WkRGzB","sm":"nemotron-3-ultra","v":0.0314285714285714},{"b":"critpt","cfg":{"effort":"max"},"dn":"Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-sonnet-4-6","oid":"critpt-bb0c3a24666ad322","rd":"2026-02-17","sid":"recfDvn5x68iGMXA8","sm":"claude-sonnet-4-6_max","v":0.0314285714285714},{"b":"critpt","dn":"NVIDIA Nemotron 3 Super 120B A12B (Reasoning)","m":"nemotron-3-super","oid":"critpt-916aef629f90e6a9","rd":"2026-03-11","sid":"reciozjK7dm0hUG57","sm":"nemotron-3-super","v":0.0314285714285714},{"b":"critpt","dn":"Kimi K2.5 (Reasoning)","m":"kimi-k2-5","oid":"critpt-9efff7e7fbbbff96","rd":"2026-01-27","sid":"recUijy8nmKKsd3NW","sm":"kimi-k2.5","v":0.0314285714285714},{"b":"critpt","cfg":{"effort":"low"},"dn":"Kimi K3 (Low)","e":"low","m":"kimi-k3","oid":"critpt-757ebd0d1ef89b70","rd":"2026-07-16","sid":"recYh162lSGUsK6qK","sm":"kimi-k3_low","v":0.0314285714285714},{"b":"critpt","cfg":{"effort":"none"},"dn":"GLM-5.2 (Non-reasoning)","e":"none","m":"glm-5-2","oid":"critpt-b538c82e12e68f04","rd":"2026-06-16","sid":"recuVPwt5tEUGNhzK","sm":"glm-5.2_none","v":0.0314285714285714},{"b":"critpt","dn":"Qwen3.6 Plus","m":"qwen3-6-plus","oid":"critpt-039c58becb4ce1e9","rd":"2026-03-31","sid":"recV1nMEdGDXiR3hS","sm":"qwen3.6-plus","v":0.0285714285714286},{"b":"critpt","cfg":{"effort":"thinking"},"dn":"DeepSeek V3.2 (Reasoning)","e":"thinking","m":"deepseek-v3-2-exp","oid":"critpt-f3c78a967e52b821","rd":"2025-09-29","sid":"recBbKGHB1LxXigNW","sm":"DeepSeek-V3.2-Exp_thinking","v":0.0285714285714286},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-5.6 Luna (Low)","e":"low","m":"gpt-5-6-luna","oid":"critpt-75f0253dfaea1330","rd":"2026-07-09","sid":"recRlD6Sc9yPJtNqv","sm":"gpt-5.6-luna_low","v":0.0257142857142857},{"b":"critpt","cfg":{"effort":"low"},"dn":"GPT-6 Luna (Low)","e":"low","m":"gpt-6-luna","oid":"critpt-809d99a9d6c92e17","rd":"2026-09-22","sid":"recLsRbGgaw2H9hK6","sm":"gpt-6-luna_low","v":0.0257142857142857},{"b":"critpt","cfg":{"effort":"high"},"dn":"Muse Glimmer (High)","e":"high","m":"muse-glimmer","oid":"critpt-de6d8bfcba773b63","rd":"2026-08-10","sid":"recmnikgUh8UV8r5D","sm":"muse-glimmer_high","v":0.0257142857142857},{"b":"critpt","dn":"Step 3.7 Flash","m":"step-3-7-flash","oid":"critpt-a68658a03f037dac","rd":"2026-05-29","sid":"receStzvN0iHAZxPM","sm":"step-3.7-flash","v":0.022857142857142902},{"b":"critpt","c":85.05,"dn":"Gemini-2.5 Pro","m":"gemini-2-5-pro","oid":"critpt-52537a10c6d9a03f","rd":"2025-06-17","sid":"rec32PvAuS31AZUEa","sm":"gemini-2.5-pro","v":0.02},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-5.6 Terra (Non-reasoning)","e":"none","m":"gpt-5-6-terra","oid":"critpt-7edbd59308712f10","rd":"2026-07-09","sid":"rec3NNgTsVrtlXXYf","sm":"gpt-5.6-terra_none","v":0.02},{"b":"critpt","dn":"DeepSeek V3.1 Terminus (Reasoning)","m":"deepseek-v3-1-terminus","oid":"critpt-1d9d79ea5919c2dd","rd":"2025-09-22","sid":"recOPY7hfm2F7tMSu","sm":"DeepSeek-V3.1-Terminus","v":0.0171428571428571},{"b":"critpt","dn":"GLM-4.7 (Reasoning)","m":"glm-4-7","oid":"critpt-46dc39e0e7eb6d30","rd":"2025-12-22","sid":"recf34WxKFtk9xprT","sm":"glm-4.7","v":0.0171020408163265},{"b":"critpt","cfg":{"effort":"high"},"dn":"gpt-oss-20b (High)","e":"high","m":"gpt-oss-20b","oid":"critpt-ceb0708d9be9d1be","rd":"2025-08-05","sid":"recONNM98iTrX4qza","sm":"gpt-oss-20b_high","v":0.0142857142857143},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-5.5 (Non-reasoning)","e":"none","m":"gpt-5-5","oid":"critpt-1aa81b867491bbab","rd":"2026-04-23","sid":"reciQD7XrIOOLRL9r","sm":"gpt-5.5_none","v":0.0142857142857143},{"b":"critpt","dn":"Gemma 4 31B (Reasoning)","m":"gemma-4-31b-it","oid":"critpt-8125934c13a1f581","rd":"2026-04-02","sid":"recw0Eh5lyCqsIELX","sm":"gemma-4-31b-it","v":0.0142857142857143},{"b":"critpt","c":52.85,"cfg":{"effort":"high"},"dn":"o3 (high)","e":"high","m":"o3-2025-04-16","oid":"critpt-58068f0d4a3d3abd","rd":"2025-04-16","sid":"recM1xbTH6LzSqYmj","sm":"o3-2025-04-16_high","v":0.013999999999999999},{"b":"critpt","dn":"Gemini 3.1 Flash-Lite","m":"gemini-3-1-flash-lite","oid":"critpt-15436d103d45cf08","rd":"2026-03-03","sid":"recVzn75nNDIaYdwg","sm":"gemini-3.1-flash-lite","v":0.0114285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"gpt-oss-120b (High)","e":"high","m":"openaigpt-oss-120b","oid":"critpt-d4a9ed7c45970e48","rd":"2025-08-05","sid":"rectdzekfUNTNey9b","sm":"openai/gpt-oss-120b_high","v":0.0114285714285714},{"b":"critpt","dn":"Claude 4.5 Sonnet (Reasoning)","m":"claude-sonnet-4-5-2025-09-29","oid":"critpt-3620f91a3f59583d","rd":"2025-09-29","sid":"recYX0ACbZcdRLldF","sm":"claude-sonnet-4-5-20250929_unknown","v":0.0114285714285714},{"b":"critpt","dn":"GLM-4.6 (Reasoning)","m":"glm-4-6","oid":"critpt-a288b4ac19a0518f","rd":"2025-09-30","sid":"recT9rNeX9mC2KIk8","sm":"glm-4.6","v":0.0114285714285714},{"b":"critpt","cfg":{"effort":"high"},"dn":"Claude Sonnet 5 (Non-reasoning, High Effort)","e":"high","m":"claude-sonnet-5","oid":"critpt-72549deeac8ca369","rd":"2026-06-30","sid":"reck73uI9KSAsqlzi","sm":"claude-sonnet-5_high","v":0.0114285714285714},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-6 Luna (Non-reasoning)","e":"none","m":"gpt-6-luna","oid":"critpt-1c9e2883bdd46b1b","rd":"2026-09-22","sid":"rec7PuqFcyNaY0M0K","sm":"gpt-6-luna_none","v":0.0114285714285714},{"b":"critpt","c":12.6,"dn":"DeepSeek R1","m":"deepseek-r1","oid":"critpt-6f88cc955f8a3963","rd":"2025-01-20","sid":"recjrHRZQEXwiWlgd","sm":"DeepSeek-R1","v":0.011000000000000001},{"b":"critpt","c":22.75,"dn":"Gemini-2.5 Flash","m":"gemini-2-5-flash","oid":"critpt-41627b2c73b113c6","rd":"2025-06-17","sid":"reck9g3jBNU8npA3a","sm":"gemini-2.5-flash","v":0.011000000000000001},{"b":"critpt","cfg":{"effort":"thinking"},"dn":"Trinity Large Thinking","e":"thinking","m":"trinity-large-thinking","oid":"critpt-797b12e15f11e231","rd":"2026-04-01","sid":"recgp04zX438rxvze","sm":"trinity-large-thinking","v":0.00857142857142857},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.6 27B (Non-reasoning)","e":"none","m":"qwen3-6-27b","oid":"critpt-b1d1d73757704da3","rd":"2026-04-22","sid":"recRVR4Aumxx6RvXz","sm":"qwen3.6-27b_none","v":0.00857142857142857},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 122B A10B (Non-reasoning)","e":"none","m":"qwen3-5-122b-a10b","oid":"critpt-3f14f42cbd385149","rd":"2026-02-24","sid":"recwij8ov95MolYot","sm":"qwen3.5-122b-a10b_none","v":0.00857142857142857},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 397B A17B (Non-reasoning)","e":"none","m":"qwen3-5-397b-a17b","oid":"critpt-d01fcbaf2605055d","rd":"2026-02-13","sid":"recGhRiMFpa9mWNcp","sm":"qwen3.5-397b-a17b_none","v":0.00857142857142857},{"b":"critpt","dn":"Mercury 2","m":"mercury-2","oid":"critpt-1e84297f014634fb","sid":"recqA73Q42rXIKFtE","sm":"mercury-2","v":0.00848979591836735},{"b":"critpt","c":27.65,"cfg":{"effort":"high"},"dn":"o4-mini (high)","e":"high","m":"o4-mini-2025-04-16","oid":"critpt-5447e61a83aa7603","rd":"2025-04-16","sid":"recnRgPtlRgPlW8OH","sm":"o4-mini-2025-04-16_high","v":0.006},{"b":"critpt","dn":"MiniMax-M2.7","m":"minimax-m2-7","oid":"critpt-6d8cab700d7ab00c","rd":"2026-03-18","sid":"rec9VpZGD78XTqjeO","sm":"MiniMax-M2.7","v":0.00571428571428571},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 35B A3B (Non-reasoning)","e":"none","m":"qwen3-5-35b-a3b","oid":"critpt-985030040c6a48e5","rd":"2026-02-24","sid":"recQbNCfoJuInAraz","sm":"qwen3.5-35b-a3b_none","v":0.00571428571428571},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 9B (Non-reasoning)","e":"none","m":"qwen3-5-9b","oid":"critpt-0478ed8e8c863186","rd":"2026-02-24","sid":"recVf4KC2kYm9fsAl","sm":"qwen3.5-9B_none","v":0.00571428571428571},{"b":"critpt","c":351.75,"dn":"Claude Opus 4","m":"claude-opus-4-2025-05-14","oid":"critpt-c1a1f011ce8659f6","rd":"2025-05-22","sid":"recyg7DC8tfdBXril","sm":"claude-opus-4-20250514_unknown","v":0.003},{"b":"critpt","dn":"Qwen3.5 9B (Reasoning)","m":"qwen3-5-9b","oid":"critpt-326cc149cdd7a041","rd":"2026-02-24","sid":"recL4KLpCFVwt4nhZ","sm":"qwen3.5-9B","v":0.00285714285714286},{"b":"critpt","dn":"Qwen3.6 35B A3B (Reasoning)","m":"qwen3-6-35b-a3b","oid":"critpt-8a6a85dcd718ad06","rd":"2026-04-14","sid":"recdtQpqo4eGSmwc7","sm":"qwen3.6-35b-a3b","v":0.00285714285714286},{"b":"critpt","dn":"Claude 4 Sonnet (Reasoning)","m":"claude-sonnet-4-2025-05-14","oid":"critpt-cf79188824a8503e","rd":"2025-05-22","sid":"recTks93nzrmB6EZE","sm":"claude-sonnet-4-20250514_unknown","v":0.00285714285714286},{"b":"critpt","dn":"Qwen3 32B (Reasoning)","m":"qwen3-32b","oid":"critpt-b36d90a99797712b","rd":"2025-04-29","sid":"recc4YhohoxvIPwpX","sm":"Qwen3-32B","v":0.00285714285714286},{"b":"critpt","dn":"Magistral Medium 1.2","m":"magistral-medium-2509","oid":"critpt-ab90028bee009621","rd":"2025-09-18","sid":"recFi7UEbKxSJUg28","sm":"magistral-medium-2509","v":0.00285714285714286},{"b":"critpt","dn":"Magistral Small 1.2","m":"magistral-small-2509","oid":"critpt-f20d23259546fcb4","rd":"2025-09-18","sid":"recWoGzZKOAstKwFi","sm":"magistral-small-2509","v":0.00285714285714286},{"b":"critpt","dn":"Command A+","m":"command-a-plus-05-2026","oid":"critpt-6fb370770e27dced","rd":"2026-05-20","sid":"recemIOg9vwrJJpyZ","sm":"command-a-plus-05-2026","v":0.00285714285714286},{"b":"critpt","cfg":{"effort":"high"},"dn":"o3-mini (High)","e":"high","m":"o3-mini-2025-01-31","oid":"critpt-8ae09a795ff35e1c","rd":"2025-01-31","sid":"recV8NFrww4OvPhlW","sm":"o3-mini-2025-01-31_high","v":0.00285714285714286},{"b":"critpt","cfg":{"effort":"none"},"dn":"GPT-5.6 Luna (Non-reasoning)","e":"none","m":"gpt-5-6-luna","oid":"critpt-38eee200306be884","rd":"2026-07-09","sid":"recypos8nKcQXV1SU","sm":"gpt-5.6-luna_none","v":0.00285714285714286},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.8 27B (Non-reasoning)","e":"none","m":"qwen3-8-27b","oid":"critpt-fa70cce32eba7099","rd":"2026-08-14","sid":"recV0ZAFyx5cMeKSc","sm":"qwen3.8-27b_none","v":0.00285714285714286},{"b":"critpt","dn":"Qwen3 30B A3B 2507 (Reasoning)","m":"qwen3-30b-a3b-thinking-2507","oid":"critpt-9b7d6a70a16dd4f9","rd":"2025-07-30","sid":"reciFGSV91TNrClEc","sm":"qwen3-30b-a3b-thinking-2507","v":0.00285714285714286},{"b":"critpt","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash (Non-reasoning)","e":"none","m":"deepseek-v4-1-flash","oid":"critpt-de8c1824134d810c","rd":"2026-09-09","sid":"recZEah5EFMAlM9BJ","sm":"deepseek-v4.1-flash_none","v":0.00285714285714286},{"b":"critpt","c":0.35,"dn":"Llama-4 Maverick","m":"chutesllama-4-maverick-17b-128e-instruct","oid":"critpt-27b6073b67c769d9","rd":"2025-04-06","sid":"recKStGDdDXYw1DQX","sm":"chutes/Llama-4-Maverick-17B-128E-Instruct","v":0},{"b":"critpt","c":8.4,"cfg":{"effort":"minimal"},"dn":"GPT-5 (minimal)","e":"minimal","m":"gpt-5-2025-08-07","oid":"critpt-a8cc27138afb915f","rd":"2025-08-07","sid":"reczlPmC8okHG1O3W","sm":"gpt-5-2025-08-07_minimal","v":0},{"b":"critpt","c":4.2,"dn":"GPT-4o","m":"gpt-4o-2024-11-20","oid":"critpt-d908873afce12ede","rd":"2024-11-20","sid":"recmsyeCu6fQhd1kq","sm":"gpt-4o-2024-11-20","v":0},{"b":"critpt","dn":"Llama 4 Scout","m":"llama-4-scout-17b-16e-instruct","oid":"critpt-53075f6219345cbe","rd":"2025-04-05","sid":"recikhUEGyub8XyDp","sm":"Llama-4-Scout-17B-16E-Instruct","v":0},{"b":"critpt","dn":"Llama 4 Maverick","m":"llama-4-maverick-17b-128e-instruct","oid":"critpt-ac98e0b8b0c97319","rd":"2025-04-05","sid":"recquOOgVnO80LfMY","sm":"Llama-4-Maverick-17B-128E-Instruct","v":0},{"b":"critpt","dn":"Claude 4.5 Haiku (Reasoning)","m":"claude-haiku-4-5-2025-10-01","oid":"critpt-43798d4ed57ec499","rd":"2025-10-15","sid":"rec2iJ4OmQbUaUA19","sm":"claude-haiku-4-5-20251001_unknown","v":0},{"b":"critpt","dn":"GPT-4.1 nano","m":"gpt-4-1-nano-2025-04-14","oid":"critpt-59db8cf861dbcbbc","rd":"2025-04-14","sid":"recFOYuzyW9KvjrYn","sm":"gpt-4.1-nano-2025-04-14","v":0},{"b":"critpt","dn":"MiMo-V2-Flash (Non-reasoning)","m":"mimo-v2-flash","oid":"critpt-0f82dd18d39f46b7","sid":"recIO10647zOjjl02","sm":"mimo-v2-flash","v":0},{"b":"critpt","dn":"Gemma 4 26B A4B (Reasoning)","m":"gemma-4-26b-a4b","oid":"critpt-b875b2715af26012","rd":"2026-04-02","sid":"recjfVFnTDM7svSTj","sm":"gemma-4-26b-a4b","v":0},{"b":"critpt","dn":"GPT-4.1 mini","m":"gpt-4-1-mini-2025-04-14","oid":"critpt-b4cdcfcdcd12d6bf","rd":"2025-04-14","sid":"rec8hhHFwNWNI8ugy","sm":"gpt-4.1-mini-2025-04-14","v":0},{"b":"critpt","dn":"Gemma 3 27B Instruct","m":"gemma-3-27b-it","oid":"critpt-7880d40a37f0952b","rd":"2025-03-12","sid":"recMuWO4PqAudohyp","sm":"gemma-3-27b-it","v":0},{"b":"critpt","dn":"Llama 3.1 Instruct 8B","m":"llama-3-1-8b-instruct","oid":"critpt-542a3c92591459f9","rd":"2024-07-23","sid":"recgkMeSayiDfcuOs","sm":"Llama-3.1-8B-Instruct","v":0},{"b":"critpt","dn":"Claude 3.5 Haiku","m":"claude-3-5-haiku-2024-10-22","oid":"critpt-8abb103b757fd149","rd":"2024-10-22","sid":"recs8Q9ZFgd0oHh2G","sm":"claude-3-5-haiku-20241022","v":0},{"b":"critpt","dn":"Mistral Small 3.1","m":"mistral-small-2503","oid":"critpt-c883eec123e24428","rd":"2025-03-17","sid":"rec1aCUq1EZVvw6oB","sm":"mistral-small-2503","v":0},{"b":"critpt","dn":"DeepSeek V3 (Dec '24)","m":"deepseek-v3","oid":"critpt-d2ee6c7c895c361e","rd":"2024-12-26","sid":"recZuquIpWlS5svDE","sm":"DeepSeek-V3","v":0},{"b":"critpt","dn":"Mistral Medium 3.1","m":"mistral-medium-2508","oid":"critpt-83a71150cca61ac5","rd":"2025-08-12","sid":"rechLLiNO5R81n2Av","sm":"mistral-medium-2508","v":0},{"b":"critpt","dn":"DeepSeek V3 0324","m":"deepseek-v3-0324","oid":"critpt-4cfcd48c0df1a7ca","rd":"2025-03-24","sid":"recmVYFrbBzF0wFtT","sm":"DeepSeek-V3-0324","v":0},{"b":"critpt","dn":"Gemini 3.5 Flash-Lite","m":"gemini-3-5-flash-lite","oid":"critpt-7103d86bed2e24b8","rd":"2026-07-21","sid":"rec9uJ0Cdk6gX8Q2p","sm":"gemini-3.5-flash-lite","v":0},{"b":"critpt","dn":"Llama 3.3 Instruct 70B","m":"llama-3-3-70b-instruct","oid":"critpt-397ceac2e91058ba","rd":"2024-12-06","sid":"reccPSgbUrVYpwQPP","sm":"Llama-3.3-70B-Instruct","v":0},{"b":"critpt","dn":"Devstral Small 2","m":"devstral-small-2512","oid":"critpt-e7c87da7310f59f1","rd":"2025-12-09","sid":"recqnbiwXvV1A93F1","sm":"devstral-small-2512","v":0},{"b":"critpt","dn":"Mistral Medium 3.5","m":"mistral-medium-2604","oid":"critpt-ea4e790369ff180f","rd":"2026-04-28","sid":"rec145Hb8G6EopI7y","sm":"mistral-medium-2604","v":0},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Nova 2.0 Pro Preview (Medium)","e":"medium","m":"nova-2-0-pro-preview","oid":"critpt-4e1802c006a8b06d","rd":"2025-12-02","sid":"recQk3naCr3jtTWtY","sm":"nova-2.0-pro-preview_medium","v":0},{"b":"critpt","dn":"Mistral Large 3","m":"mistral-large-2512","oid":"critpt-bc9b87ecbf1119b5","rd":"2025-12-02","sid":"recmTJMuwU1Hk5nk3","sm":"mistral-large-2512","v":0},{"b":"critpt","dn":"Grok 4.3 (Non-reasoning)","m":"grok-4-3","oid":"critpt-407bd3d219297e50","rd":"2026-04-17","sid":"recpyQ2IiB91kvrNf","sm":"grok-4.3_unknown","v":0},{"b":"critpt","dn":"Solar Pro 3","m":"solar-pro-3","oid":"critpt-e654a72870b3b008","rd":"2026-01-27","sid":"rec5S2focD5wAvXCh","sm":"solar-pro-3","v":0},{"b":"critpt","dn":"Phi-4 Mini Instruct","m":"phi-4-mini-instruct","oid":"critpt-4e4765506bcd7a70","rd":"2025-02-27","sid":"rec5b8acGO32INjIs","sm":"Phi-4-mini-instruct","v":0},{"b":"critpt","dn":"Qwen3 Coder Next","m":"qwen3-coder-next","oid":"critpt-39dbdf321d6a09d3","rd":"2026-02-02","sid":"recHbJSpurIdjDsK6","sm":"qwen3-coder-next","v":0},{"b":"critpt","dn":"Granite 4.1 30B","m":"granite-4-1-30b","oid":"critpt-83b161dfec43acde","rd":"2026-04-29","sid":"recm39ZQ95Zzc6oEr","sm":"granite-4.1-30b","v":0},{"b":"critpt","dn":"Mistral Small 3.2","m":"mistral-small-2506","oid":"critpt-4b6f0049ca5b3180","rd":"2025-06-20","sid":"rec9rP1j91BTUXIbd","sm":"mistral-small-2506","v":0},{"b":"critpt","dn":"Gemma 3 12B Instruct","m":"gemma-3-12b-it","oid":"critpt-aa22f9acfd36c3bf","rd":"2025-03-12","sid":"recEipNbGuHMCR5zw","sm":"gemma-3-12b-it","v":0},{"b":"critpt","dn":"Qwen3 235B A22B 2507 (Reasoning)","m":"qwenqwen3-235b-a22b-thinking-2507","oid":"critpt-5caccc01786c8790","rd":"2025-07-25","sid":"recJhEv9yEFCgBq9r","sm":"qwen/qwen3-235b-a22b-thinking-2507","v":0},{"b":"critpt","dn":"GPT-5 mini (high)","m":"gpt-5-mini-2025-08-07","oid":"critpt-f238a709c87e89e1","rd":"2025-08-07","sid":"recf8XWekVD7VWo95","sm":"gpt-5-mini-2025-08-07_unknown","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Nova 2.0 Pro Preview (Non-reasoning)","e":"none","m":"nova-2-0-pro-preview","oid":"critpt-68541ed382302f44","rd":"2025-12-02","sid":"rec3nGWaL1V4GC7Ox","sm":"nova-2.0-pro-preview_none","v":0},{"b":"critpt","cfg":{"effort":"low"},"dn":"Nova 2.0 Pro Preview (Low)","e":"low","m":"nova-2-0-pro-preview","oid":"critpt-e6386ab8df5dc705","rd":"2025-12-02","sid":"recIrIlZAJIFqA05F","sm":"nova-2.0-pro-preview_low","v":0},{"b":"critpt","dn":"GPT-5.5 Instant (June 2026)","m":"gpt-5-5-instant","oid":"critpt-f40351783eb3b4b6","rd":"2026-05-05","sid":"recTZz0IysSQlVQBG","sm":"gpt-5.5-instant","v":0},{"b":"critpt","dn":"Cogito v2.1 (Reasoning)","m":"cogito-671b-v2-1","oid":"critpt-9029c745e1f1d3bb","rd":"2025-11-19","sid":"recUm9YUU4tv4QYl4","sm":"cogito-671b-v2.1","v":0},{"b":"critpt","cfg":{"effort":"low"},"dn":"gpt-oss-120b (Low)","e":"low","m":"openaigpt-oss-120b","oid":"critpt-961dc7b7501381f2","rd":"2025-08-05","sid":"recqRZmPB2nweO1S7","sm":"openai/gpt-oss-120b_low","v":0},{"b":"critpt","cfg":{"effort":"medium"},"dn":"Qwen3.8 27B (Medium)","e":"medium","m":"qwen3-8-27b","oid":"critpt-0e9191998fa5ad47","rd":"2026-08-14","sid":"rec0L5JX4l98UW6uf","sm":"qwen3.8-27b_medium","v":0},{"b":"critpt","cfg":{"effort":"low"},"dn":"Qwen3.8 27B (Low)","e":"low","m":"qwen3-8-27b","oid":"critpt-7d34b632fc659cb7","rd":"2026-08-14","sid":"rec0LIT4IY1rcfCu4","sm":"qwen3.8-27b_low","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Grok 4.3 (Non-reasoning)","e":"none","m":"grok-4-3","oid":"critpt-e7bdc66eece156f3","rd":"2026-04-17","sid":"recReTZrRXi8ndlYR","sm":"grok-4.3_none","v":0},{"b":"critpt","dn":"Qwen3 8B (Reasoning)","m":"qwen3-8b","oid":"critpt-7f40b685a7c328e2","rd":"2025-04-28","sid":"recb788jTlPfrMXig","sm":"qwen3-8b","v":0},{"b":"critpt","dn":"Qwen3 14B (Reasoning)","m":"qwen3-14b","oid":"critpt-fc37e97ff4c87e5e","rd":"2025-04-29","sid":"reccUsj3yqYhvgvUC","sm":"qwen3-14b","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.6 35B A3B (Non-reasoning)","e":"none","m":"qwen3-6-35b-a3b","oid":"critpt-ebce8b02bbc5fa54","rd":"2026-04-14","sid":"rectaMSlhWPQdKvlu","sm":"qwen3.6-35b-a3b_none","v":0},{"b":"critpt","cfg":{"effort":"high"},"dn":"GPT-5 mini (High)","e":"high","m":"gpt-5-mini-2025-08-07","oid":"critpt-b9002fb35ea47dcf","rd":"2025-08-07","sid":"recEJeMZnvjXpLM9g","sm":"gpt-5-mini-2025-08-07_high","v":0},{"b":"critpt","dn":"Mercury 2.5","m":"mercury-2-5","oid":"critpt-97f95232d62dfa32","rd":"2026-09-08","sid":"recVvxSpw8dmPufkZ","sm":"mercury-2.5_unknown","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"DeepSeek V4 Pro 0813 (Non-reasoning)","e":"none","m":"deepseek-v4-pro-0813","oid":"critpt-1b3e9eeb1ce0cf0c","rd":"2026-08-13","sid":"recs2UizzIXPf4Ams","sm":"deepseek-v4-pro-0813_none","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 4B (Non-reasoning)","e":"none","m":"qwen3-5-4b","oid":"critpt-e4e5fe8b0e06956e","rd":"2026-02-24","sid":"rec1Ozik5rPguvpRw","sm":"qwen3.5-4b_none","v":0},{"b":"critpt","dn":"Gemma 3 270M","m":"gemma-3-270m-it","oid":"critpt-73966a0432c40f1e","rd":"2025-08-14","sid":"recDV3p8F6vK24N6p","sm":"gemma-3-270m-it","v":0},{"b":"critpt","dn":"Nova Micro","m":"amazon-nova-micro-v10","oid":"critpt-9d9cf375688e3a2e","rd":"2024-12-03","sid":"recSJlXhtSJsdN8LM","sm":"amazon.nova-micro-v1:0","v":0},{"b":"critpt","cfg":{"effort":"low"},"dn":"gpt-oss-20b (Low)","e":"low","m":"gpt-oss-20b","oid":"critpt-32eab4b377ac29bc","rd":"2025-08-05","sid":"recfBk71N0GDSAdcS","sm":"gpt-oss-20b_low","v":0},{"b":"critpt","dn":"Granite 4.0 Micro","m":"granite-4-0-micro","oid":"critpt-ba76eab9677532df","rd":"2025-10-02","sid":"rechivFJHLaCpdneq","sm":"granite-4.0-micro","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 2B (Non-reasoning)","e":"none","m":"qwen3-5-2b","oid":"critpt-aeb2c1832dc63348","rd":"2026-02-24","sid":"recpVyDQ69ao8ZVmx","sm":"qwen3.5-2b_none","v":0},{"b":"critpt","dn":"Granite 4.0 350M","m":"granite-4-0-350m","oid":"critpt-db21ceda94d48288","rd":"2025-10-28","sid":"rec4RF7qojbmcO1on","sm":"granite-4.0-350m","v":0},{"b":"critpt","dn":"Command A","m":"c4ai-command-a-03-2025","oid":"critpt-a87925ea258faaae","rd":"2025-03-13","sid":"rec4rsp6EYIL1ZVdt","sm":"c4ai-command-a-03-2025","v":0},{"b":"critpt","dn":"Qwen3.5 0.8B (Reasoning)","m":"qwen3-5-0-8b","oid":"critpt-b50cbe89c1acd71a","rd":"2026-02-24","sid":"recWDIAaYnV8xhSzO","sm":"qwen3.5-0.8b","v":0},{"b":"critpt","dn":"Llama 3.2 Instruct 11B (Vision)","m":"llama-3-2-11b-vision-instruct","oid":"critpt-07a466155d9e1ef6","rd":"2024-09-24","sid":"recYjd5kKh2MmzAH2","sm":"Llama-3.2-11B-Vision-Instruct","v":0},{"b":"critpt","dn":"Llama 3.1 Instruct 405B","m":"llama-3-1-405b-instruct","oid":"critpt-b9e82b70d2c40811","rd":"2024-07-23","sid":"reccxjg6wWheLrQE6","sm":"Llama-3.1-405B-Instruct","v":0},{"b":"critpt","cfg":{"effort":"none"},"dn":"Qwen3.5 0.8B (Non-reasoning)","e":"none","m":"qwen3-5-0-8b","oid":"critpt-39c63e24244eff0d","rd":"2026-02-24","sid":"rech4v2kar8Pap9IJ","sm":"qwen3.5-0.8b_none","v":0},{"b":"critpt","dn":"NVIDIA Nemotron Nano 9B V2 (Reasoning)","m":"nvidia-nemotron-nano-9b-v2","oid":"critpt-eb47c85fcb8cdf76","rd":"2025-08-18","sid":"recKjgaYJ5GXlGbJW","sm":"nvidia-nemotron-nano-9b-v2","v":0},{"b":"critpt","dn":"Reka Flash 3","m":"reka-flash-3","oid":"critpt-ef8beb04283993c2","rd":"2025-03-10","sid":"recP5YgoHRtlJ6toS","sm":"reka-flash-3","v":0},{"b":"critpt","dn":"Phi-4","m":"phi-4","oid":"critpt-0e5f615eb3466acd","rd":"2024-12-12","sid":"recX6iQ5Xv3BwxVDz","sm":"phi-4","v":0},{"b":"cursorbench","c":17.28,"cfg":{"effort":"max"},"dn":"Fable 5.1 Max","e":"max","m":"claude-fable-5-1","metrics":{"steps":128,"total_tokens":117236},"oid":"cursorbench-cb403654967cdc9d","rd":"2026-09-01","sm":"claude-fable-5-1_max","src":"https://cursor.com/cursorbench","v":0.518},{"b":"cursorbench","c":13.01,"cfg":{"effort":"xhigh"},"dn":"Fable 5.1 Extra High","e":"xhigh","m":"claude-fable-5-1","metrics":{"steps":101,"total_tokens":87294},"oid":"cursorbench-08960a4f603f5fa9","rd":"2026-09-01","sm":"claude-fable-5-1_xhigh","src":"https://cursor.com/cursorbench","v":0.516},{"b":"cursorbench","c":9.08,"cfg":{"effort":"high"},"dn":"Fable 5.1 High","e":"high","m":"claude-fable-5-1","metrics":{"steps":77,"total_tokens":58438},"oid":"cursorbench-119049dc9a8f4ab6","rd":"2026-09-01","sm":"claude-fable-5-1_high","src":"https://cursor.com/cursorbench","v":0.49200000000000005},{"b":"cursorbench","c":7.05,"cfg":{"effort":"medium"},"dn":"Fable 5.1 Medium","e":"medium","m":"claude-fable-5-1","metrics":{"steps":63,"total_tokens":45411},"oid":"cursorbench-13dab237a2ffdf59","rd":"2026-09-01","sm":"claude-fable-5-1_medium","src":"https://cursor.com/cursorbench","v":0.46799999999999997},{"b":"cursorbench","c":11.95,"cfg":{"effort":"max"},"dn":"Opus 5 Max","e":"max","m":"claude-opus-5","metrics":{"steps":106,"total_tokens":85384},"oid":"cursorbench-c28f3b3404fba8ad","rd":"2026-07-24","sm":"claude-opus-5_max","src":"https://cursor.com/cursorbench","v":0.466},{"b":"cursorbench","c":6.01,"cfg":{"effort":"xhigh"},"dn":"Grok 4.7 Extra High","e":"xhigh","m":"grok-4-7","metrics":{"steps":88,"total_tokens":70141},"oid":"cursorbench-28d7424b81fea2a6","rd":"2026-09-21","sm":"grok-4.7_xhigh","src":"https://cursor.com/cursorbench","v":0.46299999999999997},{"b":"cursorbench","c":11.43,"cfg":{"effort":"xhigh"},"dn":"Opus 5 Extra High","e":"xhigh","m":"claude-opus-5","metrics":{"steps":103,"total_tokens":80094},"oid":"cursorbench-37c785c761e10e0f","rd":"2026-07-24","sm":"claude-opus-5_xhigh","src":"https://cursor.com/cursorbench","v":0.461},{"b":"cursorbench","c":5.44,"cfg":{"effort":"low"},"dn":"Fable 5.1 Low","e":"low","m":"claude-fable-5-1","metrics":{"steps":51,"total_tokens":34795},"oid":"cursorbench-aa6f599f77d80e80","rd":"2026-09-01","sm":"claude-fable-5-1_low","src":"https://cursor.com/cursorbench","v":0.451},{"b":"cursorbench","c":9,"cfg":{"effort":"high"},"dn":"Opus 5 High","e":"high","m":"claude-opus-5","metrics":{"steps":86,"total_tokens":61405},"oid":"cursorbench-b4e829d445b18664","rd":"2026-07-24","sm":"claude-opus-5_high","src":"https://cursor.com/cursorbench","v":0.44700000000000006},{"b":"cursorbench","c":4.69,"cfg":{"effort":"high"},"dn":"Grok 4.7 High","e":"high","m":"grok-4-7","metrics":{"steps":71,"total_tokens":56382},"oid":"cursorbench-e7c002936275c4d6","rd":"2026-09-21","sm":"grok-4.7_high","src":"https://cursor.com/cursorbench","v":0.439},{"b":"cursorbench","c":6.94,"cfg":{"effort":"medium"},"dn":"Opus 5 Medium","e":"medium","m":"claude-opus-5","metrics":{"steps":72,"total_tokens":45272},"oid":"cursorbench-e68380f968752fe4","rd":"2026-07-24","sm":"claude-opus-5_medium","src":"https://cursor.com/cursorbench","v":0.433},{"b":"cursorbench","c":8.23,"cfg":{"effort":"max"},"dn":"GPT-5.6 Sol Max","e":"max","m":"gpt-5-6-sol","metrics":{"steps":99,"total_tokens":42944},"oid":"cursorbench-6ceca89b52f5b9d6","rd":"2026-07-09","sm":"gpt-5.6-sol_max","src":"https://cursor.com/cursorbench","v":0.41700000000000004},{"b":"cursorbench","c":3.49,"cfg":{"effort":"medium"},"dn":"Grok 4.7 Medium","e":"medium","m":"grok-4-7","metrics":{"steps":60,"total_tokens":36683},"oid":"cursorbench-781dafd1fb823c53","rd":"2026-09-21","sm":"grok-4.7_medium","src":"https://cursor.com/cursorbench","v":0.41600000000000004},{"b":"cursorbench","c":2.64,"cfg":{"effort":"max"},"dn":"Muse Spark 1.3 Max","e":"max","m":"muse-spark-1-3","metrics":{"steps":98,"total_tokens":52005},"oid":"cursorbench-91a2bb31804cb468","rd":"2026-09-02","sm":"muse-spark-1.3_max","src":"https://cursor.com/cursorbench","v":0.41600000000000004},{"b":"cursorbench","c":6.1,"cfg":{"effort":"xhigh"},"dn":"Grok 4.6 Extra High","e":"xhigh","m":"grok-4-6","metrics":{"steps":56,"total_tokens":49814},"oid":"cursorbench-7c4b284defd074fe","rd":"2026-08-12","sm":"grok-4.6_xhigh","src":"https://cursor.com/cursorbench","v":0.414},{"b":"cursorbench","c":5.14,"cfg":{"effort":"max"},"dn":"GPT-5.6 Terra Max","e":"max","m":"gpt-5-6-terra","metrics":{"steps":107,"total_tokens":60814},"oid":"cursorbench-f9f4b2b8e6af6f2a","rd":"2026-07-09","sm":"gpt-5.6-terra_max","src":"https://cursor.com/cursorbench","v":0.413},{"b":"cursorbench","c":4.87,"cfg":{"effort":"low"},"dn":"Opus 5 Low","e":"low","m":"claude-opus-5","metrics":{"steps":57,"total_tokens":31995},"oid":"cursorbench-f4efbf69041bbd02","rd":"2026-07-24","sm":"claude-opus-5_low","src":"https://cursor.com/cursorbench","v":0.40700000000000003},{"b":"cursorbench","c":5.2,"cfg":{"effort":"high"},"dn":"Grok 4.6 High","e":"high","m":"grok-4-6","metrics":{"steps":48,"total_tokens":41387},"oid":"cursorbench-c978fc98fe3e9207","rd":"2026-08-12","sm":"grok-4.6_high","src":"https://cursor.com/cursorbench","v":0.40399999999999997},{"b":"cursorbench","c":4.7,"cfg":{"effort":"high"},"dn":"Gemini 3.8 Flash High","e":"high","m":"gemini-3-8-flash","metrics":{"steps":324,"total_tokens":162565},"oid":"cursorbench-2f2de5dbb22e223c","rd":"2026-09-02","sm":"gemini-3.8-flash_high","src":"https://cursor.com/cursorbench","v":0.396},{"b":"cursorbench","c":4.4,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Sol Extra High","e":"xhigh","m":"gpt-5-6-sol","metrics":{"steps":55,"total_tokens":24729},"oid":"cursorbench-15934035fa17f4da","rd":"2026-07-09","sm":"gpt-5.6-sol_xhigh","src":"https://cursor.com/cursorbench","v":0.37700000000000006},{"b":"cursorbench","c":2.1,"cfg":{"effort":"xhigh"},"dn":"Muse Spark 1.3 Extra High","e":"xhigh","m":"muse-spark-1-3","metrics":{"steps":83,"total_tokens":40891},"oid":"cursorbench-cbdadf6a1e0df609","rd":"2026-09-02","sm":"muse-spark-1.3_xhigh","src":"https://cursor.com/cursorbench","v":0.375},{"b":"cursorbench","c":4.06,"cfg":{"effort":"medium"},"dn":"Gemini 3.8 Flash Medium","e":"medium","m":"gemini-3-8-flash","metrics":{"steps":290,"total_tokens":128364},"oid":"cursorbench-eb944474a7b0e404","rd":"2026-09-02","sm":"gemini-3.8-flash_medium","src":"https://cursor.com/cursorbench","v":0.373},{"b":"cursorbench","c":3.48,"cfg":{"effort":"medium"},"dn":"Grok 4.6 Medium","e":"medium","m":"grok-4-6","metrics":{"steps":40,"total_tokens":24893},"oid":"cursorbench-d668f522401c808e","rd":"2026-08-12","sm":"grok-4.6_medium","src":"https://cursor.com/cursorbench","v":0.36100000000000004},{"b":"cursorbench","c":1.03,"cfg":{"effort":"max"},"dn":"GPT-5.6 Luna Max","e":"max","m":"gpt-5-6-luna","metrics":{"steps":208,"total_tokens":87284},"oid":"cursorbench-a2333733178a7034","rd":"2026-07-09","sm":"gpt-5.6-luna_max","src":"https://cursor.com/cursorbench","v":0.359},{"b":"cursorbench","c":2.85,"cfg":{"effort":"high"},"dn":"GPT-5.6 Sol High","e":"high","m":"gpt-5-6-sol","metrics":{"steps":41,"total_tokens":16174},"oid":"cursorbench-a4e4bd9251596e3f","rd":"2026-07-09","sm":"gpt-5.6-sol_high","src":"https://cursor.com/cursorbench","v":0.35700000000000004},{"b":"cursorbench","c":7.17,"cfg":{"effort":"max"},"dn":"Sonnet 5 Max","e":"max","m":"claude-sonnet-5","metrics":{"steps":140,"total_tokens":149257},"oid":"cursorbench-231d865f41934e8b","rd":"2026-06-30","sm":"claude-sonnet-5_max","src":"https://cursor.com/cursorbench","v":0.341},{"b":"cursorbench","c":1.81,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Terra Extra High","e":"xhigh","m":"gpt-5-6-terra","metrics":{"steps":43,"total_tokens":23436},"oid":"cursorbench-da01cfa4a3b82ad1","rd":"2026-07-09","sm":"gpt-5.6-terra_xhigh","src":"https://cursor.com/cursorbench","v":0.336},{"b":"cursorbench","c":2.25,"cfg":{"effort":"low"},"dn":"Grok 4.6 Low","e":"low","m":"grok-4-6","metrics":{"steps":32,"total_tokens":16307},"oid":"cursorbench-d572653a13b82692","rd":"2026-08-12","sm":"grok-4.6_low","src":"https://cursor.com/cursorbench","v":0.334},{"b":"cursorbench","c":1.66,"cfg":{"effort":"high"},"dn":"Muse Spark 1.3 High","e":"high","m":"muse-spark-1-3","metrics":{"steps":69,"total_tokens":30654},"oid":"cursorbench-1e21b3d5987bd9ee","rd":"2026-09-02","sm":"muse-spark-1.3_high","src":"https://cursor.com/cursorbench","v":0.334},{"b":"cursorbench","c":1.58,"cfg":{"effort":"low"},"dn":"Grok 4.7 Low","e":"low","m":"grok-4-7","metrics":{"steps":40,"total_tokens":15677},"oid":"cursorbench-68f5598867ef8e7c","rd":"2026-09-21","sm":"grok-4.7_low","src":"https://cursor.com/cursorbench","v":0.331},{"b":"cursorbench","c":0.44,"cfg":{"effort":"xhigh"},"dn":"GPT-5.6 Luna Extra High","e":"xhigh","m":"gpt-5-6-luna","metrics":{"steps":98,"total_tokens":40598},"oid":"cursorbench-28a8001ef757bef7","rd":"2026-07-09","sm":"gpt-5.6-luna_xhigh","src":"https://cursor.com/cursorbench","v":0.33},{"b":"cursorbench","c":1.49,"cfg":{"effort":"medium"},"dn":"Muse Spark 1.3 Medium","e":"medium","m":"muse-spark-1-3","metrics":{"steps":64,"total_tokens":27255},"oid":"cursorbench-320ea82a4a3875b8","rd":"2026-09-02","sm":"muse-spark-1.3_medium","src":"https://cursor.com/cursorbench","v":0.326},{"b":"cursorbench","c":4.55,"cfg":{"effort":"xhigh"},"dn":"Sonnet 5 Extra High","e":"xhigh","m":"claude-sonnet-5","metrics":{"steps":102,"total_tokens":83373},"oid":"cursorbench-c2a09cf26e87c54a","rd":"2026-06-30","sm":"claude-sonnet-5_xhigh","src":"https://cursor.com/cursorbench","v":0.32},{"b":"cursorbench","c":1.77,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Sol Medium","e":"medium","m":"gpt-5-6-sol","metrics":{"steps":32,"total_tokens":10111},"oid":"cursorbench-8c8ecde8111c5a07","rd":"2026-07-09","sm":"gpt-5.6-sol_medium","src":"https://cursor.com/cursorbench","v":0.311},{"b":"cursorbench","c":3.48,"cfg":{"effort":"high"},"dn":"Sonnet 5 High","e":"high","m":"claude-sonnet-5","metrics":{"steps":85,"total_tokens":61146},"oid":"cursorbench-14a68771c8b75372","rd":"2026-06-30","sm":"claude-sonnet-5_high","src":"https://cursor.com/cursorbench","v":0.308},{"b":"cursorbench","c":1.11,"cfg":{"effort":"high"},"dn":"GPT-5.6 Terra High","e":"high","m":"gpt-5-6-terra","metrics":{"steps":33,"total_tokens":13162},"oid":"cursorbench-674f80f4dc09934f","rd":"2026-07-09","sm":"gpt-5.6-terra_high","src":"https://cursor.com/cursorbench","v":0.307},{"b":"cursorbench","c":0.25,"cfg":{"effort":"high"},"dn":"GPT-5.6 Luna High","e":"high","m":"gpt-5-6-luna","metrics":{"steps":64,"total_tokens":23368},"oid":"cursorbench-59bad8a2880c30eb","rd":"2026-07-09","sm":"gpt-5.6-luna_high","src":"https://cursor.com/cursorbench","v":0.294},{"b":"cursorbench","c":0.93,"cfg":{"effort":"low"},"dn":"Muse Spark 1.3 Low","e":"low","m":"muse-spark-1-3","metrics":{"steps":47,"total_tokens":17483},"oid":"cursorbench-a1728ddf13d6f597","rd":"2026-09-02","sm":"muse-spark-1.3_low","src":"https://cursor.com/cursorbench","v":0.29300000000000004},{"b":"cursorbench","c":2.31,"cfg":{"effort":"medium"},"dn":"Sonnet 5 Medium","e":"medium","m":"claude-sonnet-5","metrics":{"steps":65,"total_tokens":39114},"oid":"cursorbench-a66e96074eb9540f","rd":"2026-06-30","sm":"claude-sonnet-5_medium","src":"https://cursor.com/cursorbench","v":0.28},{"b":"cursorbench","c":0.68,"dn":"Composer 2.5","m":"composer-2-5","metrics":{"steps":41,"total_tokens":17347},"oid":"cursorbench-3c4d47a94f8d055e","rd":"2026-05-18","sm":"Composer 2.5","src":"https://cursor.com/cursorbench","v":0.277},{"b":"cursorbench","c":0.64,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Terra Medium","e":"medium","m":"gpt-5-6-terra","metrics":{"steps":25,"total_tokens":7307},"oid":"cursorbench-33fb9daefc868950","rd":"2026-07-09","sm":"gpt-5.6-terra_medium","src":"https://cursor.com/cursorbench","v":0.276},{"b":"cursorbench","c":0.52,"cfg":{"effort":"low"},"dn":"GPT-5.6 Terra Low","e":"low","m":"gpt-5-6-terra","metrics":{"steps":23,"total_tokens":5914},"oid":"cursorbench-93e6860c8b08a2a8","rd":"2026-07-09","sm":"gpt-5.6-terra_low","src":"https://cursor.com/cursorbench","v":0.252},{"b":"cursorbench","c":0.87,"cfg":{"effort":"low"},"dn":"GPT-5.6 Sol Low","e":"low","m":"gpt-5-6-sol","metrics":{"steps":21,"total_tokens":4885},"oid":"cursorbench-1dcf884e43d547d9","rd":"2026-07-09","sm":"gpt-5.6-sol_low","src":"https://cursor.com/cursorbench","v":0.24600000000000002},{"b":"cursorbench","c":0.56,"cfg":{"effort":"minimal"},"dn":"Muse Spark 1.3 Minimal","e":"minimal","m":"muse-spark-1-3","metrics":{"steps":34,"total_tokens":10620},"oid":"cursorbench-be40a69b5d7a51a5","rd":"2026-09-02","sm":"muse-spark-1.3_minimal","src":"https://cursor.com/cursorbench","v":0.24300000000000002},{"b":"cursorbench","c":1.39,"cfg":{"effort":"low"},"dn":"Sonnet 5 Low","e":"low","m":"claude-sonnet-5","metrics":{"steps":46,"total_tokens":23772},"oid":"cursorbench-5a28b2cc093bffdf","rd":"2026-06-30","sm":"claude-sonnet-5_low","src":"https://cursor.com/cursorbench","v":0.24100000000000002},{"b":"cursorbench","c":0.08,"cfg":{"effort":"medium"},"dn":"GPT-5.6 Luna Medium","e":"medium","m":"gpt-5-6-luna","metrics":{"steps":32,"total_tokens":7642},"oid":"cursorbench-86ec2385191c189e","rd":"2026-07-09","sm":"gpt-5.6-luna_medium","src":"https://cursor.com/cursorbench","v":0.222},{"b":"cursorbench","c":0.03,"cfg":{"effort":"low"},"dn":"GPT-5.6 Luna Low","e":"low","m":"gpt-5-6-luna","metrics":{"steps":18,"total_tokens":3288},"oid":"cursorbench-e7d37fe8f799f4ae","rd":"2026-07-09","sm":"gpt-5.6-luna_low","src":"https://cursor.com/cursorbench","v":0.16},{"b":"cursorbench","c":13.43,"cfg":{"effort":"max"},"dn":"Opus 5.5 Max","e":"max","m":"claude-opus-5-5","metrics":{"steps":185,"total_tokens":218363},"oid":"cursorbench-a16516934f2e1339","rd":"2026-09-22","sm":"claude-opus-5-5_max","src":"https://cursor.com/cursorbench","v":0.578},{"b":"cursorbench","c":6.98,"cfg":{"effort":"xhigh"},"dn":"Opus 5.5 Extra High","e":"xhigh","m":"claude-opus-5-5","metrics":{"steps":109,"total_tokens":101083},"oid":"cursorbench-5c9c584c455d657f","rd":"2026-09-22","sm":"claude-opus-5-5_xhigh","src":"https://cursor.com/cursorbench","v":0.56},{"b":"cursorbench","c":7.05,"cfg":{"effort":"max"},"dn":"Sonnet 5.5 Max","e":"max","m":"claude-sonnet-5-5","metrics":{"steps":170,"total_tokens":271920},"oid":"cursorbench-7b6ecc3496482f81","rd":"2026-09-28","sm":"claude-sonnet-5-5_max","src":"https://cursor.com/cursorbench","v":0.555},{"b":"cursorbench","c":2.81,"cfg":{"effort":"xhigh"},"dn":"Sonnet 5.5 Extra High","e":"xhigh","m":"claude-sonnet-5-5","metrics":{"steps":78,"total_tokens":100158},"oid":"cursorbench-abc0b0be7b6f65b4","rd":"2026-09-28","sm":"claude-sonnet-5-5_xhigh","src":"https://cursor.com/cursorbench","v":0.531},{"b":"cursorbench","c":1.2,"cfg":{"effort":"high"},"dn":"Sonnet 5.5 High","e":"high","m":"claude-sonnet-5-5","metrics":{"steps":41,"total_tokens":37391},"oid":"cursorbench-6bf50ffeb851e2c7","rd":"2026-09-28","sm":"claude-sonnet-5-5_high","src":"https://cursor.com/cursorbench","v":0.478},{"b":"cursorbench","c":0.52,"cfg":{"effort":"medium"},"dn":"Sonnet 5.5 Medium","e":"medium","m":"claude-sonnet-5-5","metrics":{"steps":22,"total_tokens":16036},"oid":"cursorbench-98d9bdc0e92cbfa6","rd":"2026-09-28","sm":"claude-sonnet-5-5_medium","src":"https://cursor.com/cursorbench","v":0.392},{"b":"cursorbench","c":0.37,"cfg":{"effort":"low"},"dn":"Sonnet 5.5 Low","e":"low","m":"claude-sonnet-5-5","metrics":{"steps":18,"total_tokens":11668},"oid":"cursorbench-224f1d6de7dd93e4","rd":"2026-09-28","sm":"claude-sonnet-5-5_low","src":"https://cursor.com/cursorbench","v":0.358},{"b":"cursorbench","c":3.97,"cfg":{"effort":"high"},"dn":"Opus 5.5 High","e":"high","m":"claude-opus-5-5","metrics":{"steps":68,"total_tokens":53078},"oid":"cursorbench-95860d295e572797","rd":"2026-09-22","sm":"claude-opus-5-5_high","src":"https://cursor.com/cursorbench","v":0.56},{"b":"cursorbench","c":2.91,"cfg":{"effort":"medium"},"dn":"Opus 5.5 Medium","e":"medium","m":"claude-opus-5-5","metrics":{"steps":54,"total_tokens":37954},"oid":"cursorbench-078c3cdd1d0aa7f8","rd":"2026-09-22","sm":"claude-opus-5-5_medium","src":"https://cursor.com/cursorbench","v":0.525},{"b":"cursorbench","c":1.17,"cfg":{"effort":"low"},"dn":"Opus 5.5 Low","e":"low","m":"claude-opus-5-5","metrics":{"steps":28,"total_tokens":15811},"oid":"cursorbench-73a24c8d1dee8606","rd":"2026-09-22","sm":"claude-opus-5-5_low","src":"https://cursor.com/cursorbench","v":0.43700000000000006},{"b":"cursorbench","c":5.05,"cfg":{"effort":"max"},"dn":"GLM 5.3 Max","e":"max","m":"glm-5-3","metrics":{"steps":166,"total_tokens":96387},"oid":"cursorbench-1c84fbd06846c86a","rd":"2026-08-14","sm":"glm-5.3_max","src":"https://cursor.com/cursorbench","v":0.42600000000000005},{"b":"cursorbench","c":3.24,"cfg":{"effort":"high"},"dn":"GLM 5.3 High","e":"high","m":"glm-5-3","metrics":{"steps":114,"total_tokens":60031},"oid":"cursorbench-f4d75eb4ec21607d","rd":"2026-08-14","sm":"glm-5.3_high","src":"https://cursor.com/cursorbench","v":0.38},{"b":"cursorbench","c":0.39,"cfg":{"effort":"max"},"dn":"GLM 5.3 Flash Max","e":"max","m":"glm-5-3-flash","metrics":{"steps":118,"total_tokens":56410},"oid":"cursorbench-09de38082f7e9ed8","rd":"2026-08-20","sm":"glm-5.3-flash_max","src":"https://cursor.com/cursorbench","v":0.368},{"b":"cursorbench","c":2.04,"cfg":{"effort":"low"},"dn":"GLM 5.3 Low","e":"low","m":"glm-5-3","metrics":{"steps":81,"total_tokens":31983},"oid":"cursorbench-ee199e180138ddad","rd":"2026-08-14","sm":"glm-5.3_low","src":"https://cursor.com/cursorbench","v":0.33299999999999996},{"b":"cursorbench","c":0.25,"cfg":{"effort":"high"},"dn":"GLM 5.3 Flash High","e":"high","m":"glm-5-3-flash","metrics":{"steps":84,"total_tokens":35104},"oid":"cursorbench-178f737cd5c4c5ed","rd":"2026-08-20","sm":"glm-5.3-flash_high","src":"https://cursor.com/cursorbench","v":0.311},{"b":"cursorbench","c":0.15,"cfg":{"effort":"low"},"dn":"GLM 5.3 Flash Low","e":"low","m":"glm-5-3-flash","metrics":{"steps":58,"total_tokens":17831},"oid":"cursorbench-7740ae06e94bedd9","rd":"2026-08-20","sm":"glm-5.3-flash_low","src":"https://cursor.com/cursorbench","v":0.269},{"b":"cursorbench","c":4.57,"cfg":{"effort":"high"},"dn":"Gemini 4 Argon High","e":"high","m":"gemini-4-argon","metrics":{"steps":118,"total_tokens":109500},"oid":"cursorbench-3e7e84e43d1f44c2","rd":"2026-09-30","sm":"gemini-4-argon_high","src":"https://cursor.com/cursorbench","v":0.529},{"b":"cursorbench","c":4.39,"cfg":{"effort":"medium"},"dn":"Gemini 4 Argon Medium","e":"medium","m":"gemini-4-argon","metrics":{"steps":115,"total_tokens":103600},"oid":"cursorbench-5debaf1e7b155bd4","rd":"2026-09-30","sm":"gemini-4-argon_medium","src":"https://cursor.com/cursorbench","v":0.528},{"b":"cursorbench","c":4.09,"cfg":{"effort":"low"},"dn":"Gemini 4 Argon Low","e":"low","m":"gemini-4-argon","metrics":{"steps":110,"total_tokens":94200},"oid":"cursorbench-33b6f0a989cbcd96","rd":"2026-09-30","sm":"gemini-4-argon_low","src":"https://cursor.com/cursorbench","v":0.506},{"b":"cursorbench","c":3.54,"cfg":{"effort":"minimal"},"dn":"Gemini 4 Argon Minimal","e":"minimal","m":"gemini-4-argon","metrics":{"steps":102,"total_tokens":39400},"oid":"cursorbench-bcbb27c4ee84847a","rd":"2026-09-30","sm":"gemini-4-argon_minimal","src":"https://cursor.com/cursorbench","v":0.488},{"b":"cursorbench","c":1.12,"cfg":{"effort":"max"},"dn":"Haiku 5.5 Max","e":"max","m":"claude-haiku-5-5","metrics":{"steps":163,"total_tokens":325934},"oid":"cursorbench-f61bff75ff6cfed4","rd":"2026-10-07","sm":"claude-haiku-5-5_max","src":"https://cursor.com/cursorbench","v":0.484},{"b":"cursorbench","c":0.56,"cfg":{"effort":"xhigh"},"dn":"Haiku 5.5 Extra High","e":"xhigh","m":"claude-haiku-5-5","metrics":{"steps":93,"total_tokens":143813},"oid":"cursorbench-cab8c9806e88411c","rd":"2026-10-07","sm":"claude-haiku-5-5_xhigh","src":"https://cursor.com/cursorbench","v":0.443},{"b":"cursorbench","c":0.32,"cfg":{"effort":"high"},"dn":"Haiku 5.5 High","e":"high","m":"claude-haiku-5-5","metrics":{"steps":59,"total_tokens":77057},"oid":"cursorbench-78933f8c545ffcc3","rd":"2026-10-07","sm":"claude-haiku-5-5_high","src":"https://cursor.com/cursorbench","v":0.423},{"b":"cursorbench","c":0.17,"cfg":{"effort":"medium"},"dn":"Haiku 5.5 Medium","e":"medium","m":"claude-haiku-5-5","metrics":{"steps":39,"total_tokens":42659},"oid":"cursorbench-5288c0a27b28c9d1","rd":"2026-10-07","sm":"claude-haiku-5-5_medium","src":"https://cursor.com/cursorbench","v":0.369},{"b":"cursorbench","c":0.08,"cfg":{"effort":"low"},"dn":"Haiku 5.5 Low","e":"low","m":"claude-haiku-5-5","metrics":{"steps":25,"total_tokens":23382},"oid":"cursorbench-6f0cd9c236dfdcfa","rd":"2026-10-07","sm":"claude-haiku-5-5_low","src":"https://cursor.com/cursorbench","v":0.309},{"b":"cursorbench","c":5.25,"cfg":{"effort":"max"},"dn":"Muse Wonder Max","e":"max","m":"muse-wonder-1-0","metrics":{"steps":114,"total_tokens":49692},"oid":"cursorbench-3dd588edcb79ab87","rd":"2026-10-12","sm":"muse-wonder-1.0_max","src":"https://cursor.com/cursorbench","v":0.504},{"b":"cursorbench","c":4.24,"cfg":{"effort":"xhigh"},"dn":"Muse Wonder Extra High","e":"xhigh","m":"muse-wonder-1-0","metrics":{"steps":96,"total_tokens":39564},"oid":"cursorbench-6fcb75907eb20e40","rd":"2026-10-12","sm":"muse-wonder-1.0_xhigh","src":"https://cursor.com/cursorbench","v":0.48200000000000004},{"b":"cursorbench","c":3.51,"cfg":{"effort":"high"},"dn":"Muse Wonder High","e":"high","m":"muse-wonder-1-0","metrics":{"steps":84,"total_tokens":33612},"oid":"cursorbench-033a0d10eaf9c373","rd":"2026-10-12","sm":"muse-wonder-1.0_high","src":"https://cursor.com/cursorbench","v":0.461},{"b":"cursorbench","c":2.82,"cfg":{"effort":"medium"},"dn":"Muse Wonder Medium","e":"medium","m":"muse-wonder-1-0","metrics":{"steps":75,"total_tokens":27124},"oid":"cursorbench-50a1015083e47d78","rd":"2026-10-12","sm":"muse-wonder-1.0_medium","src":"https://cursor.com/cursorbench","v":0.452},{"b":"cursorbench","c":1.91,"cfg":{"effort":"low"},"dn":"Muse Wonder Low","e":"low","m":"muse-wonder-1-0","metrics":{"steps":56,"total_tokens":18880},"oid":"cursorbench-e15689f5867750d4","rd":"2026-10-12","sm":"muse-wonder-1.0_low","src":"https://cursor.com/cursorbench","v":0.41700000000000004},{"b":"cursorbench","c":0.84,"cfg":{"effort":"minimal"},"dn":"Muse Wonder Minimal","e":"minimal","m":"muse-wonder-1-0","metrics":{"steps":33,"total_tokens":9761},"oid":"cursorbench-3689b101d492cf10","rd":"2026-10-12","sm":"muse-wonder-1.0_minimal","src":"https://cursor.com/cursorbench","v":0.342},{"b":"cursorbench","c":0.98,"cfg":{"effort":"max"},"dn":"DeepSeek V4.1 Flash 1017 Max","e":"max","m":"deepseek-v4-1-flash-1017","metrics":{"steps":187,"total_tokens":537047},"oid":"cursorbench-22ddd9c4e9b115c4","rd":"2026-10-17","sm":"deepseek-v4.1-flash-1017_max","src":"https://cursor.com/cursorbench","v":0.47700000000000004},{"b":"cursorbench","c":0.8,"cfg":{"effort":"medium"},"dn":"DeepSeek V4.1 Flash 1017 Medium","e":"medium","m":"deepseek-v4-1-flash-1017","metrics":{"steps":189,"total_tokens":398255},"oid":"cursorbench-82478b65cd1bf0f8","rd":"2026-10-17","sm":"deepseek-v4.1-flash-1017_medium","src":"https://cursor.com/cursorbench","v":0.46299999999999997},{"b":"cursorbench","c":0.77,"cfg":{"effort":"high"},"dn":"DeepSeek V4.1 Flash 1017 High","e":"high","m":"deepseek-v4-1-flash-1017","metrics":{"steps":186,"total_tokens":385196},"oid":"cursorbench-6db24e3d15a6e3fc","rd":"2026-10-17","sm":"deepseek-v4.1-flash-1017_high","src":"https://cursor.com/cursorbench","v":0.456},{"b":"cursorbench","c":0.6,"cfg":{"effort":"low"},"dn":"DeepSeek V4.1 Flash 1017 Low","e":"low","m":"deepseek-v4-1-flash-1017","metrics":{"steps":161,"total_tokens":290651},"oid":"cursorbench-b44588c9d1c7bb09","rd":"2026-10-17","sm":"deepseek-v4.1-flash-1017_low","src":"https://cursor.com/cursorbench","v":0.449},{"b":"cybench","m":"claude-opus-4-6","notes":"Results from Opus 4.6 system card, also reported on the official Cybench leaderboard. The card states that Opus 4.6 gets ~100% for pass@30.","oid":"cybench-82ef5716abfad7f3","rd":"2026-02-05","sid":"recWoHNahp8uxAsGZ","sm":"claude-opus-4-6_unknown","src":"https://www-cdn.anthropic.com/0dd865075ad3132672ee0ab40b05a53f14cf5288.pdf","v":0.93},{"b":"cybench","m":"claude-opus-4-5-2025-11-01","notes":"Results from Opus 4.5 system card, also reported on the official Cybench leaderboard","oid":"cybench-231eb66a912f686e","rd":"2025-11-24","sid":"recJjYw1K1NV8Z47n","sm":"claude-opus-4-5-20251101_unknown","src":"https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf","v":0.82},{"b":"cybench","m":"claude-sonnet-4-5-2025-09-29","notes":"Results from Opus 4.5 system card, also reported on the official Cybench leaderboard","oid":"cybench-1893555a8858b74a","rd":"2025-09-29","sid":"recbFGPB81lSzLlEv","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://assets.anthropic.com/m/64823ba7485345a7/Claude-Opus-4-5-System-Card.pdf","v":0.6},{"b":"cybench","m":"claude-sonnet-4-5-2025-09-29","notes":"Thinking level unknown. \"Results for Claude 4.5 Sonnet, Claude 4.1 Opus, Claude 4 Opus, and Claude 4 Sonnet are from the Claude 4.5 Sonnet System Card on a subset of 37 problems. Percentages are estimates based on probability of success on 1 trial.\"","oid":"cybench-e6950ea7f1f246c8","rd":"2025-09-29","sid":"recpAkk0ysXYhjhJu","sm":"claude-sonnet-4-5-20250929","src":"https://cybench.github.io/","v":0.55},{"b":"cybench","m":"grok-4-0709","notes":"Results from Grok 4 model card. Also reported on official Cybench leaderboard.","oid":"cybench-4831efbe5087c0b9","rd":"2025-07-09","sid":"rec77oHCMurVWgzvd","sm":"grok-4-0709","src":"https://data.x.ai/2025-08-20-grok-4-model-card.pdf","v":0.43},{"b":"cybench","m":"claude-opus-4-1-2025-08-05","notes":"Thinking level unknown. \"Results for Claude 4.5 Sonnet, Claude 4.1 Opus, Claude 4 Opus, and Claude 4 Sonnet are from the Claude 4.5 Sonnet System Card on a subset of 37 problems. Percentages are estimates based on probability of success on 1 trial.\"","oid":"cybench-23093a60981175ae","rd":"2025-08-05","sid":"recCu9LKRKI8DDSPv","sm":"claude-opus-4-1-20250805","src":"https://cybench.github.io/","v":0.42},{"b":"cybench","m":"grok-4-1","notes":"Thinking version. Results from Grok 4.1 Model card, also reported on official Cybench leaderboard.","oid":"cybench-d58e4268e9ac0ed3","rd":"2025-11-17","sid":"recd57Ck0JOqmhCSh","sm":"grok-4-1","src":"https://data.x.ai/2025-11-17-grok-4-1-model-card.pdf","v":0.39},{"b":"cybench","m":"claude-opus-4-2025-05-14","notes":"Thinking level unknown. \"Results for Claude 4.5 Sonnet, Claude 4.1 Opus, Claude 4 Opus, and Claude 4 Sonnet are from the Claude 4.5 Sonnet System Card on a subset of 37 problems. Percentages are estimates based on probability of success on 1 trial.\"","oid":"cybench-2af622b6d0958bdd","rd":"2025-05-22","sid":"recHS1d8FpTudBN0R","sm":"claude-opus-4-20250514","src":"https://cybench.github.io/","v":0.38},{"b":"cybench","m":"claude-sonnet-4-2025-05-14","notes":"Thinking level unknown. \"Results for Claude 4.5 Sonnet, Claude 4.1 Opus, Claude 4 Opus, and Claude 4 Sonnet are from the Claude 4.5 Sonnet System Card on a subset of 37 problems. Percentages are estimates based on probability of success on 1 trial.\"","oid":"cybench-d225e79a7ad1db3e","rd":"2025-05-22","sid":"recrjRfouZg28oDuU","sm":"claude-sonnet-4-20250514","src":"https://cybench.github.io/","v":0.35},{"b":"cybench","m":"grok-4-fast","notes":"Results from Grok 4 Fast Model card, also reported on official Cybench leaderboard.","oid":"cybench-2697936beaaabf26","rd":"2025-09-19","sid":"recJ2fztLtK3WB0yS","sm":"grok-4-fast","src":"https://data.x.ai/2025-09-19-grok-4-fast-model-card.pdf","v":0.3},{"b":"cybench","cfg":{"effort":"medium"},"e":"medium","m":"o3-mini-2025-01-31","notes":"\"* Results from HAL leaderboard evaluation\n\n† The scores for OpenAI o3-mini and OpenAI o1-mini are inflated because HAL likely ran on a fork of the Inspect framework that leaked the answer to a task that both models completed successfully. Their Unguided % Solved scores have been adjusted downward by 2.5% (to 22.5% and 10% respectively) and their FSTs have been updated to reflect their most difficult tasks solved excluding the leaked task (42 min for o3-mini and 11 min for o1-mini).\"\n\nAssuming medium reasoning effort.","oid":"cybench-fda5c00ff551e899","rd":"2025-01-31","sid":"recZdBmmrzod4SvRl","sm":"o3-mini-2025-01-31_medium","src":"https://cybench.github.io/","v":0.225},{"b":"cybench","m":"claude-3-7-sonnet-2025-02-19","notes":"\"* Results from HAL leaderboard evaluation\"","oid":"cybench-c015ce73e6b3191a","rd":"2025-02-24","sid":"reccK2H4ppj3E4QcM","sm":"claude-3-7-sonnet-20250219","src":"https://cybench.github.io/","v":0.2},{"b":"cybench","m":"gpt-4-5-preview-2025-02-27","notes":"\"* Results from HAL leaderboard evaluation\"","oid":"cybench-7178281bbc952f27","rd":"2025-02-27","sid":"rec7SnJJ5vaTPzZMS","sm":"gpt-4.5-preview-2025-02-27","src":"https://cybench.github.io/","v":0.175},{"b":"cybench","m":"claude-3-5-sonnet-2024-06-20","oid":"cybench-bf61c0c36746ede1","rd":"2024-06-20","sid":"recnDZnKntPQUMNMP","sm":"claude-3-5-sonnet-20240620","src":"https://cybench.github.io/","v":0.175},{"b":"cybench","m":"gpt-4o-2024-11-20","notes":"Assuming latest version","oid":"cybench-bd7bca5b88084ae9","rd":"2024-11-20","sid":"reca4ghOZAGJnaIQP","sm":"gpt-4o-2024-11-20","src":"https://cybench.github.io/","v":0.125},{"b":"cybench","m":"claude-3-opus-2024-02-29","oid":"cybench-1100b7294f48f775","rd":"2024-02-29","sid":"recBEOU9mpDvDjB0J","sm":"claude-3-opus-20240229","src":"https://cybench.github.io/","v":0.1},{"b":"cybench","cfg":{"effort":"medium"},"e":"medium","m":"o1-mini-2024-09-12","notes":"\"* Results from HAL leaderboard evaluation\n\n† The scores for OpenAI o3-mini and OpenAI o1-mini are inflated because HAL likely ran on a fork of the Inspect framework that leaked the answer to a task that both models completed successfully. Their Unguided % Solved scores have been adjusted downward by 2.5% (to 22.5% and 10% respectively) and their FSTs have been updated to reflect their most difficult tasks solved excluding the leaked task (42 min for o3-mini and 11 min for o1-mini).\"\n\nAssuming medium reasoning effort.","oid":"cybench-5e48961295ea8728","rd":"2024-09-12","sid":"recDdu0Wr7UcjeILN","sm":"o1-mini-2024-09-12_medium","src":"https://cybench.github.io/","v":0.1},{"b":"cybench","m":"o1-preview-2024-09-12","oid":"cybench-4941a8bbae21506f","rd":"2024-09-12","sid":"recrEGfmnK38cjFjn","sm":"o1-preview-2024-09-12","src":"https://cybench.github.io/","v":0.1},{"b":"cybench","m":"mixtral-8x22b-instruct-v0-1","oid":"cybench-60bbc6da55fb46bd","rd":"2024-04-17","sid":"recFsRheuwg1wunwf","sm":"Mixtral-8x22B-Instruct-v0.1","src":"https://cybench.github.io/","v":0.075},{"b":"cybench","m":"llama-3-1-405b-instruct","oid":"cybench-876a0b7c35cd6359","rd":"2024-07-23","sid":"recHTQDbjXmORU18G","sm":"Llama-3.1-405B-Instruct","src":"https://cybench.github.io/","v":0.075},{"b":"cybench","m":"gemini-1-5-pro-001-feb24","oid":"cybench-5310e5cdf0c01c0f","rd":"2024-02-15","sid":"reco8bWKr3VP7mW4o","sm":"gemini-1.5-pro-001-feb24","src":"https://cybench.github.io/","v":0.075},{"b":"cybench","m":"meta-llama-3-70b-instruct","oid":"cybench-5003b6b743bd4176","rd":"2024-04-18","sid":"recqUzjd97hyzZN2f","sm":"Meta-Llama-3-70B-Instruct","src":"https://cybench.github.io/","v":0.05},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"claude-opus-4-6","oid":"deepresearchbench-270d74e081f9c576","rd":"2026-02-05","sid":"reccVU3SIqoNBQoc4","sm":"claude-opus-4-6_high","src":"https://evals.futuresearch.ai/#drb","v":0.5529999999999999},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"claude-sonnet-4-6","oid":"deepresearchbench-f8f1e851e11b741a","rd":"2026-02-17","sid":"rec8802UWGtedVgJY","sm":"claude-sonnet-4-6_high","src":"https://evals.futuresearch.ai/#drb","v":0.549},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"claude-opus-4-5-2025-11-01","oid":"deepresearchbench-fdf03169d1e64799","rd":"2025-11-24","sid":"recHz66Mx92wWOqhz","sm":"claude-opus-4-5-20251101_high","src":"https://evals.futuresearch.ai/#drb","v":0.5479999999999999},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-5","oid":"deepresearchbench-4c3bfa416fc63468","rd":"2026-04-23","sid":"recheEJW36S0Dmd4M","sm":"gpt-5.5_high","src":"https://evals.futuresearch.ai/#drb","v":0.54},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-opus-4-5-2025-11-01","oid":"deepresearchbench-aa657ea6dfacba8a","rd":"2025-11-24","sid":"recfm3eBfDzsH1AjW","sm":"claude-opus-4-5-20251101_low","src":"https://evals.futuresearch.ai/#drb","v":0.537},{"b":"deepresearchbench","cfg":{"effort":"medium"},"e":"medium","m":"claude-opus-4-6","oid":"deepresearchbench-0c047d22b4c425d5","rd":"2026-02-05","sid":"recV41P9TO0UOXBrz","sm":"claude-opus-4-6_medium","src":"https://evals.futuresearch.ai/#drb","v":0.532},{"b":"deepresearchbench","m":"claude-sonnet-4-5-20250929-2k","oid":"deepresearchbench-25e7a27c8fa7e0bb","rd":"2025-09-29","sid":"recWmaxS0ycQLxFMa","sm":"claude-sonnet-4-5-20250929_2K","src":"https://drb.futuresearch.ai","v":0.526},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-opus-4-6","oid":"deepresearchbench-9b2a7ae84b4f2a60","rd":"2026-02-05","sid":"recHblA277kTWDIAv","sm":"claude-opus-4-6_low","src":"https://evals.futuresearch.ai/#drb","v":0.514},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-sonnet-4-6","oid":"deepresearchbench-f0d547eb5317cf09","rd":"2026-02-17","sid":"recqP7OyQXrgSWLtr","sm":"claude-sonnet-4-6_low","src":"https://evals.futuresearch.ai/#drb","v":0.504},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"claude-opus-4-8","oid":"deepresearchbench-a69305ded80d6255","rd":"2026-05-28","sid":"recH15wI9KuYl4sCc","sm":"claude-opus-4-8_high","src":"https://evals.futuresearch.ai/#drb","v":0.502},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gemini-3-flash-preview","oid":"deepresearchbench-4745e4476f2414f8","rd":"2025-12-17","sid":"rec3iKvPvr8Wl8Ani","sm":"gemini-3-flash-preview_low","src":"https://evals.futuresearch.ai/#drb","v":0.498},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-2025-08-07","oid":"deepresearchbench-882c7590ef562a26","rd":"2025-08-07","sid":"recIbyRHhHFxDYfXq","sm":"gpt-5-2025-08-07_low","src":"https://drb.futuresearch.ai","v":0.49600000000000005},{"b":"deepresearchbench","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-5","oid":"deepresearchbench-e62681564889334d","rd":"2026-04-23","sid":"rec6hR81BMyiPDlmS","sm":"gpt-5.5_medium","src":"https://evals.futuresearch.ai/#drb","v":0.49600000000000005},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-opus-4-8","oid":"deepresearchbench-cc0d0e3545273905","rd":"2026-05-28","sid":"recuiSzLJrl2Y4Id1","sm":"claude-opus-4-8_low","src":"https://evals.futuresearch.ai/#drb","v":0.493},{"b":"deepresearchbench","cfg":{"effort":"minimal"},"e":"minimal","m":"gemini-3-flash-preview","oid":"deepresearchbench-f858ee702a048675","rd":"2025-12-17","sid":"recA41zMYRSwfZqfb","sm":"gemini-3-flash-preview_minimal","src":"https://evals.futuresearch.ai/#drb","v":0.49},{"b":"deepresearchbench","cfg":{"effort":"minimal"},"e":"minimal","m":"gpt-5-2025-08-07","oid":"deepresearchbench-4a20a13562edaf2f","rd":"2025-08-07","sid":"recHggeQMAdJvIEz9","sm":"gpt-5-2025-08-07_minimal","src":"https://drb.futuresearch.ai","v":0.489},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-5","oid":"deepresearchbench-60ffdb19ecb14097","rd":"2026-04-23","sid":"reccXAd2b5mJk5VO0","sm":"gpt-5.5_low","src":"https://evals.futuresearch.ai/#drb","v":0.48700000000000004},{"b":"deepresearchbench","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-2025-08-07","oid":"deepresearchbench-8825382dd65b266c","rd":"2025-08-07","sid":"recsWpC0ObN7DsE4t","sm":"gpt-5-2025-08-07_medium","src":"https://drb.futuresearch.ai","v":0.48600000000000004},{"b":"deepresearchbench","m":"claude-opus-4-1-2025-08-05","oid":"deepresearchbench-266fa76b700ffb92","rd":"2025-08-05","sid":"recAiUkL2JKnr7mnB","sm":"claude-opus-4-1-20250805","src":"https://drb.futuresearch.ai","v":0.483},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-2025-08-07","oid":"deepresearchbench-fb9dad606cd6a594","rd":"2025-08-07","sid":"recQCpfcr9yYqfBqe","sm":"gpt-5-2025-08-07_high","src":"https://drb.futuresearch.ai","v":0.48100000000000004},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"gemini-3-flash-preview","oid":"deepresearchbench-1118ceb6affddd1d","rd":"2025-12-17","sid":"recNPE5j9yefOcUKF","sm":"gemini-3-flash-preview_high","src":"https://evals.futuresearch.ai/#drb","v":0.479},{"b":"deepresearchbench","cfg":{"effort":"high"},"e":"high","m":"gemini-3-1-pro-preview","oid":"deepresearchbench-4c835a9f0c3ce084","rd":"2026-02-19","sid":"recJkzTFEs7dhz33o","sm":"gemini-3.1-pro-preview_high","src":"https://evals.futuresearch.ai/#drb","v":0.478},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-sonnet-4-5-2025-09-29","oid":"deepresearchbench-ce7089c3a678bb85","rd":"2025-09-29","sid":"recXVi3rUmxYccZiP","sm":"claude-sonnet-4-5-20250929_low","src":"https://evals.futuresearch.ai/#drb","v":0.47500000000000003},{"b":"deepresearchbench","cfg":{"effort":"medium"},"e":"medium","m":"claude-opus-4-8","oid":"deepresearchbench-0f3e87bff5a94284","rd":"2026-05-28","sid":"recHfprP05gmb8wdt","sm":"claude-opus-4-8_medium","src":"https://evals.futuresearch.ai/#drb","v":0.474},{"b":"deepresearchbench","cfg":{"agent":"True"},"m":"grok-4-0709","oid":"deepresearchbench-3805c8c355fbf660","rd":"2025-07-09","sid":"recAThc100QSltAYp","sm":"grok-4-0709","src":"https://drb.futuresearch.ai","v":0.473},{"b":"deepresearchbench","m":"claude-opus-4-20250514-2k","notes":"2048 thinking token budget","oid":"deepresearchbench-191c8cb799c2ae5a","rd":"2025-05-22","sid":"recR3uOS0BLHqkwxL","sm":"claude-opus-4-20250514_2K","src":"https://drb.futuresearch.ai","v":0.46799999999999997},{"b":"deepresearchbench","m":"claude-sonnet-4-20250514-2k","notes":"2048 thinking token budget","oid":"deepresearchbench-01a2712b62c10523","rd":"2025-05-22","sid":"reciPR9ZrmNzGMR41","sm":"claude-sonnet-4-20250514_2K","src":"https://drb.futuresearch.ai","v":0.466},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gemini-3-pro-preview","oid":"deepresearchbench-324697fc18d7fa26","rd":"2025-11-18","sid":"recGVos7wKpDUVAmE","sm":"gemini-3-pro-preview_low","src":"https://evals.futuresearch.ai/#drb","v":0.46299999999999997},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"claude-haiku-4-5-2025-10-01","oid":"deepresearchbench-167acecd239e77ea","rd":"2025-10-15","sid":"recoGOlFp9CTraLus","sm":"claude-haiku-4-5-20251001_low","src":"https://evals.futuresearch.ai/#drb","v":0.455},{"b":"deepresearchbench","cfg":{"effort":"medium"},"e":"medium","m":"o3-2025-04-16","oid":"deepresearchbench-d0d894e0ad02ec12","rd":"2025-04-16","sid":"recZOsBdFIcclcWLC","sm":"o3-2025-04-16_medium","src":"https://drb.futuresearch.ai","v":0.452},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gemini-3-1-pro-preview","oid":"deepresearchbench-78c13b4254ec19da","rd":"2026-02-19","sid":"recsrVC4sX3BYtuHB","sm":"gemini-3.1-pro-preview_low","src":"https://evals.futuresearch.ai/#drb","v":0.445},{"b":"deepresearchbench","m":"claude-3-7-sonnet-20250219-2k","notes":"2048 thinking token budget","oid":"deepresearchbench-9b14691a7f6744ae","rd":"2025-02-24","sid":"rec7sfVScDTbadXOF","sm":"claude-3-7-sonnet-20250219_2K","src":"https://drb.futuresearch.ai","v":0.436},{"b":"deepresearchbench","m":"gemini-2-5-pro-preview-06-05","oid":"deepresearchbench-0d3f0f0f3c599332","rd":"2025-06-05","sid":"recXizbzd3b0y22vq","sm":"gemini-2.5-pro-preview-06-05","src":"https://drb.futuresearch.ai","v":0.428},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-1-2025-11-13","oid":"deepresearchbench-6a784659ba0f939e","rd":"2025-11-13","sid":"recvo8p3ckrliO2hR","sm":"gpt-5.1-2025-11-13_low","src":"https://evals.futuresearch.ai/#drb","v":0.428},{"b":"deepresearchbench","m":"gemini-2-5-pro","oid":"deepresearchbench-1f19c1bc20244d55","rd":"2025-06-17","sid":"recsIby4p4nNFRUmI","sm":"gemini-2.5-pro","src":"https://evals.futuresearch.ai/#drb","v":0.41500000000000004},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-2-2025-12-11","oid":"deepresearchbench-219f47c5bc39ecb2","rd":"2025-12-11","sid":"recfoBxQ21KfVCtxp","sm":"gpt-5.2-2025-12-11_low","src":"https://evals.futuresearch.ai/#drb","v":0.41100000000000003},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gemini-3-1-flash-lite","oid":"deepresearchbench-4a3c68f3e399108a","rd":"2026-03-03","sid":"recKtO8twbgOvKF7Q","sm":"gemini-3.1-flash-lite_low","src":"https://evals.futuresearch.ai/#drb","v":0.373},{"b":"deepresearchbench","m":"gemini-3-1-flash-lite","oid":"deepresearchbench-4bcb6cc759e4e8e1","rd":"2026-03-03","sid":"recOzFMcdH2YxAhUF","sm":"gemini-3.1-flash-lite","src":"https://evals.futuresearch.ai/#drb","v":0.364},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-4-mini-2026-03-17","oid":"deepresearchbench-3c9976e97ec3e835","rd":"2026-03-17","sid":"recFSLojghJGcBU6X","sm":"gpt-5.4-mini-2026-03-17_low","src":"https://evals.futuresearch.ai/#drb","v":0.363},{"b":"deepresearchbench","cfg":{"effort":"low"},"e":"low","m":"gpt-5-4-2026-03-05","oid":"deepresearchbench-1427150a80cd37dc","rd":"2026-03-05","sid":"recJ7rkEreGH8afLs","sm":"gpt-5.4-2026-03-05_low","src":"https://evals.futuresearch.ai/#drb","v":0.35100000000000003},{"b":"deepresearchbench","m":"deepseek-r1-0528","oid":"deepresearchbench-336f73386aa58d98","rd":"2025-05-28","sid":"rec8Lh9Jnh2v0o4mA","sm":"DeepSeek-R1-0528","src":"https://drb.futuresearch.ai","v":0.351},{"b":"deepswe","c":8.386436346666667,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.02829822249837175,"dn":"gpt-5-6-sol (max)","e":"max","m":"gpt-5-6-sol","metrics":{"output_tokens":60013.6444,"pass_at_4":0.8584,"runs":4,"steps":61.2533},"oid":"deepswe-def5797ca2c0a8d3","rd":"2026-07-09","sm":"gpt-5.6-sol_max","src":"https://deepswe.datacurve.ai/","v":0.7266666666666667},{"b":"deepswe","c":4.703663860310422,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.008191117217343103,"dn":"gpt-5-6-sol (xhigh)","e":"xhigh","m":"gpt-5-6-sol","metrics":{"output_tokens":40744.592,"pass_at_4":0.8584,"runs":4,"steps":44},"oid":"deepswe-7fd60e5afd5d8088","rd":"2026-07-09","sm":"gpt-5.6-sol_xhigh","src":"https://deepswe.datacurve.ai/","v":0.7073170731707318},{"b":"deepswe","c":13.414521495535714,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.03244917575471319,"dn":"claude-fable-5 (xhigh)","e":"xhigh","m":"claude-fable-5","metrics":{"output_tokens":80352.1704,"pass_at_4":0.885,"runs":4,"steps":68.4004},"oid":"deepswe-6991e351a895e610","rd":"2026-06-09","sm":"claude-fable-5_xhigh","src":"https://deepswe.datacurve.ai/","v":0.6991150442477876},{"b":"deepswe","c":21.634702092592594,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.04033474250807936,"dn":"claude-fable-5 (max)","e":"max","m":"claude-fable-5","metrics":{"output_tokens":118592.7729,"pass_at_4":0.8407,"runs":4,"steps":88.4289},"oid":"deepswe-70f65ed41d20e308","rd":"2026-06-09","sm":"claude-fable-5_max","src":"https://deepswe.datacurve.ai/","v":0.6972477064220184},{"b":"deepswe","c":4.945847263858093,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.025569389504458404,"dn":"gpt-5-6-terra (max)","e":"max","m":"gpt-5-6-terra","metrics":{"output_tokens":71938.6253,"pass_at_4":0.885,"runs":4,"steps":75.929},"oid":"deepswe-03f0740308fd47ae","rd":"2026-07-09","sm":"gpt-5.6-terra_max","src":"https://deepswe.datacurve.ai/","v":0.6962305986696231},{"b":"deepswe","c":3.4698331108647453,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.014321452203772357,"dn":"gpt-5-6-sol (high)","e":"high","m":"gpt-5-6-sol","metrics":{"output_tokens":28450.3193,"pass_at_4":0.8673,"runs":4,"steps":36.8891},"oid":"deepswe-f070e5b49d28cfbd","rd":"2026-07-09","sm":"gpt-5.6-sol_high","src":"https://deepswe.datacurve.ai/","v":0.6940133037694013},{"b":"deepswe","c":9.177638214788733,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.011208427174303877,"dn":"claude-fable-5 (high)","e":"high","m":"claude-fable-5","metrics":{"output_tokens":57287.0698,"pass_at_4":0.8673,"runs":4,"steps":58.7372},"oid":"deepswe-94f1fa21c43605c1","rd":"2026-06-09","sm":"claude-fable-5_high","src":"https://deepswe.datacurve.ai/","v":0.6860465116279071},{"b":"deepswe","c":3.028116810267857,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.03992001235671055,"dn":"gpt-5-6-luna (max)","e":"max","m":"gpt-5-6-luna","metrics":{"output_tokens":73399.7076,"pass_at_4":0.9027,"runs":4,"steps":101.6808},"oid":"deepswe-35c9547e9f4f647e","rd":"2026-07-09","sm":"gpt-5.6-luna_max","src":"https://deepswe.datacurve.ai/","v":0.671875},{"b":"deepswe","c":7.226236674778761,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.06465646340908864,"dn":"gpt-5-5 (xhigh)","e":"xhigh","m":"gpt-5-5","metrics":{"output_tokens":46294.7235,"pass_at_4":0.885,"runs":4,"steps":82.0155},"oid":"deepswe-b7313575b22c6646","rd":"2026-04-23","sm":"gpt-5.5_xhigh","src":"https://deepswe.datacurve.ai/","v":0.6703539823008849},{"b":"deepswe","c":5.100437004424779,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.031168426495054677,"dn":"gpt-5-5 (high)","e":"high","m":"gpt-5-5","metrics":{"output_tokens":31159.4978,"pass_at_4":0.9027,"runs":4,"steps":61.9248},"oid":"deepswe-4f05894851a9f2a0","rd":"2026-04-23","sm":"gpt-5.5_high","src":"https://deepswe.datacurve.ai/","v":0.6438053097345133},{"b":"deepswe","c":1.8620318075221238,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.01583357649307215,"dn":"gpt-5-6-sol (medium)","e":"medium","m":"gpt-5-6-sol","metrics":{"output_tokens":18425.2168,"pass_at_4":0.8053,"runs":4,"steps":30.9137},"oid":"deepswe-aac909d46542a039","rd":"2026-07-09","sm":"gpt-5.6-sol_medium","src":"https://deepswe.datacurve.ai/","v":0.6106194690265486},{"b":"deepswe","c":2.127191725663717,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.021242972019271257,"dn":"gpt-5-6-terra (xhigh)","e":"xhigh","m":"gpt-5-6-terra","metrics":{"output_tokens":39616.5398,"pass_at_4":0.8053,"runs":4,"steps":43.0664},"oid":"deepswe-7cadcf0f92e9aae6","rd":"2026-07-09","sm":"gpt-5.6-terra_xhigh","src":"https://deepswe.datacurve.ai/","v":0.6017699115044248},{"b":"deepswe","c":13.222583593240094,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.01764815347429254,"dn":"claude-opus-4-8 (max)","e":"max","m":"claude-opus-4-8","metrics":{"output_tokens":135031.6713,"pass_at_4":0.7928,"runs":4,"steps":120},"oid":"deepswe-19d5062155a9a2d1","rd":"2026-05-28","sm":"claude-opus-4-8_max","src":"https://deepswe.datacurve.ai/","v":0.5897435897435898},{"b":"deepswe","c":1.5356192685840708,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.021681017528097975,"dn":"gpt-5-6-luna (xhigh)","e":"xhigh","m":"gpt-5-6-luna","metrics":{"output_tokens":44677.9004,"pass_at_4":0.7876,"runs":4,"steps":71.0996},"oid":"deepswe-5483f2ee98304a11","rd":"2026-07-09","sm":"gpt-5.6-luna_xhigh","src":"https://deepswe.datacurve.ai/","v":0.5685840707964602},{"b":"deepswe","c":8.006355704138702,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.03713430628094476,"dn":"claude-opus-4-8 (xhigh)","e":"xhigh","m":"claude-opus-4-8","metrics":{"output_tokens":86088.7875,"pass_at_4":0.8053,"runs":4,"steps":94.6443},"oid":"deepswe-244ab034c4071744","rd":"2026-05-28","sm":"claude-opus-4-8_xhigh","src":"https://deepswe.datacurve.ai/","v":0.5436241610738255},{"b":"deepswe","c":2.749342734513274,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.02553087495290979,"dn":"gpt-5-5 (medium)","e":"medium","m":"gpt-5-5","metrics":{"output_tokens":19625.4336,"pass_at_4":0.7788,"runs":4,"steps":45.9823},"oid":"deepswe-95859bad5c80b357","rd":"2026-04-23","sm":"gpt-5.5_medium","src":"https://deepswe.datacurve.ai/","v":0.5398230088495575},{"b":"deepswe","c":26.399858950791852,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.04236916470174386,"dn":"claude-sonnet-5 (max)","e":"max","m":"claude-sonnet-5","metrics":{"output_tokens":214117.5498,"pass_at_4":0.7876,"runs":4,"steps":268.4548},"oid":"deepswe-e74928bc9d507f5b","rd":"2026-06-30","sm":"claude-sonnet-5_max","src":"https://deepswe.datacurve.ai/","v":0.5384615384615384},{"b":"deepswe","c":1.1343762765486725,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.043289704672135526,"dn":"gpt-5-6-terra (high)","e":"high","m":"gpt-5-6-terra","metrics":{"output_tokens":21517.0398,"pass_at_4":0.8053,"runs":4,"steps":33.5133},"oid":"deepswe-b1e5fdeed2d37a1f","rd":"2026-07-09","sm":"gpt-5.6-terra_high","src":"https://deepswe.datacurve.ai/","v":0.5376106194690266},{"b":"deepswe","c":2.361142644469026,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.030353424539337117,"dn":"muse-spark-1-1 (xhigh)","e":"xhigh","m":"muse-spark-1-1","metrics":{"output_tokens":74008.4204,"pass_at_4":0.7965,"runs":4,"steps":95.8252},"oid":"deepswe-f50e0107034b9190","rd":"2026-07-09","sm":"muse-spark-1.1","src":"https://deepswe.datacurve.ai/","v":0.5331858407079646},{"b":"deepswe","c":4.281940017699115,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.04561609145755844,"dn":"claude-opus-4-8 (high)","e":"high","m":"claude-opus-4-8","metrics":{"output_tokens":50063.5265,"pass_at_4":0.7788,"runs":4,"steps":72.5},"oid":"deepswe-b5f8f6f9973ff07f","rd":"2026-05-28","sm":"claude-opus-4-8_high","src":"https://deepswe.datacurve.ai/","v":0.5176991150442478},{"b":"deepswe","c":5.6524577245575225,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.015021049567382833,"dn":"gpt-5-4 (xhigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","metrics":{"output_tokens":71408.8739,"pass_at_4":0.7788,"runs":4,"steps":70.4712},"oid":"deepswe-ae212a85a6221983","rd":"2026-03-05","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://deepswe.datacurve.ai/","v":0.5176991150442478},{"b":"deepswe","c":11.890634540022173,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.034545489916778645,"dn":"claude-sonnet-5 (xhigh)","e":"xhigh","m":"claude-sonnet-5","metrics":{"output_tokens":120698.592,"pass_at_4":0.7522,"runs":4,"steps":185.5322},"oid":"deepswe-253c1d8b94b6a88a","rd":"2026-06-30","sm":"claude-sonnet-5_xhigh","src":"https://deepswe.datacurve.ai/","v":0.49667405764966743},{"b":"deepswe","c":3.4438571028761062,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.02239205861737452,"dn":"claude-opus-4-8 (medium)","e":"medium","m":"claude-opus-4-8","metrics":{"output_tokens":41313.4513,"pass_at_4":0.7611,"runs":4,"steps":65.573},"oid":"deepswe-0a0f14b224f44fcc","rd":"2026-05-28","sm":"claude-opus-4-8_medium","src":"https://deepswe.datacurve.ai/","v":0.48672566371681414},{"b":"deepswe","c":7.425560161197339,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.045063148702148406,"dn":"claude-sonnet-5 (high)","e":"high","m":"claude-sonnet-5","metrics":{"output_tokens":87294.7561,"pass_at_4":0.7965,"runs":4,"steps":146.5765},"oid":"deepswe-32320aaed3f35b20","rd":"2026-06-30","sm":"claude-sonnet-5_high","src":"https://deepswe.datacurve.ai/","v":0.4823008849557522},{"b":"deepswe","c":1.0742664646017699,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.0238819467145892,"dn":"gpt-5-6-sol (low)","e":"low","m":"gpt-5-6-sol","metrics":{"output_tokens":10579.1416,"pass_at_4":0.7168,"runs":4,"steps":23.3584},"oid":"deepswe-70971c97e28ca779","rd":"2026-07-09","sm":"gpt-5.6-sol_low","src":"https://deepswe.datacurve.ai/","v":0.45353982300884954},{"b":"deepswe","c":0.7779007499999999,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.029195672479165317,"dn":"gpt-5-6-luna (high)","e":"high","m":"gpt-5-6-luna","metrics":{"output_tokens":25778.2743,"pass_at_4":0.7522,"runs":4,"steps":49.0221},"oid":"deepswe-9f1017ed817c7746","rd":"2026-07-09","sm":"gpt-5.6-luna_high","src":"https://deepswe.datacurve.ai/","v":0.4424778761061947},{"b":"deepswe","c":3.9199075743111114,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.01725629620580811,"dn":"glm-5-2 (max)","e":"max","m":"glm-5-2","metrics":{"output_tokens":78175.3067,"pass_at_4":0.7699,"runs":4,"steps":129.1267},"oid":"deepswe-5c4bd4251e1e2b09","rd":"2026-06-16","sm":"glm-5.2_max","src":"https://deepswe.datacurve.ai/","v":0.4377777777777778},{"b":"deepswe","c":2.293369216740577,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.014635847585691517,"dn":"claude-opus-4-8 (low)","e":"low","m":"claude-opus-4-8","metrics":{"output_tokens":28922.7361,"pass_at_4":0.6814,"runs":4,"steps":53.9778},"oid":"deepswe-29f09fb79385396d","rd":"2026-05-28","sm":"claude-opus-4-8_low","src":"https://deepswe.datacurve.ai/","v":0.4079822616407982},{"b":"deepswe","c":2.8355164725663715,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.047500729479216554,"dn":"glm-5-2 (high)","e":"high","m":"glm-5-2","metrics":{"output_tokens":54245.5044,"pass_at_4":0.6814,"runs":4,"steps":121.8827},"oid":"deepswe-82a6b196e894cfad","rd":"2026-06-16","sm":"glm-5.2_high","src":"https://deepswe.datacurve.ai/","v":0.36283185840707965},{"b":"deepswe","c":0.5832493688888889,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.03381964303406632,"dn":"gpt-5-6-terra (medium)","e":"medium","m":"gpt-5-6-terra","metrics":{"output_tokens":11746.56,"pass_at_4":0.6018,"runs":4,"steps":25.1467},"oid":"deepswe-8025d71b919a1945","rd":"2026-07-09","sm":"gpt-5.6-terra_medium","src":"https://deepswe.datacurve.ai/","v":0.3511111111111111},{"b":"deepswe","c":2.815536243274336,"cfg":{"harness":"mini-swe-agent"},"ci":0.0050070165224609135,"dn":"kimi-k2-7-code","m":"kimi-k2-7-code","metrics":{"output_tokens":59297.2478,"pass_at_4":0.6106,"runs":4,"steps":149.115},"oid":"deepswe-52dd790a63447bc0","rd":"2026-06-12","sm":"kimi-k2.7-code","src":"https://deepswe.datacurve.ai/","v":0.3053097345132743},{"b":"deepswe","c":5.5223517987804875,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.04092141269189136,"dn":"claude-sonnet-4-6 (high)","e":"high","m":"claude-sonnet-4-6","metrics":{"output_tokens":76160.3126,"pass_at_4":0.5664,"runs":4,"steps":133.6585},"oid":"deepswe-f09f22ce8cf4d1af","rd":"2026-02-17","sm":"claude-sonnet-4-6_high","src":"https://deepswe.datacurve.ai/","v":0.29933481152993346},{"b":"deepswe","c":1.2002025442477877,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.02294503222007181,"dn":"gpt-5-5 (low)","e":"low","m":"gpt-5-5","metrics":{"output_tokens":9442.7743,"pass_at_4":0.4779,"runs":4,"steps":28.0664},"oid":"deepswe-45f374091c6e259d","rd":"2026-04-23","sm":"gpt-5.5_low","src":"https://deepswe.datacurve.ai/","v":0.26991150442477874},{"b":"deepswe","c":0.4277452605790646,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.007767613669447833,"dn":"gpt-5-6-terra (low)","e":"low","m":"gpt-5-6-terra","metrics":{"output_tokens":8572.2628,"pass_at_4":0.4425,"runs":4,"steps":21.4566},"oid":"deepswe-4ce285c5904079c6","rd":"2026-07-09","sm":"gpt-5.6-terra_low","src":"https://deepswe.datacurve.ai/","v":0.24053452115812918},{"b":"deepswe","c":2.1434045568888886,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.01481095461108845,"dn":"gemini-3-1-pro-preview (high)","e":"high","m":"gemini-3-1-pro-preview","metrics":{"output_tokens":28368.8844,"pass_at_4":0.2832,"runs":4,"steps":75.5644},"oid":"deepswe-ead5099cfcb2039a","rd":"2026-02-19","sm":"gemini-3.1-pro-preview","src":"https://deepswe.datacurve.ai/","v":0.1172566371681416},{"b":"deepswe","c":0.21630978982300886,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.008303197562056514,"dn":"gpt-5-6-luna (medium)","e":"medium","m":"gpt-5-6-luna","metrics":{"output_tokens":8179.5708,"pass_at_4":0.2743,"runs":4,"steps":23.677},"oid":"deepswe-6121315be75bee0a","rd":"2026-07-09","sm":"gpt-5.6-luna_medium","src":"https://deepswe.datacurve.ai/","v":0.11283185840707964},{"b":"deepswe","c":0.07240623938053097,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.008303197562056514,"dn":"gpt-5-6-luna (low)","e":"low","m":"gpt-5-6-luna","metrics":{"output_tokens":3127.7544,"pass_at_4":0.0442,"runs":4,"steps":12.4646},"oid":"deepswe-d32a42185fbc1beb","rd":"2026-07-09","sm":"gpt-5.6-luna_low","src":"https://deepswe.datacurve.ai/","v":0.015486725663716814},{"b":"deepswe","c":11.837583271396396,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.03872310426371729,"dn":"claude-opus-5 (max)","e":"max","m":"claude-opus-5","metrics":{"output_tokens":117565.6937,"pass_at_4":0.885,"runs":4,"steps":99.0428},"oid":"deepswe-94a0ab4ad8b71607","rd":"2026-07-24","sm":"claude-opus-5_max","src":"https://deepswe.datacurve.ai/","v":0.7364864864864865},{"b":"deepswe","c":9.07220232326622,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.030609818216384643,"dn":"claude-opus-5 (xhigh)","e":"xhigh","m":"claude-opus-5","metrics":{"output_tokens":91672.0157,"pass_at_4":0.8584,"runs":4,"steps":88.7248},"oid":"deepswe-9eb536f7e54beddc","rd":"2026-07-24","sm":"claude-opus-5_xhigh","src":"https://deepswe.datacurve.ai/","v":0.7315436241610739},{"b":"deepswe","c":6.076084489977728,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.019454321571601495,"dn":"claude-opus-5 (high)","e":"high","m":"claude-opus-5","metrics":{"output_tokens":64207.4343,"pass_at_4":0.8761,"runs":4,"steps":72.922},"oid":"deepswe-cedae8e0e4968948","rd":"2026-07-24","sm":"claude-opus-5_high","src":"https://deepswe.datacurve.ai/","v":0.7282850779510023},{"b":"deepswe","c":3.28978764541387,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.011725586658930944,"dn":"claude-opus-5 (medium)","e":"medium","m":"claude-opus-5","metrics":{"output_tokens":36981.5302,"pass_at_4":0.8938,"runs":4,"steps":52.2953},"oid":"deepswe-e80a19aa56fc8e04","rd":"2026-07-24","sm":"claude-opus-5_medium","src":"https://deepswe.datacurve.ai/","v":0.6890380313199106},{"b":"deepswe","c":4.654682129933482,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.045370140792823206,"dn":"kimi-k3 (max)","e":"max","m":"kimi-k3","metrics":{"output_tokens":81499.8426,"pass_at_4":0.8938,"runs":4,"steps":97.5876},"oid":"deepswe-812fc8bf1153b050","rd":"2026-07-16","sm":"kimi-k3_max","src":"https://deepswe.datacurve.ai/","v":0.6851441241685144},{"b":"deepswe","c":6.088186528935186,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.044219288792381656,"dn":"claude-fable-5 (medium)","e":"medium","m":"claude-fable-5","metrics":{"output_tokens":40201.3532,"pass_at_4":0.8319,"runs":4,"steps":48.367},"oid":"deepswe-17114d9a5f39a090","rd":"2026-06-09","sm":"claude-fable-5_medium","src":"https://deepswe.datacurve.ai/","v":0.6536697247706422},{"b":"deepswe","c":3.757873125874126,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.027940054451297294,"dn":"claude-fable-5 (low)","e":"low","m":"claude-fable-5","metrics":{"output_tokens":25242.8337,"pass_at_4":0.8142,"runs":4,"steps":37.7991},"oid":"deepswe-d6b5bbef3a00dff4","rd":"2026-06-09","sm":"claude-fable-5_low","src":"https://deepswe.datacurve.ai/","v":0.5958429561200924},{"b":"deepswe","c":1.6626223919821828,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.023306879466842727,"dn":"claude-opus-5 (low)","e":"low","m":"claude-opus-5","metrics":{"output_tokens":19884.314,"pass_at_4":0.8496,"runs":4,"steps":35.6414},"oid":"deepswe-4538562c7c5d3696","rd":"2026-07-24","sm":"claude-opus-5_low","src":"https://deepswe.datacurve.ai/","v":0.5812917594654788},{"b":"deepswe","c":2.4157481725663716,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.0228080457287792,"dn":"grok-4-5 (high)","e":"high","m":"grok-4-5","metrics":{"output_tokens":35525.3319,"pass_at_4":0.7788,"runs":4,"steps":61.3319},"oid":"deepswe-c57ddb046e96390f","rd":"2026-07-08","sm":"grok-4.5_high","src":"https://deepswe.datacurve.ai/","v":0.5376106194690266},{"b":"deepswe","c":4.419013277333334,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.037048538992472776,"dn":"gemini-3-6-flash (high)","e":"high","m":"gemini-3-6-flash","metrics":{"output_tokens":95844.8622,"pass_at_4":0.7522,"runs":4,"steps":116.7333},"oid":"deepswe-a4cb6747a315c966","rd":"2026-07-21","sm":"gemini-3.6-flash_high","src":"https://deepswe.datacurve.ai/","v":0.4668141592920354},{"b":"deepswe","c":4.079036229,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.03127576206329727,"dn":"claude-sonnet-5 (medium)","e":"medium","m":"claude-sonnet-5","metrics":{"output_tokens":56816.7533,"pass_at_4":0.646,"runs":4,"steps":107.6111},"oid":"deepswe-4ffab9dbb210ad77","rd":"2026-06-30","sm":"claude-sonnet-5_medium","src":"https://deepswe.datacurve.ai/","v":0.3977777777777778},{"b":"deepswe","c":7.341823985508849,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.01787862506784315,"dn":"gemini-3-5-flash (medium)","e":"medium","m":"gemini-3-5-flash","metrics":{"output_tokens":275777.5088,"pass_at_4":0.6637,"runs":4,"steps":85.7168},"oid":"deepswe-78f0dcc36b0ffd3e","rd":"2026-05-19","sm":"gemini-3.5-flash_medium","src":"https://deepswe.datacurve.ai/","v":0.37389380530973454},{"b":"deepswe","c":2.1865550832962137,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.011260175414427497,"dn":"claude-sonnet-5 (low)","e":"low","m":"claude-sonnet-5","metrics":{"output_tokens":35595.0824,"pass_at_4":0.5752,"runs":4,"steps":76.8886},"oid":"deepswe-25c0ff287a52e6e1","rd":"2026-06-30","sm":"claude-sonnet-5_low","src":"https://deepswe.datacurve.ai/","v":0.3051224944320713},{"b":"deepswe","c":3.4489837522123894,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.0228080457287792,"dn":"grok-4-6 (medium)","e":"medium","m":"grok-4-6","metrics":{"output_tokens":49763.9978,"pass_at_4":0.8407,"runs":4,"steps":70.2898},"oid":"deepswe-0922dc3ecc2e1162","rd":"2026-08-12","sm":"grok-4.6_medium","src":"https://deepswe.datacurve.ai/","v":0.6747787610619469},{"b":"deepswe","c":5.497668745011087,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.021756842244050494,"dn":"grok-4-6 (xhigh)","e":"xhigh","m":"grok-4-6","metrics":{"output_tokens":71403.5432,"pass_at_4":0.8496,"runs":4,"steps":87.2195},"oid":"deepswe-287aea25a6f3b441","rd":"2026-08-12","sm":"grok-4.6_xhigh","src":"https://deepswe.datacurve.ai/","v":0.6674057649667405},{"b":"deepswe","c":4.3848732239467845,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.015336008186572951,"dn":"grok-4-6 (high)","e":"high","m":"grok-4-6","metrics":{"output_tokens":61160.9446,"pass_at_4":0.8496,"runs":4,"steps":78.9601},"oid":"deepswe-369224e3fe55b8d3","rd":"2026-08-12","sm":"grok-4.6_high","src":"https://deepswe.datacurve.ai/","v":0.6518847006651884},{"b":"deepswe","c":3.7290646500445437,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.026627926725075454,"dn":"qwen3-8-max (xhigh)","e":"xhigh","m":"qwen3-8-max","metrics":{"output_tokens":95075.1826,"pass_at_4":0.8319,"runs":4,"steps":111.3385},"oid":"deepswe-d0dad1ece7b9a712","rd":"2026-08-02","sm":"qwen3.8-max_xhigh","src":"https://deepswe.datacurve.ai/","v":0.5746102449888641},{"b":"deepswe","c":1.0423604231625836,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.023186517644299014,"dn":"grok-4-6 (low)","e":"low","m":"grok-4-6","metrics":{"output_tokens":16458.343,"pass_at_4":0.6903,"runs":4,"steps":44.1871},"oid":"deepswe-8335d498e59e5268","rd":"2026-08-12","sm":"grok-4.6_low","src":"https://deepswe.datacurve.ai/","v":0.41648106904231624},{"b":"deepswe","c":3.4467114588691796,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.03966303010520439,"dn":"gemini-3-5-flash (high)","e":"high","m":"gemini-3-5-flash","metrics":{"output_tokens":75730.1929,"pass_at_4":0.6372,"runs":4,"steps":105.3038},"oid":"deepswe-80f5273df7203de1","rd":"2026-05-19","sm":"gemini-3.5-flash_high","src":"https://deepswe.datacurve.ai/","v":0.3606194690265487},{"b":"deepswe","c":2.0251351826880533,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.030865322764155222,"dn":"gemini-3-7-flash (medium)","e":"medium","m":"gemini-3-7-flash","metrics":{"output_tokens":93990.8518,"pass_at_4":0.8319,"runs":4,"steps":117.3695},"oid":"deepswe-cdf5583e61608389","rd":"2026-08-13","sm":"gemini-3.7-flash_medium","src":"https://deepswe.datacurve.ai/","v":0.6548672566371682},{"b":"deepswe","c":2.1762860351216813,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.017878625067843174,"dn":"gemini-3-7-flash (high)","e":"high","m":"gemini-3-7-flash","metrics":{"output_tokens":107248.3031,"pass_at_4":0.823,"runs":4,"steps":124.5155},"oid":"deepswe-7729d217a0f1fb10","rd":"2026-08-13","sm":"gemini-3.7-flash_high","src":"https://deepswe.datacurve.ai/","v":0.6526548672566371},{"b":"deepswe","c":1.8322645629424779,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.02589649081831869,"dn":"gemini-3-7-flash (low)","e":"low","m":"gemini-3-7-flash","metrics":{"output_tokens":73364.9558,"pass_at_4":0.7788,"runs":4,"steps":130.3761},"oid":"deepswe-bc6a5b2b2200bb01","rd":"2026-08-13","sm":"gemini-3.7-flash_low","src":"https://deepswe.datacurve.ai/","v":0.5376106194690266},{"b":"deepswe","c":3.9933584893126386,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.030186512715587,"dn":"glm-5-3 (max)","e":"max","m":"glm-5-3","metrics":{"output_tokens":80435.6098,"pass_at_4":0.8761,"runs":4,"steps":124.4723},"oid":"deepswe-eec3b03e472d65b5","rd":"2026-08-14","sm":"glm-5.3_max","src":"https://deepswe.datacurve.ai/","v":0.6895787139689579},{"b":"deepswe","c":3.6955048180309733,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.02124297201927127,"dn":"muse-spark-1-2 (xhigh)","e":"xhigh","m":"muse-spark-1-2","metrics":{"output_tokens":99226.3805,"pass_at_4":0.8142,"runs":4,"steps":100.7588},"oid":"deepswe-59c9c5515ac92dcc","rd":"2026-08-05","sm":"muse-spark-1.2_xhigh","src":"https://deepswe.datacurve.ai/","v":0.5486725663716814},{"b":"deepswe","c":2.362349413758389,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.014173132066358783,"dn":"gemini-3-8-flash (high)","e":"high","m":"gemini-3-8-flash","metrics":{"output_tokens":143242.6644,"pass_at_4":0.8584,"runs":4,"steps":166.3132},"oid":"deepswe-da2282f4c35d1cae","rd":"2026-09-02","sm":"gemini-3.8-flash_high","src":"https://deepswe.datacurve.ai/","v":0.738255033557047},{"b":"deepswe","c":1.9671024789823006,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.02280804572877925,"dn":"gemini-3-8-flash (medium)","e":"medium","m":"gemini-3-8-flash","metrics":{"output_tokens":124684.3628,"pass_at_4":0.8319,"runs":4,"steps":147.3009},"oid":"deepswe-015d61de4152c105","rd":"2026-09-02","sm":"gemini-3.8-flash_medium","src":"https://deepswe.datacurve.ai/","v":0.7101769911504425},{"b":"deepswe","c":0.48196371245535713,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.04377970751693644,"dn":"glm-5-3-flash (max)","e":"max","m":"glm-5-3-flash","metrics":{"output_tokens":72829.7701,"pass_at_4":0.8496,"runs":4,"steps":122.8906},"oid":"deepswe-daff9751cf68a646","rd":"2026-08-20","sm":"glm-5.3-flash_max","src":"https://deepswe.datacurve.ai/","v":0.6339285714285714},{"b":"deepswe","c":4.429117203539823,"cfg":{"effort":"xhigh","harness":"mini-swe-agent"},"ci":0.02865396174075141,"dn":"gpt-6-astra (xhigh)","e":"xhigh","m":"gpt-6-astra","metrics":{"output_tokens":29557.3274,"pass_at_4":0.8053,"runs":4,"steps":28.7544},"oid":"deepswe-c774c4287114148d","rd":"2026-09-03","sm":"gpt-6-astra_xhigh","src":"https://deepswe.datacurve.ai/","v":0.7411504424778761},{"b":"deepswe","c":3.9237237909292038,"cfg":{"effort":"high","harness":"mini-swe-agent"},"ci":0.03423496057873005,"dn":"gpt-6-astra (high)","e":"high","m":"gpt-6-astra","metrics":{"output_tokens":26505.854,"pass_at_4":0.823,"runs":4,"steps":27.4248},"oid":"deepswe-dcd36fbd78dd3105","rd":"2026-09-03","sm":"gpt-6-astra_high","src":"https://deepswe.datacurve.ai/","v":0.7323008849557522},{"b":"deepswe","c":7.497827996681416,"cfg":{"effort":"max","harness":"mini-swe-agent"},"ci":0.008303197562056526,"dn":"gpt-6-astra (max)","e":"max","m":"gpt-6-astra","metrics":{"output_tokens":61148.5066,"pass_at_4":0.7965,"runs":4,"steps":28.4535},"oid":"deepswe-3a4c082ce8bbbaad","rd":"2026-09-03","sm":"gpt-6-astra_max","src":"https://deepswe.datacurve.ai/","v":0.7323008849557522},{"b":"deepswe","c":3.0754742975663714,"cfg":{"effort":"medium","harness":"mini-swe-agent"},"ci":0.025896490818318695,"dn":"gpt-6-astra (medium)","e":"medium","m":"gpt-6-astra","metrics":{"output_tokens":20361.8097,"pass_at_4":0.823,"runs":4,"steps":26.031},"oid":"deepswe-d7bd2e096fed6296","rd":"2026-09-03","sm":"gpt-6-astra_medium","src":"https://deepswe.datacurve.ai/","v":0.7278761061946902},{"b":"deepswe","c":1.595223439159292,"cfg":{"effort":"low","harness":"mini-swe-agent"},"ci":0.013008610516858731,"dn":"gpt-6-astra (low)","e":"low","m":"gpt-6-astra","metrics":{"output_tokens":10579.5398,"pass_at_4":0.7965,"runs":4,"steps":19.5376},"oid":"deepswe-10ebf2e5d5d465bc","rd":"2026-09-03","sm":"gpt-6-astra_low","src":"https://deepswe.datacurve.ai/","v":0.6703539823008849},{"b":"enigma_eval","dn":"gpt-5.4-pro-2026-03-05","hi":0.2625,"lo":0.21389999999999998,"m":"gpt-5-4-pro-2026-03-05","oid":"enigma_eval-ee95d6996cd97327","rd":"2026-03-05","sid":"rec9I30aNqMMOAvz9","sm":"gpt-5.4-pro-2026-03-05_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.2382},{"b":"enigma_eval","dn":"gemini-3.1-pro-preview-high","hi":0.39490000000000003,"lo":0.3407,"m":"gemini-3-1-pro-preview","oid":"enigma_eval-c60b9965d029e873","rd":"2026-02-19","sid":"reczkTTqNDkpHLfBw","sm":"gemini-3.1-pro-preview","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.3678},{"b":"enigma_eval","dn":"gpt-5-pro-2025-10-06","hi":0.2097,"lo":0.1653,"m":"gpt-5-pro-2025-10-06","oid":"enigma_eval-f2b2d67867a9ceae","rd":"2025-10-07","sid":"recZv3rrIn0Hh0n7f","sm":"gpt-5-pro-2025-10-06_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.1875},{"b":"enigma_eval","dn":"gemini-3-pro-preview","hi":0.20439999999999997,"lo":0.1604,"m":"gemini-3-pro-preview","oid":"enigma_eval-8dc28978cdc8b22e","rd":"2025-11-18","sid":"recrkWj7aOGNnfZvJ","sm":"gemini-3-pro-preview","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.18239999999999998},{"b":"enigma_eval","cfg":{"effort":"xhigh"},"dn":"gpt-5.4-2026-03-05 (xhigh thinking)","e":"xhigh","hi":0.18050000000000002,"lo":0.13870000000000002,"m":"gpt-5-4-2026-03-05","oid":"enigma_eval-b0b12a5f6b0f7f08","rd":"2026-03-05","sid":"rech9nRz3pgAcfRAc","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.15960000000000002},{"b":"enigma_eval","cfg":{"effort":"medium"},"dn":"o3 (medium) (April 2025)","e":"medium","hi":0.15009999999999998,"lo":0.1117,"m":"o3-2025-04-16","oid":"enigma_eval-8a1e5acb14952bb4","rd":"2025-04-16","sid":"receahmxqAfNp04Hz","sm":"o3-2025-04-16_medium","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.1309},{"b":"enigma_eval","dn":"claude-opus-4-5-20251101-thinking","hi":0.1376,"lo":0.1006,"m":"claude-opus-4-5-2025-11-01","oid":"enigma_eval-0c3d0be58e84a6be","rd":"2025-11-24","sid":"recBHK1OObzEbFZGv","sm":"claude-opus-4-5-20251101_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.1191},{"b":"enigma_eval","cfg":{"effort":"high"},"dn":"o3 (high) (April 2025)","e":"high","hi":0.1376,"lo":0.1006,"m":"o3-2025-04-16","oid":"enigma_eval-33ef28b843e7935e","rd":"2025-04-16","sid":"recyQKxSx0xbZ5JCh","sm":"o3-2025-04-16_high","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.1191},{"b":"enigma_eval","dn":"gpt-5.1-thinking","hi":0.13030000000000003,"lo":0.09430000000000001,"m":"gpt-5-1-2025-11-13","oid":"enigma_eval-6217af9f9760d829","rd":"2025-11-13","sid":"recireO7DjaB7TZSb","sm":"gpt-5.1-2025-11-13_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.11230000000000001},{"b":"enigma_eval","dn":"gpt-5-2025-08-07","hi":0.12210000000000001,"lo":0.08730000000000002,"m":"gpt-5-2025-08-07","oid":"enigma_eval-7f5fd94ebeb9cfa5","rd":"2025-08-07","sid":"recTM0QtWkCOOzx2n","sm":"gpt-5-2025-08-07_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.10470000000000002},{"b":"enigma_eval","dn":"gpt-5.2-2025-12-11","hi":0.1213,"lo":0.08650000000000001,"m":"gpt-5-2-2025-12-11","oid":"enigma_eval-5cb7e702ffedfd71","rd":"2025-12-11","sid":"recesPyjn6f8vbk41","sm":"gpt-5.2-2025-12-11_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.1039},{"b":"enigma_eval","cfg":{"effort":"high"},"dn":"o4-mini (high) (April 2025)","e":"high","hi":0.10860000000000002,"lo":0.07560000000000001,"m":"o4-mini-2025-04-16","oid":"enigma_eval-246076e6d99fc6cc","rd":"2025-04-16","sid":"recwiwCIzlm1eaSZx","sm":"o4-mini-2025-04-16_high","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.09210000000000002},{"b":"enigma_eval","dn":"gpt-5-mini-2025-08-07","hi":0.09752600000000002,"lo":0.06632600000000001,"m":"gpt-5-mini-2025-08-07","oid":"enigma_eval-1f39b519c4267d68","rd":"2025-08-07","sid":"rech2WN5NSlMILTVU","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.08192600000000001},{"b":"enigma_eval","cfg":{"effort":"max"},"dn":"claude-opus-4-6-thinking-max","e":"max","hi":0.0911,"lo":0.060899999999999996,"m":"claude-opus-4-6","oid":"enigma_eval-30dd02364f4cd7d2","rd":"2026-02-05","sid":"recyLSf4pFkZBDI8M","sm":"claude-opus-4-6_max","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.076},{"b":"enigma_eval","dn":"claude-opus-4-1-20250805-thinking","hi":0.08649100000000001,"lo":0.05709100000000001,"m":"claude-opus-4-1-2025-08-05","oid":"enigma_eval-fa84df60d26b78dd","rd":"2025-08-05","sid":"recIHAQiSKe37QlrV","sm":"claude-opus-4-1-20250805_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.07179100000000001},{"b":"enigma_eval","dn":"claude-opus-4-6 (Non-Thinking)","hi":0.0828,"lo":0.054000000000000006,"m":"claude-opus-4-6","oid":"enigma_eval-2306ff519c5566e5","rd":"2026-02-05","sid":"reckakrWHvDEXZCCn","sm":"claude-opus-4-6","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0684},{"b":"enigma_eval","cfg":{"effort":"medium"},"dn":"o4-mini (medium) (April 2025)","e":"medium","hi":0.0764,"lo":0.05979999999999999,"m":"o4-mini-2025-04-16","oid":"enigma_eval-a1548c6dcc988c32","rd":"2025-04-16","sid":"reczXOhQYxfQ4z6UI","sm":"o4-mini-2025-04-16_medium","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0681},{"b":"enigma_eval","dn":"o1 Pro (March 2025)","hi":0.0751,"lo":0.04769999999999999,"m":"o1-pro-2025-03-19","oid":"enigma_eval-5955d875d0267c35","rd":"2025-03-19","sid":"recn9NeslnlYNPpi6","sm":"o1-pro-2025-03-19","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.061399999999999996},{"b":"enigma_eval","dn":"claude-sonnet-4-5-20250929-thinking","hi":0.073466,"lo":0.04646599999999999,"m":"claude-sonnet-4-5-2025-09-29","oid":"enigma_eval-9e6719b7e2f75069","rd":"2025-09-29","sid":"rec9ylDuv236QEdRz","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.059966},{"b":"enigma_eval","dn":"o1 (December 2024)","hi":0.0697,"lo":0.0433,"m":"o1-2024-12-17","oid":"enigma_eval-016ee571d67155c4","rd":"2024-12-17","sid":"reczEK6cTdemcEFoY","sm":"o1-2024-12-17_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0565},{"b":"enigma_eval","dn":"Claude Opus 4 (Thinking)","hi":0.0688,"lo":0.042600000000000006,"m":"claude-opus-4-2025-05-14","oid":"enigma_eval-746abeb4bc3bc5c6","rd":"2025-05-22","sid":"recajp4Xzqdl1hWVE","sm":"claude-opus-4-20250514_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.055700000000000006},{"b":"enigma_eval","dn":"gemini-2.5-pro-preview-06-05","hi":0.0688,"lo":0.042600000000000006,"m":"gemini-2-5-pro-preview-06-05","oid":"enigma_eval-b5405a08305e7297","rd":"2025-06-05","sid":"recDbzX7JurywUXag","sm":"gemini-2.5-pro-preview-06-05","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.055700000000000006},{"b":"enigma_eval","dn":"Claude 3.7 Sonnet Thinking (Feb 2025)","hi":0.054000000000000006,"lo":0.030600000000000002,"m":"claude-3-7-sonnet-2025-02-19","oid":"enigma_eval-a416527d44eb95d8","rd":"2025-02-24","sid":"recqgAd66eaZUdytK","sm":"claude-3-7-sonnet-20250219_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.042300000000000004},{"b":"enigma_eval","dn":"Gemini 2.5 Pro Experimental (March 2025)","hi":0.053,"lo":0.0298,"m":"gemini-2-5-pro-exp-03-25","oid":"enigma_eval-c98cc7dec239c261","rd":"2025-03-25","sid":"recHs6Sc5m0hdRAQI","sm":"gemini-2.5-pro-exp-03-25","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0414},{"b":"enigma_eval","dn":"kimi-k2.5","hi":0.0441,"lo":0.023499999999999997,"m":"kimi-k2-5","oid":"enigma_eval-b39538fd4991efeb","rd":"2026-01-27","sid":"recYB20EsagJiFNSS","sm":"kimi-k2.5","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0338},{"b":"enigma_eval","dn":"GPT-4.5 Preview (February 2025)","hi":0.042,"lo":0.0216,"m":"gpt-4-5-preview-2025-02-27","oid":"enigma_eval-fa292c786f6275a5","rd":"2025-02-27","sid":"recptVRLzhlKqqAbC","sm":"gpt-4.5-preview-2025-02-27","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0318},{"b":"enigma_eval","dn":"Claude Sonnet 4 (Thinking)","hi":0.041100000000000005,"lo":0.0213,"m":"claude-sonnet-4-2025-05-14","oid":"enigma_eval-b5e458320f34f516","rd":"2025-05-22","sid":"recnSlhjPi5DMIm9L","sm":"claude-sonnet-4-20250514_unknown","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.031200000000000002},{"b":"enigma_eval","dn":"gemini-3.1-flash-lite-preview","hi":0.0402,"lo":0.0206,"m":"gemini-3-1-flash-lite","oid":"enigma_eval-ef67f4e942754787","rd":"2026-03-03","sid":"rectxc47EYXrt5A4p","sm":"gemini-3.1-flash-lite","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0304},{"b":"enigma_eval","dn":"Gemini 2.5 Flash Preview (May 2025)","hi":0.0362,"lo":0.017800000000000003,"m":"gemini-2-5-flash-preview-05-20","oid":"enigma_eval-c5aae580574a21ed","rd":"2025-05-20","sid":"reclMZj4AID4W3rFB","sm":"gemini-2.5-flash-preview-05-20","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.027000000000000003},{"b":"enigma_eval","dn":"Gemini 2.5 Pro Preview (May 06 2025)","hi":0.032299999999999995,"lo":0.0149,"m":"gemini-2-5-pro-preview-05-06","oid":"enigma_eval-07f49d355c2ebe8e","rd":"2025-05-06","sid":"recl5V5gG8jovVkv0","sm":"gemini-2.5-pro-preview-05-06","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0236},{"b":"enigma_eval","dn":"Claude 3.7 Sonnet (February 2025)","hi":0.0312,"lo":0.013999999999999999,"m":"claude-3-7-sonnet-2025-02-19","oid":"enigma_eval-14d5595c6cef2598","rd":"2025-02-24","sid":"rec6zp1wPGHYVjyLu","sm":"claude-3-7-sonnet-20250219","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0226},{"b":"enigma_eval","dn":"GPT-4.1","hi":0.030100000000000002,"lo":0.013300000000000001,"m":"gpt-4-1-2025-04-14","oid":"enigma_eval-8c3c5b3083f651a3","rd":"2025-04-14","sid":"recyWpf0QCF5zacnN","sm":"gpt-4.1-2025-04-14","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0217},{"b":"enigma_eval","cfg":{"effort":"none"},"dn":"gpt-5.1-instant","e":"none","hi":0.0273,"lo":0.0115,"m":"gpt-5-1-2025-11-13","oid":"enigma_eval-6e84d3d9acb853a9","rd":"2025-11-13","sid":"recB21sSGBxuHQADI","sm":"gpt-5.1-2025-11-13_none","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0194},{"b":"enigma_eval","dn":"Gemini 2.0 Flash Thinking (January 2025)","hi":0.017,"lo":0.005000000000000001,"m":"gemini-2-0-flash-thinking-exp-01-21","oid":"enigma_eval-51762ad30b8fa798","rd":"2025-01-21","sid":"recQZPoJp4e9mxIEX","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.011000000000000001},{"b":"enigma_eval","dn":"Claude 3.5 Sonnet (October 2024)","hi":0.014600000000000002,"lo":0.0036,"m":"claude-3-5-sonnet-2024-10-22","oid":"enigma_eval-5631a9fa6dfe5e3e","rd":"2024-10-22","sid":"rectZdJnjJEcv95yF","sm":"claude-3-5-sonnet-20241022","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0091},{"b":"enigma_eval","dn":"Pixtral Large (November 2024)","hi":0.0137,"lo":0.0030999999999999995,"m":"pixtral-large-2411","oid":"enigma_eval-9f3d44529db6ce9b","rd":"2024-11-18","sid":"recrs0rEjh7yqB1lr","sm":"pixtral-large-2411","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0084},{"b":"enigma_eval","dn":"Claude 3 Opus","hi":0.0127,"lo":0.0036999999999999984,"m":"claude-3-opus-2024-02-29","oid":"enigma_eval-74feba75d3ac6a7b","rd":"2024-02-29","sid":"recOmwnjZmNcq0Pvi","sm":"claude-3-opus-20240229","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.008199999999999999},{"b":"enigma_eval","dn":"GPT-4o (November 2024)","hi":0.012400000000000001,"lo":0.0036,"m":"gpt-4o-2024-11-20","oid":"enigma_eval-bba0ab290e35250e","rd":"2024-11-20","sid":"recXjD048nIZmi1Bc","sm":"gpt-4o-2024-11-20","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.008},{"b":"enigma_eval","dn":"Gemini 2.0 Pro Experimental (Feb 2025)","hi":0.011699999999999999,"lo":0.0021000000000000003,"m":"gemini-2-0-pro-exp-02-05","oid":"enigma_eval-dc5aa978229992b4","rd":"2025-02-05","sid":"recNfiBVKOOhKY28W","sm":"gemini-2.0-pro-exp-02-05","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0069},{"b":"enigma_eval","dn":"Gemini 2.0 Flash (February 2025)","hi":0.0108,"lo":0.0017999999999999995,"m":"gemini-2-0-flash-02-05","oid":"enigma_eval-07b841d5e578b8a5","rd":"2025-02-05","sid":"recmlf6ufdqD98ZUR","sm":"gemini-2.0-flash-02-05","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0063},{"b":"enigma_eval","dn":"Llama 4 Maverick","hi":0.0101,"lo":0.0014999999999999996,"m":"llama-4-maverick-17b-128e-instruct","oid":"enigma_eval-1c1eb9562f48f7cd","rd":"2025-04-05","sid":"rec1GymcsU4urYVLi","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0058},{"b":"enigma_eval","dn":"Llama 3.2 90B Vision Instruct","hi":0.007299999999999999,"lo":0.00030000000000000035,"m":"llama-3-2-90b-vision-instruct","oid":"enigma_eval-47c9258e8d681085","rd":"2024-09-24","sid":"rec0FJEYmqS0s0k2C","sm":"Llama-3.2-90B-Vision-Instruct","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.0038},{"b":"enigma_eval","cfg":{"effort":"high"},"dn":"claude-fable-5-high","e":"high","hi":0.42080000000000006,"lo":0.3648,"m":"claude-fable-5","oid":"enigma_eval-5d9cf1be899f1046","rd":"2026-06-09","sid":"recXux9NHXeb5HwBk","sm":"claude-fable-5_high","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.39280000000000004},{"b":"enigma_eval","cfg":{"effort":"high"},"dn":"gpt-5.6-sol-high","e":"high","hi":0.3992,"lo":0.34319999999999995,"m":"gpt-5-6-sol","oid":"enigma_eval-525c29939799a774","rd":"2026-07-09","sid":"recIyRaAOijnX5cVP","sm":"gpt-5.6-sol_high","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.3712},{"b":"enigma_eval","cfg":{"effort":"high"},"dn":"gemini-3.5-flash-high","e":"high","hi":0.2786,"lo":0.2296,"m":"gemini-3-5-flash","oid":"enigma_eval-5d26c65e0bfc6699","rd":"2026-05-19","sid":"rectrabR7sN1RLhNF","sm":"gemini-3.5-flash_high","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.2541},{"b":"enigma_eval","cfg":{"effort":"xhigh"},"dn":"claude-opus-4-8-xhigh","e":"xhigh","hi":0.25920000000000004,"lo":0.21100000000000002,"m":"claude-opus-4-8","oid":"enigma_eval-8d7971c2d12eeb59","rd":"2026-05-28","sid":"recgVaZ4h79uz51GB","sm":"claude-opus-4-8_xhigh","src":"https://labs.scale.com/leaderboard/enigma_eval","v":0.23510000000000003},{"b":"exploitbench","dn":"Claude Mythos Preview (AutoNudge)","m":"claude-mythos-preview-early","metrics":{"spend_usd":36428.03},"oid":"exploitbench-bcaeac783347998f","rd":"2026-04-07","sid":"rec8ARZkKFFt70sei","sm":"claude-mythos-preview-early","v":0.738},{"b":"exploitbench","dn":"Claude Mythos Preview","m":"claude-mythos-preview-early","metrics":{"spend_usd":25083.46},"oid":"exploitbench-71ada4ab86a8b259","rd":"2026-04-07","sid":"recjEAXc8oRVGx1ir","sm":"claude-mythos-preview-early","v":0.706},{"b":"exploitbench","dn":"GPT 5.5 (AutoNudge)","m":"gpt-5-5","metrics":{"spend_usd":8134.3},"oid":"exploitbench-2a9610625fb67297","rd":"2026-04-23","sid":"rec2RjnmOh7quC9z7","sm":"gpt-5.5_unknown","v":0.474},{"b":"exploitbench","dn":"GPT 5.5","m":"gpt-5-5","metrics":{"spend_usd":1882.48},"oid":"exploitbench-93070ac581656cf9","rd":"2026-04-23","sid":"recL6VGGQlSyDupqx","sm":"gpt-5.5_unknown","v":0.418},{"b":"exploitbench","cfg":{"harness":"Codex"},"dn":"GPT 5.5 (Codex) (AutoNudge)","m":"gpt-5-5","metrics":{"spend_usd":3073.48},"oid":"exploitbench-87bb388b99bd6329","rd":"2026-04-23","sid":"recGamVsuksqNnzUS","sm":"gpt-5.5_unknown","v":0.41600000000000004},{"b":"exploitbench","cfg":{"harness":"Codex"},"dn":"GPT 5.5 (Codex)","m":"gpt-5-5","metrics":{"spend_usd":1252.35},"oid":"exploitbench-1c9333bf938b51b7","rd":"2026-04-23","sid":"rec2VxQC3W0bjMPbP","sm":"gpt-5.5_unknown","v":0.326},{"b":"exploitbench","dn":"Claude Opus 4.7 (AutoNudge)","m":"claude-opus-4-7","metrics":{"spend_usd":5630.55},"oid":"exploitbench-1eb73a064a4be705","rd":"2026-04-16","sid":"recQOfz7VF4xo8mUD","sm":"claude-opus-4-7","v":0.265},{"b":"exploitbench","dn":"Gemini 3.1 Pro Preview","m":"gemini-3-1-pro-preview","metrics":{"spend_usd":3449.53},"oid":"exploitbench-c54de3a77c7b053b","rd":"2026-02-19","sid":"rec7zLb49nemnC3vf","sm":"gemini-3.1-pro-preview","v":0.261},{"b":"exploitbench","dn":"Claude Opus 4.7","m":"claude-opus-4-7","metrics":{"spend_usd":3635.97},"oid":"exploitbench-e9a4d505158ec7da","rd":"2026-04-16","sid":"recm34ZgUETmQXqQn","sm":"claude-opus-4-7","v":0.239},{"b":"exploitbench","dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","metrics":{"spend_usd":4360.02},"oid":"exploitbench-1e0904d39cbd1330","rd":"2026-02-17","sid":"recrcDwfRSNeL1idE","sm":"claude-sonnet-4-6","v":0.23600000000000002},{"b":"exploitbench","dn":"Claude Sonnet 4.6 (AutoNudge)","m":"claude-sonnet-4-6","metrics":{"spend_usd":6314.62},"oid":"exploitbench-47277dafa9df5579","rd":"2026-02-17","sid":"receOcHylvEuyb65E","sm":"claude-sonnet-4-6","v":0.227},{"b":"exploitbench","dn":"Kimi K2.6 (AutoNudge)","m":"kimi-k2-6","metrics":{"spend_usd":889.41},"oid":"exploitbench-5f1737de23601f46","rd":"2026-04-20","sid":"rectO7hLZlegkHry6","sm":"kimi-k2.6","v":0.184},{"b":"exploitbench","dn":"Glm 5.1 (AutoNudge)","m":"glm-5-1","metrics":{"spend_usd":893.75},"oid":"exploitbench-83f5e1efcb90b749","rd":"2026-04-07","sid":"recHAQgsjFtyZCsxI","sm":"glm-5.1","v":0.18100000000000002},{"b":"exploitbench","dn":"Kimi K2.6","m":"kimi-k2-6","metrics":{"spend_usd":665.52},"oid":"exploitbench-0f721768fb1f1fa7","rd":"2026-04-20","sid":"recd8a9bPUW8ZMAJu","sm":"kimi-k2.6","v":0.163},{"b":"exploitbench","dn":"Glm 5.1","m":"glm-5-1","metrics":{"spend_usd":798.84},"oid":"exploitbench-246a471ed86834ac","rd":"2026-04-07","sid":"recLE7fyAMhy7S6YY","sm":"glm-5.1","v":0.16},{"b":"exploitbench","dn":"Gemini 3.1 Pro Preview (AutoNudge)","m":"gemini-3-1-pro-preview","metrics":{"spend_usd":2462.48},"oid":"exploitbench-3f6d6580fd927dc9","rd":"2026-02-19","sid":"recy6Awak3A1Mkmma","sm":"gemini-3.1-pro-preview","v":0.157},{"b":"exploitbench","dn":"Claude Haiku 4.5 (AutoNudge)","m":"claude-haiku-4-5-2025-10-01","metrics":{"spend_usd":338.74},"oid":"exploitbench-5347f9558899a4e0","rd":"2025-10-15","sid":"reckP3aOUme4XBsiB","sm":"claude-haiku-4-5-20251001_unknown","v":0.13699999999999998},{"b":"exploitbench","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","metrics":{"spend_usd":99.9},"oid":"exploitbench-c39563ea708a2b0c","rd":"2025-10-15","sid":"recjOeA8F3sV5Sqn6","sm":"claude-haiku-4-5-20251001_unknown","v":0.134},{"b":"exploitbench","dn":"MiniMax M2.7","m":"minimax-m2-7","metrics":{"spend_usd":94.18},"oid":"exploitbench-9f1005c292316b4e","rd":"2026-03-18","sid":"recxcMrwtWglgP5Eq","sm":"MiniMax-M2.7","v":0.133},{"b":"exploitbench","dn":"MiniMax M2.7 (AutoNudge)","m":"minimax-m2-7","metrics":{"spend_usd":200.31},"oid":"exploitbench-dea68d74c771be12","rd":"2026-03-18","sid":"rec63WcoLvj8GNKpj","sm":"MiniMax-M2.7","v":0.133},{"b":"fictionlivebench","m":"chutesllama-4-maverick-17b-128e-instruct","oid":"fictionlivebench-2ea42e503d0c361d","rd":"2025-04-06","sid":"recewrJmZKExZcwdW","sm":"chutes/Llama-4-Maverick-17B-128E-Instruct","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.364},{"b":"fictionlivebench","m":"chutesqwen3-235b-a22b-thinking-2507","oid":"fictionlivebench-e425422f2091eae3","rd":"2025-07-25","sid":"recGAZIAhonPCxfl4","sm":"chutes/Qwen3-235B-A22B-Thinking-2507","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.688},{"b":"fictionlivebench","m":"claude-3-7-sonnet-2025-02-19","oid":"fictionlivebench-ea68c5c108a5273a","rd":"2025-02-24","sid":"recGGYBroJyICDKfr","sm":"claude-3-7-sonnet-20250219","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.344},{"b":"fictionlivebench","m":"claude-3-7-sonnet-20250219-8k","oid":"fictionlivebench-51a4e7c0c282ad60","rd":"2025-02-24","sid":"recicBevhaGuqHc8k","sm":"claude-3-7-sonnet-20250219_8K","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.531},{"b":"fictionlivebench","m":"claude-opus-4-2025-05-14","oid":"fictionlivebench-26bfc42dee23ac53","rd":"2025-05-22","sid":"recplw5R3L8qJpXOV","sm":"claude-opus-4-20250514","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.375},{"b":"fictionlivebench","m":"claude-sonnet-4-2025-05-14","oid":"fictionlivebench-386eb7241acf28b8","rd":"2025-05-22","sid":"recypVCUSuzvpsExW","sm":"claude-sonnet-4-20250514","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.364},{"b":"fictionlivebench","m":"deepinfraqwen3-next-80b-a3b-instruct","oid":"fictionlivebench-558e4aa1a54339bb","rd":"2025-09-11","sid":"recNjAnkVZJOFT6ey","sm":"deepinfra/Qwen3-Next-80B-A3B-Instruct","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.625},{"b":"fictionlivebench","m":"deepseek-r1","oid":"fictionlivebench-aee6ca4f925c33d6","rd":"2025-01-20","sid":"recsl5Wju2ogMhZCb","sm":"DeepSeek-R1","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.333},{"b":"fictionlivebench","m":"gemini-2-0-flash-001","oid":"fictionlivebench-c8b0002c4816a92a","rd":"2025-02-05","sid":"recGOLP9NP6FCnY23","sm":"gemini-2.0-flash-001","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.625},{"b":"fictionlivebench","m":"gemini-2-0-flash-thinking-exp-01-21","notes":"Assuming later of two gemini-2.0-flash-thinking-exp versions","oid":"fictionlivebench-5856ed67417182d8","rd":"2025-01-21","sid":"recoYeNjjrrMQeCov","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.375},{"b":"fictionlivebench","m":"gemini-2-5-flash-lite-preview-06-17-thinking","notes":"Assuming default thinking token budget","oid":"fictionlivebench-803d428046e40068","rd":"2025-06-17","sid":"rec715RUAXp9NTaq5","sm":"gemini-2.5-flash-lite-preview-06-17-thinking","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.219},{"b":"fictionlivebench","m":"gemini-2-5-flash-preview-05-20","oid":"fictionlivebench-543c4a320f196b02","rd":"2025-05-20","sid":"recRrMD11LH6hyqH3","sm":"gemini-2.5-flash-preview-05-20","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.688},{"b":"fictionlivebench","m":"gemini-2-5-pro-exp-03-25","oid":"fictionlivebench-2996859a4328bd2c","rd":"2025-03-25","sid":"recAZiG0Os3UJJLZP","sm":"gemini-2.5-pro-exp-03-25","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.906},{"b":"fictionlivebench","m":"gemini-2-5-pro-preview-05-06","oid":"fictionlivebench-b507f7327658806a","rd":"2025-05-06","sid":"rec8fA2rBmp1u5Oyc","sm":"gemini-2.5-pro-preview-05-06","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.719},{"b":"fictionlivebench","m":"gemini-2-5-pro-preview-06-05","oid":"fictionlivebench-dd279bc7e8f8a353","rd":"2025-06-05","sid":"rec6WM2ba5tKuRU73","sm":"gemini-2.5-pro-preview-06-05","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.875},{"b":"fictionlivebench","m":"gpt-4-1-2025-04-14","oid":"fictionlivebench-fbe50843cc9c5462","rd":"2025-04-14","sid":"recsSWb66UgmmizaT","sm":"gpt-4.1-2025-04-14","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.625},{"b":"fictionlivebench","m":"gpt-4-5-preview-2025-02-27","oid":"fictionlivebench-4d5b1dd0e7950cb4","rd":"2025-02-27","sid":"recxY9ZAuaCjDhXWj","sm":"gpt-4.5-preview-2025-02-27","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.639},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-2025-08-07","oid":"fictionlivebench-435e62ff78cd6b6a","rd":"2025-08-07","sid":"rec5ebPSkPFSMrFog","sm":"gpt-5-2025-08-07_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.969},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-mini-2025-08-07","oid":"fictionlivebench-b1d73d3fc9009225","rd":"2025-08-07","sid":"recD4lezAe4h8rXLG","sm":"gpt-5-mini-2025-08-07_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.625},{"b":"fictionlivebench","m":"grok-4-0709","oid":"fictionlivebench-aa1fea9f2615ea45","rd":"2025-07-09","sid":"rec1710QvfYzhnUy3","sm":"grok-4-0709","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.969},{"b":"fictionlivebench","m":"grok-4-fast","oid":"fictionlivebench-a356734551a53613","rd":"2025-09-19","sid":"recSUyk1OUgBytYVw","sm":"grok-4-fast","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.75},{"b":"fictionlivebench","m":"kimi-k2-5","notes":"High reasoning","oid":"fictionlivebench-be3722dff1b08239","rd":"2026-01-27","sid":"recEJI420l5GzobVI","sm":"kimi-k2.5","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.781},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"o1-2024-12-17","oid":"fictionlivebench-ba98261e7e266183","rd":"2024-12-17","sid":"recGwARcoUTbSBMel","sm":"o1-2024-12-17_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.531},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"o3-2025-04-16","notes":"Assuming medium reasoning effort","oid":"fictionlivebench-a790e118609ca6b9","rd":"2025-04-16","sid":"rectq32FQAnUPr23W","sm":"o3-2025-04-16_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":1},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"o3-mini-2025-01-31","oid":"fictionlivebench-7307f2ff6344cdcd","rd":"2025-01-31","sid":"recz82I1vqPvL5SHN","sm":"o3-mini-2025-01-31_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.438},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"o3-pro-2025-06-10","notes":"Assuming medium reasoning effort","oid":"fictionlivebench-92ae97ee601b7e4c","rd":"2025-06-10","sid":"recuo6HwZhOIVNK58","sm":"o3-pro-2025-06-10_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.889},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"o4-mini-2025-04-16","notes":"Assuming medium reasoning effort","oid":"fictionlivebench-2ec83853454f7995","rd":"2025-04-16","sid":"recLBEgSpvxFhqIG7","sm":"o4-mini-2025-04-16_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.625},{"b":"fictionlivebench","m":"chatgpt-4o-01-29-2025","oid":"fictionlivebench-a05dc4d14f0d824a","rd":"2025-01-29","sid":"recbbzkiiGYbSKc0i","sm":"chatgpt-4o-01-29-2025","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.656},{"b":"fictionlivebench","m":"deepseek-v3-1","oid":"fictionlivebench-37e8bc643b3313f4","rd":"2025-08-21","sid":"recUqjDWvUSuFpARK","sm":"DeepSeek-V3.1","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.531},{"b":"fictionlivebench","m":"fireworkskimi-k2-instruct-0905","oid":"fictionlivebench-625f06fd41f06c8f","rd":"2025-09-05","sid":"recgQKMiUkKGr536O","sm":"fireworks/Kimi-K2-Instruct-0905","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.406},{"b":"fictionlivebench","m":"gemini-2-0-pro-exp-02-05","notes":"The 'free' indication doesn't change the model version here because Google is the only provider","oid":"fictionlivebench-22cbbc52df5fed30","rd":"2025-02-05","sid":"reciNLWokJjiot8GQ","sm":"gemini-2.0-pro-exp-02-05","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.375},{"b":"fictionlivebench","m":"gemini-2-5-pro-preview-03-25","oid":"fictionlivebench-845d1f03cfa30696","rd":"2025-03-31","sid":"recH0VvFyZFwdVAJM","sm":"gemini-2.5-pro-preview-03-25","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.719},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"grok-3-mini-beta","notes":"Assuming medium reasoning effort","oid":"fictionlivebench-d1a455adee54168f","rd":"2025-04-09","sid":"recCtye3NGebPCPb7","sm":"grok-3-mini-beta_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.656},{"b":"fictionlivebench","m":"minimax-m1-80k","notes":"Assuming 80k version","oid":"fictionlivebench-77113a0425616922","rd":"2025-06-13","sid":"recNoCb58YZgN4Vbk","sm":"MiniMax-M1-80k","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.594},{"b":"fictionlivebench","m":"parasail-qwen3-235b-a22b-instruct-2507","oid":"fictionlivebench-f6edbea1f0fb6829","rd":"2025-09-01","sid":"recshhfvwBqzpWiyF","sm":"parasail-qwen3-235b-a22b-instruct-2507","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.444},{"b":"fictionlivebench","m":"chutesqwen3-next-80b-a3b-instruct","oid":"fictionlivebench-c585d44e6fdf3a71","rd":"2025-09-11","sid":"recOQ8BmlPxZq94pr","sm":"chutes/Qwen3-Next-80B-A3B-Instruct","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.469},{"b":"fictionlivebench","m":"gpt-4-1-mini-2025-04-14","oid":"fictionlivebench-def4097c7e4be06f","rd":"2025-04-14","sid":"recTq2bSBrRw3Y4IQ","sm":"gpt-4.1-mini-2025-04-14","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.469},{"b":"fictionlivebench","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-nano-2025-08-07","oid":"fictionlivebench-bbcfd015b10f6a88","rd":"2025-08-07","sid":"recDspmMIr9SGBdu2","sm":"gpt-5-nano-2025-08-07_medium","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.219},{"b":"fictionlivebench","m":"grok-3-beta","oid":"fictionlivebench-15b998da53798ba5","rd":"2025-04-09","sid":"recAQD3j7HwNBWHJc","sm":"grok-3-beta","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.583},{"b":"fictionlivebench","m":"chutesllama-4-scout-17b-16e-instruct","oid":"fictionlivebench-51fe09f7e52797cd","rd":"2025-04-06","sid":"recP9MDNJ4QeSksRm","sm":"chutes/Llama-4-Scout-17B-16E Instruct","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.273},{"b":"fictionlivebench","m":"gemini-2-5-flash-preview-04-17","oid":"fictionlivebench-11f32c51b300f6fd","rd":"2025-04-17","sid":"recq3pfN7UN5ydgLW","sm":"gemini-2.5-flash-preview-04-17","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.531},{"b":"fictionlivebench","m":"gpt-4-1-nano-2025-04-14","oid":"fictionlivebench-0d76c4157663ccbf","rd":"2025-04-14","sid":"recGABpbpnWCKfE8t","sm":"gpt-4.1-nano-2025-04-14","src":"https://fiction.live/stories/Fiction-liveBench-Mar-14-2025/oQdzQvKHw8JyXbN87/home","v":0.188},{"b":"forecastbench","cfg":{"effort":"high"},"dn":"gpt-5.5-2026-04-23-high-web-search†","e":"high","hi":67.8,"lo":66.2,"m":"gpt-5-5","oid":"forecastbench-4be8ff8eadc35c01","rd":"2026-04-23","sid":"recP7svRlBedLrxYg","sm":"gpt-5.5_high","v":67},{"b":"forecastbench","cfg":{"effort":"medium"},"dn":"gpt-5.6-sol-standard-medium-web-search-code-execution†","e":"medium","hi":67.3,"lo":65.6,"m":"gpt-5-6-sol","oid":"forecastbench-d38bda87cb2aa586","rd":"2026-07-09","sid":"rec3vWjJwu8sZDGKc","sm":"gpt-5.6-sol_medium","v":66.5},{"b":"forecastbench","cfg":{"effort":"max"},"dn":"claude-opus-5-adaptive-thinking-max-web-search-code-execution-128k†","e":"max","hi":67.3,"lo":65.7,"m":"claude-opus-5","oid":"forecastbench-55bf52ba65b6c53e","rd":"2026-07-24","sid":"rec7nWNGmSFwCBnct","sm":"claude-opus-5_max","v":66.5},{"b":"forecastbench","cfg":{"effort":"max"},"dn":"claude-opus-4-8-adaptive-thinking-max-web-search-128000†","e":"max","hi":67.3,"lo":65.4,"m":"claude-opus-4-8","oid":"forecastbench-3faa2b4cbb083974","rd":"2026-05-28","sid":"recWmkiqePDvVkqQW","sm":"claude-opus-4-8_max","v":66.4},{"b":"forecastbench","cfg":{"effort":"high"},"dn":"gemini-3.7-flash-high-web-search-code-execution†","e":"high","hi":67,"lo":65.4,"m":"gemini-3-7-flash","oid":"forecastbench-74d1ef1ea34bcb7c","rd":"2026-08-13","sid":"recxfN2QaCtiYiQVJ","sm":"gemini-3.7-flash_high","v":66.2},{"b":"forecastbench","cfg":{"effort":"high"},"dn":"grok-4.5-high-web-x-search†","e":"high","hi":66.9,"lo":65.2,"m":"grok-4-5","oid":"forecastbench-12643f3d1cdc372f","rd":"2026-07-08","sid":"recal5PqdeER2y6cG","sm":"grok-4.5_high","v":66},{"b":"forecastbench","cfg":{"effort":"max"},"dn":"claude-fable-5-max-web-search-code-execution-128k†","e":"max","hi":66.7,"lo":65.3,"m":"claude-fable-5","oid":"forecastbench-d781128bd8e49427","rd":"2026-06-09","sid":"rectKJcQQAoZe8G9r","sm":"claude-fable-5_max","v":66},{"b":"forecastbench","cfg":{"effort":"medium"},"dn":"gemini-3.6-flash-medium-web-search-code-execution†","e":"medium","hi":66.7,"lo":65,"m":"gemini-3-6-flash","oid":"forecastbench-b9ec5eb4fbeecf6b","rd":"2026-07-21","sid":"recCRQyyDmO184RjA","sm":"gemini-3.6-flash_medium","v":65.9},{"b":"forecastbench","cfg":{"effort":"high"},"dn":"claude-fable-5-high-web-search-code-execution-128k†","e":"high","hi":66.4,"lo":65,"m":"claude-fable-5","oid":"forecastbench-323297da374b4d57","rd":"2026-06-09","sid":"recudWsw8oqwp2Cob","sm":"claude-fable-5_high","v":65.7},{"b":"forecastbench","cfg":{"effort":"promax"},"dn":"gpt-5.6-sol-pro-max-web-search-code-execution-128k-cost-optimized†","e":"promax","hi":66.5,"lo":64.9,"m":"gpt-5-6-sol","oid":"forecastbench-be20c02e9734a0c7","rd":"2026-07-09","sid":"recFXTxZahTdG8ouo","sm":"gpt-5.6-sol_promax","v":65.6},{"b":"forecastbench","cfg":{"effort":"high"},"dn":"gemini-3.1-pro-preview-high-web-search†","e":"high","hi":66.7,"lo":64.4,"m":"gemini-3-1-pro-preview","oid":"forecastbench-0f6fa6ce5acd5a69","rd":"2026-02-19","sid":"rec5Sst08GNYeUbKH","sm":"gemini-3.1-pro-preview_high","v":65.5},{"b":"forecastbench","cfg":{"effort":"xhigh"},"dn":"grok-4.6-xhigh-web-x-search†","e":"xhigh","hi":66.4,"lo":64.6,"m":"grok-4-6","oid":"forecastbench-a53e2bdc2a491e93","rd":"2026-08-12","sid":"rec7JGqOWQ7FQCIGx","sm":"grok-4.6_xhigh","v":65.5},{"b":"forecastbench","cfg":{"effort":"max"},"dn":"gpt-5.6-sol-standard-max-web-search-code-execution-128k†","e":"max","hi":66.3,"lo":64.7,"m":"gpt-5-6-sol","oid":"forecastbench-c1ff75f96bc8c846","rd":"2026-07-09","sid":"recRrBpuzsML0mp8e","sm":"gpt-5.6-sol_max","v":65.5},{"b":"forecastbench","dn":"gpt-5-2025-08-07†","hi":66.1,"lo":64.5,"m":"gpt-5-2025-08-07","oid":"forecastbench-738f32609ed95da2","rd":"2025-08-07","sid":"reccWyhMQkRbeiDqp","sm":"gpt-5-2025-08-07_unknown","v":65.3},{"b":"forecastbench","dn":"gemini-3-pro-preview†","hi":65.8,"lo":64.7,"m":"gemini-3-pro-preview","oid":"forecastbench-45cb74848e2b5826","rd":"2025-11-18","sid":"recuDEDrYZHnNAqIk","sm":"gemini-3-pro-preview","v":65.2},{"b":"forecastbench","dn":"grok-4-0709†","hi":65.4,"lo":64.5,"m":"grok-4-0709","oid":"forecastbench-8a7d5fae698baa89","rd":"2025-07-09","sid":"recuhguVHB0PtQHfd","sm":"grok-4-0709","v":65},{"b":"forecastbench","dn":"claude-opus-4-5-20251101†","hi":65.5,"lo":64.3,"m":"claude-opus-4-5-2025-11-01","oid":"forecastbench-994b15d4e245eae0","rd":"2025-11-24","sid":"rec8XTmcz8SApqjua","sm":"claude-opus-4-5-20251101_unknown","v":64.9},{"b":"forecastbench","dn":"grok-4-fast-reasoning†","hi":65.4,"lo":64.5,"m":"grok-4-fast","oid":"forecastbench-aac9e97edcab1e3b","rd":"2025-09-19","sid":"rec2D8SZWc6m2WXP3","sm":"grok-4-fast","v":64.9},{"b":"forecastbench","dn":"gpt-4.5-preview-2025-02-27†","hi":65.5,"lo":64,"m":"gpt-4-5-preview-2025-02-27","oid":"forecastbench-0ba7ce4816d0d36e","rd":"2025-02-27","sid":"recjT324vn2nnfnsc","sm":"gpt-4.5-preview-2025-02-27","v":64.8},{"b":"forecastbench","dn":"grok-4-1-fast-reasoning†","hi":65.3,"lo":64.3,"m":"grok-4-1-fast-reasoning","oid":"forecastbench-46dc95bf7f8d1b6f","rd":"2025-11-19","sid":"recB1NcN2xprhvbJG","sm":"grok-4-1-fast-reasoning","v":64.8},{"b":"forecastbench","dn":"claude-opus-4-1-20250805†","hi":65.1,"lo":64.2,"m":"claude-opus-4-1-2025-08-05","oid":"forecastbench-2a0597f3a3a11e91","rd":"2025-08-05","sid":"rec9eeZxYTgoo7Hkh","sm":"claude-opus-4-1-20250805_unknown","v":64.7},{"b":"forecastbench","dn":"gemini-3-flash-preview†","hi":65.2,"lo":64.1,"m":"gemini-3-flash-preview","oid":"forecastbench-e6811a34690857cd","rd":"2025-12-17","sid":"recoX9IAbj7pt6EUu","sm":"gemini-3-flash-preview","v":64.7},{"b":"forecastbench","cfg":{"effort":"max"},"dn":"kimi-k3-max-128k†","e":"max","hi":65.5,"lo":63.8,"m":"kimi-k3","oid":"forecastbench-5ef7cf550f4078a0","rd":"2026-07-16","sid":"recIcK4KoyeyHSfL5","sm":"kimi-k3_max","v":64.7},{"b":"forecastbench","dn":"o3-2025-04-16†","hi":65.3,"lo":64,"m":"o3-2025-04-16","oid":"forecastbench-da2541a35e7a1557","rd":"2025-04-16","sid":"recQzNVfRtKHwluCr","sm":"o3-2025-04-16_unknown","v":64.6},{"b":"forecastbench","dn":"gpt-5.5-2026-04-23†","hi":65.1,"lo":63.9,"m":"gpt-5-5","oid":"forecastbench-6d627e3205c51eb0","rd":"2026-04-23","sid":"recg8o8ME9jvWLuML","sm":"gpt-5.5_unknown","v":64.5},{"b":"forecastbench","dn":"grok-4.20-0309-reasoning†","hi":65.1,"lo":63.8,"m":"grok-4-20-0309-reasoning","oid":"forecastbench-6bd2adc2359dbdc9","rd":"2026-02-17","sid":"recaFseoOAYggLxIr","sm":"grok-4.20-0309-reasoning","v":64.5},{"b":"forecastbench","dn":"gemini-2.5-pro-preview-03-25†","hi":65.5,"lo":63.2,"m":"gemini-2-5-pro-preview-03-25","oid":"forecastbench-de0646e8323c0d06","rd":"2025-03-31","sid":"reckDVPANlcEHlfqK","sm":"gemini-2.5-pro-preview-03-25","v":64.4},{"b":"forecastbench","dn":"gemini-3.5-flash†","hi":65.1,"lo":63.7,"m":"gemini-3-5-flash","oid":"forecastbench-3d84b7cad40c43ea","rd":"2026-05-19","sid":"recLAoiAI4TYHgHC9","sm":"gemini-3.5-flash_unknown","v":64.4},{"b":"forecastbench","dn":"gemini-3.1-pro-preview†","hi":64.8,"lo":63.7,"m":"gemini-3-1-pro-preview","oid":"forecastbench-b7a204544a6ddc76","rd":"2026-02-19","sid":"recGrD94EBUWAPBDk","sm":"gemini-3.1-pro-preview","v":64.3},{"b":"forecastbench","dn":"claude-sonnet-4-6-adaptive-thinking-16000†","hi":64.9,"lo":63.6,"m":"claude-sonnet-4-6-16k","oid":"forecastbench-225e621b6b4e3e2e","rd":"2026-02-17","sid":"recKTCifhyUMgdLeA","sm":"claude-sonnet-4-6_16K","v":64.3},{"b":"forecastbench","dn":"minimax-m3-adaptive-thinking-12000†","hi":65.1,"lo":63.5,"m":"minimax-m3","oid":"forecastbench-ef28615c18acbd82","rd":"2026-06-01","sid":"recd6qdek1lILmjhD","sm":"MiniMax-M3","v":64.3},{"b":"forecastbench","dn":"o4-mini-2025-04-16†","hi":65.2,"lo":63.2,"m":"o4-mini-2025-04-16","oid":"forecastbench-4fab6728fbf3c46a","rd":"2025-04-16","sid":"recefG1iyWtNb8TLM","sm":"o4-mini-2025-04-16_unknown","v":64.2},{"b":"forecastbench","dn":"gemini-2.5-pro†","hi":64.7,"lo":63.7,"m":"gemini-2-5-pro","oid":"forecastbench-95be92d714504a6b","rd":"2025-06-17","sid":"rec0OxLUOambJwoDX","sm":"gemini-2.5-pro","v":64.2},{"b":"forecastbench","dn":"gpt-5.2-2025-12-11†","hi":64.8,"lo":63.6,"m":"gpt-5-2-2025-12-11","oid":"forecastbench-a2c17fd5e4445795","rd":"2025-12-11","sid":"recRfMRmNxVKx2NLd","sm":"gpt-5.2-2025-12-11_unknown","v":64.2},{"b":"forecastbench","dn":"claude-sonnet-4-20250514†","hi":64.7,"lo":63.6,"m":"claude-sonnet-4-2025-05-14","oid":"forecastbench-4d58d965ee87777c","rd":"2025-05-22","sid":"recyoRRySmkrzFAe5","sm":"claude-sonnet-4-20250514_unknown","v":64.2},{"b":"forecastbench","dn":"claude-sonnet-4-6-1024†","hi":64.7,"lo":63.7,"m":"claude-sonnet-4-6","oid":"forecastbench-8d4367fcdd899426","rd":"2026-02-17","sid":"recfxJhuHzARUsxvz","sm":"claude-sonnet-4-6_unknown","v":64.2},{"b":"forecastbench","dn":"claude-sonnet-4-5-20250929-1024†","hi":64.5,"lo":63.7,"m":"claude-sonnet-4-5-2025-09-29","oid":"forecastbench-196a169044fc0f36","rd":"2025-09-29","sid":"rec6rvuXzXJ0PYHGV","sm":"claude-sonnet-4-5-20250929_unknown","v":64.1},{"b":"forecastbench","dn":"qwen3-235b-a22b-fp8-tput†","hi":64.6,"lo":63.5,"m":"qwen3-235b-a22b","oid":"forecastbench-fb9ab9695bf23f85","rd":"2025-04-29","sid":"reczywFpfMTG2ieZL","sm":"Qwen3-235B-A22B","v":64.1},{"b":"forecastbench","dn":"claude-3-7-sonnet-20250219†","hi":64.4,"lo":63.5,"m":"claude-3-7-sonnet-2025-02-19","oid":"forecastbench-980d16910a5ef494","rd":"2025-02-24","sid":"recPDsa2ABcvje85I","sm":"claude-3-7-sonnet-20250219_unknown","v":64},{"b":"forecastbench","dn":"o3-mini-2025-01-31†","hi":64.4,"lo":63.2,"m":"o3-mini-2025-01-31","oid":"forecastbench-cce232fe467f49fc","rd":"2025-01-31","sid":"recah7EMZgPigkWwt","sm":"o3-mini-2025-01-31_unknown","v":63.8},{"b":"forecastbench","dn":"glm-5†","hi":64.7,"lo":62.8,"m":"glm-5","oid":"forecastbench-6fb8dffe9d4f5a27","rd":"2026-02-11","sid":"recM50PCrT0KbEz8d","sm":"glm-5","v":63.7},{"b":"forecastbench","dn":"claude-sonnet-5-adaptive-thinking-16000†","hi":64.3,"lo":62.7,"m":"claude-sonnet-5-16k","oid":"forecastbench-719400f385a8fc3d","rd":"2026-06-30","sid":"rec7vZ5KnWuolxLBY","sm":"claude-sonnet-5_16K","v":63.5},{"b":"forecastbench","dn":"claude-haiku-4-5-20251001-1024†","hi":63.8,"lo":63,"m":"claude-haiku-4-5-2025-10-01","oid":"forecastbench-c8f108ba27edc775","rd":"2025-10-15","sid":"recHzqsoXRNhLqILn","sm":"claude-haiku-4-5-20251001_unknown","v":63.4},{"b":"forecastbench","dn":"claude-opus-4-6-1024†","hi":64.1,"lo":62.6,"m":"claude-opus-4-6","oid":"forecastbench-8f0887dca7213fa7","rd":"2026-02-05","sid":"recBjBmXDxws7h35t","sm":"claude-opus-4-6_unknown","v":63.4},{"b":"forecastbench","dn":"grok-4.3†","hi":64.2,"lo":62.6,"m":"grok-4-3","oid":"forecastbench-8de7258dc192d2cd","rd":"2026-04-17","sid":"reciCRhY4qb9b9yIb","sm":"grok-4.3_unknown","v":63.4},{"b":"forecastbench","dn":"grok-4.20-beta-0309-reasoning†","hi":64.9,"lo":61.4,"m":"grok-4-20","oid":"forecastbench-88f3998c1c4172ee","rd":"2026-02-17","sid":"rec9suJzugDF1Jc9w","sm":"grok-4-20","v":63.3},{"b":"forecastbench","dn":"gpt-5-mini-2025-08-07†","hi":63.8,"lo":62.8,"m":"gpt-5-mini-2025-08-07","oid":"forecastbench-a090f569fdcaa4a7","rd":"2025-08-07","sid":"reckEXEY3bIZL58k6","sm":"gpt-5-mini-2025-08-07_unknown","v":63.3},{"b":"forecastbench","dn":"claude-3-5-sonnet-20241022†","hi":63.8,"lo":62.6,"m":"claude-3-5-sonnet-2024-10-22","oid":"forecastbench-86fc23c62e79bb14","rd":"2024-10-22","sid":"recaBL9Azg8jp2O7y","sm":"claude-3-5-sonnet-20241022","v":63.2},{"b":"forecastbench","dn":"gpt-4-turbo-2024-04-09†","hi":63.8,"lo":62.6,"m":"gpt-4-turbo-2024-04-09","oid":"forecastbench-25bf12ca885528d0","rd":"2024-04-09","sid":"recEOJseJ7IdZQ3Wq","sm":"gpt-4-turbo-2024-04-09","v":63.2},{"b":"forecastbench","dn":"claude-3-5-sonnet-20240620†","hi":63.6,"lo":62.6,"m":"claude-3-5-sonnet-2024-06-20","oid":"forecastbench-a4312b5e2c4667e5","rd":"2024-06-20","sid":"recGK8Hn8MRR4MOyG","sm":"claude-3-5-sonnet-20240620","v":63.1},{"b":"forecastbench","dn":"gpt-5.4-2026-03-05†","hi":63.7,"lo":62.5,"m":"gpt-5-4-2026-03-05","oid":"forecastbench-4ffc3da1b5fa1b34","rd":"2026-03-05","sid":"recZnAThj9SUNL9aZ","sm":"gpt-5.4-2026-03-05_unknown","v":63.1},{"b":"forecastbench","dn":"claude-opus-4-7-1024†","hi":63.9,"lo":62.3,"m":"claude-opus-4-7","oid":"forecastbench-3bb8b9a047237d70","rd":"2026-04-16","sid":"rec2AAzdm6T8MVMeB","sm":"claude-opus-4-7_unknown","v":63.1},{"b":"forecastbench","dn":"gemini-2.5-flash†","hi":63.7,"lo":62.2,"m":"gemini-2-5-flash","oid":"forecastbench-6d3def32e0427d21","rd":"2025-06-17","sid":"recT0z7BoGjvcxJVC","sm":"gemini-2.5-flash","v":62.9},{"b":"forecastbench","dn":"gemini-3.1-flash-lite†","hi":63.5,"lo":62,"m":"gemini-3-1-flash-lite","oid":"forecastbench-9bde206860d9d441","rd":"2026-03-03","sid":"recmEZycYSiXJSmPv","sm":"gemini-3.1-flash-lite","v":62.8},{"b":"forecastbench","dn":"gemini-2.5-flash-preview-04-17†","hi":64.6,"lo":60.5,"m":"gemini-2-5-flash-preview-04-17","oid":"forecastbench-514730b3693e3b62","rd":"2025-04-17","sid":"recZVtQT2n49S8pCK","sm":"gemini-2.5-flash-preview-04-17","v":62.7},{"b":"forecastbench","dn":"kimi-k2-instruct-0905†","hi":63.4,"lo":62,"m":"kimi-k2-instruct-0905","oid":"forecastbench-8b4de88155c515c6","rd":"2025-09-05","sid":"recU5qLUlYYpHySmf","sm":"Kimi-K2-Instruct-0905","v":62.7},{"b":"forecastbench","dn":"claude-opus-4-8-adaptive-thinking-high-24000†","hi":63.5,"lo":61.9,"m":"claude-opus-4-8-24k","oid":"forecastbench-afa2b69d21ee5c3d","rd":"2026-05-28","sid":"recbyJDhLrhkSVaOq","sm":"claude-opus-4-8_24K","v":62.7},{"b":"forecastbench","dn":"meta-llama-3.1-405b-instruct-turbo†","hi":63.3,"lo":61.8,"m":"llama-3-1-405b-instruct","oid":"forecastbench-3b82aea6445c6e28","rd":"2024-07-23","sid":"recZzSkRNYMMLfRjU","sm":"Llama-3.1-405B-Instruct","v":62.6},{"b":"forecastbench","dn":"deepseek-v3†","hi":63.1,"lo":62.1,"m":"deepseek-v3","oid":"forecastbench-eb7cabbb03bb4705","rd":"2024-12-26","sid":"recxdWqYWZD0Rgg6h","sm":"DeepSeek-V3","v":62.6},{"b":"forecastbench","dn":"claude-3-opus-20240229†","hi":63.1,"lo":61.6,"m":"claude-3-opus-2024-02-29","oid":"forecastbench-60baac64bd0508db","rd":"2024-02-29","sid":"recektsiAQBaYu3WJ","sm":"claude-3-opus-20240229","v":62.4},{"b":"forecastbench","dn":"gpt-4.1-2025-04-14†","hi":62.9,"lo":61.8,"m":"gpt-4-1-2025-04-14","oid":"forecastbench-652af2be4bfa61cc","rd":"2025-04-14","sid":"reccvLJeY6UikypFn","sm":"gpt-4.1-2025-04-14","v":62.3},{"b":"forecastbench","dn":"gpt-5.1-2025-11-13†","hi":62.9,"lo":61.7,"m":"gpt-5-1-2025-11-13","oid":"forecastbench-d5d6a70d1172e46c","rd":"2025-11-13","sid":"reccpgRjKw7XGeVii","sm":"gpt-5.1-2025-11-13_unknown","v":62.3},{"b":"forecastbench","dn":"kimi-k2-instruct†","hi":63,"lo":61.4,"m":"kimi-k2-instruct","oid":"forecastbench-a0ce2fd061bf42ef","rd":"2025-07-12","sid":"recxtbOrA1qDfCB66","sm":"Kimi-K2-Instruct","v":62.2},{"b":"forecastbench","dn":"qwen3-235b-a22b-thinking-2507†","hi":63.1,"lo":61.3,"m":"qwen3-235b-a22b-thinking-2507","oid":"forecastbench-c9c7d04e61e65a64","rd":"2025-07-25","sid":"reclGkQm8kpSA51uq","sm":"Qwen3-235B-A22B-Thinking-2507","v":62.2},{"b":"forecastbench","dn":"glm-4.5-air-fp8†","hi":62.9,"lo":61.1,"m":"glm-4-5-air","oid":"forecastbench-ba2d049f57651f01","rd":"2025-07-20","sid":"recmZZGF5TBg4C8oc","sm":"GLM-4.5-Air","v":62},{"b":"forecastbench","dn":"mistral-large-2407†","hi":62.3,"lo":61.2,"m":"mistral-large-2407","oid":"forecastbench-1192f0a7bc32b400","rd":"2024-07-24","sid":"rec1uMZr3t2gqWlpd","sm":"mistral-large-2407","v":61.7},{"b":"forecastbench","dn":"qwen2.5-72b-instruct-turbo†","hi":62.5,"lo":60.7,"m":"qwen2-5-72b-instruct","oid":"forecastbench-425762707a5c369d","rd":"2024-09-19","sid":"recg3iIRMCbHteT8S","sm":"qwen2.5-72b-instruct","v":61.6},{"b":"forecastbench","dn":"gpt-4-0613†","hi":62.7,"lo":60.3,"m":"gpt-4-0613","oid":"forecastbench-456e55f36485f9b4","rd":"2023-06-13","sid":"recD7kikgmSl3m6Db","sm":"gpt-4-0613","v":61.5},{"b":"forecastbench","dn":"mistral-large-2411†","hi":62.3,"lo":60.7,"m":"mistral-large-2411","oid":"forecastbench-f0cd14aba4bbe34b","rd":"2024-11-18","sid":"rec6ii8THDniC180j","sm":"mistral-large-2411","v":61.5},{"b":"forecastbench","dn":"claude-opus-4-8-1024†","hi":62.4,"lo":60.5,"m":"claude-opus-4-8","oid":"forecastbench-98073ba6ef7dc474","rd":"2026-05-28","sid":"recjcXxCkKLap6sxo","sm":"claude-opus-4-8_unknown","v":61.5},{"b":"forecastbench","dn":"llama-3.3-70b-instruct-turbo†","hi":61.8,"lo":60.6,"m":"llama-3-3-70b-instruct","oid":"forecastbench-207587e52e5763b1","rd":"2024-12-06","sid":"recl7RJ530qSfBxOA","sm":"Llama-3.3-70B-Instruct","v":61.3},{"b":"forecastbench","dn":"gemma-4-31b-it†","hi":63.2,"lo":59.4,"m":"gemma-4-31b-it","oid":"forecastbench-cc23c6f078a8894e","rd":"2026-04-02","sid":"recofvsKUgVR9LzoM","sm":"gemma-4-31b-it","v":61.3},{"b":"forecastbench","dn":"gpt-4o-2024-05-13†","hi":61.8,"lo":60.7,"m":"gpt-4o-2024-05-13","oid":"forecastbench-a8e4e9d5a7455e77","rd":"2024-05-13","sid":"recgp08cE6zC9FZML","sm":"gpt-4o-2024-05-13","v":61.2},{"b":"forecastbench","dn":"gpt-5.4-mini-2026-03-17†","hi":62,"lo":60.4,"m":"gpt-5-4-mini-2026-03-17","oid":"forecastbench-138d7c4c8de5e510","rd":"2026-03-17","sid":"rec0me5KEIlpxOxbI","sm":"gpt-5.4-mini-2026-03-17_unknown","v":61.2},{"b":"forecastbench","dn":"deepseek-v3.1†","hi":61.3,"lo":60.1,"m":"deepseek-v3-1","oid":"forecastbench-08c892ee641bf115","rd":"2025-08-21","sid":"reclg89DRRXzbpiUg","sm":"DeepSeek-V3.1","v":60.7},{"b":"forecastbench","dn":"deepseek-v4-pro†","hi":62.4,"lo":58.8,"m":"deepseek-v4-pro","oid":"forecastbench-7cfaa50c2ba0d88a","rd":"2026-04-24","sid":"rec1u1AW7HUycrLM5","sm":"deepseek-v4-pro_unknown","v":60.7},{"b":"forecastbench","dn":"gpt-4o-2024-11-20-web-search†","hi":62.7,"lo":58.1,"m":"gpt-4o-2024-11-20","oid":"forecastbench-aa7666ff82c8c5cd","rd":"2024-11-20","sid":"recYwspIyyVivqOfK","sm":"gpt-4o-2024-11-20","v":60.6},{"b":"forecastbench","dn":"llama-3-70b-chat-hf†","hi":61.1,"lo":59.2,"m":"meta-llama-3-70b","oid":"forecastbench-fcce9f8e6c89de67","rd":"2024-04-18","sid":"rec5MmlJoH4mV41ty","sm":"Meta-Llama-3-70B","v":60.2},{"b":"forecastbench","dn":"qwen1.5-110b-chat†","hi":60.8,"lo":58.2,"m":"qwen1-5-110b-chat","oid":"forecastbench-12a2787340971218","rd":"2024-04-25","sid":"recuqriVbFfTVrg7B","sm":"qwen1.5-110b-chat","v":59.5},{"b":"forecastbench","dn":"llama-3-8b-chat-hf†","hi":60.8,"lo":58,"m":"meta-llama-3-8b-instruct","oid":"forecastbench-d3014a8a31ed60b7","rd":"2024-04-18","sid":"recV7mrylsTwGK1pP","sm":"Meta-Llama-3-8B-Instruct","v":59.4},{"b":"forecastbench","dn":"llama-4-scout-17b-16e-instruct†","hi":59.8,"lo":58.5,"m":"llama-4-scout-17b-16e-instruct","oid":"forecastbench-f83b9ec914d7f03e","rd":"2025-04-05","sid":"recBd74YkXcInLqiw","sm":"Llama-4-Scout-17B-16E-Instruct","v":59.1},{"b":"forecastbench","dn":"gpt-5-nano-2025-08-07†","hi":59.6,"lo":58.5,"m":"gpt-5-nano-2025-08-07","oid":"forecastbench-3e3f60a34a0b8053","rd":"2025-08-07","sid":"recgH1aj3jfcm3NaL","sm":"gpt-5-nano-2025-08-07_unknown","v":59},{"b":"forecastbench","dn":"mixtral-8x22b-instruct-v0.1†","hi":60.4,"lo":57.1,"m":"mixtral-8x22b-instruct-v0-1","oid":"forecastbench-09eead15a0176542","rd":"2024-04-17","sid":"recymnaWRIEaDxdKj","sm":"Mixtral-8x22B-Instruct-v0.1","v":58.8},{"b":"forecastbench","dn":"gemini-1.5-pro†","hi":60.6,"lo":56.9,"m":"gemini-1-5-pro-001","oid":"forecastbench-3d8f2a6924e1d2bc","rd":"2024-05-14","sid":"recC5M21ztCVuHKBJ","sm":"gemini-1.5-pro-001","v":58.8},{"b":"forecastbench","dn":"claude-2.1†","hi":60,"lo":57.3,"m":"claude-2-1","oid":"forecastbench-d59ca1f90cf2383f","rd":"2023-11-21","sid":"recHplwtDHYC4w4YP","sm":"claude-2.1","v":58.7},{"b":"forecastbench","dn":"gpt-5.4-nano-2026-03-17†","hi":59.5,"lo":57.8,"m":"gpt-5-4-nano-2026-03-17","oid":"forecastbench-edc8f920ff4c2e8f","rd":"2026-03-17","sid":"recvU1Cr4BBezWzXU","sm":"gpt-5.4-nano-2026-03-17_unknown","v":58.6},{"b":"forecastbench","dn":"gemini-1.5-flash†","hi":60.6,"lo":56.6,"m":"gemini-1-5-flash-001","oid":"forecastbench-e9e02d8f6a4f6cf2","rd":"2024-05-23","sid":"recTFEQaCJsC7CqRm","sm":"gemini-1.5-flash-001","v":58.6},{"b":"forecastbench","dn":"mixtral-8x7b-instruct-v0.1†","hi":60.1,"lo":56.2,"m":"mixtral-8x7b-instruct-v0-1","oid":"forecastbench-48656cb521b634d9","rd":"2023-12-11","sid":"recPMwHmvxVRaVoHg","sm":"Mixtral-8x7B-Instruct-v0.1","v":58.2},{"b":"forecastbench","dn":"gemini-2.0-flash-lite-001†","hi":58.6,"lo":55.2,"m":"gemini-2-0-flash-lite-001","oid":"forecastbench-ae1431e4db418460","rd":"2025-02-25","sid":"recmFMI3VKGssuIRU","sm":"gemini-2.0-flash-lite-001","v":57},{"b":"forecastbench","dn":"claude-3-haiku-20240307†","hi":55.5,"lo":54.2,"m":"claude-3-haiku-2024-03-07","oid":"forecastbench-3e66d7c0893582b1","rd":"2024-03-07","sid":"rec5GC5VDz2J0wIvA","sm":"claude-3-haiku-20240307","v":54.8},{"b":"forecastbench","dn":"llama-2-70b-chat-hf†","hi":52.7,"lo":48.3,"m":"llama-2-70b-chat-hf","oid":"forecastbench-9b3f4e375c0b302c","rd":"2023-07-18","sid":"recyD6PVPwW6w4cJX","sm":"Llama-2-70b-chat-hf","v":50.4},{"b":"forecastbench","dn":"gpt-3.5-turbo-0125†","hi":48.1,"lo":44.6,"m":"gpt-3-5-turbo-0125","oid":"forecastbench-d7f0e7c05f659abe","rd":"2024-01-25","sid":"recmmQycs8kJygEul","sm":"gpt-3.5-turbo-0125","v":46.4},{"b":"frontiercode","cfg":{"effort":"medium","harness":"claude-code"},"dn":"Claude Opus 5.5","e":"medium","m":"claude-opus-5-5","oid":"frontiercode-9c26cfc49cc01f21","rd":"2026-09-22","sm":"claude-opus-5-5_medium","src":"https://cognition.com/frontiercode","v":0.5464},{"b":"frontiercode","cfg":{"effort":"xhigh","harness":"claude-code"},"dn":"Claude Fable 5","m":"claude-fable-5","oid":"frontiercode-d13a5096c324ab1c","rd":"2026-06-09","sm":"claude-fable-5_unknown","src":"https://cognition.com/frontiercode","v":0.5348},{"b":"frontiercode","cfg":{"effort":"max","harness":"claude-code"},"dn":"Claude Opus 5","e":"max","m":"claude-opus-5","oid":"frontiercode-14d4db9fd74ff2f3","rd":"2026-07-24","sm":"claude-opus-5_max","src":"https://cognition.com/frontiercode","v":0.5338},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-6 Astra","e":"max","m":"gpt-6-astra","oid":"frontiercode-7f22de70867ed013","rd":"2026-09-03","sm":"gpt-6-astra_max","src":"https://cognition.com/frontiercode","v":0.5326},{"b":"frontiercode","cfg":{"effort":"xhigh","harness":"claude-code"},"dn":"Claude Sonnet 5.5","e":"xhigh","m":"claude-sonnet-5-5","oid":"frontiercode-b453bffb9047413d","rd":"2026-09-28","sm":"claude-sonnet-5-5_xhigh","src":"https://cognition.com/frontiercode","v":0.5209},{"b":"frontiercode","cfg":{"effort":"medium","harness":"claude-code"},"dn":"Claude Fable 5.1","e":"medium","m":"claude-fable-5-1","oid":"frontiercode-7f066b27371c7d46","rd":"2026-09-01","sm":"claude-fable-5-1_medium","src":"https://cognition.com/frontiercode","v":0.5091},{"b":"frontiercode","cfg":{"effort":"medium","harness":"codex"},"dn":"GPT-6.1 Sol","e":"medium","m":"gpt-6-1-sol","oid":"frontiercode-50b7f42eb711e01c","rd":"2026-09-29","sm":"gpt-6.1-sol_medium","src":"https://cognition.com/frontiercode","v":0.5023},{"b":"frontiercode","cfg":{"effort":"max","harness":"devin"},"dn":"SWE-2","e":"max","m":"swe-2","oid":"frontiercode-56edaaf5cef56c63","rd":"2026-09-10","sm":"swe-2_max","src":"https://cognition.com/frontiercode","v":0.5},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-6 Sol","e":"max","m":"gpt-6-sol","oid":"frontiercode-1b57016bb44b6ceb","rd":"2026-09-22","sm":"gpt-6-sol_max","src":"https://cognition.com/frontiercode","v":0.4927},{"b":"frontiercode","cfg":{"effort":"high","harness":"grok-build"},"dn":"Grok 4.6","m":"grok-4-6","oid":"frontiercode-5985485550de25e0","rd":"2026-08-12","sm":"grok-4.6_unknown","src":"https://cognition.com/frontiercode","v":0.4801000000000001},{"b":"frontiercode","cfg":{"effort":"high","harness":"grok-build"},"dn":"Grok 4.7","m":"grok-4-7","oid":"frontiercode-ccdc0ecf5da3f13e","rd":"2026-09-21","sm":"grok-4.7_unknown","src":"https://cognition.com/frontiercode","v":0.4759},{"b":"frontiercode","cfg":{"effort":"medium","harness":"devin-cli"},"dn":"Gemini 4 Argon","e":"medium","m":"gemini-4-argon","oid":"frontiercode-a11393b8c3158fd1","rd":"2026-09-30","sm":"gemini-4-argon_medium","src":"https://cognition.com/frontiercode","v":0.4755},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-5.6 Sol","m":"gpt-5-6-sol","oid":"frontiercode-2277cf2d4db35649","rd":"2026-07-09","sm":"gpt-5.6-sol_unknown","src":"https://cognition.com/frontiercode","v":0.47490000000000004},{"b":"frontiercode","cfg":{"effort":"max","harness":"claude-code"},"dn":"Claude Opus 4.8","m":"claude-opus-4-8","oid":"frontiercode-c0bb65d07a46bfb9","rd":"2026-05-28","sm":"claude-opus-4-8_unknown","src":"https://cognition.com/frontiercode","v":0.465},{"b":"frontiercode","cfg":{"effort":"max","harness":"claude-code"},"dn":"Claude Haiku 5.5","e":"max","m":"claude-haiku-5-5","oid":"frontiercode-6579c1f6643b753e","rd":"2026-10-07","sm":"claude-haiku-5-5_max","src":"https://cognition.com/frontiercode","v":0.4636},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"Kimi K3","m":"kimi-k3","oid":"frontiercode-dbd7fd6b8f9e920a","rd":"2026-07-16","sm":"kimi-k3_unknown","src":"https://cognition.com/frontiercode","v":0.44170000000000004},{"b":"frontiercode","cfg":{"effort":"high","harness":"devin-cli"},"dn":"DeepSeek V4.1 Flash","e":"high","m":"deepseek-v4-1-flash","oid":"frontiercode-9c5cf9de26ed5562","rd":"2026-09-09","sm":"deepseek-v4.1-flash_high","src":"https://cognition.com/frontiercode","v":0.4387},{"b":"frontiercode","cfg":{"effort":"medium","harness":"devin-cli"},"dn":"Gemini 3.7 Flash","m":"gemini-3-7-flash","oid":"frontiercode-711ad859f6561955","rd":"2026-08-13","sm":"gemini-3.7-flash_unknown","src":"https://cognition.com/frontiercode","v":0.43590000000000007},{"b":"frontiercode","cfg":{"effort":"xhigh","harness":"codex"},"dn":"GPT-5.5","m":"gpt-5-5","oid":"frontiercode-370dd24cb196daa2","rd":"2026-04-23","sm":"gpt-5.5_unknown","src":"https://cognition.com/frontiercode","v":0.42960000000000004},{"b":"frontiercode","cfg":{"effort":"xhigh","harness":"claude-code"},"dn":"Claude Sonnet 5","m":"claude-sonnet-5","oid":"frontiercode-23d28b5469463679","rd":"2026-06-30","sm":"claude-sonnet-5_unknown","src":"https://cognition.com/frontiercode","v":0.42730000000000007},{"b":"frontiercode","cfg":{"effort":"high","harness":"grok-build"},"dn":"Grok 4.5","m":"grok-4-5","oid":"frontiercode-91788da41a462346","rd":"2026-07-08","sm":"grok-4.5_unknown","src":"https://cognition.com/frontiercode","v":0.4244},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-6 Luna","e":"max","m":"gpt-6-luna","oid":"frontiercode-eef2e41d9e7db780","rd":"2026-09-22","sm":"gpt-6-luna_max","src":"https://cognition.com/frontiercode","v":0.4242},{"b":"frontiercode","cfg":{"effort":"none","harness":"devin-cli"},"dn":"SWE-1.7","e":"none","m":"swe-1-7","oid":"frontiercode-e84a01de8e916e06","rd":"2026-07-08","sm":"swe-1.7","src":"https://cognition.com/frontiercode","v":0.41990000000000005},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-5.6 Terra","m":"gpt-5-6-terra","oid":"frontiercode-6177daccb251a148","rd":"2026-07-09","sm":"gpt-5.6-terra_unknown","src":"https://cognition.com/frontiercode","v":0.4131},{"b":"frontiercode","cfg":{"effort":"medium","harness":"devin-cli"},"dn":"Gemini 3.8 Flash","e":"medium","m":"gemini-3-8-flash","oid":"frontiercode-96a9f464d5befe94","rd":"2026-09-02","sm":"gemini-3.8-flash_medium","src":"https://cognition.com/frontiercode","v":0.4119},{"b":"frontiercode","cfg":{"effort":"max","harness":"devin-cli"},"dn":"GLM 5.3","e":"max","m":"glm-5-3","oid":"frontiercode-364ced9fd16b2e03","rd":"2026-08-14","sm":"glm-5.3_max","src":"https://cognition.com/frontiercode","v":0.40140000000000003},{"b":"frontiercode","cfg":{"effort":"max","harness":"codex"},"dn":"GPT-5.6 Luna","m":"gpt-5-6-luna","oid":"frontiercode-1a6fba45d0579a9c","rd":"2026-07-09","sm":"gpt-5.6-luna_unknown","src":"https://cognition.com/frontiercode","v":0.3981},{"b":"frontiercode","cfg":{"effort":"high","harness":"chisel"},"dn":"Mistral Large 4 (Preview)","e":"high","m":"mistral-large-4-0","oid":"frontiercode-7d5fabfa72003b6b","rd":"2026-10-06","sm":"mistral-large-4-0_high","src":"https://cognition.com/frontiercode","v":0.3943},{"b":"frontiercode","cfg":{"effort":"max","harness":"claude-code"},"dn":"Claude Opus 4.7","m":"claude-opus-4-7","oid":"frontiercode-d8579e176ad06377","rd":"2026-04-16","sm":"claude-opus-4-7_unknown","src":"https://cognition.com/frontiercode","v":0.3854},{"b":"frontiercode","cfg":{"effort":"medium","harness":"devin-cli"},"dn":"Gemini 3.6 Flash","m":"gemini-3-6-flash","oid":"frontiercode-881d5e0f2c4b7597","rd":"2026-07-21","sm":"gemini-3.6-flash_unknown","src":"https://cognition.com/frontiercode","v":0.3437},{"b":"frontiercode","cfg":{"effort":"max","harness":"devin-cli"},"dn":"GLM 5.3 Flash","e":"max","m":"glm-5-3-flash","oid":"frontiercode-8440fdae862f24a8","rd":"2026-08-20","sm":"glm-5.3-flash_max","src":"https://cognition.com/frontiercode","v":0.3183},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"Kimi K2.7","e":"none","m":"kimi-k2-7-code","oid":"frontiercode-2a0311ff139f4329","rd":"2026-06-12","sm":"kimi-k2.7-code","src":"https://cognition.com/frontiercode","v":0.3006},{"b":"frontiercode","cfg":{"effort":"high","harness":"devin-cli"},"dn":"DeepSeek V4 Pro 0813","e":"high","m":"deepseek-v4-pro-0813","oid":"frontiercode-5dbb6100c52be8cf","rd":"2026-08-13","sm":"deepseek-v4-pro-0813_high","src":"https://cognition.com/frontiercode","v":0.2855},{"b":"frontiercode","cfg":{"effort":"xhigh","harness":"codex"},"dn":"GPT-5.4-mini","m":"gpt-5-4-mini-2026-03-17","oid":"frontiercode-8228df8c9c7a743f","rd":"2026-03-17","sm":"gpt-5.4-mini-2026-03-17_unknown","src":"https://cognition.com/frontiercode","v":0.2704},{"b":"frontiercode","cfg":{"effort":"high","harness":"claude-code"},"dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"frontiercode-caa2f25031f137d0","rd":"2026-02-05","sm":"claude-opus-4-6_unknown","src":"https://cognition.com/frontiercode","v":0.2664},{"b":"frontiercode","cfg":{"effort":"none","harness":"cursor-cli"},"dn":"Composer 2.5","e":"none","m":"composer-2-5","oid":"frontiercode-6217f70efae00243","rd":"2026-05-18","sm":"Composer 2.5","src":"https://cognition.com/frontiercode","v":0.2564},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"GLM 5.2","e":"none","m":"glm-5-2","oid":"frontiercode-c18963311d23df0a","rd":"2026-06-16","sm":"glm-5.2_none","src":"https://cognition.com/frontiercode","v":0.245},{"b":"frontiercode","cfg":{"effort":"max","harness":"claude-code"},"dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","oid":"frontiercode-1f2e0d4a43a779f9","rd":"2026-02-17","sm":"claude-sonnet-4-6_unknown","src":"https://cognition.com/frontiercode","v":0.24310000000000004},{"b":"frontiercode","cfg":{"effort":"high","harness":"devin-cli"},"dn":"DeepSeek V4 Flash 0731","m":"deepseek-v4-flash-0731","oid":"frontiercode-6d1151915d944c0b","rd":"2026-07-31","sm":"deepseek-v4-flash-0731_unknown","src":"https://cognition.com/frontiercode","v":0.188},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"DeepSeek V4 Pro","e":"none","m":"deepseek-v4-pro","oid":"frontiercode-1d1697a8c42a7c2f","rd":"2026-04-24","sm":"deepseek-v4-pro_none","src":"https://cognition.com/frontiercode","v":0.1764},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"MiniMax M3","e":"none","m":"minimax-m3","oid":"frontiercode-9d2c94a0faf68a98","rd":"2026-06-01","sm":"MiniMax-M3","src":"https://cognition.com/frontiercode","v":0.147},{"b":"frontiercode","cfg":{"effort":"0.99","harness":"mini-swe-agent"},"dn":"Inkling","e":"0.99","m":"inkling","oid":"frontiercode-251bac39db0088ba","rd":"2026-07-15","sm":"Inkling","src":"https://cognition.com/frontiercode","v":0.14},{"b":"frontiercode","cfg":{"effort":"none","harness":"devin-cli"},"dn":"Nemotron 3 Ultra","e":"none","m":"nemotron-3-ultra","oid":"frontiercode-77f63e1fe869c5ff","rd":"2026-06-04","sm":"nemotron-3-ultra","src":"https://cognition.com/frontiercode","v":0.1361},{"b":"frontiercode","cfg":{"effort":"none","harness":"mini-swe-agent"},"dn":"Qwen 3.7 Plus","e":"none","m":"qwen3-7-plus","oid":"frontiercode-a365d44cc3326566","rd":"2026-06-02","sm":"qwen3.7-plus","src":"https://cognition.com/frontiercode","v":0.102},{"b":"frontiercode","cfg":{"effort":"none","harness":"devin-cli"},"dn":"SWE-1.6","e":"none","m":"swe-1-6","oid":"frontiercode-39648cc89e3a1ad1","rd":"2026-04-07","sm":"swe-1.6","src":"https://cognition.com/frontiercode","v":0.09390000000000001},{"b":"frontiercode","cfg":{"effort":"none","harness":"devin-cli"},"dn":"Mistral 3.5 Medium","e":"none","m":"mistral-medium-2604","oid":"frontiercode-c8cf09182dee85a9","rd":"2026-04-28","sm":"mistral-medium-2604","src":"https://cognition.com/frontiercode","v":0.08},{"b":"frontiermath","cfg":{"effort":"max"},"d":"2026-06-08","e":"max","evaluated_at":"2026-06-08T00:20:40.000Z","m":"claude-opus-4-8","oid":"frontiermath-bf76c8eeea5a0582","rd":"2026-05-28","se":0.029366965952163048,"sid":"6dUEQ3YfeMZzSm9ytgiuDi","sm":"claude-opus-4-8_max","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6dUEQ3YfeMZzSm9ytgiuDi.eval","v":0.4724137931034483},{"b":"frontiermath","d":"2026-05-27","evaluated_at":"2026-05-27T22:28:21.000Z","m":"qwen3-6-max-preview","oid":"frontiermath-086897542db09546","rd":"2026-04-20","se":0.02479378972186167,"sid":"T6yJ47dEheZTjRVDS6Bgzt","sm":"qwen3.6-max-preview","v":0.23103448275862068},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2026-05-22","e":"high","evaluated_at":"2026-05-22T16:42:34.000Z","m":"gemini-3-5-flash","oid":"frontiermath-7dab4b47a056c47d","rd":"2026-05-19","se":0.028686590614000477,"sid":"Ectc9Fcn4xSKZFQwj9Xn7R","sm":"gemini-3.5-flash_high","v":0.3896551724137931},{"b":"frontiermath","d":"2026-05-14","evaluated_at":"2026-05-14T12:48:49.000Z","m":"qwen3-5-plus","oid":"frontiermath-9ec5ac9894d0243c","rd":"2026-02-16","se":0.02397373715108675,"sid":"Kbc5vW47MpsnYaqeBuoJoQ","sm":"qwen3.5-plus","v":0.2103448275862069},{"b":"frontiermath","d":"2026-05-12","evaluated_at":"2026-05-12T16:08:00.000Z","m":"qwen3-6-plus","oid":"frontiermath-37e98aa128cc455d","rd":"2026-03-31","se":0.025868232968648857,"sid":"Vxki26PKxMtCWsahLMmBHB","sm":"qwen3.6-plus","v":0.2620689655172414},{"b":"frontiermath","d":"2026-05-12","evaluated_at":"2026-05-12T15:03:26.000Z","m":"qwen3-6-flash","oid":"frontiermath-f3fd6c668d0ab200","rd":"2026-04-27","se":0.01791432224407271,"sid":"n4BkffZyi6KAx8HajDU4ZK","sm":"qwen3.6-flash","v":0.10344827586206896},{"b":"frontiermath","d":"2026-05-12","evaluated_at":"2026-05-12T12:27:22.000Z","m":"qwen3-5-flash","oid":"frontiermath-627504eae57fb60e","rd":"2026-02-25","se":0.014192986356621421,"sid":"VHC8c6kTi5zjkHU67QSkZW","sm":"qwen3.5-flash","v":0.06206896551724138},{"b":"frontiermath","d":"2026-05-11","evaluated_at":"2026-05-11T13:39:21.000Z","m":"glm-5-1","oid":"frontiermath-7e46e5083f30578f","rd":"2026-04-07","se":0.027753489933368777,"sid":"4DNcqcs4R97ZWNTMDrXSZN","sm":"glm-5.1","v":0.33448275862068966},{"b":"frontiermath","d":"2026-05-07","evaluated_at":"2026-05-07T21:21:29.000Z","m":"kimi-k2-6","oid":"frontiermath-29ef4969ffc63996","rd":"2026-04-20","se":0.028686590614000477,"sid":"4StdTsQNqgsKc8RFdauiF2","sm":"kimi-k2.6","v":0.3896551724137931},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2026-04-23","e":"xhigh","evaluated_at":"2026-04-23T20:51:52.501Z","m":"gpt-5-5-pro-pre-release","oid":"frontiermath-5972d2c4aab8cd48","rd":"2026-04-23","se":0.029,"sid":"gpt-5-5-pro-pre-release-xhigh-t13","sm":"gpt-5.5-pro-pre-release_xhigh","v":0.51},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2026-04-23","e":"high","evaluated_at":"2026-04-23T20:51:39.890Z","m":"gpt-5-5-pro-pre-release","oid":"frontiermath-e1aca4aa34415f22","rd":"2026-04-23","se":0.029,"sid":"gpt-5-5-pro-pre-release-high-t13","sm":"gpt-5.5-pro-pre-release_high","v":0.524},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2026-04-23","e":"xhigh","evaluated_at":"2026-04-23T20:51:32.704Z","m":"gpt-5-5-pre-release","oid":"frontiermath-4bcf68c664ec34bc","rd":"2026-04-23","se":0.029,"sid":"gpt-5-5-pre-release-xhigh-t13","sm":"gpt-5.5-pre-release_xhigh","v":0.517},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2026-04-17","e":"xhigh","evaluated_at":"2026-04-17T11:24:50.000Z","m":"claude-opus-4-7","oid":"frontiermath-f078cf1c42323d40","rd":"2026-04-16","se":0.02918426387376174,"sid":"VFJMk4X6U8Yx5FwtQh4jbk","sm":"claude-opus-4-7_xhigh","v":0.4379310344827586},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T13:54:39.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"frontiermath-665c3aee0b6187c4","rd":"2026-03-17","se":0.02065,"sid":"7oFhmdG6DwLVLAenx8Wu4z","sm":"gpt-5.4-mini-2026-03-17_high","v":0.2828},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T09:11:20.668Z","m":"gpt-5-4-nano-2026-03-17","oid":"frontiermath-c15b5797b8845b00","rd":"2026-03-17","se":0.0258,"sid":"SZpkpKjZLWJvmXWPQsMpbv","sm":"gpt-5.4-nano-2026-03-17_high","v":0.2586},{"b":"frontiermath","d":"2026-04-08","evaluated_at":"2026-04-08T16:30:10.530Z","m":"muse-spark","oid":"frontiermath-4320090199c7ecdb","rd":"2026-04-08","se":0.029,"sid":"muse-spark-pre-release-tiers-1-3","sm":"muse-spark","v":0.39},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2026-03-06","e":"xhigh","evaluated_at":"2026-03-06T03:53:08.691Z","m":"gpt-5-4-2026-03-05","oid":"frontiermath-9f1698aa137f794c","rd":"2026-03-05","se":0.029,"sid":"pre-release-gpt-5-4-xhigh-fm-t13","sm":"gpt-5.4-2026-03-05_xhigh","v":0.476},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2026-03-06","e":"xhigh","evaluated_at":"2026-03-06T03:47:23.261Z","m":"gpt-5-4-pro-2026-03-05","oid":"frontiermath-6b0e04abd2441d07","rd":"2026-03-05","se":0.029,"sid":"pre-release-gpt-5-4-pro-fm-t13","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.5},{"b":"frontiermath","d":"2026-02-20","evaluated_at":"2026-02-20T14:53:58.744Z","m":"claude-sonnet-4-6-16k","oid":"frontiermath-b0b87f9c60a5beb6","rd":"2026-02-17","se":0.028121636040639858,"sid":"ewFjUhQhYidjkErDdLvcQL","sm":"claude-sonnet-4-6_16K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ewFjUhQhYidjkErDdLvcQL.eval","v":0.324},{"b":"frontiermath","d":"2026-02-19","evaluated_at":"2026-02-19T19:17:36.176Z","m":"glm-5","oid":"frontiermath-879ef7c450950629","rd":"2026-02-11","se":0.021951258107494095,"sid":"MEXifXdpy4M5JH6EoyF35G","sm":"glm-5","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MEXifXdpy4M5JH6EoyF35G.eval","v":0.16433566433566432},{"b":"frontiermath","d":"2026-02-19","evaluated_at":"2026-02-19T18:54:42.570Z","m":"gemini-3-1-pro-preview","oid":"frontiermath-7668abecb03dbf49","rd":"2026-02-19","se":0.028,"sid":"CZ5mNHNivNueD6boJnxq6H","sm":"gemini-3.1-pro-preview","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/CZ5mNHNivNueD6boJnxq6H.eval","v":0.369},{"b":"frontiermath","cfg":{"effort":"max"},"d":"2026-02-12","e":"max","evaluated_at":"2026-02-12T17:17:42.489Z","m":"claude-opus-4-6","oid":"frontiermath-3fafed3e7d142008","rd":"2026-02-05","se":0.029,"sid":"Y4von2jMmwvEK5MuNw9pSt","sm":"claude-opus-4-6_max","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Y4von2jMmwvEK5MuNw9pSt.eval","v":0.407},{"b":"frontiermath","d":"2026-02-06","evaluated_at":"2026-02-06T15:39:35.993Z","m":"claude-opus-4-6-64k","oid":"frontiermath-6b360697a45afe97","rd":"2026-02-05","se":0.028775377199008325,"sid":"QSNvJqsvYdNbb85DyL9AXn","sm":"claude-opus-4-6_64K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QSNvJqsvYdNbb85DyL9AXn.eval","v":0.39655172413793105},{"b":"frontiermath","d":"2026-02-06","evaluated_at":"2026-02-06T12:46:04.911Z","m":"claude-opus-4-6-32k","oid":"frontiermath-6dea86ddd328df5a","rd":"2026-02-05","se":0.028817526385684453,"sid":"b2iuVcuCPEau2ryhBsjhS9","sm":"claude-opus-4-6_32K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/b2iuVcuCPEau2ryhBsjhS9.eval","v":0.4},{"b":"frontiermath","d":"2026-02-05","evaluated_at":"2026-02-05T18:49:51.181Z","m":"claude-opus-4-6","oid":"frontiermath-9fe923af9963ff04","rd":"2026-02-05","se":0.02859177289764447,"sid":"nFxCq7MqCfKjhxKr7hT9qr","sm":"claude-opus-4-6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nFxCq7MqCfKjhxKr7hT9qr.eval","v":0.38275862068965516},{"b":"frontiermath","d":"2026-02-03","evaluated_at":"2026-02-03T10:31:18.169Z","m":"fireworkskimi-k2p5","oid":"frontiermath-97fa1dd34535bfb0","rd":"2026-01-27","se":0.026,"sid":"manual_kimi-k2.5_fmprivate_frankenstein","sm":"fireworks/kimi-k2p5","v":0.279},{"b":"frontiermath","d":"2026-01-30","evaluated_at":"2026-01-30T03:56:27.987Z","m":"zai-orgglm-4-7","oid":"frontiermath-cfcc1ae700750c99","rd":"2025-12-22","se":0.009121434048862995,"sid":"GJNr6pF9QUonmhqw5dXWpd","sm":"zai-org/GLM-4.7","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GJNr6pF9QUonmhqw5dXWpd.eval","v":0.024390243902439025},{"b":"frontiermath","d":"2025-12-22","evaluated_at":"2025-12-22T16:13:27.305Z","m":"fireworksdeepseek-v3p2","oid":"frontiermath-c110492dcc2647ca","rd":"2025-12-01","se":0.041,"sid":"manual_deepseek-v3.2_frontiermath_tiers_13_private_frankenstein_run_incomplete","sm":"fireworks/deepseek-v3p2","v":0.221},{"b":"frontiermath","d":"2025-12-18","evaluated_at":"2025-12-18T18:23:15.700Z","m":"gemini-2-5-flash","oid":"frontiermath-cdf03553b1c41289","rd":"2025-06-17","se":0.012651332738454618,"sid":"bTbDNLFqsUogG3fejZEAZc","sm":"gemini-2.5-flash","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bTbDNLFqsUogG3fejZEAZc.eval","v":0.04844290657439446},{"b":"frontiermath","d":"2025-12-17","evaluated_at":"2025-12-17T22:40:29.699Z","m":"gemini-3-flash-preview","oid":"frontiermath-767952c269195b3a","rd":"2025-12-17","se":0.028221558784583784,"sid":"R7dSNsq2ciVRVHB7nir69v","sm":"gemini-3-flash-preview","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/R7dSNsq2ciVRVHB7nir69v.eval","v":0.356401384083045},{"b":"frontiermath","cfg":{"effort":"xhigh"},"d":"2025-12-13","e":"xhigh","evaluated_at":"2025-12-13T13:45:00.000Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath-067b07340aa1bcf3","rd":"2025-12-11","se":0.029,"sid":"hYCViU82i8tejydKA4Pcuu","sm":"gpt-5.2-2025-12-11_xhigh","v":0.407},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T20:49:17.862Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath-4311289f7a1c90ce","rd":"2025-12-11","se":0.028858188319787523,"sid":"dcQtgoUnQbkqTT6Cueo8Re","sm":"gpt-5.2-2025-12-11_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/dcQtgoUnQbkqTT6Cueo8Re.eval","v":0.40344827586206894},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-12-11","e":"medium","evaluated_at":"2025-12-11T20:49:17.514Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath-fd95a294d95a5a6d","rd":"2025-12-11","se":0.02838379845926905,"sid":"BWGVnFerZCyKTuc86w7c5q","sm":"gpt-5.2-2025-12-11_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BWGVnFerZCyKTuc86w7c5q.eval","v":0.3689655172413793},{"b":"frontiermath","cfg":{"effort":"low"},"d":"2025-12-11","e":"low","evaluated_at":"2025-12-11T20:24:40.964Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath-d307ec0b178b505a","rd":"2025-12-11","se":0.02597695517914575,"sid":"NghuUppaqcCUtT8BqRgUL6","sm":"gpt-5.2-2025-12-11_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NghuUppaqcCUtT8BqRgUL6.eval","v":0.2655172413793103},{"b":"frontiermath","d":"2025-12-11","evaluated_at":"2025-12-11T17:22:27.962Z","m":"qwenqwen3-235b-a22b-thinking-2507","oid":"frontiermath-e1bb2e31a7679a0c","rd":"2025-07-25","se":0.016589928458951776,"sid":"TLsGMRt4tYtWnheLm6jA5s","sm":"Qwen/Qwen3-235B-A22B-Thinking-2507","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/TLsGMRt4tYtWnheLm6jA5s.eval","v":0.08480565371024736},{"b":"frontiermath","d":"2025-12-08","evaluated_at":"2025-12-08T20:08:02.803Z","m":"zai-orgglm-4-6","oid":"frontiermath-674e6d899684a524","rd":"2025-09-30","se":0.011313651347924142,"sid":"MPJBHaxt8mXcKBWa94rsXj","sm":"zai-org/GLM-4.6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MPJBHaxt8mXcKBWa94rsXj.eval","v":0.03819444444444445},{"b":"frontiermath","d":"2025-12-05","evaluated_at":"2025-12-05T16:22:32.448Z","m":"moonshotaikimi-k2-thinking","oid":"frontiermath-ad48b2096a0c8221","rd":"2025-11-06","se":0.02433800055326239,"sid":"FmXuaTot9QsNdFYwwFHuEG","sm":"moonshotai/Kimi-K2-Thinking","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FmXuaTot9QsNdFYwwFHuEG.eval","v":0.21403508771929824},{"b":"frontiermath","cfg":{"effort":"none"},"d":"2025-11-25","e":"none","evaluated_at":"2025-11-25T22:22:28.656Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath-a0ffbe820c4b7c99","rd":"2025-11-13","se":0.008489188512823553,"sid":"XNbh4FSUJeFAe2auBg5Bdi","sm":"gpt-5.1-2025-11-13_none","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XNbh4FSUJeFAe2auBg5Bdi.eval","v":0.02097902097902098},{"b":"frontiermath","cfg":{"effort":"low"},"d":"2025-11-25","e":"low","evaluated_at":"2025-11-25T15:12:29.299Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath-eee983496cceb79e","rd":"2025-11-13","se":0.022288963139763994,"sid":"7o6HCQaKUrxWjrf2vhH7hV","sm":"gpt-5.1-2025-11-13_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/7o6HCQaKUrxWjrf2vhH7hV.eval","v":0.17301038062283736},{"b":"frontiermath","d":"2025-11-25","evaluated_at":"2025-11-25T04:02:38.737Z","m":"claude-opus-4-5-2025-11-01","oid":"frontiermath-d9d196cc3e3b8235","rd":"2025-11-24","se":0.02382827611454509,"sid":"9nctB7cH6kcLT24UbScEMa","sm":"claude-opus-4-5-20251101","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9nctB7cH6kcLT24UbScEMa.eval","v":0.20689655172413793},{"b":"frontiermath","d":"2025-11-25","evaluated_at":"2025-11-25T01:11:04.094Z","m":"claude-opus-4-5-20251101-32k","oid":"frontiermath-3f1046e1fc654672","rd":"2025-11-24","se":0.02382827611454509,"sid":"DVjCqcKsfQ4kJbwucCiuH7","sm":"claude-opus-4-5-20251101_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DVjCqcKsfQ4kJbwucCiuH7.eval","v":0.20689655172413793},{"b":"frontiermath","d":"2025-11-25","evaluated_at":"2025-11-25T01:10:48.342Z","m":"claude-opus-4-5-20251101-16k","oid":"frontiermath-79212256230b1e85","rd":"2025-11-24","se":0.023680184210831092,"sid":"Xvp9LtJuwPtSab63ZWwkTC","sm":"claude-opus-4-5-20251101_16K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Xvp9LtJuwPtSab63ZWwkTC.eval","v":0.20344827586206896},{"b":"frontiermath","d":"2025-11-24","evaluated_at":"2025-11-24T02:37:34.534Z","m":"gemini-2-5-pro","oid":"frontiermath-2b4469b1ce50b13a","rd":"2025-06-17","se":0.020494847099131453,"sid":"cA9LKKKzDVxiLRcegKSb2b","sm":"gemini-2.5-pro","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cA9LKKKzDVxiLRcegKSb2b.eval","v":0.1413793103448276},{"b":"frontiermath","d":"2025-11-21","evaluated_at":"2025-11-21T12:00:00.000Z","m":"gemini-3-pro-preview","oid":"frontiermath-17f7d469b813a8f0","rd":"2025-11-18","se":0.0284,"sid":"manual_gemini_3_fmprivate_frankenstein","sm":"gemini-3-pro-preview","v":0.376},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-11-17","e":"medium","evaluated_at":"2025-11-17T21:28:57.104Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath-66e1c18f891049f6","rd":"2025-11-13","se":0.026083646905766342,"sid":"7c2hHEgF9JomG9X7BLP7By","sm":"gpt-5.1-2025-11-13_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/7c2hHEgF9JomG9X7BLP7By.eval","v":0.2689655172413793},{"b":"frontiermath","cfg":{"effort":"low"},"d":"2025-11-17","e":"low","evaluated_at":"2025-11-17T01:17:15.370Z","m":"o3-2025-04-16","oid":"frontiermath-85f7b680f6ff05e9","rd":"2025-04-16","se":0.017487682248157596,"sid":"DKLrszPeTvFUGq39AaTniX","sm":"o3-2025-04-16_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DKLrszPeTvFUGq39AaTniX.eval","v":0.09722222222222222},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-16","e":"high","evaluated_at":"2025-11-16T20:35:07.675Z","m":"o3-2025-04-16","oid":"frontiermath-5f1c498e3b8c3ddd","rd":"2025-04-16","se":0.022968716344899227,"sid":"XQ46BgPhHaYZsrPQuTvz8J","sm":"o3-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XQ46BgPhHaYZsrPQuTvz8J.eval","v":0.18685121107266436},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-11-16","e":"medium","evaluated_at":"2025-11-16T20:35:03.993Z","m":"o3-2025-04-16","oid":"frontiermath-def625a3c15b1445","rd":"2025-04-16","se":0.022042438716799215,"sid":"XwNqEvxaaCQ8M2NpKJGoZG","sm":"o3-2025-04-16_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XwNqEvxaaCQ8M2NpKJGoZG.eval","v":0.16896551724137931},{"b":"frontiermath","d":"2025-11-16","evaluated_at":"2025-11-16T12:36:38.474Z","m":"claude-sonnet-4-5-2025-09-29","oid":"frontiermath-bba6f119596ecb8a","rd":"2025-09-29","se":0.017092785280147835,"sid":"n26C8XHigC9Xfsdb24JZtd","sm":"claude-sonnet-4-5-20250929","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/n26C8XHigC9Xfsdb24JZtd.eval","v":0.09310344827586207},{"b":"frontiermath","d":"2025-11-16","evaluated_at":"2025-11-16T12:36:27.931Z","m":"claude-sonnet-4-5-20250929-32k","oid":"frontiermath-d0be7b8eaf95d5cb","rd":"2025-09-29","se":0.021169728769293714,"sid":"LUpSd7DzWf3Fw4iwXb3MPV","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/LUpSd7DzWf3Fw4iwXb3MPV.eval","v":0.1522491349480969},{"b":"frontiermath","cfg":{"effort":"low"},"d":"2025-11-16","e":"low","evaluated_at":"2025-11-16T03:21:42.143Z","m":"o4-mini-2025-04-16","oid":"frontiermath-448a221da25f25c1","rd":"2025-04-16","se":0.018175392977158443,"sid":"82dL9ukqav96KZ8aHfqPpq","sm":"o4-mini-2025-04-16_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/82dL9ukqav96KZ8aHfqPpq.eval","v":0.10689655172413794},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-16","e":"high","evaluated_at":"2025-11-16T01:50:26.218Z","m":"o3-mini-2025-01-31","oid":"frontiermath-d060547e760fea47","rd":"2025-01-31","se":0.019396402576684655,"sid":"W2YTmg94XqVbH7p86TAqJT","sm":"o3-mini-2025-01-31_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/W2YTmg94XqVbH7p86TAqJT.eval","v":0.12413793103448276},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T20:36:07.951Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath-52dbea0a5bcc89c8","rd":"2025-11-13","se":0.02721380905679257,"sid":"nveMCdbvDijDTBN9JPWNve","sm":"gpt-5.1-2025-11-13_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nveMCdbvDijDTBN9JPWNve.eval","v":0.3103448275862069},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T19:38:13.991Z","m":"gpt-5-2025-08-07","oid":"frontiermath-33b0802afdfc0cfa","rd":"2025-08-07","se":0.027532461664839192,"sid":"JwfCZuni4TRRnA77UWwpyg","sm":"gpt-5-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JwfCZuni4TRRnA77UWwpyg.eval","v":0.32413793103448274},{"b":"frontiermath","d":"2025-11-13","evaluated_at":"2025-11-13T15:54:58.691Z","m":"claude-sonnet-4-5-20250929-59k","oid":"frontiermath-3b503e98b77f31f3","rd":"2025-09-29","se":0.020132987375676092,"sid":"3GzYKQW2K4iijm6a73rVeh","sm":"claude-sonnet-4-5-20250929_59K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3GzYKQW2K4iijm6a73rVeh.eval","v":0.13494809688581316},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T15:54:56.299Z","m":"gpt-5-mini-2025-08-07","oid":"frontiermath-af52a3c507c0e602","rd":"2025-08-07","se":0.02618833296520288,"sid":"DkMXbLvTgWJDWnUQVuBA8J","sm":"gpt-5-mini-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DkMXbLvTgWJDWnUQVuBA8J.eval","v":0.27241379310344827},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-11-13","e":"medium","evaluated_at":"2025-11-13T15:54:51.979Z","m":"o4-mini-2025-04-16","oid":"frontiermath-3b849dcf00cd1409","rd":"2025-04-16","se":0.023060480732052885,"sid":"XPCVZtFEC6BYKAwhLauMSD","sm":"o4-mini-2025-04-16_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XPCVZtFEC6BYKAwhLauMSD.eval","v":0.1896551724137931},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T15:54:49.087Z","m":"o4-mini-2025-04-16","oid":"frontiermath-1bf0818a12a92d4e","rd":"2025-04-16","se":0.02541251077219606,"sid":"Z7hiWJnEkkLmQXviEiTdHt","sm":"o4-mini-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z7hiWJnEkkLmQXviEiTdHt.eval","v":0.2482758620689655},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-11-13","e":"medium","evaluated_at":"2025-11-13T15:54:46.227Z","m":"gpt-5-mini-2025-08-07","oid":"frontiermath-75f267f1b0440307","rd":"2025-08-07","se":0.023680184210831092,"sid":"VPHGtT9EdPS6YxbzDAt98q","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VPHGtT9EdPS6YxbzDAt98q.eval","v":0.20344827586206896},{"b":"frontiermath","d":"2025-11-13","evaluated_at":"2025-11-13T15:54:39.607Z","m":"grok-4-0709","oid":"frontiermath-f6d352a334b5b929","rd":"2025-07-09","se":0.023375906908472147,"sid":"oN54TKXxcn9gofYBASTVCY","sm":"grok-4-0709","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/oN54TKXxcn9gofYBASTVCY.eval","v":0.19655172413793104},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-11-13","e":"medium","evaluated_at":"2025-11-13T15:54:39.070Z","m":"gpt-5-2025-08-07","oid":"frontiermath-ec7005ac21a26ed5","rd":"2025-08-07","se":0.02618833296520288,"sid":"hrJBSLQUe5hrHECkuySSJr","sm":"gpt-5-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hrJBSLQUe5hrHECkuySSJr.eval","v":0.27241379310344827},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T16:20:13.144Z","m":"gpt-5-nano-2025-08-07","oid":"frontiermath-14364119a0416337","rd":"2025-08-07","se":0.01620688385836769,"sid":"2eMHpXXR9h7P2oNPDQkTBY","sm":"gpt-5-nano-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2eMHpXXR9h7P2oNPDQkTBY.eval","v":0.08275862068965517},{"b":"frontiermath","d":"2025-10-22","evaluated_at":"2025-10-22T12:29:30.662Z","m":"claude-haiku-4-5-20251001-32k","oid":"frontiermath-8a228025126ba406","rd":"2025-10-15","se":0.013911554772795832,"sid":"fdNzD5mp8r2bHxy2PXx6Gh","sm":"claude-haiku-4-5-20251001_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/fdNzD5mp8r2bHxy2PXx6Gh.eval","v":0.059027777777777776},{"b":"frontiermath","d":"2025-10-16","evaluated_at":"2025-10-16T09:34:31.339Z","m":"claude-haiku-4-5-2025-10-01","oid":"frontiermath-cfc06bdbd8c00313","rd":"2025-10-15","se":0.011715642254112493,"sid":"8sTJmYZ7MSk8obK9vjUjHm","sm":"claude-haiku-4-5-20251001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8sTJmYZ7MSk8obK9vjUjHm.eval","v":0.041379310344827586},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T18:43:21.321Z","m":"gpt-5-nano-2025-08-07","oid":"frontiermath-f3cf615c5267405a","rd":"2025-08-07","se":0.015245401561146893,"sid":"dHQzrNGjuR5kiHKtNxG62X","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/dHQzrNGjuR5kiHKtNxG62X.eval","v":0.07241379310344828},{"b":"frontiermath","d":"2025-08-05","evaluated_at":"2025-08-05T23:13:11.048Z","m":"claude-opus-4-20250514-27k","oid":"frontiermath-fdcff32de34ccefc","rd":"2025-05-22","se":0.011715642254112493,"sid":"ZKroUnGN6qoNssoSQbtzzz","sm":"claude-opus-4-20250514_27K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZKroUnGN6qoNssoSQbtzzz.eval","v":0.041379310344827586},{"b":"frontiermath","d":"2025-08-05","evaluated_at":"2025-08-05T19:56:41.845Z","m":"claude-opus-4-1-20250805-27k","oid":"frontiermath-0a39a42be10c252d","rd":"2025-08-05","se":0.015245401561146893,"sid":"ezJ5C38svWJkQMjxTcrgg7","sm":"claude-opus-4-1-20250805_27K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ezJ5C38svWJkQMjxTcrgg7.eval","v":0.07241379310344828},{"b":"frontiermath","d":"2025-08-05","evaluated_at":"2025-08-05T19:56:40.259Z","m":"claude-opus-4-1-2025-08-05","oid":"frontiermath-53557bbc784ec7cd","rd":"2025-08-05","se":0.013818435156390344,"sid":"TQQnqmugDk3GX5hkr8HMWz","sm":"claude-opus-4-1-20250805","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/TQQnqmugDk3GX5hkr8HMWz.eval","v":0.05862068965517241},{"b":"frontiermath","d":"2025-07-04","evaluated_at":"2025-07-04T10:42:31.483Z","m":"claude-opus-4-2025-05-14","oid":"frontiermath-bf9588badfbb88eb","rd":"2025-05-22","se":0.012172075609474642,"sid":"GKD2ELzaXHx3giARevEetr","sm":"claude-opus-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GKD2ELzaXHx3giARevEetr.eval","v":0.04482758620689655},{"b":"frontiermath","d":"2025-07-04","evaluated_at":"2025-07-04T10:41:31.264Z","m":"claude-sonnet-4-2025-05-14","oid":"frontiermath-543a795f8d5ceb22","rd":"2025-05-22","se":0.011715642254112493,"sid":"UcfxxJmo4ftRzPEnXrRdSw","sm":"claude-sonnet-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/UcfxxJmo4ftRzPEnXrRdSw.eval","v":0.041379310344827586},{"b":"frontiermath","d":"2025-07-03","evaluated_at":"2025-07-03T13:57:21.637Z","m":"gemini-2-5-pro-preview-06-05","oid":"frontiermath-408bdf4c11d010fc","rd":"2025-06-05","se":0.01791432224407271,"sid":"hNLQNC6TASXNNzat8kVS77","sm":"gemini-2.5-pro-preview-06-05","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hNLQNC6TASXNNzat8kVS77.eval","v":0.10344827586206896},{"b":"frontiermath","d":"2025-05-14","evaluated_at":"2025-05-14T13:10:51.812Z","m":"qwen-plus-2025-04-28","oid":"frontiermath-25aeda5f61dd0c2d","rd":"2025-04-28","se":0.007657032895812118,"sid":"bVFEkuHiUSxjoh8SQm74jo","sm":"qwen-plus-2025-04-28","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bVFEkuHiUSxjoh8SQm74jo.eval","v":0.017241379310344827},{"b":"frontiermath","d":"2025-05-07","evaluated_at":"2025-05-07T17:33:33.886Z","m":"mistral-medium-2505","oid":"frontiermath-840a1a6b8a1c84da","rd":"2025-05-07","se":0.003460207612456747,"sid":"PZRX6ME25bn26B6BTRWx6W","sm":"mistral-medium-2505","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PZRX6ME25bn26B6BTRWx6W.eval","v":0.0034602076124567475},{"b":"frontiermath","d":"2025-04-14","evaluated_at":"2025-04-14T22:51:44.342Z","m":"gpt-4-1-nano-2025-04-14","oid":"frontiermath-ca8b91f26a0ad002","rd":"2025-04-14","se":0.0059518867144507945,"sid":"chtQvebcoaE33KkhAjbCnw","sm":"gpt-4.1-nano-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/chtQvebcoaE33KkhAjbCnw.eval","v":0.010344827586206896},{"b":"frontiermath","d":"2025-04-14","evaluated_at":"2025-04-14T22:49:44.216Z","m":"gpt-4-1-mini-2025-04-14","oid":"frontiermath-8addea285387eb96","rd":"2025-04-14","se":0.01217207560947464,"sid":"ephMMUGDANc9iVTNNuuoZx","sm":"gpt-4.1-mini-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ephMMUGDANc9iVTNNuuoZx.eval","v":0.04482758620689655},{"b":"frontiermath","d":"2025-04-14","evaluated_at":"2025-04-14T22:40:01.283Z","m":"gpt-4-1-2025-04-14","oid":"frontiermath-aba20a8d1f62eb1a","rd":"2025-04-14","se":0.013430381628597847,"sid":"NPeiYpjbscreYnFkHPvcNB","sm":"gpt-4.1-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NPeiYpjbscreYnFkHPvcNB.eval","v":0.05517241379310345},{"b":"frontiermath","cfg":{"effort":"low"},"d":"2025-04-10","e":"low","evaluated_at":"2025-04-10T20:30:21.697Z","m":"grok-3-mini-beta","oid":"frontiermath-9d9e36900bfecd23","rd":"2025-04-09","se":0.009634354634513523,"sid":"9FTMS2t6Ksm43CduHeqtJr","sm":"grok-3-mini-beta_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9FTMS2t6Ksm43CduHeqtJr.eval","v":0.027586206896551724},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-04-10","e":"high","evaluated_at":"2025-04-10T18:02:17.992Z","m":"grok-3-mini-beta","oid":"frontiermath-4a556c3426b1f895","rd":"2025-04-09","se":0.013818435156390344,"sid":"Z83F6bJ7Ne5rXaXwgR2xWT","sm":"grok-3-mini-beta_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z83F6bJ7Ne5rXaXwgR2xWT.eval","v":0.05862068965517241},{"b":"frontiermath","d":"2025-04-10","evaluated_at":"2025-04-10T17:57:03.446Z","m":"grok-3-beta","oid":"frontiermath-abdad4953c8bebf6","rd":"2025-04-09","se":0.011237029601999628,"sid":"DzjFQArKFTZHioQVwKYVtX","sm":"grok-3-beta","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DzjFQArKFTZHioQVwKYVtX.eval","v":0.03793103448275862},{"b":"frontiermath","d":"2025-04-08","evaluated_at":"2025-04-08T10:17:16.175Z","m":"llama-4-maverick-17b-128e-instruct-fp8","oid":"frontiermath-359b14aecb90df05","rd":"2025-04-05","se":0.004868154158215009,"sid":"QjvBS34fsngfBBMZUirSyh","sm":"Llama-4-Maverick-17B-128E-Instruct-FP8","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QjvBS34fsngfBBMZUirSyh.eval","v":0.006896551724137931},{"b":"frontiermath","d":"2025-04-08","evaluated_at":"2025-04-08T10:14:56.486Z","m":"llama-4-scout-17b-16e-instruct","oid":"frontiermath-d74b605e0f7550ae","rd":"2025-04-05","se":0,"sid":"bhjLDA5A7gddsX5cAYKma8","sm":"Llama-4-Scout-17B-16E-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bhjLDA5A7gddsX5cAYKma8.eval","v":0},{"b":"frontiermath","d":"2025-04-02","evaluated_at":"2025-04-02T16:55:37.430Z","m":"qwen-max-2025-01-25","oid":"frontiermath-0dc90777524b8a01","rd":"2025-01-25","se":0.005951886714450794,"sid":"F77vDPynTKJbz7m2bt7Py8","sm":"qwen-max-2025-01-25","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/F77vDPynTKJbz7m2bt7Py8.eval","v":0.010344827586206896},{"b":"frontiermath","d":"2025-03-13","evaluated_at":"2025-03-13T16:49:59.754Z","m":"claude-3-7-sonnet-20250219-64k","oid":"frontiermath-02546fc7fe52cde9","rd":"2025-02-24","se":0.01020064175308735,"sid":"DoU54Xy93J3MxztC4uxwYy","sm":"claude-3-7-sonnet-20250219_64K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DoU54Xy93J3MxztC4uxwYy.eval","v":0.03103448275862069},{"b":"frontiermath","d":"2025-03-13","evaluated_at":"2025-03-13T09:34:11.446Z","m":"claude-3-7-sonnet-20250219-32k","oid":"frontiermath-c26b6affa085d1be","rd":"2025-02-24","se":0.010733271038801586,"sid":"aZqVpPa4SWHapN9QHEoEm3","sm":"claude-3-7-sonnet-20250219_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/aZqVpPa4SWHapN9QHEoEm3.eval","v":0.034482758620689655},{"b":"frontiermath","d":"2025-03-09","evaluated_at":"2025-03-09T17:18:49.858Z","m":"gemini-2-0-flash-001","oid":"frontiermath-579629814f483302","rd":"2025-02-05","se":0.007657032895812118,"sid":"5RpSr8gJZfcZn586aV2hDW","sm":"gemini-2.0-flash-001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5RpSr8gJZfcZn586aV2hDW.eval","v":0.017241379310344827},{"b":"frontiermath","d":"2025-03-07","evaluated_at":"2025-03-07T23:36:27.507Z","m":"deepseek-v3","oid":"frontiermath-a00c85a900644539","rd":"2024-12-26","se":0.007657032895812118,"sid":"aXcfCytWFYNk7E5AZGMzLb","sm":"DeepSeek-V3","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/aXcfCytWFYNk7E5AZGMzLb.eval","v":0.017241379310344827},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-03-07","e":"high","evaluated_at":"2025-03-07T19:46:02.827Z","m":"o1-2024-12-17","oid":"frontiermath-6b6fdcb61cc0cc20","rd":"2024-12-17","se":0.017092785280147835,"sid":"bynpttv6jLvbXTY7ktnBpk","sm":"o1-2024-12-17_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bynpttv6jLvbXTY7ktnBpk.eval","v":0.09310344827586207},{"b":"frontiermath","d":"2025-03-07","evaluated_at":"2025-03-07T18:56:32.370Z","m":"gpt-4o-2024-08-06","oid":"frontiermath-05a636f56c05d0ba","rd":"2024-08-06","se":0.003448275862068965,"sid":"SmT4ukjkxsXZvFExMMVFgW","sm":"gpt-4o-2024-08-06","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SmT4ukjkxsXZvFExMMVFgW.eval","v":0.0034482758620689655},{"b":"frontiermath","d":"2025-03-07","evaluated_at":"2025-03-07T18:54:11.499Z","m":"claude-3-5-sonnet-2024-06-20","oid":"frontiermath-e767ecedec8011e6","rd":"2024-06-20","se":0.0059518867144507945,"sid":"h7f9jZVWzZNDHNxP5eewNH","sm":"claude-3-5-sonnet-20240620","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/h7f9jZVWzZNDHNxP5eewNH.eval","v":0.010344827586206896},{"b":"frontiermath","d":"2025-03-07","evaluated_at":"2025-03-07T18:40:43.945Z","m":"gemini-1-5-flash-002","oid":"frontiermath-5ab9fb6b92898689","rd":"2024-09-24","se":0,"sid":"8FtfWCm8DdAqF9rnMVwv8h","sm":"gemini-1.5-flash-002","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8FtfWCm8DdAqF9rnMVwv8h.eval","v":0},{"b":"frontiermath","d":"2025-03-07","evaluated_at":"2025-03-07T18:11:31.647Z","m":"claude-3-5-haiku-2024-10-22","oid":"frontiermath-6782bbf4f04587ec","rd":"2024-10-22","se":0.003448275862068965,"sid":"JQcqpW9xPfJRKJkHne6DpK","sm":"claude-3-5-haiku-20241022","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JQcqpW9xPfJRKJkHne6DpK.eval","v":0.0034482758620689655},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T23:54:46.118Z","m":"claude-3-5-sonnet-2024-10-22","oid":"frontiermath-4c718fc0ef647187","rd":"2024-10-22","se":0.008373130807525473,"sid":"MVnDrt94wyokzx3a7ZJkph","sm":"claude-3-5-sonnet-20241022","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MVnDrt94wyokzx3a7ZJkph.eval","v":0.020689655172413793},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-03-06","e":"medium","evaluated_at":"2025-03-06T22:32:06.577Z","m":"o1-mini-2024-09-12","oid":"frontiermath-4d2e5f92410b5804","rd":"2024-09-12","se":0.007657032895812118,"sid":"hC2FY4WuKm6BLgPhbWeZDY","sm":"o1-mini-2024-09-12_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hC2FY4WuKm6BLgPhbWeZDY.eval","v":0.017241379310344827},{"b":"frontiermath","cfg":{"effort":"high"},"d":"2025-03-06","e":"high","evaluated_at":"2025-03-06T22:30:16.233Z","m":"o1-mini-2024-09-12","oid":"frontiermath-7dba16f3b6276b9e","rd":"2024-09-12","se":0.006860663093423027,"sid":"2ekT2UzaebvmGprKvmeeEa","sm":"o1-mini-2024-09-12_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2ekT2UzaebvmGprKvmeeEa.eval","v":0.013793103448275862},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T19:16:31.497Z","m":"claude-3-7-sonnet-20250219-16k","oid":"frontiermath-5e3e6cb2628632c5","rd":"2025-02-24","se":0.011715642254112493,"sid":"PU7LXbdNB8Unfvu4qCeJZF","sm":"claude-3-7-sonnet-20250219_16K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PU7LXbdNB8Unfvu4qCeJZF.eval","v":0.041379310344827586},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T18:06:22.963Z","m":"grok-2-1212","oid":"frontiermath-4e8483df6c09a1db","rd":"2024-12-12","se":0.004868154158215009,"sid":"kinyvFjqSqUiY5FyYv7tsA","sm":"grok-2-1212","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kinyvFjqSqUiY5FyYv7tsA.eval","v":0.006896551724137931},{"b":"frontiermath","cfg":{"effort":"medium"},"d":"2025-03-06","e":"medium","evaluated_at":"2025-03-06T18:06:08.042Z","m":"o3-mini-2025-01-31","oid":"frontiermath-eef523227fd42e8c","rd":"2025-01-31","se":0.012761417139125066,"sid":"U8CjtcxudQeP2cnMNixfJH","sm":"o3-mini-2025-01-31_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/U8CjtcxudQeP2cnMNixfJH.eval","v":0.08081896551724138},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T18:05:58.538Z","m":"claude-3-7-sonnet-2025-02-19","oid":"frontiermath-74ddf54b87ac0003","rd":"2025-02-24","se":0.010200641753087351,"sid":"m7X2ppkiEkweCtFppiRW8S","sm":"claude-3-7-sonnet-20250219","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/m7X2ppkiEkweCtFppiRW8S.eval","v":0.03103448275862069},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T18:05:57.178Z","m":"mistral-large-2411","oid":"frontiermath-25d777410d260ac8","rd":"2024-11-18","se":0.003448275862068965,"sid":"cEvTUVyxAqq5UDgsGQbvLM","sm":"mistral-large-2411","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cEvTUVyxAqq5UDgsGQbvLM.eval","v":0.0034482758620689655},{"b":"frontiermath","d":"2025-03-06","evaluated_at":"2025-03-06T18:03:58.731Z","m":"gpt-4o-2024-11-20","oid":"frontiermath-1164b0274f4bdb9a","rd":"2024-11-20","se":0.003448275862068965,"sid":"Z7HU95T8eema56EpLtECvG","sm":"gpt-4o-2024-11-20","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Z7HU95T8eema56EpLtECvG.eval","v":0.0034482758620689655},{"b":"frontiermath","m":"gemini-2-5-deep-think-2025-08-01-webapp","oid":"frontiermath-4882e2330d3e5f05","rd":"2025-08-01","se":0.027,"sid":"manual_gemini_2.5_deep_think_run_frontiermath_tiers_1_to_3","sm":"gemini-2.5-deep-think-2025-08-01-webapp","v":0.29},{"b":"frontiermath_tier_4","cfg":{"effort":"max"},"d":"2026-06-08","e":"max","evaluated_at":"2026-06-08T01:03:04.000Z","m":"claude-opus-4-8","oid":"frontiermath_tier_4-192b79cbdb067ac7","rd":"2026-05-28","se":0.06761023307918171,"sid":"5x5EAvuecqo3SXnZWpvrJA","sm":"claude-opus-4-8_max","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5x5EAvuecqo3SXnZWpvrJA.eval","v":0.3125},{"b":"frontiermath_tier_4","d":"2026-05-28","evaluated_at":"2026-05-28T05:51:34.000Z","m":"qwen3-6-max-preview","oid":"frontiermath_tier_4-706ee2c75b776608","rd":"2026-04-20","se":0.02914766351555603,"sid":"6G3krrJBj2Dsuunaow97uR","sm":"qwen3.6-max-preview","v":0.041666666666666664},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2026-05-25","e":"high","evaluated_at":"2026-05-25T13:24:56.000Z","m":"gemini-3-5-flash","oid":"frontiermath_tier_4-661152823626ff36","rd":"2026-05-19","se":0.05148145136372291,"sid":"icxeMK8LHGTpbEfCNdbqA4","sm":"gemini-3.5-flash_high","v":0.14583333333333334},{"b":"frontiermath_tier_4","d":"2026-05-15","evaluated_at":"2026-05-15T05:49:39.000Z","m":"qwen3-5-plus","oid":"frontiermath_tier_4-ad5156b8aa9be855","rd":"2026-02-16","se":0.020833333333333332,"sid":"ErcX9xv4Pwcgg63Dkrn2zM","sm":"qwen3.5-plus","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2026-05-12","evaluated_at":"2026-05-12T19:59:41.000Z","m":"qwen3-6-plus","oid":"frontiermath_tier_4-c9317316d69e531a","rd":"2026-03-31","se":0.04031495390390805,"sid":"NuKcKB4qgQwXDPaDEnpqjo","sm":"qwen3.6-plus","v":0.08333333333333333},{"b":"frontiermath_tier_4","d":"2026-05-12","evaluated_at":"2026-05-12T16:05:39.000Z","m":"qwen3-6-flash","oid":"frontiermath_tier_4-0164e6293010b76f","rd":"2026-04-27","se":0,"sid":"4tmJcpwFZKJrmHacCFtPYV","sm":"qwen3.6-flash","v":0},{"b":"frontiermath_tier_4","d":"2026-05-12","evaluated_at":"2026-05-12T13:06:53.000Z","m":"qwen3-5-flash","oid":"frontiermath_tier_4-58f412334630bc48","rd":"2026-02-25","se":0,"sid":"cpbqvSFES6AdJqou8FP8ZD","sm":"qwen3.5-flash","v":0},{"b":"frontiermath_tier_4","d":"2026-05-12","evaluated_at":"2026-05-12T05:39:46.000Z","m":"glm-5-1","oid":"frontiermath_tier_4-8bab68b9194834c0","rd":"2026-04-07","se":0.04824031156174749,"sid":"8vKCYPopFbKzWZ47oLDDSo","sm":"glm-5.1","v":0.125},{"b":"frontiermath_tier_4","d":"2026-05-08","evaluated_at":"2026-05-08T13:03:56.784Z","m":"gdm-ai-co-mathematician","oid":"frontiermath_tier_4-15f303cd7544b0a2","rd":"2026-05-08","se":0.072,"sid":"gdm-ai-co-mathematician-manual-run","sm":"gdm-ai-co-mathematician","v":0.479},{"b":"frontiermath_tier_4","d":"2026-05-08","evaluated_at":"2026-05-08T10:29:10.963Z","m":"kimi-k2-6","oid":"frontiermath_tier_4-a10a80979945060c","rd":"2026-04-20","se":0.0515,"sid":"kimi-k2.6-frontiermth-tier4-frankenstein","sm":"kimi-k2.6","v":0.1458},{"b":"frontiermath_tier_4","cfg":{"effort":"xhigh"},"d":"2026-04-23","e":"xhigh","evaluated_at":"2026-04-23T20:52:04.600Z","m":"gpt-5-5-pro-pre-release","oid":"frontiermath_tier_4-7823b27a0162817a","rd":"2026-04-23","se":0.071,"sid":"gpt-5-5-pro-pre-release-xhigh-t4","sm":"gpt-5.5-pro-pre-release_xhigh","v":0.396},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2026-04-23","e":"high","evaluated_at":"2026-04-23T20:51:42.498Z","m":"gpt-5-5-pro-pre-release","oid":"frontiermath_tier_4-a04bf7976cf925c6","rd":"2026-04-23","se":0.071,"sid":"gpt-5-5-pro-pre-release-high-t4","sm":"gpt-5.5-pro-pre-release_high","v":0.396},{"b":"frontiermath_tier_4","cfg":{"effort":"xhigh"},"d":"2026-04-23","e":"xhigh","evaluated_at":"2026-04-23T20:51:35.635Z","m":"gpt-5-5-pre-release","oid":"frontiermath_tier_4-3cf8de8e5ed25042","rd":"2026-04-23","se":0.069,"sid":"gpt-5-5-pre-release-xhigh-t4","sm":"gpt-5.5-pre-release_xhigh","v":0.354},{"b":"frontiermath_tier_4","cfg":{"effort":"xhigh"},"d":"2026-04-17","e":"xhigh","evaluated_at":"2026-04-17T12:51:00.000Z","m":"claude-opus-4-7","oid":"frontiermath_tier_4-ab003c9b8216409a","rd":"2026-04-16","se":0.06130657275434798,"sid":"9oFmdEcfQNyYqA5PMmbJxV","sm":"claude-opus-4-7_xhigh","v":0.22916666666666666},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T13:54:39.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"frontiermath_tier_4-15860b43c3ccd8e9","rd":"2026-03-17","se":0.0208,"sid":"hPP9Lrd5W3wVyUq9uYuYm9","sm":"gpt-5.4-mini-2026-03-17_high","v":0.0208},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T09:12:42.664Z","m":"gpt-5-4-nano-2026-03-17","oid":"frontiermath_tier_4-4512a001128e40d4","rd":"2026-03-17","se":0.0353,"sid":"ZKPz9sjRBK6goz5vzYwtuV","sm":"gpt-5.4-nano-2026-03-17_high","v":0.0625},{"b":"frontiermath_tier_4","d":"2026-04-08","evaluated_at":"2026-04-08T16:30:14.687Z","m":"muse-spark","oid":"frontiermath_tier_4-7592bbfe06bf7823","rd":"2026-04-08","se":0.051,"sid":"muse-spark-pre-release-tier-4","sm":"muse-spark","v":0.146},{"b":"frontiermath_tier_4","cfg":{"effort":"xhigh"},"d":"2026-03-06","e":"xhigh","evaluated_at":"2026-03-06T03:53:12.418Z","m":"gpt-5-4-2026-03-05","oid":"frontiermath_tier_4-2a0cff3be1fe3d51","rd":"2026-03-05","se":0.064,"sid":"pre-release-gpt-5-4-xhigh-fm-t4","sm":"gpt-5.4-2026-03-05_xhigh","v":0.271},{"b":"frontiermath_tier_4","d":"2026-03-06","evaluated_at":"2026-03-06T03:51:41.153Z","m":"gpt-5-4-pro-2026-03-05-web-app","oid":"frontiermath_tier_4-f4a1c009869fffcd","rd":"2026-03-05","se":0.07,"sid":"pre-release-gpt-5-4-pro-fm-t4-web-app","sm":"gpt-5.4-pro-2026-03-05-web-app","v":0.375},{"b":"frontiermath_tier_4","d":"2026-02-21","evaluated_at":"2026-02-21T20:36:46.695Z","m":"claude-sonnet-4-6-16k","oid":"frontiermath_tier_4-7c6253ccef78f825","rd":"2026-02-17","se":0.04,"sid":"RTfHBvZX3Bh5EPPj6Cf3ab","sm":"claude-sonnet-4-6_16K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RTfHBvZX3Bh5EPPj6Cf3ab.eval","v":0.083},{"b":"frontiermath_tier_4","d":"2026-02-19","evaluated_at":"2026-02-19T18:11:11.812Z","m":"gemini-3-1-pro-preview","oid":"frontiermath_tier_4-8727bb44d7cf579d","rd":"2026-02-19","se":0.054,"sid":"ZZrMMV7TirCbEsHm9pSH27","sm":"gemini-3.1-pro-preview","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZZrMMV7TirCbEsHm9pSH27.eval","v":0.167},{"b":"frontiermath_tier_4","d":"2026-02-19","evaluated_at":"2026-02-19T17:25:47.564Z","m":"glm-5","oid":"frontiermath_tier_4-be46db694f43528e","rd":"2026-02-11","se":0.021,"sid":"RomrkYdVrg67NirntgRrDL","sm":"glm-5","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RomrkYdVrg67NirntgRrDL.eval","v":0.021},{"b":"frontiermath_tier_4","cfg":{"effort":"max"},"d":"2026-02-12","e":"max","evaluated_at":"2026-02-12T16:30:15.979Z","m":"claude-opus-4-6","oid":"frontiermath_tier_4-b3ce9886193a5aad","rd":"2026-02-05","se":0.061,"sid":"SfehtojAZdZxu7BLgPLDYA","sm":"claude-opus-4-6_max","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SfehtojAZdZxu7BLgPLDYA.eval","v":0.229},{"b":"frontiermath_tier_4","d":"2026-02-06","evaluated_at":"2026-02-06T14:57:13.045Z","m":"claude-opus-4-6-64k","oid":"frontiermath_tier_4-b0aed115b3fb8919","rd":"2026-02-05","se":0.05923816975988693,"sid":"mAeSEYBeiSU5vUhrnsP6Uj","sm":"claude-opus-4-6_64K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/mAeSEYBeiSU5vUhrnsP6Uj.eval","v":0.20833333333333334},{"b":"frontiermath_tier_4","d":"2026-02-06","evaluated_at":"2026-02-06T12:46:05.800Z","m":"claude-opus-4-6-32k","oid":"frontiermath_tier_4-fa8828901111bb46","rd":"2026-02-05","se":0.05923816975988693,"sid":"UVXUd6ZFnQTci66adkFepy","sm":"claude-opus-4-6_32K","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/UVXUd6ZFnQTci66adkFepy.eval","v":0.20833333333333334},{"b":"frontiermath_tier_4","d":"2026-02-05","evaluated_at":"2026-02-05T20:01:07.029Z","m":"claude-opus-4-6","oid":"frontiermath_tier_4-ac49945c9ed438dd","rd":"2026-02-05","se":0.05148145136372291,"sid":"nvXhiYKL9RDVDfHWw53zFT","sm":"claude-opus-4-6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nvXhiYKL9RDVDfHWw53zFT.eval","v":0.14583333333333334},{"b":"frontiermath_tier_4","d":"2026-02-02","evaluated_at":"2026-02-02T10:58:21.119Z","m":"fireworkskimi-k2p5","oid":"frontiermath_tier_4-8afb0f620c794f88","rd":"2026-01-27","se":0.029,"sid":"F2Fn4eKsSE8ifoE6GUpxC6","sm":"fireworks/kimi-k2p5","v":0.042},{"b":"frontiermath_tier_4","d":"2026-01-30","evaluated_at":"2026-01-30T03:58:10.181Z","m":"zai-orgglm-4-7","oid":"frontiermath_tier_4-f1e50a64cc451daf","rd":"2025-12-22","se":0,"sid":"fgfsjZ6NrexyT8kA6arR3B","sm":"zai-org/GLM-4.7","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/fgfsjZ6NrexyT8kA6arR3B.eval","v":0},{"b":"frontiermath_tier_4","d":"2025-12-24","evaluated_at":"2025-12-24T06:00:00.000Z","m":"gpt-5-2-pro-2025-12-11-webapp","oid":"frontiermath_tier_4-dbb9b57a194515c7","rd":"2025-12-11","se":0.067,"sid":"manual_gpt_5.2_pro_run_frontiermath_tier_4","sm":"gpt-5.2-pro-2025-12-11-webapp","v":0.313},{"b":"frontiermath_tier_4","d":"2025-12-18","evaluated_at":"2025-12-18T18:23:16.725Z","m":"gemini-2-5-flash","oid":"frontiermath_tier_4-92ae5c613cb34b7e","rd":"2025-06-17","se":0.02914766351555603,"sid":"8qSepnktUjAsynVccET4Mw","sm":"gemini-2.5-flash","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8qSepnktUjAsynVccET4Mw.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-12-17","evaluated_at":"2025-12-17T22:40:29.476Z","m":"gemini-3-flash-preview","oid":"frontiermath_tier_4-04f1bdcc81842a97","rd":"2025-12-17","se":0.02914766351555603,"sid":"natXTgCj8Y7mMbV5623YDd","sm":"gemini-3-flash-preview","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/natXTgCj8Y7mMbV5623YDd.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-12-16","evaluated_at":"2025-12-16T15:14:53.837Z","m":"fireworksdeepseek-v3p2","oid":"frontiermath_tier_4-849c4adec4d91ba1","rd":"2025-12-01","se":0.021,"sid":"manual_deepseek-v3.2_frontiermath_tier_4_private_frankenstein_run","sm":"fireworks/deepseek-v3p2","v":0.021},{"b":"frontiermath_tier_4","cfg":{"effort":"xhigh"},"d":"2025-12-14","e":"xhigh","evaluated_at":"2025-12-14T10:55:00.000Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath_tier_4-1049988e49ba000b","rd":"2025-12-11","se":0.056,"sid":"XLLXdo2tmmvqRwYxbn3kmo","sm":"gpt-5.2-2025-12-11_xhigh","v":0.188},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T20:49:18.565Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath_tier_4-39d3006cdc8910aa","rd":"2025-12-11","se":0.056,"sid":"TjaWpQnjPC8SNJH8EDcXSS","sm":"gpt-5.2-2025-12-11_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/TjaWpQnjPC8SNJH8EDcXSS.eval","v":0.188},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-12-11","e":"medium","evaluated_at":"2025-12-11T20:49:17.097Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath_tier_4-21304186832d6fbc","rd":"2025-12-11","se":0.054,"sid":"92XEociAUsCM7WC9knviuo","sm":"gpt-5.2-2025-12-11_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/92XEociAUsCM7WC9knviuo.eval","v":0.167},{"b":"frontiermath_tier_4","cfg":{"effort":"low"},"d":"2025-12-11","e":"low","evaluated_at":"2025-12-11T20:22:58.448Z","m":"gpt-5-2-2025-12-11","oid":"frontiermath_tier_4-ee22077c6468281e","rd":"2025-12-11","se":0.03530829267912701,"sid":"BvgEDgU5ahio7wQBUMD3oE","sm":"gpt-5.2-2025-12-11_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BvgEDgU5ahio7wQBUMD3oE.eval","v":0.0625},{"b":"frontiermath_tier_4","d":"2025-12-11","evaluated_at":"2025-12-11T04:59:19.473Z","m":"qwenqwen3-235b-a22b-thinking-2507","oid":"frontiermath_tier_4-9852e5aa46c3f1da","rd":"2025-07-25","se":0,"sid":"BLw6H5xovDe5FTysit7dNp","sm":"Qwen/Qwen3-235B-A22B-Thinking-2507","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BLw6H5xovDe5FTysit7dNp.eval","v":0},{"b":"frontiermath_tier_4","d":"2025-12-08","evaluated_at":"2025-12-08T19:26:56.763Z","m":"zai-orgglm-4-6","oid":"frontiermath_tier_4-9ea4b5d517ae0e81","rd":"2025-09-30","se":0.02127659574468085,"sid":"PDFNt4rT82hpKcKKsA7DEd","sm":"zai-org/GLM-4.6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PDFNt4rT82hpKcKKsA7DEd.eval","v":0.02127659574468085},{"b":"frontiermath_tier_4","d":"2025-12-04","evaluated_at":"2025-12-04T16:53:31.304Z","m":"moonshotaikimi-k2-thinking","oid":"frontiermath_tier_4-d7a454fe5417a7f1","rd":"2025-11-06","se":0,"sid":"cnujSZS7UAAQeDu2U5Xfdw","sm":"moonshotai/Kimi-K2-Thinking","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cnujSZS7UAAQeDu2U5Xfdw.eval","v":0},{"b":"frontiermath_tier_4","d":"2025-11-25","evaluated_at":"2025-11-25T04:02:24.517Z","m":"claude-opus-4-5-2025-11-01","oid":"frontiermath_tier_4-dc190bf3a8ce3f56","rd":"2025-11-24","se":0.02914766351555603,"sid":"eAqv27skC4aGnAguQEhUgg","sm":"claude-opus-4-5-20251101","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eAqv27skC4aGnAguQEhUgg.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-11-25","evaluated_at":"2025-11-25T01:11:03.782Z","m":"claude-opus-4-5-20251101-16k","oid":"frontiermath_tier_4-b5203519f408cefd","rd":"2025-11-24","se":0.020833333333333332,"sid":"eteYQLCEzxb65iDj6U8din","sm":"claude-opus-4-5-20251101_16K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eteYQLCEzxb65iDj6U8din.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-11-25","evaluated_at":"2025-11-25T01:10:48.990Z","m":"claude-opus-4-5-20251101-32k","oid":"frontiermath_tier_4-1453e66abedc7479","rd":"2025-11-24","se":0.02914766351555603,"sid":"BaZfJQ3QBVD9F5cMp7mRdF","sm":"claude-opus-4-5-20251101_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BaZfJQ3QBVD9F5cMp7mRdF.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-11-21","evaluated_at":"2025-11-21T12:00:00.000Z","m":"gemini-3-pro-preview","oid":"frontiermath_tier_4-9689a364c5330459","rd":"2025-11-18","se":0.05693291125,"sid":"manual_gemini_3_fmtier4_frankenstein","sm":"gemini-3-pro-preview","v":0.1875},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-11-17","e":"medium","evaluated_at":"2025-11-17T21:28:56.133Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath_tier_4-7fe4e06b55dc9a1e","rd":"2025-11-13","se":0.029147663515556026,"sid":"S2W9ETQSRNc4xR3cmUjKbF","sm":"gpt-5.1-2025-11-13_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/S2W9ETQSRNc4xR3cmUjKbF.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T20:36:08.563Z","m":"gpt-5-1-2025-11-13","oid":"frontiermath_tier_4-782f3e7487002c51","rd":"2025-11-13","se":0.04824031156174749,"sid":"ee2YzVv39Tk6tmA7DNjTG5","sm":"gpt-5.1-2025-11-13_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ee2YzVv39Tk6tmA7DNjTG5.eval","v":0.125},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T16:22:42.856Z","m":"gpt-5-mini-2025-08-07","oid":"frontiermath_tier_4-718058fe9e2db00e","rd":"2025-08-07","se":0.03530829267912701,"sid":"5rh6TyPLbkJmQZs8iY6msJ","sm":"gpt-5-mini-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5rh6TyPLbkJmQZs8iY6msJ.eval","v":0.0625},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T16:22:14.713Z","m":"gpt-5-nano-2025-08-07","oid":"frontiermath_tier_4-b67211f23e8a412a","rd":"2025-08-07","se":0,"sid":"CvK572G5SSd5Yk8ZGikDR6","sm":"gpt-5-nano-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/CvK572G5SSd5Yk8ZGikDR6.eval","v":0},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T02:14:04.513Z","m":"gpt-5-2025-08-07","oid":"frontiermath_tier_4-3aabd420c16c0d8e","rd":"2025-08-07","se":0.04824031156174749,"sid":"6LwyJZsWEzuD8mhSRUF2xn","sm":"gpt-5-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6LwyJZsWEzuD8mhSRUF2xn.eval","v":0.125},{"b":"frontiermath_tier_4","d":"2025-10-22","evaluated_at":"2025-10-22T15:24:54.352Z","m":"claude-haiku-4-5-20251001-32k","oid":"frontiermath_tier_4-a156c186f6de3f95","rd":"2025-10-15","se":0.020833333333333332,"sid":"YRVwm4B88qK6aUEFBdNQmP","sm":"claude-haiku-4-5-20251001_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YRVwm4B88qK6aUEFBdNQmP.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-10-22","evaluated_at":"2025-10-22T14:19:03.957Z","m":"claude-sonnet-4-5-20250929-32k","oid":"frontiermath_tier_4-02e748b665d775a7","rd":"2025-09-29","se":0.029147663515556026,"sid":"d8WWwzb6b57mWqddhBXuDi","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/d8WWwzb6b57mWqddhBXuDi.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-10-09","evaluated_at":"2025-10-09T16:00:00.000Z","m":"grok-4-heavy-web-app","oid":"frontiermath_tier_4-6a12855ff7ff8d09","rd":"2025-07-10","se":0.021,"sid":"grok-4-heavy-manual-run-frontiermath","sm":"grok-4-heavy-web-app","v":0.0208},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-10-09","e":"high","evaluated_at":"2025-10-09T13:40:00.000Z","m":"gpt-5-pro-2025-10-06","oid":"frontiermath_tier_4-8eda3306f49d36cd","rd":"2025-10-07","se":0.051,"sid":"VshpC5TEp4F72cutXRRmhs","sm":"gpt-5-pro-2025-10-06_high","v":0.146},{"b":"frontiermath_tier_4","d":"2025-09-29","evaluated_at":"2025-09-29T17:41:54.543Z","m":"claude-sonnet-4-5-2025-09-29","oid":"frontiermath_tier_4-b39d67175002e4d4","rd":"2025-09-29","se":0.020833333333333332,"sid":"2o8XZ8V6W5jKD7TuY4frux","sm":"claude-sonnet-4-5-20250929","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2o8XZ8V6W5jKD7TuY4frux.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-08-11","evaluated_at":"2025-08-11T12:47:20.407Z","m":"grok-4-0709","oid":"frontiermath_tier_4-d88d0d94e61ff22b","rd":"2025-07-09","se":0.020833333333333332,"sid":"Cqdzvvh9tdrdmDjvu8bntM","sm":"grok-4-0709","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Cqdzvvh9tdrdmDjvu8bntM.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T22:00:02.124Z","m":"o4-mini-2025-04-16","oid":"frontiermath_tier_4-d323916eced70624","rd":"2025-04-16","se":0.020833333333333332,"sid":"oBjz3oZSvAkw5ArLtZMwWW","sm":"o4-mini-2025-04-16_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/oBjz3oZSvAkw5ArLtZMwWW.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T18:43:40.577Z","m":"gpt-5-2025-08-07","oid":"frontiermath_tier_4-888a09eb8034c770","rd":"2025-08-07","se":0.03530829267912701,"sid":"RrmHCttQmYWNaADyThwFwf","sm":"gpt-5-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RrmHCttQmYWNaADyThwFwf.eval","v":0.0625},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T18:43:35.509Z","m":"gpt-5-mini-2025-08-07","oid":"frontiermath_tier_4-8abaa0583b0d58c1","rd":"2025-08-07","se":0.02914766351555603,"sid":"EAqUKKCdvdkYTGegACdean","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/EAqUKKCdvdkYTGegACdean.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T18:43:33.699Z","m":"gpt-5-nano-2025-08-07","oid":"frontiermath_tier_4-75cbcada9090d620","rd":"2025-08-07","se":0.020833333333333332,"sid":"F6Tw2b9fDKde6N9eBu6ory","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/F6Tw2b9fDKde6N9eBu6ory.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-08-05","evaluated_at":"2025-08-05T22:14:34.193Z","m":"claude-opus-4-1-20250805-27k","oid":"frontiermath_tier_4-70cb8e69a8f9432e","rd":"2025-08-05","se":0.029147663515556026,"sid":"a6VpLAmwU23BmxsqrZTGZg","sm":"claude-opus-4-1-20250805_27K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/a6VpLAmwU23BmxsqrZTGZg.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-07-03","evaluated_at":"2025-07-03T15:44:08.541Z","m":"gemini-2-5-pro","oid":"frontiermath_tier_4-767fb3a31e79d6ae","rd":"2025-06-17","se":0.02914766351555603,"sid":"XtEW42hsVgroBKHK465wqA","sm":"gemini-2.5-pro","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XtEW42hsVgroBKHK465wqA.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-07-03","evaluated_at":"2025-07-03T13:56:46.043Z","m":"gemini-2-5-pro-preview-06-05","oid":"frontiermath_tier_4-982bd8d0a0ea07b2","rd":"2025-06-05","se":0.020833333333333332,"sid":"iwLqUvjgt378uQoKiCdSXG","sm":"gemini-2.5-pro-preview-06-05","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iwLqUvjgt378uQoKiCdSXG.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:38.433Z","m":"gpt-4-1-2025-04-14","oid":"frontiermath_tier_4-486d71f44b1948a4","rd":"2025-04-14","se":0,"sid":"oDrEMnoqz3AapWeyrxZd6C","sm":"gpt-4.1-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/oDrEMnoqz3AapWeyrxZd6C.eval","v":0},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:23.211Z","m":"claude-opus-4-2025-05-14","oid":"frontiermath_tier_4-d113e50a3d4fe738","rd":"2025-05-22","se":0,"sid":"MUYxn78tdpBMLETSQg5rgN","sm":"claude-opus-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MUYxn78tdpBMLETSQg5rgN.eval","v":0},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-07-01","e":"high","evaluated_at":"2025-07-01T17:05:19.927Z","m":"o3-2025-04-16","oid":"frontiermath_tier_4-a7e0afc0d4626a31","rd":"2025-04-16","se":0.020833333333333332,"sid":"a9fSMpBem4uhc6pxuEiQqM","sm":"o3-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/a9fSMpBem4uhc6pxuEiQqM.eval","v":0.020833333333333332},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:16.422Z","m":"claude-3-5-sonnet-2024-06-20","oid":"frontiermath_tier_4-62db3afe5372839a","rd":"2024-06-20","se":0,"sid":"ELsuPViQBj3UPywy6WyB5H","sm":"claude-3-5-sonnet-20240620","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ELsuPViQBj3UPywy6WyB5H.eval","v":0},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-07-01","e":"high","evaluated_at":"2025-07-01T17:05:14.302Z","m":"o3-mini-2025-01-31","oid":"frontiermath_tier_4-a9cc8de5ab3c3bc6","rd":"2025-01-31","se":0.029147663515556026,"sid":"YZ6Ts7tSVrvzjSRGxJowdn","sm":"o3-mini-2025-01-31_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YZ6Ts7tSVrvzjSRGxJowdn.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:12.990Z","m":"claude-opus-4-20250514-27k","oid":"frontiermath_tier_4-02db2d3cad9b6bbe","rd":"2025-05-22","se":0.029147663515556026,"sid":"Ee6YnzHKyAUxmYeL23TL9s","sm":"claude-opus-4-20250514_27K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ee6YnzHKyAUxmYeL23TL9s.eval","v":0.041666666666666664},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:10.267Z","m":"claude-3-5-sonnet-2024-10-22","oid":"frontiermath_tier_4-7a3ed2caa48f31a7","rd":"2024-10-22","se":0,"sid":"8mhf6cwsHtrRkAwxZSBgfV","sm":"claude-3-5-sonnet-20241022","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8mhf6cwsHtrRkAwxZSBgfV.eval","v":0},{"b":"frontiermath_tier_4","cfg":{"effort":"high"},"d":"2025-07-01","e":"high","evaluated_at":"2025-07-01T17:05:06.108Z","m":"o4-mini-2025-04-16","oid":"frontiermath_tier_4-3e5b0de689c16d9e","rd":"2025-04-16","se":0.03530829267912701,"sid":"WgEkfyLvjHYXfPje5Z2CPX","sm":"o4-mini-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WgEkfyLvjHYXfPje5Z2CPX.eval","v":0.0625},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:05:01.787Z","m":"grok-3-beta","oid":"frontiermath_tier_4-395dadeaf3323f73","rd":"2025-04-09","se":0,"sid":"ZyHLq7azxVkZJZRLTj3kMB","sm":"grok-3-beta","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZyHLq7azxVkZJZRLTj3kMB.eval","v":0},{"b":"frontiermath_tier_4","d":"2025-07-01","evaluated_at":"2025-07-01T17:04:25.401Z","m":"claude-sonnet-4-2025-05-14","oid":"frontiermath_tier_4-be4def632dd9dd1d","rd":"2025-05-22","se":0,"sid":"XMYUtk8tkqDFgohY8iYfmy","sm":"claude-sonnet-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XMYUtk8tkqDFgohY8iYfmy.eval","v":0},{"b":"frontiermath_tier_4","m":"gemini-2-5-deep-think-2025-08-01-webapp","oid":"frontiermath_tier_4-ea257a07ff4746ba","rd":"2025-08-01","se":0.044,"sid":"manual_gemini_2.5_deep_think_run_frontiermath_tier_4","sm":"gemini-2.5-deep-think-2025-08-01-webapp","v":0.104},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Opus 5","m":"claude-opus-5","metrics":{"checkpoints":84,"total_tokens":8710000,"wall_clock_hours":24.0833},"oid":"gbaeval-2d159919d2a114f2","rd":"2026-07-24","sm":"claude-opus-5_unknown","src":"https://gbaeval.com/model/claude-opus-5","v":0.7959852171423216},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Fable 5","m":"claude-fable-5","metrics":{"checkpoints":77,"total_tokens":41710000,"wall_clock_hours":23.8333},"oid":"gbaeval-0374448ed428db90","rd":"2026-06-09","sm":"claude-fable-5_unknown","src":"https://gbaeval.com/model/claude-fable-5","v":0.7447198828312318},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Opus 4.8","m":"claude-opus-4-8","metrics":{"checkpoints":59,"total_tokens":57080000,"wall_clock_hours":24.15},"oid":"gbaeval-2b5df689f645b3cb","rd":"2026-05-28","sm":"claude-opus-4-8_unknown","src":"https://gbaeval.com/model/claude-opus-4-8","v":0.7090450062635527},{"b":"gbaeval","cfg":{"provider":"xAI"},"dn":"Grok 4.5","m":"grok-4-5","metrics":{"checkpoints":93,"total_tokens":0,"wall_clock_hours":24.0167},"oid":"gbaeval-8ba93eec92400f87","rd":"2026-07-08","sm":"grok-4.5_unknown","src":"https://gbaeval.com/model/grok-4-5","v":0.6541267565853209},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Sonnet 5","m":"claude-sonnet-5","metrics":{"checkpoints":51,"total_tokens":54810000,"wall_clock_hours":24.05},"oid":"gbaeval-9d2affbd9bb45509","rd":"2026-06-30","sm":"claude-sonnet-5_unknown","src":"https://gbaeval.com/model/claude-sonnet-5","v":0.6526299757455419},{"b":"gbaeval","cfg":{"provider":"OpenAI"},"dn":"GPT-5.5","m":"gpt-5-5","metrics":{"checkpoints":70,"total_tokens":240910000,"wall_clock_hours":23.9639},"oid":"gbaeval-4d2d7de23a92060d","rd":"2026-04-23","sm":"gpt-5.5_unknown","src":"https://gbaeval.com/model/gpt-5-5","v":0.5322037178151383},{"b":"gbaeval","cfg":{"provider":"OpenAI"},"dn":"GPT-5.6 Sol","m":"gpt-5-6-sol","metrics":{"checkpoints":94,"total_tokens":14010000,"wall_clock_hours":24},"oid":"gbaeval-57d68e98eaa8db63","rd":"2026-07-09","sm":"gpt-5.6-sol_unknown","src":"https://gbaeval.com/model/codex-gpt-5-6-sol","v":0.5259227864005247},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","metrics":{"checkpoints":54,"total_tokens":13090000,"wall_clock_hours":17.0542},"oid":"gbaeval-13d9b615f03a7480","rd":"2026-02-17","sm":"claude-sonnet-4-6_unknown","src":"https://gbaeval.com/model/claude-sonnet","v":0.4875663483204475},{"b":"gbaeval","cfg":{"provider":"Moonshot AI"},"dn":"Kimi K3","m":"kimi-k3","metrics":{"checkpoints":92,"total_tokens":10590000,"wall_clock_hours":24.1},"oid":"gbaeval-28a4406fa77651ed","rd":"2026-07-16","sm":"kimi-k3_unknown","src":"https://gbaeval.com/model/kimi-k3","v":0.48346855379537595},{"b":"gbaeval","cfg":{"provider":"OpenAI"},"dn":"GPT-5.4","m":"gpt-5-4-2026-03-05","metrics":{"checkpoints":69,"total_tokens":22430000,"wall_clock_hours":23.2958},"oid":"gbaeval-f5361dd82551a24d","rd":"2026-03-05","sm":"gpt-5.4-2026-03-05_unknown","src":"https://gbaeval.com/model/gpt-5-4","v":0.4514817826662626},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Opus 4.6","m":"claude-opus-4-6","metrics":{"checkpoints":75,"total_tokens":11270000,"wall_clock_hours":21.25},"oid":"gbaeval-37345b76476ab1de","rd":"2026-02-05","sm":"claude-opus-4-6_unknown","src":"https://gbaeval.com/model/claude-opus-4-6","v":0.4411559216788414},{"b":"gbaeval","cfg":{"provider":"Anthropic"},"dn":"Claude Opus 4.7","m":"claude-opus-4-7","metrics":{"checkpoints":68,"total_tokens":80090000,"wall_clock_hours":19.3022},"oid":"gbaeval-647cee091bb4454b","rd":"2026-04-16","sm":"claude-opus-4-7_unknown","src":"https://gbaeval.com/model/claude-opus","v":0.43811361876085897},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"Muse Spark 1.1","m":"muse-spark-1-1","metrics":{"checkpoints":91,"total_tokens":449590000,"wall_clock_hours":24},"oid":"gbaeval-6730849586f8bae7","rd":"2026-07-09","sm":"muse-spark-1.1","src":"https://gbaeval.com/model/goose-muse-spark-1-1","v":0.07917257714954119},{"b":"gbaeval","cfg":{"provider":"Google"},"dn":"Gemini 3.5 Flash","m":"gemini-3-5-flash","metrics":{"checkpoints":46,"total_tokens":2352560000,"wall_clock_hours":20.7333},"oid":"gbaeval-fa6f1fe37d2cd128","rd":"2026-05-19","sm":"gemini-3.5-flash_unknown","src":"https://gbaeval.com/model/goose-gemini-3-5-flash","v":0.06701366402557545},{"b":"gbaeval","cfg":{"provider":"xAI"},"dn":"Grok Build 0.1","m":"grok-build-0-1","metrics":{"checkpoints":42,"total_tokens":0,"wall_clock_hours":23.9167},"oid":"gbaeval-70bf358ced100c6a","rd":"2026-05-29","sm":"grok-build-0.1","src":"https://gbaeval.com/model/grok","v":0.023812535358844113},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"MiniMax M3","m":"minimax-m3","metrics":{"checkpoints":90,"total_tokens":1528030000,"wall_clock_hours":24.0333},"oid":"gbaeval-c9e2db2d7f75ea55","rd":"2026-06-01","sm":"MiniMax-M3","src":"https://gbaeval.com/model/goose-minimax-m3","v":0.009136320401503086},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"Kimi K2.6","m":"kimi-k2-6","metrics":{"checkpoints":133,"total_tokens":59970000,"wall_clock_hours":24},"oid":"gbaeval-56df7d731ab5d942","rd":"2026-04-20","sm":"kimi-k2.6","src":"https://gbaeval.com/model/goose-kimi-k2-6","v":0.008624932115315736},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"Kimi K2.7","m":"kimi-k2-7-code","metrics":{"checkpoints":100,"total_tokens":788350000,"wall_clock_hours":23.9833},"oid":"gbaeval-311892311d1428ad","rd":"2026-06-12","sm":"kimi-k2.7-code","src":"https://gbaeval.com/model/goose-kimi-k2-7","v":0.008526563108968812},{"b":"gbaeval","cfg":{"provider":"Google"},"dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","metrics":{"checkpoints":1,"total_tokens":0,"wall_clock_hours":0},"oid":"gbaeval-1faba664f017d39e","rd":"2026-02-19","sm":"gemini-3.1-pro-preview","src":"https://gbaeval.com/model/gemini-pro","v":0.008462790979968672},{"b":"gbaeval","cfg":{"effort":"max","provider":"OpenRouter via goose"},"dn":"Qwen 3.7 Max","e":"max","m":"qwen3-7-max","metrics":{"checkpoints":92,"total_tokens":949140000,"wall_clock_hours":24.1},"oid":"gbaeval-327fa048b9dd7a00","rd":"2026-05-19","sm":"qwen3.7-max","src":"https://gbaeval.com/model/goose-qwen3-7-max","v":0.004066608893369503},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"GLM 5.2","m":"glm-5-2","metrics":{"checkpoints":96,"total_tokens":1394790000,"wall_clock_hours":23.5333},"oid":"gbaeval-9d49ed1ca8311c6b","rd":"2026-06-16","sm":"glm-5.2_unknown","src":"https://gbaeval.com/model/goose-glm-5-2","v":0.00017218231881747017},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"MiniMax M2.7","m":"minimax-m2-7","metrics":{"checkpoints":0,"total_tokens":191970000,"wall_clock_hours":24},"oid":"gbaeval-bb4904dd67f3d4a8","rd":"2026-03-18","sm":"MiniMax-M2.7","src":"https://gbaeval.com/model/goose-minimax-m2-7","v":0},{"b":"gbaeval","cfg":{"provider":"OpenRouter via goose"},"dn":"GLM 5.1","m":"glm-5-1","metrics":{"checkpoints":0,"total_tokens":5680000,"wall_clock_hours":24},"oid":"gbaeval-872458d8ab255b67","rd":"2026-04-07","sm":"glm-5.1","src":"https://gbaeval.com/model/goose-glm-5-1","v":0},{"b":"gdp_pdf","cfg":{"provider":"Anthropic"},"dn":"Claude Fable 5 / Mythos 5","m":"claude-fable-5","oid":"gdp_pdf-d31e2d0662efc25b","rd":"2026-06-09","sid":"rec7bg6dxysw8NR5Q","sm":"claude-fable-5_unknown","src":"https://surgehq.ai/leaderboards/gdp-pdf","v":0.3},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT 5.5 (xHigh reasoning)","e":"xhigh","m":"gpt-5-5","oid":"gdp_pdf-7ff0bcc2c9797f3a","rd":"2026-04-23","sid":"recTQqSxqF0lILDYj","sm":"gpt-5.5_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.26},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 4.8 (Adaptive/Max)","e":"max","m":"claude-opus-4-8","oid":"gdp_pdf-b8f01e787a20636a","rd":"2026-05-28","sid":"recRnDhgeDepMekVw","sm":"claude-opus-4-8_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.24},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 4.7 (Adaptive/Max)","e":"max","m":"claude-opus-4-7","oid":"gdp_pdf-f7a5d43f5b9280db","rd":"2026-04-16","sid":"recpPhcZvnB3KuwPz","sm":"claude-opus-4-7_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.21},{"b":"gdp_pdf","cfg":{"provider":"Google"},"dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"gdp_pdf-152702ee834be202","rd":"2026-02-19","sid":"recVfCuiq4mi3E3ME","sm":"gemini-3.1-pro-preview","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.17},{"b":"gdp_pdf","cfg":{"provider":"Google"},"dn":"Gemini 3.5 Flash (default)","m":"gemini-3-5-flash","oid":"gdp_pdf-943226d1d23444f1","rd":"2026-05-19","sid":"recYTXDosKPzreEXS","sm":"gemini-3.5-flash_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"provider":"Kimi"},"dn":"Kimi K2.6 (Thinking on)","m":"kimi-k2-6","oid":"gdp_pdf-6a87dd0e5d5e5763","rd":"2026-04-20","sid":"recFT788w1BzHc72m","sm":"kimi-k2.6","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.12},{"b":"gdp_pdf","cfg":{"provider":"Google"},"dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"gdp_pdf-ea8dde58684cc6b6","rd":"2025-12-17","sid":"rec82iUSNxO9xcFP6","sm":"gemini-3-flash-preview","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.1},{"b":"gdp_pdf","cfg":{"provider":"Grok"},"dn":"Grok 4.3 (High)","m":"grok-4-3","oid":"gdp_pdf-3f24b72b896d836e","rd":"2026-04-17","sid":"recGGgc1Xoii0Henf","sm":"grok-4.3_unknown","src":"https://surgehq.ai/leaderboards/gdp-pdf","v":0.08},{"b":"gdp_pdf","cfg":{"provider":"OpenAI"},"dn":"GPT 5.6 Sol","m":"gpt-5-6-sol","oid":"gdp_pdf-50fa605bce57318f","rd":"2026-07-09","sid":"recEoezFUpx5r9plm","sm":"gpt-5.6-sol_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.307},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Fable 5 (Adaptive/Max)","e":"max","m":"claude-fable-5","oid":"gdp_pdf-f3bc0ec3dfb217c4","rd":"2026-06-09","sid":"reciidUSVW7Il7V97","sm":"claude-fable-5_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.298},{"b":"gdp_pdf","cfg":{"provider":"OpenAI"},"dn":"GPT 5.6 Terra (default)","m":"gpt-5-6-terra","oid":"gdp_pdf-7501f8b22c34eb4d","rd":"2026-07-09","sid":"recOLBqmPqK8cfKsl","sm":"gpt-5.6-terra_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.247},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive/Max)","e":"max","m":"claude-opus-5","oid":"gdp_pdf-2eb04b1badba114c","rd":"2026-07-24","sid":"rec2cs33laSLNj6Lw","sm":"claude-opus-5_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.24},{"b":"gdp_pdf","cfg":{"provider":"OpenAI"},"dn":"GPT 5.6 Luna (default)","m":"gpt-5-6-luna","oid":"gdp_pdf-409ea88b6ca259db","rd":"2026-07-09","sid":"reckqeB1VdbggHoL6","sm":"gpt-5.6-luna_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.227},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Kimi"},"dn":"Kimi K3 (Max reasoning)","e":"max","m":"kimi-k3","oid":"gdp_pdf-e1ea48b27eafbc85","rd":"2026-07-16","sid":"recfxhDzFdV53LQ0m","sm":"kimi-k3_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.19},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Sonnet 4.6 (Adaptive/Max)","e":"max","m":"claude-sonnet-4-6","oid":"gdp_pdf-9bdb3373e04f4e83","rd":"2026-02-17","sid":"rec4j6mC3UodTLvOe","sm":"claude-sonnet-4-6_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.18},{"b":"gdp_pdf","cfg":{"provider":"Meta"},"dn":"Muse Spark 1.1 (default)","m":"muse-spark-1-1","oid":"gdp_pdf-0a0a155e2057dad2","rd":"2026-07-09","sid":"recpuh3fWrfdSwpl1","sm":"muse-spark-1.1","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.15},{"b":"gdp_pdf","cfg":{"provider":"Google"},"dn":"Gemini 3.6 Flash","m":"gemini-3-6-flash","oid":"gdp_pdf-8b504577a69d2703","rd":"2026-07-21","sid":"recHSD7QMCu8KGrOm","sm":"gemini-3.6-flash_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Grok"},"dn":"Grok 4.5 (High reasoning)","e":"high","m":"grok-4-5","oid":"gdp_pdf-e0682e6a6b15e661","rd":"2026-07-08","sid":"reclaaCsvnf5P4Uks","sm":"grok-4.5_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.5 Flash-Lite (High reasoning)","e":"high","m":"gemini-3-5-flash-lite","oid":"gdp_pdf-7a78fc32221cc14e","rd":"2026-07-21","sid":"recimKbn874aJpFRT","sm":"gemini-3.5-flash-lite_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.1},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Grok"},"dn":"Grok 4.3 (High reasoning)","e":"high","m":"grok-4-3","oid":"gdp_pdf-728b79d50a5adfd5","rd":"2026-04-17","sid":"recC0RypB4OulCpYT","sm":"grok-4.3_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.08},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT 5.6 Sol (Max reasoning)","e":"max","m":"gpt-5-6-sol","oid":"gdp_pdf-423609ac027c4862","rd":"2026-07-09","sid":"recqNsF9PYv7pQ7El","sm":"gpt-5.6-sol_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.307},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.6 Flash (High reasoning)","e":"high","m":"gemini-3-6-flash","oid":"gdp_pdf-668316c1d9585305","rd":"2026-07-21","sid":"recpPtfC4TRWfB0tr","sm":"gemini-3.6-flash_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"provider":"Amazon"},"dn":"Nova 2 Pro (default)","m":"nova-2-0-pro-preview","oid":"gdp_pdf-ec87cbc1e607d30c","rd":"2025-12-02","sid":"reclqb9TCU8e50nr1","sm":"nova-2.0-pro-preview_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.02},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark 1.2 (xHigh reasoning)","e":"xhigh","m":"muse-spark-1-2","oid":"gdp_pdf-66d926db8e8a384e","rd":"2026-08-05","sid":"recsDYKciDC3SjmIk","sm":"muse-spark-1.2_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.16},{"b":"gdp_pdf","cfg":{"provider":"Meta"},"dn":"Muse Spark 1.2 (default)","m":"muse-spark-1-2","oid":"gdp_pdf-d7dcf25707522bca","rd":"2026-08-05","sid":"recS9aC08eujgqAAC","sm":"muse-spark-1.2","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.12},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT 5.6 Terra (Medium reasoning)","e":"medium","m":"gpt-5-6-terra","oid":"gdp_pdf-156446b5a3f66f3c","rd":"2026-07-09","sid":"rec6Csv08fAnLjfOP","sm":"gpt-5.6-terra_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.247},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.7 Flash (High reasoning)","e":"high","m":"gemini-3-7-flash","oid":"gdp_pdf-5371671c56680c7e","rd":"2026-08-13","sid":"rechYJ35wK010IkOL","sm":"gemini-3.7-flash_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.23800000000000002},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Qwen"},"dn":"Qwen 3.8 Max (xHigh reasoning)","e":"xhigh","m":"qwen3-8-max","oid":"gdp_pdf-6b627e730758c48b","rd":"2026-08-02","sid":"recIpso0sS1nvaLml","sm":"qwen3.8-max_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.23199999999999998},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT 5.6 Luna (Medium reasoning)","e":"medium","m":"gpt-5-6-luna","oid":"gdp_pdf-109dce12fc9e0892","rd":"2026-07-09","sid":"recOkb3yKojzovCBt","sm":"gpt-5.6-luna_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.227},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"Google"},"dn":"Gemini 3.7 Flash (Medium reasoning)","e":"medium","m":"gemini-3-7-flash","oid":"gdp_pdf-7d315e8bdc853462","rd":"2026-08-13","sid":"recmCleSF4T8iXGOv","sm":"gemini-3.7-flash_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.218},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Grok"},"dn":"Grok 4.6 (xHigh reasoning)","e":"xhigh","m":"grok-4-6","oid":"gdp_pdf-6b90cf3dbdcf06d5","rd":"2026-08-12","sid":"rec0nleN6rWiEZurZ","sm":"grok-4.6_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.172},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.1 Pro (High reasoning)","e":"high","m":"gemini-3-1-pro-preview","oid":"gdp_pdf-d6b60d7c0d33eec6","rd":"2026-02-19","sid":"rec2YmuDdJfXpQrU9","sm":"gemini-3.1-pro-preview_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.17},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Grok"},"dn":"Grok 4.6 (High reasoning)","e":"high","m":"grok-4-6","oid":"gdp_pdf-e419a602d80c0997","rd":"2026-08-12","sid":"recYB4E6PiXIYbMNt","sm":"grok-4.6_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.16},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"Meta"},"dn":"Muse Spark 1.1 (Medium reasoning)","e":"medium","m":"muse-spark-1-1","oid":"gdp_pdf-aac3906e2314a65b","rd":"2026-07-09","sid":"recGACkW4vtU4ZX85","sm":"muse-spark-1.1_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.15},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"Google"},"dn":"Gemini 3.5 Flash (Medium reasoning)","e":"medium","m":"gemini-3-5-flash","oid":"gdp_pdf-9feb321cfd8a334a","rd":"2026-05-19","sid":"rec7amX7V5Mz3c4J7","sm":"gemini-3.5-flash_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Z.ai"},"dn":"GLM 5.3 Flash (Max reasoning)","e":"max","m":"glm-5-3-flash","oid":"gdp_pdf-66f1f6891054c8b2","rd":"2026-08-20","sid":"recdTmA4PKhCjBi1v","sm":"glm-5.3-flash_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.14},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3 Flash (High reasoning)","e":"high","m":"gemini-3-flash-preview","oid":"gdp_pdf-a8d34ef232a465b6","rd":"2025-12-17","sid":"recXoLxrkglvzgyMB","sm":"gemini-3-flash-preview_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.1},{"b":"gdp_pdf","cfg":{"effort":"none","provider":"Amazon"},"dn":"Nova 2 Pro (No reasoning)","e":"none","m":"nova-2-0-pro-preview","oid":"gdp_pdf-4588097cebf6a1a9","rd":"2025-12-02","sid":"recEUIh2qUm0io3np","sm":"nova-2.0-pro-preview_none","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.02},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Adaptive/High)","e":"high","m":"claude-fable-5-1","oid":"gdp_pdf-80af1edd3aab531d","rd":"2026-09-01","sid":"recrTn9skaQVIvmPM","sm":"claude-fable-5-1_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.29600000000000004},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Adaptive/Max)","e":"max","m":"claude-fable-5-1","oid":"gdp_pdf-2bab8043f4890549","rd":"2026-09-01","sid":"recSvpisHXMgkbvmf","sm":"claude-fable-5-1_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.276},{"b":"gdp_pdf","cfg":{"provider":"Meta"},"dn":"Muse Spark 1.3 (Auto reasoning)","m":"muse-spark-1-3","oid":"gdp_pdf-3a4ac4f1c74c2e13","rd":"2026-09-02","sid":"recq3Specht6SLPz1","sm":"muse-spark-1.3_unknown","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.276},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark 1.3 (xHigh reasoning)","e":"xhigh","m":"muse-spark-1-3","oid":"gdp_pdf-88f6d7b6bf7f3e1b","rd":"2026-09-02","sid":"recFcshC1mHYEL3yJ","sm":"muse-spark-1.3_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.276},{"b":"gdp_pdf","cfg":{"effort":"medium","provider":"Google"},"dn":"Gemini 3.8 Flash (Medium reasoning)","e":"medium","m":"gemini-3-8-flash","oid":"gdp_pdf-2049e4236d117354","rd":"2026-09-02","sid":"recNOvVVe7jvpspUr","sm":"gemini-3.8-flash_medium","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.23399999999999999},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.8 Flash (High reasoning)","e":"high","m":"gemini-3-8-flash","oid":"gdp_pdf-702bf751f3969e4c","rd":"2026-09-02","sid":"recf9BdIGPShsbpnb","sm":"gemini-3.8-flash_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.23199999999999998},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Qwen"},"dn":"Qwen 3.8 Flash (xHigh reasoning)","e":"xhigh","m":"qwen3-8-flash","oid":"gdp_pdf-51690fe1757c267a","rd":"2026-08-26","sid":"recrz0p5LnyKdILVL","sm":"qwen3.8-flash_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.166},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 5.5 (Adaptive/Max)","e":"max","m":"claude-opus-5-5","oid":"gdp_pdf-a12e6a5be560964d","rd":"2026-09-22","sid":"rec7dEXMmBbh7vcYv","sm":"claude-opus-5-5_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.306},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT 6 Sol (Max reasoning)","e":"max","m":"gpt-6-sol","oid":"gdp_pdf-60d09267b5eece55","rd":"2026-09-22","sid":"recNioYpcVH6vDVRS","sm":"gpt-6-sol_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.264},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT 6 Luna (Max reasoning)","e":"max","m":"gpt-6-luna","oid":"gdp_pdf-00fc8b4e6e5511b9","rd":"2026-09-22","sid":"rec1mcsu2XY89LcY7","sm":"gpt-6-luna_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.23},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT 6 Astra (Max reasoning)","e":"max","m":"gpt-6-astra","oid":"gdp_pdf-7d25ffe6f4a850b9","rd":"2026-09-03","sid":"receclu0CyBBVUHTZ","sm":"gpt-6-astra_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.342},{"b":"gdp_pdf","cfg":{"effort":"xhigh","provider":"Grok"},"dn":"Grok 4.7 (xHigh reasoning)","e":"xhigh","m":"grok-4-7","oid":"gdp_pdf-c8355f18e6741988","rd":"2026-09-21","sid":"recMxIOts56Sl9JEB","sm":"grok-4.7_xhigh","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.228},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4.1 Flash (Max reasoning)","e":"max","m":"deepseek-v4-1-flash","oid":"gdp_pdf-8eabb414bb7c8219","rd":"2026-09-09","sid":"recEaIQoSO4PJp1wW","sm":"deepseek-v4.1-flash_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.198},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT 6.1 Sol (Max reasoning)","e":"max","m":"gpt-6-1-sol","oid":"gdp_pdf-5d3de4bc0b08fdcd","rd":"2026-09-29","sid":"recHHzUvSpGHKsoJK","sm":"gpt-6.1-sol_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.35000000000000003},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (Adaptive/Max)","e":"max","m":"claude-sonnet-5-5","oid":"gdp_pdf-dd49d43d90b88ff1","rd":"2026-09-28","sid":"rec0UVUbE2GVRELuC","sm":"claude-sonnet-5-5_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.314},{"b":"gdp_pdf","cfg":{"effort":"max","provider":"Meta"},"dn":"Muse Wonder (Max reasoning)","e":"max","m":"muse-wonder-1-0","oid":"gdp_pdf-6ec9af6593336eb0","rd":"2026-10-12","sid":"recQYavqr4GkieYHV","sm":"muse-wonder-1.0_max","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.308},{"b":"gdp_pdf","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 4 Argon (High reasoning)","e":"high","m":"gemini-4-argon","oid":"gdp_pdf-9cbb8c5139858ee5","rd":"2026-09-30","sid":"recbmgyzrQsktiT3t","sm":"gemini-4-argon_high","src":"https://surgehq.ai/benchmarks/gdp-pdf","v":0.284},{"b":"gdpval","cfg":{"effort":"none"},"dn":"GPT-5.2","e":"none","m":"gpt-5-2-2025-12-11","oid":"gdpval-2d9b656296c8a363","rd":"2025-12-11","sid":"rechPytR66Sfja2jE","sm":"gpt-5.2-2025-12-11_none","v":0.49700000000000005},{"b":"gdpval","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"gdpval-4332211978d83917","rd":"2025-11-24","sid":"recnf779wFJBtP8ZX","sm":"claude-opus-4-5-20251101","v":0.455},{"b":"gdpval","dn":"Claude Opus 4.1","m":"claude-opus-4-1-2025-08-05","oid":"gdpval-c9641538f6ac1f34","rd":"2025-08-05","sid":"rechImbYYM7BaW9f2","sm":"claude-opus-4-1-20250805","v":0.436},{"b":"gdpval","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"gdpval-2d7da58c4fec88c0","rd":"2025-09-29","sid":"recgGv7NYJkLHPSjn","sm":"claude-sonnet-4-5-20250929","v":0.425},{"b":"gdpval","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"gdpval-40e607048b1bf5b4","rd":"2025-11-18","sid":"recpJDfwotqNLiCaw","sm":"gemini-3-pro-preview","v":0.40299999999999997},{"b":"gdpval","cfg":{"effort":"medium"},"dn":"GPT-5","e":"medium","m":"gpt-5-2025-08-07","oid":"gdpval-70fb72a22ef2b552","rd":"2025-08-07","sid":"recA9mPzDrm2XCNZc","sm":"gpt-5-2025-08-07_medium","v":0.348},{"b":"gdpval","cfg":{"effort":"medium"},"dn":"o3","e":"medium","m":"o3-2025-04-16","oid":"gdpval-74fbcb14201b2f03","rd":"2025-04-16","sid":"rec92NYmhzWbgq0aG","sm":"o3-2025-04-16_medium","v":0.308},{"b":"gdpval","cfg":{"effort":"high"},"dn":"o4-mini high","e":"high","m":"o4-mini-2025-04-16","oid":"gdpval-cc2d98555edc6531","rd":"2025-04-16","sid":"recN2m297LeySJIAW","sm":"o4-mini-2025-04-16_high","v":0.253},{"b":"gdpval","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"gdpval-5cf0384cda45edd4","rd":"2025-06-17","sid":"recL5axz7UL23Yt69","sm":"gemini-2.5-pro","v":0.233},{"b":"gdpval","cfg":{"effort":"high"},"dn":"Grok 4","e":"high","m":"grok-4-0709","oid":"gdpval-2d220a5edd4ca09d","rd":"2025-07-09","sid":"reccXcktDNtWmiFOI","sm":"grok-4-0709_high","v":0.21100000000000002},{"b":"gdpval","dn":"GPT-4o","m":"gpt-4o-2024-11-20","oid":"gdpval-61e7b5a53ff1eaaa","rd":"2024-11-20","sid":"recHDSF8S2VhrANTu","sm":"gpt-4o-2024-11-20","v":0.099},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-3-flash-preview","oid":"geobench-1148a95b413916a7","rd":"2025-12-17","sid":"rec4g5alUjBYlfrX0","sm":"gemini-3-flash-preview","v":4333},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-3-pro-preview","oid":"geobench-ecf22dd7116bba14","rd":"2025-11-18","sid":"recwbNbVo3KioFg1v","sm":"gemini-3-pro-preview","src":"https://geobench.org/","v":4145},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-2-5-pro-preview-05-06","oid":"geobench-eab9dc1b7e5fbddb","rd":"2025-05-06","sid":"recVj2Vj168vkJXaT","sm":"gemini-2.5-pro-preview-05-06","src":"https://geobench.org/","v":4085},{"b":"geobench","cfg":{"tools":"Search"},"m":"gemini-2-5-pro-exp-03-25","oid":"geobench-e46ffb38cf4e3e84","rd":"2025-03-25","sid":"recKwlVhpDLw3jftZ","sm":"gemini-2.5-pro-exp-03-25","src":"https://geobench.org/","v":4019},{"b":"geobench","cfg":{"tools":"Search"},"m":"gemini-2-5-pro-preview-05-06","oid":"geobench-59f6b6aa97ad06b1","rd":"2025-05-06","sid":"recEx1qcKpzEhL29G","sm":"gemini-2.5-pro-preview-05-06","src":"https://geobench.org/","v":3983},{"b":"geobench","cfg":{"effort":"medium","tools":"None"},"e":"medium","m":"gpt-5-2025-08-07","oid":"geobench-0b811df4bf0d89b2","rd":"2025-08-07","sid":"recFgJeRfhlSbQeu9","sm":"gpt-5-2025-08-07_medium","src":"https://geobench.org/","v":3937},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-1-5-flash-002","notes":"Assuming latest Flash version","oid":"geobench-8a65ce6ba007a56b","rd":"2024-09-24","sid":"recZO3tP9x1pQsytO","sm":"gemini-1.5-flash-002","src":"https://geobench.org/","v":3934},{"b":"geobench","cfg":{"effort":"high","tools":"None"},"e":"high","m":"o3-2025-04-16","oid":"geobench-2a648661b3d820ba","rd":"2025-04-16","sid":"recMNCiUt6YNqVDKL","sm":"o3-2025-04-16_high","src":"https://geobench.org/","v":3930},{"b":"geobench","cfg":{"effort":"medium","tools":"None"},"e":"medium","m":"o1-2024-12-17","notes":"Assuming medium reasoning effort.","oid":"geobench-d21d50382a58b122","rd":"2024-12-17","sid":"recu2g66G4TcuHV94","sm":"o1-2024-12-17_medium","src":"https://geobench.org/","v":3925},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-2-0-flash-001","oid":"geobench-76798b283fb90236","rd":"2025-02-05","sid":"rec7kPZI1YDSo2DgB","sm":"gemini-2.0-flash-001","src":"https://geobench.org/","v":3897},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-opus-4-5-2025-11-01","oid":"geobench-f37859c7c60e92b7","rd":"2025-11-24","sid":"recPrjrIZ4Da3oEFc","sm":"claude-opus-4-5-20251101","v":3877},{"b":"geobench","cfg":{"effort":"medium","tools":"None"},"e":"medium","m":"o3-2025-04-16","oid":"geobench-c9d6b4ea01f80137","rd":"2025-04-16","sid":"recyDcak8Z17j2dz8","sm":"o3-2025-04-16_medium","src":"https://geobench.org/","v":3858},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-2-5-flash-preview-04-17","notes":"Assuming \"Gemini 2.5 Flash Experimental\" refers to gemini-2.5-flash-preview-04-17 since there is no version with experimental in the name","oid":"geobench-c141ad6500c5777b","rd":"2025-04-17","sid":"recZQKTTSvZulfgVB","sm":"gemini-2.5-flash-preview-04-17","src":"https://geobench.org/","v":3854},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-3-7-sonnet-20250219-15k","oid":"geobench-3729d9a614fe0c0b","rd":"2025-02-24","sid":"recavCfmmQ6tMEwtu","sm":"claude-3-7-sonnet-20250219_15K","src":"https://geobench.org/","v":3844},{"b":"geobench","cfg":{"tools":"None"},"m":"gpt-4-1-2025-04-14","oid":"geobench-c5eef1fe2c77e67d","rd":"2025-04-14","sid":"rec85xrmzw8WaUL5H","sm":"gpt-4.1-2025-04-14","src":"https://geobench.org/","v":3791},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-3-7-sonnet-2025-02-19","oid":"geobench-69eed74398057cfd","rd":"2025-02-24","sid":"recz7nkHBBRKXIWzd","sm":"claude-3-7-sonnet-20250219","src":"https://geobench.org/","v":3791},{"b":"geobench","cfg":{"tools":"None"},"m":"gemini-2-5-flash-preview-05-20","oid":"geobench-7764428b414af557","rd":"2025-05-20","sid":"recjQCnP7PGnXclzu","sm":"gemini-2.5-flash-preview-05-20","src":"https://geobench.org/","v":3786},{"b":"geobench","cfg":{"effort":"high","tools":"None"},"e":"high","m":"o4-mini-2025-04-16","oid":"geobench-1e32e39a1a55368b","rd":"2025-04-16","sid":"rec1VvZVwbTipNWEC","sm":"o4-mini-2025-04-16_high","src":"https://geobench.org/","v":3717},{"b":"geobench","cfg":{"tools":"None"},"m":"gpt-4o-2024-11-20","notes":"Assuming latest vesrsion of gpt-4o","oid":"geobench-9d77aa08c1e69466","rd":"2024-11-20","sid":"rec2Ind0RGchMKj3j","sm":"gpt-4o-2024-11-20","src":"https://geobench.org/","v":3696},{"b":"geobench","cfg":{"effort":"medium","tools":"None"},"e":"medium","m":"o4-mini-2025-04-16","oid":"geobench-4831bda3defaf70b","rd":"2025-04-16","sid":"recUod4Wgb4Jaz8cA","sm":"o4-mini-2025-04-16_medium","src":"https://geobench.org/","v":3651},{"b":"geobench","cfg":{"tools":"None"},"m":"qwen2-5-vl-72b-instruct","oid":"geobench-bac635fa6b1eae51","rd":"2024-09-19","sid":"recYbzPfMGsVvfzsa","sm":"Qwen2.5-VL-72B-Instruct","src":"https://geobench.org/","v":3448},{"b":"geobench","cfg":{"tools":"None"},"m":"gpt-4o-mini-2024-07-18","oid":"geobench-e0ee9a2a1efca172","rd":"2024-07-18","sid":"recpyh8ly8Rnrqvgj","sm":"gpt-4o-mini-2024-07-18","src":"https://geobench.org/","v":3401},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-3-5-sonnet-2024-10-22","notes":"Assuming latest Sonnet model","oid":"geobench-6a838fbf603b7201","rd":"2024-10-22","sid":"rec3IW78fmDClmevf","sm":"claude-3-5-sonnet-20241022","src":"https://geobench.org/","v":3327},{"b":"geobench","cfg":{"tools":"None"},"m":"gemma-3-27b-it","oid":"geobench-d21912374680fcd2","rd":"2025-03-12","sid":"recNtW91J7ujwVEd3","sm":"gemma-3-27b-it","src":"https://geobench.org/","v":3087},{"b":"geobench","cfg":{"tools":"None"},"m":"llama-4-maverick-17b-128e-instruct","oid":"geobench-e08cde6acbbbdde2","rd":"2025-04-05","sid":"recZ3e3Hb3wbvXFIV","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://geobench.org/","v":3069},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-opus-4-20250514-32k","notes":"Thinking token budget was 31,448. Rounded to 32k to reference existing model version.","oid":"geobench-fbccc466e849c920","rd":"2025-05-22","sid":"rec9ARZdPBfnWVjCp","sm":"claude-opus-4-20250514_32K","src":"https://geobench.org/","v":2915},{"b":"geobench","cfg":{"tools":"None"},"m":"llama-3-2-90b-vision-instruct","oid":"geobench-4119b736c6d66212","rd":"2024-09-24","sid":"recuaxYkSjCYJAD6e","sm":"Llama-3.2-90B-Vision-Instruct","src":"https://geobench.org/","v":2798},{"b":"geobench","cfg":{"tools":"None"},"m":"grok-4-0709","oid":"geobench-5b053db056aec5b8","rd":"2025-07-09","sid":"reckc8skUG0WI9i4f","sm":"grok-4-0709","src":"https://geobench.org/","v":2719},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-3-5-haiku-2024-10-22","oid":"geobench-1a3baadb2037593e","rd":"2024-10-22","sid":"recno5e6hTQGMAJmt","sm":"claude-3-5-haiku-20241022","src":"https://geobench.org/","v":2507},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-sonnet-4-2025-05-14","oid":"geobench-1cf103b67363f43a","rd":"2025-05-22","sid":"recc6aFqMfrhbx2q0","sm":"claude-sonnet-4-20250514","src":"https://geobench.org/","v":2468},{"b":"geobench","cfg":{"tools":"None"},"m":"claude-sonnet-4-20250514-32k","notes":"Thinking token budget was 31,448. Rounded to 32k to reference existing model version.","oid":"geobench-33382b81485a004c","rd":"2025-05-22","sid":"rec633RXeiU9CsPRR","sm":"claude-sonnet-4-20250514_32K","src":"https://geobench.org/","v":2369},{"b":"geobench","cfg":{"tools":"None"},"m":"pixtral-12b-2409","oid":"geobench-d46db662018052d5","rd":"2024-09-17","sid":"rectZf8KZ5UNsoaMx","sm":"Pixtral-12B-2409","src":"https://geobench.org/","v":2131},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-11-06","e":"max","evaluated_at":"2026-11-06T00:59:23.000Z","m":"kimi-k3-1-pro-together","oid":"gpqa_diamond-7900cb9fc043a422","rd":"2026-10-23","se":0.01439446786793499,"sid":"2bhYPjF6VRWc5CeERGq8g5","sm":"kimi-k3.1-pro-together_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2bhYPjF6VRWc5CeERGq8g5.eval","v":0.9419191919191919},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T20:57:35.000Z","m":"doubao-seed-evolving","oid":"gpqa_diamond-228204b80fdb5fda","rd":"2026-11-02","se":0.015670135540928182,"sid":"KWK3fWVchTtVXd4kAEhwK5","sm":"doubao-seed-evolving_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKWK3fWVchTtVXd4kAEhwK5.eval","v":0.9311868686868687},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T13:47:17.000Z","m":"mistral-large-4-0-oss","oid":"gpqa_diamond-98383b43f2ebf804","rd":"2026-10-29","se":0.018113832926930726,"sid":"brFy5AcCtkxoccgJToWYmR","sm":"mistral-large-4-0-oss_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbrFy5AcCtkxoccgJToWYmR.eval","v":0.9015151515151515},{"b":"gpqa_diamond","d":"2026-10-30","evaluated_at":"2026-10-30T18:28:17.000Z","m":"seed-1-6","oid":"gpqa_diamond-29c73dcc4a807835","rd":"2025-06-11","se":0.025273066621077898,"sid":"koZMW3Nn9vWz4vZUXdZLEC","sm":"seed-1.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkoZMW3Nn9vWz4vZUXdZLEC.eval","v":0.7758838383838383},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T12:30:09.000Z","m":"doubao-seed-2-1-turbo-260628","oid":"gpqa_diamond-62e07bb760b9c2ac","rd":"2026-06-23","se":0.017895899774362865,"sid":"eMBLN26MgdzXLL6XgwmKB9","sm":"doubao-seed-2.1-turbo-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeMBLN26MgdzXLL6XgwmKB9.eval","v":0.8958333333333334},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:56.000Z","m":"doubao-seed-2-0-lite-260428","oid":"gpqa_diamond-0d3be77173ff3f1a","rd":"2026-04-28","se":0.01944079830983592,"sid":"TJUDw6vaSyDFKZ2UacuD5L","sm":"doubao-seed-2.0-lite-260428_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTJUDw6vaSyDFKZ2UacuD5L.eval","v":0.8813131313131313},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:39.000Z","m":"doubao-seed-2-0-pro-260215","oid":"gpqa_diamond-37ececf8e047151a","rd":"2026-02-14","se":0.019429399729068986,"sid":"2z8ecBVZbXKHtpyFyxpJyB","sm":"doubao-seed-2.0-pro-260215_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2z8ecBVZbXKHtpyFyxpJyB.eval","v":0.8819444444444444},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:29.000Z","m":"doubao-seed-2-1-pro-260628","oid":"gpqa_diamond-810714305e76eab7","rd":"2026-06-23","se":0.018417155503921614,"sid":"7RKgYb6sU9XB5RVBRQ3nCB","sm":"doubao-seed-2.1-pro-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7RKgYb6sU9XB5RVBRQ3nCB.eval","v":0.8933080808080808},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-10-26","e":"max","evaluated_at":"2026-10-26T20:00:37.000Z","m":"gpt-6-1-terra","oid":"gpqa_diamond-7508c2115b023839","rd":"2026-10-26","se":0.014911672683567374,"sid":"5Jm2Rvv9sNx7M8nodinFAp","sm":"gpt-6.1-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5Jm2Rvv9sNx7M8nodinFAp.eval","v":0.9431818181818182},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-10-19","e":"max","evaluated_at":"2026-10-19T22:38:32.000Z","m":"deepseek-v4-1-flash-1017","oid":"gpqa_diamond-a3cac8e5836283c0","rd":"2026-10-17","se":0.015671362011907143,"sid":"ZK85PJtsCSeQJgV8dA4R4G","sm":"deepseek-v4.1-flash-1017_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZK85PJtsCSeQJgV8dA4R4G.eval","v":0.9318181818181818},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-10-15","e":"xhigh","evaluated_at":"2026-10-15T18:52:04.000Z","m":"muse-spark-open-816b-a42b","oid":"gpqa_diamond-f6c6b889bcfa585e","rd":"2026-10-15","se":0.018706209910489154,"sid":"JnKpor8c3EKWGvmyjLFVzH","sm":"Muse-Spark-Open-816B-A42B_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJnKpor8c3EKWGvmyjLFVzH.eval","v":0.8857323232323232},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:15:08.000Z","m":"gemini-4-argon","oid":"gpqa_diamond-f7865636d4bfae27","rd":"2026-09-30","se":0.011918129589602694,"sid":"QEouTvczNsxgkR4i7i8wBF","sm":"gemini-4-argon_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQEouTvczNsxgkR4i7i8wBF.eval","v":0.9640151515151515},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-10-12","e":"high","evaluated_at":"2026-10-12T21:38:22.000Z","m":"mistral-large-4-0","oid":"gpqa_diamond-7e22cb9add76defa","rd":"2026-10-06","se":0.019328132475292338,"sid":"Ekx5phukyiJhAcQodMfS2p","sm":"mistral-large-4-0_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEkx5phukyiJhAcQodMfS2p.eval","v":0.8806818181818182},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-10-09","e":"max","evaluated_at":"2026-10-09T14:38:01.000Z","m":"deepseek-v4-1-flash","oid":"gpqa_diamond-9386f6f437a9a161","rd":"2026-09-09","se":0.017287759372442393,"sid":"ZU5ebpDaDPiDZLgv7cG8B9","sm":"deepseek-v4.1-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZU5ebpDaDPiDZLgv7cG8B9.eval","v":0.8983585858585859},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-10-09","e":"max","evaluated_at":"2026-10-09T14:35:53.000Z","m":"claude-haiku-5-5","oid":"gpqa_diamond-e8a59d53e51c424b","rd":"2026-10-07","se":0.019774484382483127,"sid":"ZEKWTGJiWFscHimMkiUmec","sm":"claude-haiku-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZEKWTGJiWFscHimMkiUmec.eval","v":0.8964646464646465},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T22:00:34.000Z","m":"gpt-6-1-sol","oid":"gpqa_diamond-fa72beaff59f3045","rd":"2026-09-29","se":0.013780748450364692,"sid":"LspviLnbeZ82WCngWp6rwR","sm":"gpt-6.1-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLspviLnbeZ82WCngWp6rwR.eval","v":0.9539141414141414},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T15:15:14.000Z","m":"claude-sonnet-5-5","oid":"gpqa_diamond-cdd5e23e41e2b8e5","rd":"2026-09-28","se":0.013710320714521202,"sid":"QPk8jbJvo4986sbWtdte6J","sm":"claude-sonnet-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQPk8jbJvo4986sbWtdte6J.eval","v":0.9558080808080808},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T22:14:05.000Z","m":"gpt-6-luna","oid":"gpqa_diamond-2bfc948a168fe8b2","rd":"2026-09-22","se":0.018537378631915503,"sid":"7F3QFMAkwnCV62HcvxP96q","sm":"gpt-6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7F3QFMAkwnCV62HcvxP96q.eval","v":0.9046717171717171},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-09-22","e":"xhigh","evaluated_at":"2026-09-22T21:49:22.000Z","m":"grok-4-7","oid":"gpqa_diamond-67ba4cf583e1274e","rd":"2026-09-21","se":0.015505824295534454,"sid":"hxAhYJfXdPNanxatXpG5hC","sm":"grok-4.7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhxAhYJfXdPNanxatXpG5hC.eval","v":0.92739898989899},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T19:59:24.000Z","m":"gpt-6-sol","oid":"gpqa_diamond-c3af663424446b66","rd":"2026-09-22","se":0.014938512008159717,"sid":"aJD7ZfFftu2cZoToJhhzF7","sm":"gpt-6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaJD7ZfFftu2cZoToJhhzF7.eval","v":0.9431818181818182},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T18:27:39.000Z","m":"claude-opus-5-5","oid":"gpqa_diamond-b7a1e3830a3615c8","rd":"2026-09-22","se":0.019434397190192135,"sid":"6gmbahPSxHf9xVhAYqpnhi","sm":"claude-opus-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6gmbahPSxHf9xVhAYqpnhi.eval","v":0.9059343434343434},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-09-02","e":"xhigh","evaluated_at":"2026-09-02T18:04:58.000Z","m":"qwen3-8-max-0902","oid":"gpqa_diamond-54786d00350e7fc5","rd":"2026-09-01","se":0.017249976996605708,"sid":"VzeQXKBH9maR2nDcZJ8b7S","sm":"qwen3.8-max-0902_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVzeQXKBH9maR2nDcZJ8b7S.eval","v":0.922979797979798},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-09-02","e":"high","evaluated_at":"2026-09-02T17:15:36.000Z","m":"gemini-3-8-flash","oid":"gpqa_diamond-2c42d31d04e9723f","rd":"2026-09-02","se":0.01401136318210652,"sid":"Br3mjs2GrYcCwX9AMwcihU","sm":"gemini-3.8-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBr3mjs2GrYcCwX9AMwcihU.eval","v":0.9539141414141414},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:46:42.000Z","m":"granite-4-0-micro","oid":"gpqa_diamond-fbdd7239cc019563","rd":"2025-10-02","se":0.018237830822136196,"sid":"SadaSwnSdMpYRstc8zvgjJ","sm":"granite-4.0-micro","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSadaSwnSdMpYRstc8zvgjJ.eval","v":0.2828282828282828},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:46:22.000Z","m":"seed-oss-36b-instruct","oid":"gpqa_diamond-9ab3811fe9e2e8d5","rd":"2025-08-20","se":0.02741123270979468,"sid":"BVat526nF6NUGoVaLCp7DT","sm":"seed-oss-36b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBVat526nF6NUGoVaLCp7DT.eval","v":0.7152777777777778},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:45:08.000Z","m":"qwen3-32b","oid":"gpqa_diamond-4dba835a956d75e9","rd":"2025-04-29","se":0.028146234201602498,"sid":"EXpHB6kFqDjRfG9A5LsDf6","sm":"Qwen3-32B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEXpHB6kFqDjRfG9A5LsDf6.eval","v":0.5410353535353535},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:44:24.000Z","m":"qwen3-30b-a3b-thinking-2507","oid":"gpqa_diamond-ec5f6fcfc53f57b4","rd":"2025-07-30","se":0.028327897613545493,"sid":"2gzNF2d53h2oCKDnojLtSP","sm":"qwen3-30b-a3b-thinking-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2gzNF2d53h2oCKDnojLtSP.eval","v":0.7007575757575758},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:44:04.000Z","m":"qwen3-30b-a3b-instruct-2507","oid":"gpqa_diamond-09d7d63a28a8c034","rd":"2025-07-29","se":0.028844434239616544,"sid":"X3HfncZtnLKUPoUuRc9YsR","sm":"qwen3-30b-a3b-instruct-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX3HfncZtnLKUPoUuRc9YsR.eval","v":0.5561868686868687},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:43:32.000Z","m":"qwen3-30b-a3b","oid":"gpqa_diamond-081c65ce8f287c8a","rd":"2025-04-29","se":0.02827514183647295,"sid":"DGSmhVxWsi7chsuKS64VLg","sm":"Qwen3-30B-A3B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDGSmhVxWsi7chsuKS64VLg.eval","v":0.5037878787878788},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:43:15.000Z","m":"qwen3-30b-a3b","oid":"gpqa_diamond-81b5014f69326a93","rd":"2025-04-29","se":0.029727711361608088,"sid":"BnX67AoZzgVbHgaM8F8Brz","sm":"Qwen3-30B-A3B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBnX67AoZzgVbHgaM8F8Brz.eval","v":0.6167929292929293},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:42:14.000Z","m":"qwen2-5-7b-instruct","oid":"gpqa_diamond-48fd3ebfc566236d","rd":"2024-09-19","se":0.02291485990511179,"sid":"6jkd5nJzspGZJtvZynZZTD","sm":"qwen2.5-7b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6jkd5nJzspGZJtvZynZZTD.eval","v":0.3547979797979798},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:40:44.000Z","m":"mistral-small-3-1-2503","oid":"gpqa_diamond-4ffca21be566e1a4","rd":"2025-03-17","se":0.02348827340432996,"sid":"DvbGrBaybHM3UhJwWhE3LR","sm":"mistral-small-3.1-2503","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDvbGrBaybHM3UhJwWhE3LR.eval","v":0.41919191919191917},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:40:23.000Z","m":"mistral-small-3-2501","oid":"gpqa_diamond-6625cb1dd854962c","rd":"2025-01-30","se":0.027888880246470463,"sid":"YJPwhEJa6qQWB23HQjByxt","sm":"mistral-small-3-2501","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYJPwhEJa6qQWB23HQjByxt.eval","v":0.47285353535353536},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:39:36.000Z","m":"magistral-small-2509","oid":"gpqa_diamond-6d6664d3a21c878e","rd":"2025-09-18","se":0.025993462698453398,"sid":"3UbYVQELDHAcQgu7sTTnRf","sm":"magistral-small-2509","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3UbYVQELDHAcQgu7sTTnRf.eval","v":0.476010101010101},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:39:11.000Z","m":"llama-3-2-1b-instruct","oid":"gpqa_diamond-353c32ab2fc8aa02","rd":"2024-09-24","se":0.016278803912773237,"sid":"FGEsLTxzZ7fE9aq7DhdqZb","sm":"Llama-3.2-1B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFGEsLTxzZ7fE9aq7DhdqZb.eval","v":0.23926767676767677},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:37:57.000Z","m":"glm-4-7-flash","oid":"gpqa_diamond-c5ed0a551afb9447","se":0.021751012010888356,"sid":"GSde7u2Mb6GZGpubJjjPam","sm":"glm-4.7-flash_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGSde7u2Mb6GZGpubJjjPam.eval","v":0.4513888888888889},{"b":"gpqa_diamond","d":"2026-08-30","evaluated_at":"2026-08-30T22:37:35.000Z","m":"glm-4-7-flash","oid":"gpqa_diamond-1571842ac404730b","se":0.02934798760577577,"sid":"GL7htq7XRnbzqpAVK5f862","sm":"glm-4.7-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGL7htq7XRnbzqpAVK5f862.eval","v":0.6054292929292929},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-30","e":"max","evaluated_at":"2026-08-30T14:57:43.000Z","m":"gpt-6-astra","oid":"gpqa_diamond-21d85be475df8592","rd":"2026-09-03","se":0.013696812097020311,"sid":"3h8STJpG2VzY9LiBUzwotv","sm":"gpt-6-astra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3h8STJpG2VzY9LiBUzwotv.eval","v":0.9577020202020202},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T19:33:14.000Z","m":"granite-4-0-1b","oid":"gpqa_diamond-23231d49b72c92e4","rd":"2025-10-28","se":0.01619451031298072,"sid":"oWiE4CniV7k5adL5wDh5r8","sm":"granite-4.0-1b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoWiE4CniV7k5adL5wDh5r8.eval","v":0.2398989898989899},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T18:43:36.000Z","m":"qwen3-5-35b-a3b","oid":"gpqa_diamond-e7c918a9a30b6196","rd":"2026-02-24","se":0.023198629229768078,"sid":"KKm5DbPUgXo88eYr64g4SK","sm":"qwen3.5-35b-a3b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKKm5DbPUgXo88eYr64g4SK.eval","v":0.8118686868686869},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T18:38:38.000Z","m":"qwen3-5-35b-a3b","oid":"gpqa_diamond-f773f5badecadffc","rd":"2026-02-24","se":0.02179018174848238,"sid":"RWjuCuVGvaRd4vDjJJ6aP2","sm":"qwen3.5-35b-a3b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRWjuCuVGvaRd4vDjJJ6aP2.eval","v":0.8345959595959596},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T17:47:35.000Z","m":"qwen3-4b-instruct-2507","oid":"gpqa_diamond-3d5b98cf6002a053","rd":"2025-08-06","se":0.028502193235395257,"sid":"VhQBa9AViRt2UKSNpEAyjN","sm":"qwen3-4b-instruct-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVhQBa9AViRt2UKSNpEAyjN.eval","v":0.4583333333333333},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T17:41:08.000Z","m":"qwen3-4b","oid":"gpqa_diamond-efb2a669f06fc407","rd":"2025-04-29","se":0.026892896738900395,"sid":"SLJyD8yew4TxcmprJfVvuE","sm":"qwen3-4b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSLJyD8yew4TxcmprJfVvuE.eval","v":0.4356060606060606},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T17:40:19.000Z","m":"qwen3-4b","oid":"gpqa_diamond-c59a2ba3fb7964ca","rd":"2025-04-29","se":0.030102242301033352,"sid":"G3dinPvwrci2Nu3m9pby2i","sm":"qwen3-4b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FG3dinPvwrci2Nu3m9pby2i.eval","v":0.5233585858585859},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T17:33:22.000Z","m":"qwen3-32b","oid":"gpqa_diamond-baa4bf088b067851","rd":"2025-04-29","se":0.027890331058144285,"sid":"Ffoxqm7YsZr3FimCGsGAJv","sm":"Qwen3-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFfoxqm7YsZr3FimCGsGAJv.eval","v":0.6571969696969697},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T16:54:57.000Z","m":"qwen3-14b","oid":"gpqa_diamond-6270182bab2376d6","rd":"2025-04-29","se":0.02752462335115392,"sid":"GRQV9Q8gHoHnSEPWofiQuq","sm":"qwen3-14b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGRQV9Q8gHoHnSEPWofiQuq.eval","v":0.5340909090909091},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T16:53:49.000Z","m":"qwen3-14b","oid":"gpqa_diamond-9b9565b89766f05d","rd":"2025-04-29","se":0.02931953778086544,"sid":"XGK9DjJHHVWr2LyT298ooL","sm":"qwen3-14b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXGK9DjJHHVWr2LyT298ooL.eval","v":0.6376262626262627},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T16:47:40.000Z","m":"qwen3-1-7b","oid":"gpqa_diamond-58537efea25b095a","rd":"2025-04-29","se":0.022564006554294688,"sid":"GdfzaHmiZht2DKPZDtQAET","sm":"qwen3-1.7b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGdfzaHmiZht2DKPZDtQAET.eval","v":0.3061868686868687},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T16:47:04.000Z","m":"qwen3-1-7b","oid":"gpqa_diamond-1776300574d1e02c","rd":"2025-04-29","se":0.02748387280745701,"sid":"F3CtjUceBrMi9iYmjULx7J","sm":"qwen3-1.7b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF3CtjUceBrMi9iYmjULx7J.eval","v":0.380050505050505},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T16:15:28.000Z","m":"mistral-small-3-2-2506","oid":"gpqa_diamond-21f605301ca42370","rd":"2025-06-20","se":0.027717443261958633,"sid":"V2CVKXcEdLnEXmppjUEB9M","sm":"mistral-small-3.2-2506","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV2CVKXcEdLnEXmppjUEB9M.eval","v":0.49053030303030304},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T12:37:41.000Z","m":"granite-4-0-350m","oid":"gpqa_diamond-407272c372f734c9","rd":"2025-10-28","se":0.009083571075486867,"sid":"F6xZSehzKjpybEh8JkGwni","sm":"granite-4.0-350m","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF6xZSehzKjpybEh8JkGwni.eval","v":0.11237373737373738},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T08:35:26.000Z","m":"gemma-3-4b-it","oid":"gpqa_diamond-2feae513d8e9773e","rd":"2025-03-12","se":0.018690142324718148,"sid":"McJyXLHBzx5VjAqXjmrrsb","sm":"gemma-3-4b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMcJyXLHBzx5VjAqXjmrrsb.eval","v":0.23232323232323232},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T08:27:48.000Z","m":"gemma-3-1b-it","oid":"gpqa_diamond-31697b7674a2c37b","rd":"2025-03-12","se":0.01521201537793133,"sid":"Pc6CyTEYKbcdDUDKzGZnS9","sm":"gemma-3-1b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPc6CyTEYKbcdDUDKzGZnS9.eval","v":0.1994949494949495},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T08:22:17.000Z","m":"gemma-3-12b-it","oid":"gpqa_diamond-ce0ca169591ebf1a","rd":"2025-03-12","se":0.025254327795483048,"sid":"KfQKr2ibCFMx5xBELoiM3Y","sm":"gemma-3-12b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKfQKr2ibCFMx5xBELoiM3Y.eval","v":0.39457070707070707},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T08:15:19.000Z","m":"deepseek-r1-distill-qwen-32b","oid":"gpqa_diamond-cff4538862bf06b1","rd":"2025-01-20","se":0.027534177016461762,"sid":"dwyb2yzFewGT6w6oFdVMAc","sm":"DeepSeek-R1-Distill-Qwen-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fdwyb2yzFewGT6w6oFdVMAc.eval","v":0.6414141414141414},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T07:52:16.000Z","m":"deepseek-r1-distill-qwen-1-5b","oid":"gpqa_diamond-13e07193f10e6dbf","rd":"2025-01-20","se":0.020592418168039042,"sid":"ESnXA8LqMqhjcsoWQaMLVQ","sm":"DeepSeek-R1-Distill-Qwen-1.5B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FESnXA8LqMqhjcsoWQaMLVQ.eval","v":0.33585858585858586},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T07:48:47.000Z","m":"deepseek-r1-0528-qwen3-8b","oid":"gpqa_diamond-784754fdc4e76acf","rd":"2025-05-28","se":0.010663485573693416,"sid":"dszRh7ELyCEukWnwYnUxMp","sm":"deepseek-r1-0528-qwen3-8b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdszRh7ELyCEukWnwYnUxMp.eval","v":0.0928030303030303},{"b":"gpqa_diamond","d":"2026-08-28","evaluated_at":"2026-08-28T07:20:47.000Z","m":"qwq-32b","oid":"gpqa_diamond-a3a91ee25f6ff993","rd":"2025-03-05","se":0.0281399081227253,"sid":"PUXMxQiRY6XaG6NjwsEDLy","sm":"QwQ-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPUXMxQiRY6XaG6NjwsEDLy.eval","v":0.6534090909090909},{"b":"gpqa_diamond","d":"2026-08-27","evaluated_at":"2026-08-27T16:30:31.000Z","m":"llama-3-1-8b-instruct","oid":"gpqa_diamond-778fa452b34e310e","rd":"2024-07-23","se":0.018583813167150035,"sid":"fxwRTtt8bLV4DNutwtMEox","sm":"Llama-3.1-8B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfxwRTtt8bLV4DNutwtMEox.eval","v":0.26957070707070707},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T14:30:46.000Z","m":"qwen3-5-9b","oid":"gpqa_diamond-c2b6603a42f67354","rd":"2026-02-24","se":0.023888485629911964,"sid":"AczFCvkWR5SmdbV7eXruqj","sm":"qwen3.5-9B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAczFCvkWR5SmdbV7eXruqj.eval","v":0.7891414141414141},{"b":"gpqa_diamond","d":"2026-08-27","evaluated_at":"2026-08-27T14:30:17.000Z","m":"qwen3-5-9b","oid":"gpqa_diamond-0527875996297b58","rd":"2026-02-24","se":0.02365568030958479,"sid":"o6migB5WP27uLr9nUWpCiM","sm":"qwen3.5-9B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo6migB5WP27uLr9nUWpCiM.eval","v":0.7897727272727273},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T14:21:49.000Z","m":"qwen3-8b","oid":"gpqa_diamond-39fa6c7800e3a818","rd":"2025-04-28","se":0.02623286175358367,"sid":"LcttjHHzTyAW6MqNE29xiK","sm":"qwen3-8b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLcttjHHzTyAW6MqNE29xiK.eval","v":0.47474747474747475},{"b":"gpqa_diamond","d":"2026-08-27","evaluated_at":"2026-08-27T14:21:43.000Z","m":"qwen3-8b","oid":"gpqa_diamond-53b9d2b13b3cb738","rd":"2025-04-28","se":0.029327713464985333,"sid":"jRiWmBxtBAyiXiRBRMY8wX","sm":"qwen3-8b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjRiWmBxtBAyiXiRBRMY8wX.eval","v":0.5675505050505051},{"b":"gpqa_diamond","d":"2026-08-27","evaluated_at":"2026-08-27T14:17:43.000Z","m":"gemma-3-27b-it","oid":"gpqa_diamond-dc7f48ccf0975fe4","rd":"2025-03-12","se":0.026982120891168943,"sid":"jdFH2zZdssYKDVJQPKXvNB","sm":"gemma-3-27b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjdFH2zZdssYKDVJQPKXvNB.eval","v":0.4772727272727273},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T14:06:50.000Z","m":"gpt-oss-20b","oid":"gpqa_diamond-d2aecd52c138c1fe","rd":"2025-08-05","se":0.024864059770258008,"sid":"GEwyLqZT93Hp55fdtHrJhm","sm":"gpt-oss-20b_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGEwyLqZT93Hp55fdtHrJhm.eval","v":0.6079545454545454},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T14:05:49.000Z","m":"gpt-oss-20b","oid":"gpqa_diamond-7052948f490a82cb","rd":"2025-08-05","se":0.02502509361882172,"sid":"DCJQMqipndGxwMXe8BDGeg","sm":"gpt-oss-20b_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDCJQMqipndGxwMXe8BDGeg.eval","v":0.5315656565656566},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-26","e":"max","evaluated_at":"2026-08-26T22:26:55.000Z","m":"glm-5-3-flash","oid":"gpqa_diamond-efbeea0b1d858aa0","rd":"2026-08-20","se":0.017183231147411448,"sid":"Jvt5BNr4ERbE9QGgMcEbmT","sm":"glm-5.3-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJvt5BNr4ERbE9QGgMcEbmT.eval","v":0.9015151515151515},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-24","e":"max","evaluated_at":"2026-08-24T19:44:18.000Z","m":"glm-5-3","oid":"gpqa_diamond-11bf10fa22a3fba5","rd":"2026-08-14","se":0.015851578202142853,"sid":"dLDBahYBTGRNjbsXA5pyr3","sm":"glm-5.3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdLDBahYBTGRNjbsXA5pyr3.eval","v":0.9090909090909091},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-18","e":"max","evaluated_at":"2026-08-18T21:03:23.000Z","m":"deepseek-v4-pro-0813","oid":"gpqa_diamond-43762d4d6614c96a","rd":"2026-08-13","se":0.015242573629934338,"sid":"nEJGtXGN588zynPkynZkxz","sm":"deepseek-v4-pro-0813_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnEJGtXGN588zynPkynZkxz.eval","v":0.9166666666666666},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-14","e":"high","evaluated_at":"2026-08-14T18:29:12.000Z","m":"gemini-3-7-flash","oid":"gpqa_diamond-83e3843a50a2f0b0","rd":"2026-08-13","se":0.013456525292246259,"sid":"XyVL3VffJ6UboY97vxpohL","sm":"gemini-3.7-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXyVL3VffJ6UboY97vxpohL.eval","v":0.9482323232323232},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-08-14","e":"xhigh","evaluated_at":"2026-08-14T14:19:47.000Z","m":"inkling-small","oid":"gpqa_diamond-03ffb5f7b543df47","rd":"2026-07-15","se":0.019100012831823398,"sid":"jTHw5SET8usd7emYPjEDjS","sm":"inkling-small_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjTHw5SET8usd7emYPjEDjS.eval","v":0.88510101010101},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-08-14","e":"xhigh","evaluated_at":"2026-08-14T13:47:42.000Z","m":"grok-4-6","oid":"gpqa_diamond-829edc04876f8099","rd":"2026-08-12","se":0.015204299687218694,"sid":"9YJf2nviwjM6WVhw8qeJta","sm":"grok-4.6_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9YJf2nviwjM6WVhw8qeJta.eval","v":0.9318181818181818},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-12","e":"high","evaluated_at":"2026-08-12T19:39:52.000Z","m":"grok-4-6","oid":"gpqa_diamond-abd420716f7e6a01","rd":"2026-08-12","se":0.014480018812911388,"sid":"hnP6Z88x8bPEBRZ2PW6kps","sm":"grok-4.6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhnP6Z88x8bPEBRZ2PW6kps.eval","v":0.9400252525252525},{"b":"gpqa_diamond","d":"2026-08-10","evaluated_at":"2026-08-10T17:46:52.000Z","m":"nemotron-3-ultra","oid":"gpqa_diamond-657c19049459d05e","rd":"2026-06-04","se":0.025190921114603887,"sid":"bpZy8tAbh7UV6A6U7XZtBg","sm":"nemotron-3-ultra","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbpZy8tAbh7UV6A6U7XZtBg.eval","v":0.8535353535353535},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-10","e":"none","evaluated_at":"2026-08-10T17:46:29.000Z","m":"minimax-m3","oid":"gpqa_diamond-04a5734b5486e7ce","rd":"2026-06-01","se":0.027772533334218946,"sid":"KM4G49tQZB8ce67RxYGyMy","sm":"MiniMax-M3_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKM4G49tQZB8ce67RxYGyMy.eval","v":0.8131313131313131},{"b":"gpqa_diamond","d":"2026-08-10","evaluated_at":"2026-08-10T17:46:03.000Z","m":"minimax-m3","oid":"gpqa_diamond-b171b2dbb8160912","rd":"2026-06-01","se":0.020482086775424225,"sid":"hDESmyGFrbZDomjff3rqzD","sm":"MiniMax-M3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhDESmyGFrbZDomjff3rqzD.eval","v":0.9090909090909091},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-10","e":"none","evaluated_at":"2026-08-10T17:45:32.000Z","m":"glm-5-2","oid":"gpqa_diamond-8001344231f28223","rd":"2026-06-16","se":0.03225883512300997,"sid":"YDQJAMgMjLaikPkeVDbAjr","sm":"glm-5.2_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYDQJAMgMjLaikPkeVDbAjr.eval","v":0.7121212121212122},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-10","e":"low","evaluated_at":"2026-08-10T17:45:10.000Z","m":"glm-5-2","oid":"gpqa_diamond-4a00560f452adc28","rd":"2026-06-16","se":0.02325315795194205,"sid":"SB6Qj6rCBDk8jZmFcCTzWc","sm":"glm-5.2_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSB6Qj6rCBDk8jZmFcCTzWc.eval","v":0.8787878787878788},{"b":"gpqa_diamond","d":"2026-08-10","evaluated_at":"2026-08-10T17:44:34.000Z","m":"glm-5-1","oid":"gpqa_diamond-7320275c56bf7f32","rd":"2026-04-07","se":0.021469735576055356,"sid":"CBbeMKRRSeRsXnvSfn7Png","sm":"glm-5.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCBbeMKRRSeRsXnvSfn7Png.eval","v":0.898989898989899},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-08-07","e":"xhigh","evaluated_at":"2026-08-07T01:00:11.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"gpqa_diamond-1869d6241e74dd67","rd":"2026-03-17","se":0.02406315641682249,"sid":"gKB2rCD7mHzFN2MCt2YCni","sm":"gpt-5.4-mini-2026-03-17_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgKB2rCD7mHzFN2MCt2YCni.eval","v":0.8686868686868687},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T01:00:09.000Z","m":"gpt-5-6-sol","oid":"gpqa_diamond-b1571fb7a3ba92be","rd":"2026-07-09","se":0.026869716187429872,"sid":"DDfYPyKQW7vgDtGgkwcKSi","sm":"gpt-5.6-sol_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDDfYPyKQW7vgDtGgkwcKSi.eval","v":0.8282828282828283},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:57:11.000Z","m":"gpt-5-5","oid":"gpqa_diamond-3a3f0f63d29d09de","rd":"2026-04-23","se":0.029857515673386438,"sid":"KesKa5EXtFG7H4UMBFzgCh","sm":"gpt-5.5_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKesKa5EXtFG7H4UMBFzgCh.eval","v":0.7727272727272727},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:54:16.000Z","m":"kimi-k3","oid":"gpqa_diamond-1b0ad52d94340c1c","rd":"2026-07-16","se":0.025545650426603637,"sid":"mvNb3hNgSZ2gS6jHMtvCdk","sm":"kimi-k3_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmvNb3hNgSZ2gS6jHMtvCdk.eval","v":0.8484848484848485},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-07","e":"high","evaluated_at":"2026-08-07T00:53:51.000Z","m":"kimi-k3","oid":"gpqa_diamond-4236506d0ac33740","rd":"2026-07-16","se":0.019417681889724515,"sid":"AoAD5uVtgVDnXmwA7STuno","sm":"kimi-k3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAoAD5uVtgVDnXmwA7STuno.eval","v":0.9191919191919192},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:50:06.000Z","m":"gpt-5-6-terra","oid":"gpqa_diamond-99da14a9fb6323d1","rd":"2026-07-09","se":0.029857515673386438,"sid":"F7v3JtKFb27j2YVmsndLM5","sm":"gpt-5.6-terra_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF7v3JtKFb27j2YVmsndLM5.eval","v":0.7727272727272727},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:46:46.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"gpqa_diamond-1d01952cdad50f24","rd":"2026-03-17","se":0.034169036403915276,"sid":"Qyz2Qt5x4P2XwB3w4h4jFR","sm":"gpt-5.4-mini-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQyz2Qt5x4P2XwB3w4h4jFR.eval","v":0.6414141414141414},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:46:39.000Z","m":"qwen3-6-max-preview","oid":"gpqa_diamond-bf87c9466e77415e","rd":"2026-04-20","se":0.023664359402880177,"sid":"Jv2hJvjpvA8xJW3Bzv7bfw","sm":"qwen3.6-max-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJv2hJvjpvA8xJW3Bzv7bfw.eval","v":0.8737373737373737},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:46:32.000Z","m":"gpt-5-6-sol","oid":"gpqa_diamond-8109704f83d220b3","rd":"2026-07-09","se":0.021469735576055356,"sid":"6cmgkNAQZekBr6e6rVdLe9","sm":"gpt-5.6-sol_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6cmgkNAQZekBr6e6rVdLe9.eval","v":0.898989898989899},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:46:28.000Z","m":"qwen3-7-max","oid":"gpqa_diamond-d2115fa31d27be9c","rd":"2026-05-19","se":0.020482086775424225,"sid":"W6mTanyjvXhasFBzHukv4S","sm":"qwen3.7-max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FW6mTanyjvXhasFBzHukv4S.eval","v":0.9090909090909091},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2026-08-07","e":"medium","evaluated_at":"2026-08-07T00:42:14.000Z","m":"o3-2025-04-16","oid":"gpqa_diamond-9db12ee790d2933d","rd":"2025-04-16","se":0.028057791672989062,"sid":"TBYA7JWneLUCy9kyktFBbt","sm":"o3-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTBYA7JWneLUCy9kyktFBbt.eval","v":0.8080808080808081},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:38:38.000Z","m":"gpt-5-1-2025-11-13","oid":"gpqa_diamond-28f0ff101a7aff57","rd":"2025-11-13","se":0.033586181457325226,"sid":"UT5fVL4EGvSTZBKMTPy9YN","sm":"gpt-5.1-2025-11-13_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUT5fVL4EGvSTZBKMTPy9YN.eval","v":0.6666666666666666},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:38:29.000Z","m":"kimi-k2-7-code","oid":"gpqa_diamond-15e83808219bd26c","rd":"2026-06-12","se":0.02325315795194205,"sid":"XneEig6pACkAWeekvgCwNk","sm":"kimi-k2.7-code","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXneEig6pACkAWeekvgCwNk.eval","v":0.8787878787878788},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:35:08.000Z","m":"gpt-5-6-luna","oid":"gpqa_diamond-d272268b8a997485","rd":"2026-07-09","se":0.03427308652999934,"sid":"bDKKx7zTdW5WwMCfwqsLi4","sm":"gpt-5.6-luna_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbDKKx7zTdW5WwMCfwqsLi4.eval","v":0.6363636363636364},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:35:04.000Z","m":"gpt-5-6-terra","oid":"gpqa_diamond-7fcbd7b4113872c6","rd":"2026-07-09","se":0.023664359402880177,"sid":"JcuoAWhcnngPEZBPmiUyLF","sm":"gpt-5.6-terra_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJcuoAWhcnngPEZBPmiUyLF.eval","v":0.8737373737373737},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:31:39.000Z","m":"qwen3-6-plus","oid":"gpqa_diamond-1f18b5d1e13bf410","rd":"2026-03-31","se":0.022828881775249357,"sid":"f6ZdMtRpdfskGbrBCzrKSn","sm":"qwen3.6-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ff6ZdMtRpdfskGbrBCzrKSn.eval","v":0.8838383838383839},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2026-08-07","e":"medium","evaluated_at":"2026-08-07T00:24:28.000Z","m":"o4-mini-2025-04-16","oid":"gpqa_diamond-6d14a0b14e1df9b4","rd":"2025-04-16","se":0.02962022787479049,"sid":"noAGevebNp5W5yAnmbEYaz","sm":"o4-mini-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnoAGevebNp5W5yAnmbEYaz.eval","v":0.7777777777777778},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:24:27.000Z","m":"qwen3-6-27b","oid":"gpqa_diamond-82484ad82342d4c4","rd":"2026-04-22","se":0.024825909793343353,"sid":"buLf8Ne29xmzMZBRQkvrQG","sm":"qwen3.6-27b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbuLf8Ne29xmzMZBRQkvrQG.eval","v":0.8585858585858586},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:24:17.000Z","m":"qwen3-6-27b","oid":"gpqa_diamond-3701fd3ebd9877df","rd":"2026-04-22","se":0.02554565042660364,"sid":"WeP7Q27N5PzxSgg4ybgijX","sm":"qwen3.6-27b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWeP7Q27N5PzxSgg4ybgijX.eval","v":0.8484848484848485},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:24:06.000Z","m":"qwen3-5-397b-a17b","oid":"gpqa_diamond-bc721c228903a09b","rd":"2026-02-13","se":0.024825909793343353,"sid":"gZMFRbVKnBmcnZU4YqqKMN","sm":"qwen3.5-397b-a17b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgZMFRbVKnBmcnZU4YqqKMN.eval","v":0.8585858585858586},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:24:02.000Z","m":"qwen3-5-397b-a17b","oid":"gpqa_diamond-c5c244f02b2d95e1","rd":"2026-02-13","se":0.02445015597318985,"sid":"bXrmQqzF9sXbuZfRrSc7NP","sm":"qwen3.5-397b-a17b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbXrmQqzF9sXbuZfRrSc7NP.eval","v":0.8636363636363636},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:20:32.000Z","m":"qwen3-5-plus","oid":"gpqa_diamond-a14d2da83471419c","rd":"2026-02-16","se":0.025545650426603637,"sid":"mGXYSUNCMisvaRzhQp6g2L","sm":"qwen3.5-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmGXYSUNCMisvaRzhQp6g2L.eval","v":0.8484848484848485},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-07","e":"minimal","evaluated_at":"2026-08-07T00:17:01.000Z","m":"gemini-3-6-flash","oid":"gpqa_diamond-e3fbbb040163bc47","rd":"2026-07-21","se":0.024825909793343353,"sid":"aPmR8fV49N98a6NVKpASAf","sm":"gemini-3.6-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaPmR8fV49N98a6NVKpASAf.eval","v":0.8585858585858586},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:12:45.000Z","m":"qwen3-6-35b-a3b","oid":"gpqa_diamond-c10e10cef2eeb4d7","rd":"2026-04-14","se":0.02622591986362927,"sid":"EWAyxFsRUvuu4NpmmLqWkn","sm":"qwen3.6-35b-a3b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEWAyxFsRUvuu4NpmmLqWkn.eval","v":0.8383838383838383},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:12:39.000Z","m":"qwen3-6-35b-a3b","oid":"gpqa_diamond-4a02956d03137401","rd":"2026-04-14","se":0.02554565042660364,"sid":"9kErpmgQ42wTvgJaFn478v","sm":"qwen3.6-35b-a3b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9kErpmgQ42wTvgJaFn478v.eval","v":0.8484848484848485},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:11:57.000Z","m":"qwen3-6-flash","oid":"gpqa_diamond-8fe895d78d2a2901","rd":"2026-04-27","se":0.026552207828215258,"sid":"H2FvRXagQRQmiNa7WrNMU2","sm":"qwen3.6-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH2FvRXagQRQmiNa7WrNMU2.eval","v":0.8333333333333334},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:08:31.000Z","m":"gpt-5-6-luna","oid":"gpqa_diamond-3c2eec88a82dbc50","rd":"2026-07-09","se":0.027178752639044908,"sid":"nW39ZzuDQ3xTAGVoEfFG95","sm":"gpt-5.6-luna_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnW39ZzuDQ3xTAGVoEfFG95.eval","v":0.8232323232323232},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:07:25.000Z","m":"qwen3-7-plus","oid":"gpqa_diamond-768a821f63a1eaf3","rd":"2026-06-02","se":0.02325315795194205,"sid":"URoWjw9Htc6QPwABVcrybw","sm":"qwen3.7-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FURoWjw9Htc6QPwABVcrybw.eval","v":0.8787878787878788},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:03:42.000Z","m":"qwen3-7-plus","oid":"gpqa_diamond-43e5857bbe70eb02","rd":"2026-06-02","se":0.027479603010538825,"sid":"BWz8uig4QyqPtke59pM88z","sm":"qwen3.7-plus_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBWz8uig4QyqPtke59pM88z.eval","v":0.8181818181818182},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:03:33.000Z","m":"qwen3-7-flash","oid":"gpqa_diamond-52b1b1508f25ed8b","rd":"2026-07-27","se":0.027178752639044908,"sid":"i6i3gQf3ng2aAipvowPpws","sm":"qwen3.7-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi6i3gQf3ng2aAipvowPpws.eval","v":0.8232323232323232},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:03:20.000Z","m":"qwen3-7-flash","oid":"gpqa_diamond-26d9c9191105a9d5","rd":"2026-07-27","se":0.028057791672989062,"sid":"Yhds8GYkCXWgRQhuLiURua","sm":"qwen3.7-flash_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYhds8GYkCXWgRQhuLiURua.eval","v":0.8080808080808081},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:01:43.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"gpqa_diamond-b5c2006c8cd8013f","rd":"2026-03-17","se":0.035402943770953675,"sid":"gLewKMc7ttCD2moUrgAyYX","sm":"gpt-5.4-nano-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgLewKMc7ttCD2moUrgAyYX.eval","v":0.5555555555555556},{"b":"gpqa_diamond","d":"2026-08-07","evaluated_at":"2026-08-07T00:00:59.000Z","m":"qwen3-5-flash","oid":"gpqa_diamond-3d80a32e30860b74","rd":"2026-02-25","se":0.027178752639044908,"sid":"S7wbot7JR53BAjM8F9gaws","sm":"qwen3.5-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FS7wbot7JR53BAjM8F9gaws.eval","v":0.8232323232323232},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-07","e":"minimal","evaluated_at":"2026-08-07T00:00:44.000Z","m":"gpt-5-mini-2025-08-07","oid":"gpqa_diamond-6a9b0634e8dbd7d7","rd":"2025-08-07","se":0.032087795587867514,"sid":"hYSstHWNUNwYoZdmZpYQqK","sm":"gpt-5-mini-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhYSstHWNUNwYoZdmZpYQqK.eval","v":0.7171717171717171},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:00:23.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"gpqa_diamond-bcc187ca8c98716c","rd":"2026-03-17","se":0.03191178226713548,"sid":"Nq7w5iKALDSuT5oQdn5m8f","sm":"gpt-5.4-nano-2026-03-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNq7w5iKALDSuT5oQdn5m8f.eval","v":0.7222222222222222},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T23:59:38.000Z","m":"gemma-4-31b-it","oid":"gpqa_diamond-6558fb4bb3c49a3d","rd":"2026-04-02","se":0.03053289223393202,"sid":"P5L8qbXo42vTvbU3UvTNjk","sm":"gemma-4-31b-it_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FP5L8qbXo42vTvbU3UvTNjk.eval","v":0.7575757575757576},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T23:59:25.000Z","m":"gemma-4-26b-a4b-it","oid":"gpqa_diamond-bc4078def35688bf","rd":"2026-04-02","se":0.031544498882702825,"sid":"Usx9KaiXvHZU4qrzgc4sLg","sm":"gemma-4-26b-a4b-it_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUsx9KaiXvHZU4qrzgc4sLg.eval","v":0.7323232323232324},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-06","e":"none","evaluated_at":"2026-08-06T23:58:34.000Z","m":"deepseek-v4-pro","oid":"gpqa_diamond-848fc5b12b843409","rd":"2026-04-24","se":0.031544498882702825,"sid":"FtGueFbnFnKBAuLmEEufFg","sm":"deepseek-v4-pro_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFtGueFbnFnKBAuLmEEufFg.eval","v":0.7323232323232324},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T23:58:27.000Z","m":"deepseek-v4-pro","oid":"gpqa_diamond-fcfac29db8f49817","rd":"2026-04-24","se":0.020482086775424225,"sid":"TPdRwHTJtaJYSu2Cxc5Czn","sm":"deepseek-v4-pro_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTPdRwHTJtaJYSu2Cxc5Czn.eval","v":0.9090909090909091},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T23:57:58.000Z","m":"claude-opus-4-6","oid":"gpqa_diamond-5479c3fff9b45119","rd":"2026-02-05","se":0.02282888177524936,"sid":"JcQMgXdZKU54u52B8TGQKp","sm":"claude-opus-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJcQMgXdZKU54u52B8TGQKp.eval","v":0.8838383838383839},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:50:56.000Z","m":"gpt-oss-20b","oid":"gpqa_diamond-8c036080144853f1","rd":"2025-08-05","se":0.03550702465131341,"sid":"hYATFPeUnMq7Std2nLUVAJ","sm":"gpt-oss-20b_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhYATFPeUnMq7Std2nLUVAJ.eval","v":0.4595959595959596},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:36:03.000Z","m":"gemini-3-6-flash","oid":"gpqa_diamond-68f3599caa315c01","rd":"2026-07-21","se":0.02445015597318985,"sid":"JoraLb9y55QXpMg4Rrmr5x","sm":"gemini-3.6-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJoraLb9y55QXpMg4Rrmr5x.eval","v":0.8636363636363636},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:35:12.000Z","m":"gemini-3-5-flash-lite","oid":"gpqa_diamond-d9b5d3b667dfb4c3","rd":"2026-07-21","se":0.030532892233932022,"sid":"dgSJTFQ4UtuQKEhB9ss6K7","sm":"gemini-3.5-flash-lite_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdgSJTFQ4UtuQKEhB9ss6K7.eval","v":0.7575757575757576},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T22:35:11.000Z","m":"gemini-3-5-flash-lite","oid":"gpqa_diamond-bc72be42d7759635","rd":"2026-07-21","se":0.031156269519646826,"sid":"7aAKcnsCXoSumfAu8mr2un","sm":"gemini-3.5-flash-lite_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7aAKcnsCXoSumfAu8mr2un.eval","v":0.7424242424242424},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:35:04.000Z","m":"gemini-3-5-flash-lite","oid":"gpqa_diamond-2ca6821992284cc4","rd":"2026-07-21","se":0.026552207828215258,"sid":"chbh6XstiqtUa5W9zuyfjS","sm":"gemini-3.5-flash-lite_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fchbh6XstiqtUa5W9zuyfjS.eval","v":0.8333333333333334},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:30:55.000Z","m":"gemini-3-5-flash","oid":"gpqa_diamond-9f642b19f19b4371","rd":"2026-05-19","se":0.02239078763821682,"sid":"eiTYduCDNq3HgwTg4ZhEw6","sm":"gemini-3.5-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeiTYduCDNq3HgwTg4ZhEw6.eval","v":0.8888888888888888},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:30:37.000Z","m":"gemini-3-1-pro-preview","oid":"gpqa_diamond-12ffe3294adff3a8","rd":"2026-02-19","se":0.016319950700767385,"sid":"BgmrBFsbsaD8rNpebv49Da","sm":"gemini-3.1-pro-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBgmrBFsbsaD8rNpebv49Da.eval","v":0.9444444444444444},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T22:26:14.000Z","m":"gemini-3-1-flash-lite","oid":"gpqa_diamond-3421b171e4bb2316","rd":"2026-03-03","se":0.031353050095330834,"sid":"CmHy8cVECGkvQTwakfjRo2","sm":"gemini-3.1-flash-lite_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCmHy8cVECGkvQTwakfjRo2.eval","v":0.7373737373737373},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:26:07.000Z","m":"gemini-3-1-flash-lite","oid":"gpqa_diamond-751c941a6a0a14f0","rd":"2026-03-03","se":0.027479603010538825,"sid":"nE4hmJo3QV6X5jeGWAeAze","sm":"gemini-3.1-flash-lite_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnE4hmJo3QV6X5jeGWAeAze.eval","v":0.8181818181818182},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:25:58.000Z","m":"gemini-3-1-flash-lite","oid":"gpqa_diamond-a8be02e6842927c0","rd":"2026-03-03","se":0.03115626951964683,"sid":"99ys2jgQiPQ5UwrqnazfqA","sm":"gemini-3.1-flash-lite_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F99ys2jgQiPQ5UwrqnazfqA.eval","v":0.7424242424242424},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:22:58.000Z","m":"gemini-3-flash-preview","oid":"gpqa_diamond-7cb96c23346d2fbc","rd":"2025-12-17","se":0.021938047738853106,"sid":"gVdraDT3pXmaExP9VbqME4","sm":"gemini-3-flash-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgVdraDT3pXmaExP9VbqME4.eval","v":0.8939393939393939},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:21:47.000Z","m":"claude-sonnet-5","oid":"gpqa_diamond-e275e4b65424fa28","rd":"2026-06-30","se":0.028335609732463334,"sid":"FNXAt76ZUS3sWhQDFMga7U","sm":"claude-sonnet-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFNXAt76ZUS3sWhQDFMga7U.eval","v":0.803030303030303},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:21:09.000Z","m":"claude-sonnet-4-6","oid":"gpqa_diamond-1a4af64a1cba8d2e","rd":"2026-02-17","se":0.029126522834586777,"sid":"SEePRKRcv8oqn5HpD2KFxB","sm":"claude-sonnet-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSEePRKRcv8oqn5HpD2KFxB.eval","v":0.7878787878787878},{"b":"gpqa_diamond","d":"2026-08-06","evaluated_at":"2026-08-06T22:21:07.000Z","m":"claude-opus-5","oid":"gpqa_diamond-22c843d3ab5b1125","rd":"2026-07-24","se":0.018263105420199485,"sid":"mLmUnBuX6DYKtb6x46GteM","sm":"claude-opus-5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmLmUnBuX6DYKtb6x46GteM.eval","v":0.9292929292929293},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:21:06.000Z","m":"claude-opus-5","oid":"gpqa_diamond-3a83d56cc12f678a","rd":"2026-07-24","se":0.02325315795194205,"sid":"WfaJS4L7Xdz5ZBXgJMbRGb","sm":"claude-opus-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWfaJS4L7Xdz5ZBXgJMbRGb.eval","v":0.8787878787878788},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-08-06","e":"none","evaluated_at":"2026-08-06T22:19:27.000Z","m":"claude-opus-4-8","oid":"gpqa_diamond-f4589e04576dfee1","rd":"2026-05-28","se":0.025190921114603887,"sid":"ckbZdkrXPwbFdjn6bmUQsi","sm":"claude-opus-4-8_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FckbZdkrXPwbFdjn6bmUQsi.eval","v":0.8535353535353535},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:19:23.000Z","m":"claude-opus-4-8","oid":"gpqa_diamond-1d0d82d7629c2827","rd":"2026-05-28","se":0.022828881775249357,"sid":"GThtce6GAmRAS8srQyGQoA","sm":"claude-opus-4-8_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGThtce6GAmRAS8srQyGQoA.eval","v":0.8838383838383839},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:19:01.000Z","m":"claude-opus-4-7","oid":"gpqa_diamond-a4680268b9c40feb","rd":"2026-04-16","se":0.02445015597318985,"sid":"Ju7pUkkwYVj6ugi9Ri5Jjp","sm":"claude-opus-4-7_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJu7pUkkwYVj6ugi9Ri5Jjp.eval","v":0.8636363636363636},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:17:56.000Z","m":"claude-fable-5","oid":"gpqa_diamond-dc5c709c957f1ec7","rd":"2026-06-09","se":0.024825909793343353,"sid":"S5pm7ctk9zgPfbyHrbD9AV","sm":"claude-fable-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FS5pm7ctk9zgPfbyHrbD9AV.eval","v":0.8585858585858586},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:17:37.000Z","m":"claude-fable-5","oid":"gpqa_diamond-f6b62e6b92eb5bb0","rd":"2026-06-09","se":0.029126522834586777,"sid":"U5PsUdX2ak4kJ2hgnQkWnX","sm":"claude-fable-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FU5PsUdX2ak4kJ2hgnQkWnX.eval","v":0.7878787878787878},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:17:31.000Z","m":"claude-fable-5","oid":"gpqa_diamond-8e8beff6ad72219e","rd":"2026-06-09","se":0.026552207828215258,"sid":"d23hYvSNGFo7rniLFyBWiu","sm":"claude-fable-5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd23hYvSNGFo7rniLFyBWiu.eval","v":0.8333333333333334},{"b":"gpqa_diamond","d":"2026-08-06","evaluated_at":"2026-08-06T22:17:01.000Z","m":"magistral-small-2506","oid":"gpqa_diamond-dee162de2220c1b7","rd":"2025-06-10","se":0.0353608594752948,"sid":"dwGLob8Vhrwx4Wn376jVmt","sm":"magistral-small-2506","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdwGLob8Vhrwx4Wn376jVmt.eval","v":0.5606060606060606},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-08-05","e":"xhigh","evaluated_at":"2026-08-05T18:55:39.000Z","m":"inkling","oid":"gpqa_diamond-6c2fd6106de0394a","rd":"2026-07-15","se":0.019355903133033585,"sid":"dfmiZmjAcNkLuAJL4z65eb","sm":"Inkling_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdfmiZmjAcNkLuAJL4z65eb.eval","v":0.8825757575757576},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-08-04","e":"xhigh","evaluated_at":"2026-08-04T16:28:46.000Z","m":"qwen3-8-max","oid":"gpqa_diamond-f830df2a0e4bb4e6","rd":"2026-08-02","se":0.01686498021450146,"sid":"TEVNDC9QXBh7PQvX9kHZip","sm":"qwen3.8-max_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTEVNDC9QXBh7PQvX9kHZip.eval","v":0.9267676767676768},{"b":"gpqa_diamond","d":"2026-08-02","evaluated_at":"2026-08-02T02:46:37.000Z","m":"gpt-5-5-instant","oid":"gpqa_diamond-6eb229793225b632","rd":"2026-05-05","se":0.022532782662252126,"sid":"WPVhgi8NT9grAT7kfipT9G","sm":"gpt-5.5-instant","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWPVhgi8NT9grAT7kfipT9G.eval","v":0.8251262626262627},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-08-02","e":"max","evaluated_at":"2026-08-02T00:22:31.000Z","m":"deepseek-v4-flash-0731","oid":"gpqa_diamond-527f503f892269a2","rd":"2026-07-31","se":0.017655466909172534,"sid":"EDFusDaaFkh5n2MqKNxMBD","sm":"deepseek-v4-flash-0731_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEDFusDaaFkh5n2MqKNxMBD.eval","v":0.9103535353535354},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-08-02","e":"high","evaluated_at":"2026-08-02T00:21:27.000Z","m":"gemini-3-6-flash","oid":"gpqa_diamond-656aa4cf86e738e4","rd":"2026-07-21","se":0.014028967942064968,"sid":"86SY5JbPB49xXhHFQW6tHu","sm":"gemini-3.6-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F86SY5JbPB49xXhHFQW6tHu.eval","v":0.9412878787878788},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-07-24","e":"max","evaluated_at":"2026-07-24T18:40:39.000Z","m":"claude-opus-5","oid":"gpqa_diamond-840e70300ff44b55","rd":"2026-07-24","se":0.014782015056089346,"sid":"D3mtasTumfvjR4fjmnHWRy","sm":"claude-opus-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FD3mtasTumfvjR4fjmnHWRy.eval","v":0.9387626262626263},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-20","e":"low","evaluated_at":"2026-07-20T18:29:17.000Z","m":"o1-2024-12-17","oid":"gpqa_diamond-37cf2de03d18e19b","rd":"2024-12-17","se":0.031156269519646826,"sid":"hP6P5e9zuHPToaQTTQC9A2","sm":"o1-2024-12-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhP6P5e9zuHPToaQTTQC9A2.eval","v":0.7424242424242424},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-07-20","e":"minimal","evaluated_at":"2026-07-20T18:29:01.000Z","m":"gpt-5-2025-08-07","oid":"gpqa_diamond-93e7c8747b41bcec","rd":"2025-08-07","se":0.032087795587867514,"sid":"n4wyt7wHjhDdjrGacE9smk","sm":"gpt-5-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn4wyt7wHjhDdjrGacE9smk.eval","v":0.7171717171717171},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-20","e":"low","evaluated_at":"2026-07-20T18:28:59.000Z","m":"o3-mini-2025-01-31","oid":"gpqa_diamond-ec224e58d3676a6c","rd":"2025-01-31","se":0.0331847733384533,"sid":"hJf7Hxg4tgdftQ2QDNHMJA","sm":"o3-mini-2025-01-31_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhJf7Hxg4tgdftQ2QDNHMJA.eval","v":0.6818181818181818},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-07-16","e":"max","evaluated_at":"2026-07-16T21:59:40.000Z","m":"kimi-k3","oid":"gpqa_diamond-d53de97d6a9d1927","rd":"2026-07-16","se":0.014937225364194858,"sid":"B8MaPUe3EYwUZASsNqE7C9","sm":"kimi-k3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FB8MaPUe3EYwUZASsNqE7C9.eval","v":0.9311868686868687},{"b":"gpqa_diamond","d":"2026-07-16","evaluated_at":"2026-07-16T00:10:43.000Z","m":"deepseek-chat","oid":"gpqa_diamond-380047b55d5bff07","rd":"2025-12-01","se":0.03225883512300997,"sid":"6osoAGM4euiAAksriYyLtj","sm":"deepseek-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6osoAGM4euiAAksriYyLtj.eval","v":0.7121212121212122},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-15","e":"low","evaluated_at":"2026-07-15T03:15:08.000Z","m":"o3-2025-04-16","oid":"gpqa_diamond-3707844a5e72a313","rd":"2025-04-16","se":0.028606204289229834,"sid":"AZf6N4kygJGHPsSUkYtCba","sm":"o3-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAZf6N4kygJGHPsSUkYtCba.eval","v":0.797979797979798},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2026-07-15","e":"medium","evaluated_at":"2026-07-15T03:14:58.000Z","m":"gpt-5-4-2026-03-05","oid":"gpqa_diamond-5e9ae010557ac560","rd":"2026-03-05","se":0.02239078763821682,"sid":"ManJk43Kf9VuXowNqoDrov","sm":"gpt-5.4-2026-03-05_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FManJk43Kf9VuXowNqoDrov.eval","v":0.8888888888888888},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-07-15","e":"none","evaluated_at":"2026-07-15T03:14:58.000Z","m":"gpt-5-4-2026-03-05","oid":"gpqa_diamond-6dbdada67862a785","rd":"2026-03-05","se":0.030954055470365872,"sid":"LaBuzZExRqocD4FET6yx26","sm":"gpt-5.4-2026-03-05_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLaBuzZExRqocD4FET6yx26.eval","v":0.7474747474747475},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-07-15","e":"high","evaluated_at":"2026-07-15T03:14:50.000Z","m":"gpt-5-4-2026-03-05","oid":"gpqa_diamond-f05b3e913efc3f7f","rd":"2026-03-05","se":0.021469735576055356,"sid":"FbhcT8NnJUntcyNGubJXY6","sm":"gpt-5.4-2026-03-05_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFbhcT8NnJUntcyNGubJXY6.eval","v":0.898989898989899},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-15","e":"low","evaluated_at":"2026-07-15T03:14:49.000Z","m":"gpt-5-4-2026-03-05","oid":"gpqa_diamond-453148ef6f30af11","rd":"2026-03-05","se":0.025545650426603637,"sid":"aYgDc2iSDpmHkQfu7YjCHc","sm":"gpt-5.4-2026-03-05_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaYgDc2iSDpmHkQfu7YjCHc.eval","v":0.8484848484848485},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-07-15","e":"minimal","evaluated_at":"2026-07-15T03:14:20.000Z","m":"gemini-3-5-flash","oid":"gpqa_diamond-f5709f5c5bd610f5","rd":"2026-05-19","se":0.02445015597318985,"sid":"QDZRoAVaEDtdS6nECEeE35","sm":"gemini-3.5-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQDZRoAVaEDtdS6nECEeE35.eval","v":0.8636363636363636},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-13","e":"low","evaluated_at":"2026-07-13T17:28:21.000Z","m":"gpt-5-nano-2025-08-07","oid":"gpqa_diamond-ea85700c3c30dea1","rd":"2025-08-07","se":0.03521224908841589,"sid":"mNZ34pXP8JN779JoHnjNQc","sm":"gpt-5-nano-2025-08-07_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmNZ34pXP8JN779JoHnjNQc.eval","v":0.5757575757575758},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-07-13","e":"high","evaluated_at":"2026-07-13T17:28:15.000Z","m":"claude-sonnet-4-6","oid":"gpqa_diamond-e8fa603104885621","rd":"2026-02-17","se":0.026552207828215258,"sid":"DmdKPD9Z9RXwYyATAoG4Uw","sm":"claude-sonnet-4-6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDmdKPD9Z9RXwYyATAoG4Uw.eval","v":0.8333333333333334},{"b":"gpqa_diamond","cfg":{"effort":"minimal"},"d":"2026-07-13","e":"minimal","evaluated_at":"2026-07-13T17:28:09.000Z","m":"gpt-5-nano-2025-08-07","oid":"gpqa_diamond-28d65c819a036d64","rd":"2025-08-07","se":0.03560716516531066,"sid":"hp4x7bhBYhkQqZoZFf4dTZ","sm":"gpt-5-nano-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fhp4x7bhBYhkQqZoZFf4dTZ.eval","v":0.48484848484848486},{"b":"gpqa_diamond","cfg":{"effort":"none"},"d":"2026-07-13","e":"none","evaluated_at":"2026-07-13T17:28:07.000Z","m":"gpt-5-2-2025-12-11","oid":"gpqa_diamond-271a041c576f1308","rd":"2025-12-11","se":0.031544498882702825,"sid":"4sPs53wmM9bvN8JXzsiNCk","sm":"gpt-5.2-2025-12-11_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4sPs53wmM9bvN8JXzsiNCk.eval","v":0.7323232323232324},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2026-07-13","e":"medium","evaluated_at":"2026-07-13T17:28:00.000Z","m":"claude-sonnet-4-6","oid":"gpqa_diamond-c58aea6ef05bc778","rd":"2026-02-17","se":0.026552207828215258,"sid":"e8MXnAM8HMF8eEzYtYA7xv","sm":"claude-sonnet-4-6_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fe8MXnAM8HMF8eEzYtYA7xv.eval","v":0.8333333333333334},{"b":"gpqa_diamond","d":"2026-07-13","evaluated_at":"2026-07-13T16:45:48.000Z","m":"grok-4-20-0309-reasoning","oid":"gpqa_diamond-ea47cad5d679e193","rd":"2026-02-17","se":0.01863338951496632,"sid":"STxFZiSPAPScUuENvXBWLc","sm":"grok-4.20-0309-reasoning","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSTxFZiSPAPScUuENvXBWLc.eval","v":0.8933080808080808},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-07-11","e":"low","evaluated_at":"2026-07-11T08:04:01.000Z","m":"o4-mini-2025-04-16","oid":"gpqa_diamond-1274a67c3dbbe995","rd":"2025-04-16","se":0.030746300742124484,"sid":"XwRFKnuHv5qdUp9m5uTAEb","sm":"o4-mini-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXwRFKnuHv5qdUp9m5uTAEb.eval","v":0.7525252525252525},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T17:07:54.000Z","m":"gpt-5-6-terra","oid":"gpqa_diamond-2369923739a2278c","rd":"2026-07-09","se":0.015363672147634115,"sid":"4bLcFNu5zF6YuZf3qVKoHE","sm":"gpt-5.6-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4bLcFNu5zF6YuZf3qVKoHE.eval","v":0.9330808080808081},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T13:36:49.000Z","m":"gpt-5-6-luna","oid":"gpqa_diamond-175cd9ea3f908ef0","rd":"2026-07-09","se":0.01728284355494035,"sid":"LWkDtkmd3TNmLzMhfZD5HW","sm":"gpt-5.6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLWkDtkmd3TNmLzMhfZD5HW.eval","v":0.9160353535353535},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T02:07:13.000Z","m":"gpt-5-6-sol","oid":"gpqa_diamond-2cf4e36a8d004049","rd":"2026-07-09","se":0.01570109070867593,"sid":"nuKCgZReCBsXKCkNj9bK37","sm":"gpt-5.6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnuKCgZReCBsXKCkNj9bK37.eval","v":0.9349747474747475},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-07-08","e":"high","evaluated_at":"2026-07-08T23:54:22.000Z","m":"grok-4-5","oid":"gpqa_diamond-eafc349e10ed0a89","rd":"2026-07-08","se":0.014340395485738813,"sid":"9kSgmUd9vh4CBXz3tJhctN","sm":"grok-4.5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9kSgmUd9vh4CBXz3tJhctN.eval","v":0.9343434343434344},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-07-01","e":"xhigh","evaluated_at":"2026-07-01T01:22:14.000Z","m":"claude-sonnet-5","oid":"gpqa_diamond-95d54b898ca9ac52","rd":"2026-06-30","se":0.01783832131341021,"sid":"HhiYxEx3Bf8YDPUh34K4DC","sm":"claude-sonnet-5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHhiYxEx3Bf8YDPUh34K4DC.eval","v":0.9053030303030303},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-06-24","e":"max","evaluated_at":"2026-06-24T20:55:39.000Z","m":"glm-5-2","oid":"gpqa_diamond-a3272556520d8697","rd":"2026-06-16","se":0.016123207533687733,"sid":"c5kpB4axzmygRywWqBKFmv","sm":"glm-5.2_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fc5kpB4axzmygRywWqBKFmv.eval","v":0.9185606060606061},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-06-17","e":"high","evaluated_at":"2026-06-17T16:44:33.000Z","m":"grok-4-3","oid":"gpqa_diamond-5adbbb8a027efa78","rd":"2026-04-17","se":0.01963840420048217,"sid":"Vw7spGUnJyf3vuzSjTcj2F","sm":"grok-4.3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVw7spGUnJyf3vuzSjTcj2F.eval","v":0.8882575757575758},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-06-16","e":"max","evaluated_at":"2026-06-16T20:16:55.000Z","m":"deepseek-v4-pro","oid":"gpqa_diamond-9bb2d8f69284f53d","rd":"2026-04-24","se":0.017473014662966007,"sid":"LuZUnhD5rBLjRw36DaqfvZ","sm":"deepseek-v4-pro_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLuZUnhD5rBLjRw36DaqfvZ.eval","v":0.8964646464646465},{"b":"gpqa_diamond","cfg":{"effort":"max"},"d":"2026-06-07","e":"max","evaluated_at":"2026-06-07T23:03:11.000Z","m":"claude-opus-4-8","oid":"gpqa_diamond-5818f1a107424888","rd":"2026-05-28","se":0.019157125794612026,"sid":"dqgqP2uEPswfECCVj2Kbr9","sm":"claude-opus-4-8_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdqgqP2uEPswfECCVj2Kbr9.eval","v":0.9103535353535354},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-05-22","e":"high","evaluated_at":"2026-05-22T16:02:44.000Z","m":"gemini-3-5-flash","oid":"gpqa_diamond-2553e6825fbdb26c","rd":"2026-05-19","se":0.016387002835275905,"sid":"dYYtcwENFRtm9jZTDA2p9C","sm":"gemini-3.5-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdYYtcwENFRtm9jZTDA2p9C.eval","v":0.928030303030303},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2026-05-05","e":"low","evaluated_at":"2026-05-05T18:00:18.624Z","m":"gpt-5-5","oid":"gpqa_diamond-dc11ccdf02a9caaf","rd":"2026-04-23","se":0.017532658377721708,"sid":"2CdomqtBVz2it7a2Zx2G2t","sm":"gpt-5.5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2CdomqtBVz2it7a2Zx2G2t.eval","v":0.9065656565656566},{"b":"gpqa_diamond","d":"2026-05-01","evaluated_at":"2026-05-01T17:25:36.000Z","m":"kimi-k2-6","oid":"gpqa_diamond-982160ea67c6d94b","rd":"2026-04-20","se":0.0171745162866666,"sid":"LjgJoDosTLkiGVT59smA7q","sm":"kimi-k2.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLjgJoDosTLkiGVT59smA7q.eval","v":0.9078282828282829},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-04-24","e":"xhigh","evaluated_at":"2026-04-24T16:01:38.000Z","m":"gpt-5-5-pro-pre-release","oid":"gpqa_diamond-bbb7566e31109743","rd":"2026-04-23","se":0.015832823810775415,"sid":"cBmhJzrC3SkfZ7uVgGVYZX","sm":"gpt-5.5-pro-pre-release_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcBmhJzrC3SkfZ7uVgGVYZX.eval","v":0.9392135642135643},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-04-24","e":"xhigh","evaluated_at":"2026-04-24T13:35:12.000Z","m":"gpt-5-5-pre-release","oid":"gpqa_diamond-efc624efc1c81c62","rd":"2026-04-23","se":0.015469771177277509,"sid":"FUv4EUwR78nAsdGbiwiLEr","sm":"gpt-5.5-pre-release_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFUv4EUwR78nAsdGbiwiLEr.eval","v":0.9400252525252525},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-04-17","e":"xhigh","evaluated_at":"2026-04-17T16:16:45.000Z","m":"claude-opus-4-7","oid":"gpqa_diamond-6861ae555e3deaba","rd":"2026-04-16","se":0.018464275887293338,"sid":"VJezHrJ5YPCbpui8pZgcJg","sm":"claude-opus-4-7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVJezHrJ5YPCbpui8pZgcJg.eval","v":0.9015151515151515},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T14:03:19.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"gpqa_diamond-98e338b3427e9dfd","rd":"2026-03-17","se":0.022274469189069883,"sid":"AQF46S9hBhosKPEMy3fyB9","sm":"gpt-5.4-mini-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAQF46S9hBhosKPEMy3fyB9.eval","v":0.8358585858585859},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2026-04-14","e":"high","evaluated_at":"2026-04-14T15:07:20.513Z","m":"gpt-5-4-nano-2026-03-17","oid":"gpqa_diamond-a5bbfc257fe854e4","rd":"2026-03-17","se":0.0244,"sid":"fZB3qGSohAVYzN97mniuUp","sm":"gpt-5.4-nano-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfZB3qGSohAVYzN97mniuUp.eval","v":0.7847},{"b":"gpqa_diamond","d":"2026-04-08","evaluated_at":"2026-04-08T16:29:51.099Z","m":"muse-spark","oid":"gpqa_diamond-38ddb04b68150de6","rd":"2026-04-08","se":0.022,"sid":"muse-spark-pre-release-gpqa","sm":"muse-spark","v":0.898},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-03-20","e":"xhigh","evaluated_at":"2026-03-20T00:19:39.820Z","m":"gpt-5-4-pro-2026-03-05","oid":"gpqa_diamond-0a78f3a733e07bbd","rd":"2026-03-05","se":0.016,"sid":"gpt-5-4-pro-gpqa-cloud-run","sm":"gpt-5.4-pro-2026-03-05_xhigh","v":0.946},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2026-03-06","e":"xhigh","evaluated_at":"2026-03-06T02:57:02.864Z","m":"gpt-5-4-2026-03-05","oid":"gpqa_diamond-97f3407002609c13","rd":"2026-03-05","se":0.018,"sid":"fFatyce8UvpN7ZivdmrhAy","sm":"gpt-5.4-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfFatyce8UvpN7ZivdmrhAy.eval","v":0.933},{"b":"gpqa_diamond","d":"2026-02-20","evaluated_at":"2026-02-20T17:51:37.062Z","m":"claude-sonnet-4-6-32k","oid":"gpqa_diamond-4b1a3dcbf3c5b75a","rd":"2026-02-17","se":0.020034282226331888,"sid":"58eQmyCfPa3FufhXJFvAL2","sm":"claude-sonnet-4-6_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F58eQmyCfPa3FufhXJFvAL2.eval","v":0.8737373737373737},{"b":"gpqa_diamond","d":"2026-02-20","evaluated_at":"2026-02-20T05:01:02.541Z","m":"gemini-3-1-pro-preview","oid":"gpqa_diamond-5e836b143017d26f","rd":"2026-02-19","se":0.017,"sid":"kR9F3oHBCnwQW4AiTPKwpd","sm":"gemini-3.1-pro-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkR9F3oHBCnwQW4AiTPKwpd.eval","v":0.941},{"b":"gpqa_diamond","d":"2026-02-12","evaluated_at":"2026-02-12T17:19:26.257Z","m":"glm-5","oid":"gpqa_diamond-ee8a6a33431a4d94","rd":"2026-02-11","se":0.02336331210092387,"sid":"n4hyz75SVpwPEisKVVjvNg","sm":"glm-5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn4hyz75SVpwPEisKVVjvNg.eval","v":0.8781725888324873},{"b":"gpqa_diamond","d":"2026-02-06","evaluated_at":"2026-02-06T02:17:07.741Z","m":"claude-opus-4-6-32k","oid":"gpqa_diamond-5d119c2d7b106add","rd":"2026-02-05","se":0.016916245311450143,"sid":"DiXCMhdfhcjaCUSzYWgACy","sm":"claude-opus-4-6_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDiXCMhdfhcjaCUSzYWgACy.eval","v":0.9053030303030303},{"b":"gpqa_diamond","d":"2026-02-06","evaluated_at":"2026-02-06T02:17:05.633Z","m":"claude-opus-4-6-64k","oid":"gpqa_diamond-c1b12c13a20ff641","rd":"2026-02-05","se":0.019363218328924313,"sid":"eB9e2mtd5nRwqdFyLEsz6k","sm":"claude-opus-4-6_64K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeB9e2mtd5nRwqdFyLEsz6k.eval","v":0.8876262626262627},{"b":"gpqa_diamond","d":"2026-02-02","evaluated_at":"2026-02-02T11:01:01.260Z","m":"fireworkskimi-k2p5","oid":"gpqa_diamond-9b9e9fa05869a0ff","rd":"2026-01-27","se":0.019,"sid":"FmqGUq87N4JcuSbo6Uu3U5","sm":"fireworks/kimi-k2p5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFmqGUq87N4JcuSbo6Uu3U5.eval","v":0.876},{"b":"gpqa_diamond","d":"2026-01-29","evaluated_at":"2026-01-29T20:49:26.702Z","m":"glm-4-7","oid":"gpqa_diamond-de936ea1e5e06fff","rd":"2025-12-22","se":0.02374901666263655,"sid":"6phXzd4aCbnBUSQmBsaywa","sm":"glm-4.7","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6phXzd4aCbnBUSQmBsaywa.eval","v":0.8333333333333334},{"b":"gpqa_diamond","d":"2025-12-17","evaluated_at":"2025-12-17T22:39:00.278Z","m":"gemini-3-flash-preview","oid":"gpqa_diamond-66a8ddedde3d4f1e","rd":"2025-12-17","se":0.019036353018587762,"sid":"oAu9CZSNPZgwiwKnoNpM8Q","sm":"gemini-3-flash-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoAu9CZSNPZgwiwKnoNpM8Q.eval","v":0.8320707070707071},{"b":"gpqa_diamond","d":"2025-12-16","evaluated_at":"2025-12-16T14:32:07.252Z","m":"deepseek-reasoner","oid":"gpqa_diamond-617aa5dbd24a772e","rd":"2025-12-01","se":0.0201537212608744,"sid":"CDdjbqiYYPtEBGoZLMPM3n","sm":"deepseek-reasoner","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCDdjbqiYYPtEBGoZLMPM3n.eval","v":0.8342352092352093},{"b":"gpqa_diamond","cfg":{"effort":"xhigh"},"d":"2025-12-13","e":"xhigh","evaluated_at":"2025-12-13T09:04:00.000Z","m":"gpt-5-2-2025-12-11","oid":"gpqa_diamond-f08a16128e26f8fc","rd":"2025-12-11","se":0.018,"sid":"Ejo6Fy7XYxUDcCwbejfcYX","sm":"gpt-5.2-2025-12-11_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEjo6Fy7XYxUDcCwbejfcYX.eval","v":0.914},{"b":"gpqa_diamond","d":"2025-12-11","evaluated_at":"2025-12-11T23:00:25.608Z","m":"qwen3-235b-a22b-thinking-2507","oid":"gpqa_diamond-a9aeed3c3faf2bdd","rd":"2025-07-25","se":0.026000933515631548,"sid":"gu39ndv9QxWP3NRxSUJRaX","sm":"Qwen3-235B-A22B-Thinking-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fgu39ndv9QxWP3NRxSUJRaX.eval","v":0.8005050505050505},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T22:37:44.523Z","m":"openaigpt-oss-120b","oid":"gpqa_diamond-70cc2c56100f0647","rd":"2025-08-05","se":0.027437163443883767,"sid":"LC5Rhog6wt2PT84CtJ483h","sm":"openai/gpt-oss-120b_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLC5Rhog6wt2PT84CtJ483h.eval","v":0.7575757575757576},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T20:49:22.808Z","m":"gpt-5-2-2025-12-11","oid":"gpqa_diamond-976ce4d0e450abc5","rd":"2025-12-11","se":0.01913858277633924,"sid":"3Q6osT8EYezayG7yfxkSpr","sm":"gpt-5.2-2025-12-11_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3Q6osT8EYezayG7yfxkSpr.eval","v":0.8819444444444444},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-12-11","e":"medium","evaluated_at":"2025-12-11T20:49:17.803Z","m":"gpt-5-2-2025-12-11","oid":"gpqa_diamond-c40679a0af821ca0","rd":"2025-12-11","se":0.019090689343231743,"sid":"MuhtxxvALeZZ37cSG5CVcc","sm":"gpt-5.2-2025-12-11_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMuhtxxvALeZZ37cSG5CVcc.eval","v":0.8787878787878788},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2025-12-11","e":"low","evaluated_at":"2025-12-11T20:25:14.724Z","m":"gpt-5-2-2025-12-11","oid":"gpqa_diamond-abaf2195f5f36132","rd":"2025-12-11","se":0.02265112408975899,"sid":"oD4Jpoi75KWChqTPDaN9aB","sm":"gpt-5.2-2025-12-11_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoD4Jpoi75KWChqTPDaN9aB.eval","v":0.827020202020202},{"b":"gpqa_diamond","d":"2025-11-25","evaluated_at":"2025-11-25T01:10:45.648Z","m":"claude-opus-4-5-20251101-16k","oid":"gpqa_diamond-1ebdfc35bbf71c6c","rd":"2025-11-24","se":0.021508147720723254,"sid":"cXkPywkhy3NnW992ri73yG","sm":"claude-opus-4-5-20251101_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcXkPywkhy3NnW992ri73yG.eval","v":0.8547979797979798},{"b":"gpqa_diamond","d":"2025-11-24","evaluated_at":"2025-11-24T20:14:34.565Z","m":"claude-opus-4-5-20251101-32k","oid":"gpqa_diamond-e2f4d61913caa0be","rd":"2025-11-24","se":0.020686959671393914,"sid":"Kwwo3UFcPDk6iLAeSeeRaa","sm":"claude-opus-4-5-20251101_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKwwo3UFcPDk6iLAeSeeRaa.eval","v":0.860479797979798},{"b":"gpqa_diamond","d":"2025-11-24","evaluated_at":"2025-11-24T19:57:20.151Z","m":"claude-opus-4-5-2025-11-01","oid":"gpqa_diamond-0e0dcd6a7767dacd","rd":"2025-11-24","se":0.023909479653766753,"sid":"mEkZaYpiurQ6TEMAALK3su","sm":"claude-opus-4-5-20251101","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmEkZaYpiurQ6TEMAALK3su.eval","v":0.8068181818181818},{"b":"gpqa_diamond","d":"2025-11-19","evaluated_at":"2025-11-19T15:16:10.279Z","m":"gemini-3-pro-preview","oid":"gpqa_diamond-ba74c3e3af721120","rd":"2025-11-18","se":0.016526963574741,"sid":"Cg54udQGcTCnfDGX2qePLZ","sm":"gemini-3-pro-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCg54udQGcTCnfDGX2qePLZ.eval","v":0.9261363636363636},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-11-17","e":"medium","evaluated_at":"2025-11-17T21:28:29.934Z","m":"gpt-5-1-2025-11-13","oid":"gpqa_diamond-32afc0299a077c2c","rd":"2025-11-13","se":0.021250422082249816,"sid":"V3YQtSEnQ3TPZEPPgkCKQM","sm":"gpt-5.1-2025-11-13_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FV3YQtSEnQ3TPZEPPgkCKQM.eval","v":0.8503787878787878},{"b":"gpqa_diamond","d":"2025-11-16","evaluated_at":"2025-11-16T19:00:08.124Z","m":"gemini-2-5-pro","oid":"gpqa_diamond-e85dd2b67a532f7e","rd":"2025-06-17","se":0.02117010842348584,"sid":"cqUZ6gcq2HBnShFRWvKjGr","sm":"gemini-2.5-pro","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcqUZ6gcq2HBnShFRWvKjGr.eval","v":0.8529040404040404},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T19:52:15.039Z","m":"gpt-5-1-2025-11-13","oid":"gpqa_diamond-23b9c78a436eaee5","rd":"2025-11-13","se":0.01875368281077595,"sid":"CSVtMksEVjBNsGJGjW4GcQ","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCSVtMksEVjBNsGJGjW4GcQ.eval","v":0.8762626262626263},{"b":"gpqa_diamond","d":"2025-11-11","evaluated_at":"2025-11-11T09:27:01.300Z","m":"kimi-k2-thinking-turbo","oid":"gpqa_diamond-bb5bbbe2243d19f6","rd":"2025-11-06","se":0.020879845731985747,"sid":"muGecFpAg9wHd6gexWJ99L","sm":"kimi-k2-thinking-turbo","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmuGecFpAg9wHd6gexWJ99L.eval","v":0.8421717171717171},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T16:20:27.447Z","m":"gpt-5-nano-2025-08-07","oid":"gpqa_diamond-a88e45d23c2521fe","rd":"2025-08-07","se":0.027594508998251045,"sid":"2pSDjMSGDh4HsAxkqUuNQP","sm":"gpt-5-nano-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2pSDjMSGDh4HsAxkqUuNQP.eval","v":0.6944444444444444},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T09:59:56.459Z","m":"gpt-5-mini-2025-08-07","oid":"gpqa_diamond-f6005cd296a93965","rd":"2025-08-07","se":0.022767295427113803,"sid":"jP8EyLtPFsakgnsHe2oJM6","sm":"gpt-5-mini-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjP8EyLtPFsakgnsHe2oJM6.eval","v":0.75},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-10-29","e":"high","evaluated_at":"2025-10-29T22:29:36.692Z","m":"gpt-5-2025-08-07","oid":"gpqa_diamond-c94dc61e4fb994d5","rd":"2025-08-07","se":0.020787836099851995,"sid":"9nPyNrZxwtoT7eS4DZbQF6","sm":"gpt-5-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9nPyNrZxwtoT7eS4DZbQF6.eval","v":0.8617424242424242},{"b":"gpqa_diamond","d":"2025-10-28","evaluated_at":"2025-10-28T13:46:39.393Z","m":"claude-sonnet-4-5-20250929-16k","oid":"gpqa_diamond-3d847213f35d36ee","rd":"2025-09-29","se":0.02912652283458678,"sid":"LkVuUxvsHNzpcg3sUiH7f3","sm":"claude-sonnet-4-5-20250929_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLkVuUxvsHNzpcg3sUiH7f3.eval","v":0.7878787878787878},{"b":"gpqa_diamond","d":"2025-10-28","evaluated_at":"2025-10-28T13:46:37.314Z","m":"claude-sonnet-4-5-20250929-59k","oid":"gpqa_diamond-51c571dcce076219","rd":"2025-09-29","se":0.027178752639044908,"sid":"XaGuWDxPtqVvyxqdMncKFj","sm":"claude-sonnet-4-5-20250929_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXaGuWDxPtqVvyxqdMncKFj.eval","v":0.8232323232323232},{"b":"gpqa_diamond","d":"2025-10-23","evaluated_at":"2025-10-23T12:16:00.615Z","m":"gpt-4-0314","oid":"gpqa_diamond-d9fd3c51c0301f31","rd":"2023-03-14","se":0.02363099375599328,"sid":"FkbLkW5ed8rnZGFJMevDFn","sm":"gpt-4-0314","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFkbLkW5ed8rnZGFJMevDFn.eval","v":0.3573232323232323},{"b":"gpqa_diamond","d":"2025-10-22","evaluated_at":"2025-10-22T09:43:52.881Z","m":"claude-haiku-4-5-20251001-32k","oid":"gpqa_diamond-a8bd4fe963aabfa5","rd":"2025-10-15","se":0.03225883512300997,"sid":"GEaAdb9UmoUFbEtBx7WzAR","sm":"claude-haiku-4-5-20251001_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGEaAdb9UmoUFbEtBx7WzAR.eval","v":0.7121212121212122},{"b":"gpqa_diamond","d":"2025-10-21","evaluated_at":"2025-10-21T15:15:58.761Z","m":"claude-sonnet-4-5-20250929-32k","oid":"gpqa_diamond-1ed8257443bf763f","rd":"2025-09-29","se":0.027603867020557334,"sid":"ii6JE9j57eSbuswEsJArTa","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fii6JE9j57eSbuswEsJArTa.eval","v":0.817258883248731},{"b":"gpqa_diamond","d":"2025-10-16","evaluated_at":"2025-10-16T11:58:17.749Z","m":"claude-haiku-4-5-2025-10-01","oid":"gpqa_diamond-7c3a3326d3026004","rd":"2025-10-15","se":0.028154535023020826,"sid":"cgnopvpB6mzsfb9yGnYK2z","sm":"claude-haiku-4-5-20251001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcgnopvpB6mzsfb9yGnYK2z.eval","v":0.6047979797979798},{"b":"gpqa_diamond","d":"2025-10-06","evaluated_at":"2025-10-06T16:27:30.400Z","m":"qwen3-max-2025-09-23","oid":"gpqa_diamond-2578069738bb58eb","rd":"2025-09-24","se":0.02734528103171235,"sid":"46RCssznDADYeg8MuSgvsj","sm":"qwen3-max-2025-09-23","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F46RCssznDADYeg8MuSgvsj.eval","v":0.726010101010101},{"b":"gpqa_diamond","d":"2025-09-29","evaluated_at":"2025-09-29T18:04:57.033Z","m":"claude-sonnet-4-5-2025-09-29","oid":"gpqa_diamond-e2c6f84e97f77372","rd":"2025-09-29","se":0.031353050095330834,"sid":"fsZmzw8jjyUgMQGwxodBwN","sm":"claude-sonnet-4-5-20250929","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfsZmzw8jjyUgMQGwxodBwN.eval","v":0.7373737373737373},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T19:34:31.180Z","m":"gpt-5-mini-2025-08-07","oid":"gpqa_diamond-8918a646e93c2a60","rd":"2025-08-07","se":0.023192741878083397,"sid":"CFNzodoTSVVVgp9PUXftUP","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCFNzodoTSVVVgp9PUXftUP.eval","v":0.7165404040404041},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T19:34:30.433Z","m":"gpt-5-2025-08-07","oid":"gpqa_diamond-d375801ed459d1da","rd":"2025-08-07","se":0.021314825297984473,"sid":"YQCwJRoGbGAqNCQoaL6cjk","sm":"gpt-5-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYQCwJRoGbGAqNCQoaL6cjk.eval","v":0.8535353535353535},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T19:34:29.562Z","m":"gpt-5-nano-2025-08-07","oid":"gpqa_diamond-fa364c79e95dda37","rd":"2025-08-07","se":0.026607011205052013,"sid":"ELAxkvAigvEVdMmhxiaMiy","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FELAxkvAigvEVdMmhxiaMiy.eval","v":0.6742424242424242},{"b":"gpqa_diamond","d":"2025-08-05","evaluated_at":"2025-08-05T16:53:28.370Z","m":"claude-opus-4-1-20250805-27k","oid":"gpqa_diamond-254f397c925b0c30","rd":"2025-08-05","se":0.030088629490217445,"sid":"3obcsGCfuwYeEm9wRyqzPg","sm":"claude-opus-4-1-20250805_27K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3obcsGCfuwYeEm9wRyqzPg.eval","v":0.7676767676767676},{"b":"gpqa_diamond","d":"2025-08-05","evaluated_at":"2025-08-05T16:51:47.698Z","m":"claude-opus-4-1-20250805-16k","oid":"gpqa_diamond-5a6b552625775b3a","rd":"2025-08-05","se":0.029857515673386438,"sid":"erLgyKQs2mg2memE9SJvK5","sm":"claude-opus-4-1-20250805_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FerLgyKQs2mg2memE9SJvK5.eval","v":0.7727272727272727},{"b":"gpqa_diamond","d":"2025-08-05","evaluated_at":"2025-08-05T16:45:23.948Z","m":"claude-opus-4-1-2025-08-05","oid":"gpqa_diamond-9ae3aaf897d71358","rd":"2025-08-05","se":0.031544498882702825,"sid":"j62bkFDjBSeFyVpnGzuzg4","sm":"claude-opus-4-1-20250805","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fj62bkFDjBSeFyVpnGzuzg4.eval","v":0.7323232323232324},{"b":"gpqa_diamond","d":"2025-06-05","evaluated_at":"2025-06-05T16:07:50.021Z","m":"gemini-2-5-pro-preview-06-05","oid":"gpqa_diamond-fca663d95ace07f1","rd":"2025-06-05","se":0.02554565042660364,"sid":"Q4qTzHnjLyiCCRvNpXioUq","sm":"gemini-2.5-pro-preview-06-05","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQ4qTzHnjLyiCCRvNpXioUq.eval","v":0.8484848484848485},{"b":"gpqa_diamond","d":"2025-06-03","evaluated_at":"2025-06-03T11:52:52.449Z","m":"qwen3-235b-a22b","oid":"gpqa_diamond-19ebbc20c5011a91","rd":"2025-04-29","se":0.027124209393506977,"sid":"agebmxGseTvmvN4AHN8i4m","sm":"qwen3-235b-a22b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FagebmxGseTvmvN4AHN8i4m.eval","v":0.7070707070707071},{"b":"gpqa_diamond","d":"2025-06-03","evaluated_at":"2025-06-03T09:10:11.019Z","m":"gemini-2-5-pro-preview-05-06","oid":"gpqa_diamond-1e2ac20e13d18695","rd":"2025-05-06","se":0.033586181457325226,"sid":"TT66gGiVoeYLGQMCESPrEG","sm":"gemini-2.5-pro-preview-05-06","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTT66gGiVoeYLGQMCESPrEG.eval","v":0.6666666666666666},{"b":"gpqa_diamond","d":"2025-05-29","evaluated_at":"2025-05-29T13:30:43.463Z","m":"deepseek-r1-0528","oid":"gpqa_diamond-163b167692d4c094","rd":"2025-05-28","se":0.024023818843083005,"sid":"LiRiryJsGb3h5UuxX8bmJv","sm":"DeepSeek-R1-0528","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLiRiryJsGb3h5UuxX8bmJv.eval","v":0.7632575757575758},{"b":"gpqa_diamond","d":"2025-05-26","evaluated_at":"2025-05-26T22:53:45.927Z","m":"claude-3-7-sonnet-20250219-64k","oid":"gpqa_diamond-3c8f435a5b5e6015","rd":"2025-02-24","se":0.026506883123060064,"sid":"2t26VSNdsAE5pefQEMP25x","sm":"claude-3-7-sonnet-20250219_64K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2t26VSNdsAE5pefQEMP25x.eval","v":0.7850378787878788},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-05-26","e":"high","evaluated_at":"2025-05-26T18:38:04.231Z","m":"grok-3-mini-beta","oid":"gpqa_diamond-ac677fb1ce50550e","rd":"2025-04-09","se":0.027675521820041432,"sid":"JFZMWFVGNi5R5TdZrvQUFw","sm":"grok-3-mini-beta_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJFZMWFVGNi5R5TdZrvQUFw.eval","v":0.7462121212121212},{"b":"gpqa_diamond","d":"2025-05-26","evaluated_at":"2025-05-26T18:38:03.702Z","m":"deepseek-r1","oid":"gpqa_diamond-de126cdf5388c629","rd":"2025-01-20","se":0.03070485693321271,"sid":"fFGyQR2mfDufvrSV2rzWyV","sm":"DeepSeek-R1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfFGyQR2mfDufvrSV2rzWyV.eval","v":0.6922348484848484},{"b":"gpqa_diamond","d":"2025-05-26","evaluated_at":"2025-05-26T18:38:03.090Z","m":"claude-sonnet-4-20250514-59k","oid":"gpqa_diamond-0b87b6dfc777408c","rd":"2025-05-22","se":0.02674821434321096,"sid":"J9wjcQqKJ7hCSYnnoa5E9i","sm":"claude-sonnet-4-20250514_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ9wjcQqKJ7hCSYnnoa5E9i.eval","v":0.7781991873045173},{"b":"gpqa_diamond","d":"2025-05-26","evaluated_at":"2025-05-26T17:42:41.696Z","m":"grok-3-beta","oid":"gpqa_diamond-9dce99c254874e4e","rd":"2025-04-09","se":0.026863078746340353,"sid":"FQ8Ykq4JWRSXnyF9rbmLc7","sm":"grok-3-beta","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFQ8Ykq4JWRSXnyF9rbmLc7.eval","v":0.6758207070707071},{"b":"gpqa_diamond","d":"2025-05-22","evaluated_at":"2025-05-22T23:06:11.598Z","m":"claude-sonnet-4-20250514-32k","oid":"gpqa_diamond-1a83e9398b24e5a0","rd":"2025-05-22","se":0.02937661648494561,"sid":"8hGbWtbpUjAQpebkKQfSi7","sm":"claude-sonnet-4-20250514_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8hGbWtbpUjAQpebkKQfSi7.eval","v":0.7828282828282829},{"b":"gpqa_diamond","d":"2025-05-22","evaluated_at":"2025-05-22T22:54:06.726Z","m":"claude-opus-4-20250514-16k","oid":"gpqa_diamond-2b1f0d283018ff0c","rd":"2025-05-22","se":0.03031371053819892,"sid":"CtxFusPxCraL3EeRJu2c5k","sm":"claude-opus-4-20250514_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCtxFusPxCraL3EeRJu2c5k.eval","v":0.7626262626262627},{"b":"gpqa_diamond","d":"2025-05-22","evaluated_at":"2025-05-22T22:54:05.316Z","m":"claude-sonnet-4-20250514-16k","oid":"gpqa_diamond-f6eefa022e7accb9","rd":"2025-05-22","se":0.030532892233932022,"sid":"PPLoBnbKQ9CEYZNSk3E7LE","sm":"claude-sonnet-4-20250514_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPPLoBnbKQ9CEYZNSk3E7LE.eval","v":0.7575757575757576},{"b":"gpqa_diamond","d":"2025-05-22","evaluated_at":"2025-05-22T17:26:47.959Z","m":"claude-sonnet-4-2025-05-14","oid":"gpqa_diamond-c51e6c5d6f681b8d","rd":"2025-05-22","se":0.033586181457325226,"sid":"ZF4eejw7J2KrcoNWd6hUbU","sm":"claude-sonnet-4-20250514","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZF4eejw7J2KrcoNWd6hUbU.eval","v":0.6666666666666666},{"b":"gpqa_diamond","d":"2025-05-22","evaluated_at":"2025-05-22T17:26:47.111Z","m":"claude-opus-4-2025-05-14","oid":"gpqa_diamond-4f6b5e5606d8f1d8","rd":"2025-05-22","se":0.03289477330098615,"sid":"EnjJ8bGi9vRNNQFzfkLCkQ","sm":"claude-opus-4-20250514","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEnjJ8bGi9vRNNQFzfkLCkQ.eval","v":0.6919191919191919},{"b":"gpqa_diamond","d":"2025-05-07","evaluated_at":"2025-05-07T17:32:22.166Z","m":"mistral-medium-2505","oid":"gpqa_diamond-0e587fe93b257017","rd":"2025-05-07","se":0.02824697229654645,"sid":"FNMbfvEEm9PRnrJnrqk2PH","sm":"mistral-medium-2505","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFNMbfvEEm9PRnrJnrqk2PH.eval","v":0.5953282828282829},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:16.139Z","m":"o4-mini-2025-04-16","oid":"gpqa_diamond-8ac43f8853095061","rd":"2025-04-16","se":0.024034764096121958,"sid":"9Sgsz5XCjscuZkexf7MX9c","sm":"o4-mini-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9Sgsz5XCjscuZkexf7MX9c.eval","v":0.7960858585858586},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:09.093Z","m":"o3-2025-04-16","oid":"gpqa_diamond-a4f4f0725e005953","rd":"2025-04-16","se":0.021267123846387202,"sid":"evANZQ9oQTdGYbDERw7VrD","sm":"o3-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FevANZQ9oQTdGYbDERw7VrD.eval","v":0.8181818181818182},{"b":"gpqa_diamond","d":"2025-04-14","evaluated_at":"2025-04-14T22:49:57.790Z","m":"gpt-4-1-mini-2025-04-14","oid":"gpqa_diamond-f95d05a5cd7686e5","rd":"2025-04-14","se":0.027051500965295915,"sid":"dXHddeGRX7JRVa5UNfad5C","sm":"gpt-4.1-mini-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdXHddeGRX7JRVa5UNfad5C.eval","v":0.6584595959595959},{"b":"gpqa_diamond","d":"2025-04-14","evaluated_at":"2025-04-14T22:49:56.708Z","m":"gpt-4-1-nano-2025-04-14","oid":"gpqa_diamond-6c03a74c9142a0bc","rd":"2025-04-14","se":0.024849734950435368,"sid":"azWfeFPP6zSocuQX3JYBU5","sm":"gpt-4.1-nano-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FazWfeFPP6zSocuQX3JYBU5.eval","v":0.48926767676767674},{"b":"gpqa_diamond","d":"2025-04-14","evaluated_at":"2025-04-14T18:34:33.975Z","m":"gpt-4-1-2025-04-14","oid":"gpqa_diamond-ecaa246a7219b1f0","rd":"2025-04-14","se":0.02821440093855908,"sid":"Gj8pd5gU9r4LqwDK8XZ4mL","sm":"gpt-4.1-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGj8pd5gU9r4LqwDK8XZ4mL.eval","v":0.6691919191919192},{"b":"gpqa_diamond","d":"2025-04-11","evaluated_at":"2025-04-11T15:34:21.181Z","m":"qwq-plus","oid":"gpqa_diamond-fad5a170ebbdf1e0","rd":"2025-04-08","se":0.028469238770829935,"sid":"PtxHF8BH8WuNdNfDnutGBb","sm":"qwq-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPtxHF8BH8WuNdNfDnutGBb.eval","v":0.6540404040404041},{"b":"gpqa_diamond","cfg":{"effort":"low"},"d":"2025-04-10","e":"low","evaluated_at":"2025-04-10T20:30:33.024Z","m":"grok-3-mini-beta","oid":"gpqa_diamond-c472f6b8b97f35b2","rd":"2025-04-09","se":0.03031371053819892,"sid":"McxfQdC5aypnD3STwZUBjH","sm":"grok-3-mini-beta_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMcxfQdC5aypnD3STwZUBjH.eval","v":0.7626262626262627},{"b":"gpqa_diamond","d":"2025-04-08","evaluated_at":"2025-04-08T08:34:50.470Z","m":"llama-4-scout-17b-16e-instruct","oid":"gpqa_diamond-5562b1ab12146f71","rd":"2025-04-05","se":0.031549020124994755,"sid":"MsXAXDEKP3xBKtiA7aTBBG","sm":"Llama-4-Scout-17B-16E-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMsXAXDEKP3xBKtiA7aTBBG.eval","v":0.5183080808080808},{"b":"gpqa_diamond","d":"2025-04-08","evaluated_at":"2025-04-08T08:34:49.725Z","m":"llama-4-maverick-17b-128e-instruct-fp8","oid":"gpqa_diamond-7398a44af076ec8a","rd":"2025-04-05","se":0.028244823529728173,"sid":"LyHy99ubGCqBhoaksQjkiF","sm":"Llama-4-Maverick-17B-128E-Instruct-FP8","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLyHy99ubGCqBhoaksQjkiF.eval","v":0.6698232323232324},{"b":"gpqa_diamond","d":"2025-04-07","evaluated_at":"2025-04-07T21:38:51.185Z","m":"qwen-turbo-2024-11-01","oid":"gpqa_diamond-bb7aae303bb875e8","rd":"2024-11-01","se":0.02617727508870251,"sid":"9Ke5At9bYuf56SmKJJ7Sz7","sm":"qwen-turbo-2024-11-01","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9Ke5At9bYuf56SmKJJ7Sz7.eval","v":0.41792929292929293},{"b":"gpqa_diamond","d":"2025-04-07","evaluated_at":"2025-04-07T19:28:06.394Z","m":"qwen-plus-2025-01-25","oid":"gpqa_diamond-2ca34a0ea2697f83","rd":"2025-01-25","se":0.027144936812277986,"sid":"QtraMoKiQRRpZc9txpFAYF","sm":"qwen-plus-2025-01-25","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQtraMoKiQRRpZc9txpFAYF.eval","v":0.4810606060606061},{"b":"gpqa_diamond","d":"2025-04-01","evaluated_at":"2025-04-01T19:22:56.011Z","m":"qwen-max-2025-01-25","oid":"gpqa_diamond-215694487f7d9720","rd":"2025-01-25","se":0.027858831475709448,"sid":"GBf7rqHkGGzfX87V37F9mH","sm":"qwen-max-2025-01-25","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGBf7rqHkGGzfX87V37F9mH.eval","v":0.5612373737373737},{"b":"gpqa_diamond","d":"2025-04-01","evaluated_at":"2025-04-01T14:40:59.363Z","m":"deepseek-v3-0324","oid":"gpqa_diamond-9e82977560b87810","rd":"2025-03-24","se":0.026836302525030095,"sid":"VjfLwKbT6kTprYsMGcz2YT","sm":"DeepSeek-V3-0324","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVjfLwKbT6kTprYsMGcz2YT.eval","v":0.6761363636363636},{"b":"gpqa_diamond","d":"2025-03-31","evaluated_at":"2025-03-31T16:06:24.669Z","m":"gemini-2-5-pro-exp-03-25","oid":"gpqa_diamond-e8007a4fd42bd7a6","rd":"2025-03-25","se":0.02622591986362927,"sid":"WKjzEQdfnBHnYeZoKeCWnB","sm":"gemini-2.5-pro-exp-03-25","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWKjzEQdfnBHnYeZoKeCWnB.eval","v":0.8383838383838383},{"b":"gpqa_diamond","d":"2025-03-18","evaluated_at":"2025-03-18T16:08:06.162Z","m":"mistral-small-2503","oid":"gpqa_diamond-7823580cfe98ff38","rd":"2025-03-17","se":0.027616495140012243,"sid":"S8bBDAbucsFceirHxeEDvu","sm":"mistral-small-2503","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FS8bBDAbucsFceirHxeEDvu.eval","v":0.47474747474747475},{"b":"gpqa_diamond","d":"2025-03-12","evaluated_at":"2025-03-12T23:51:44.119Z","m":"claude-3-5-haiku-2024-10-22","oid":"gpqa_diamond-36a70050f703cca6","rd":"2024-10-22","se":0.026351515390511556,"sid":"JpLxrpvthx7A75sbaHZ9Et","sm":"claude-3-5-haiku-20241022","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJpLxrpvthx7A75sbaHZ9Et.eval","v":0.3813131313131313},{"b":"gpqa_diamond","d":"2025-03-12","evaluated_at":"2025-03-12T21:23:34.734Z","m":"gemini-1-5-flash-8b-001","oid":"gpqa_diamond-28542112de9d4603","rd":"2024-10-03","se":0.02416551116211757,"sid":"gFT36W4KU3wggLQ9nNCctg","sm":"gemini-1.5-flash-8b-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgFT36W4KU3wggLQ9nNCctg.eval","v":0.32954545454545453},{"b":"gpqa_diamond","d":"2025-03-10","evaluated_at":"2025-03-10T23:03:48.213Z","m":"deepseek-r1-distill-qwen-14b","oid":"gpqa_diamond-def04aafb29a5730","rd":"2025-01-20","se":0.02863094678190378,"sid":"8UqaLobhmNJLuPGTTpeaCw","sm":"DeepSeek-R1-Distill-Qwen-14B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8UqaLobhmNJLuPGTTpeaCw.eval","v":0.44696969696969696},{"b":"gpqa_diamond","d":"2025-03-10","evaluated_at":"2025-03-10T22:05:35.890Z","m":"claude-3-7-sonnet-20250219-32k","oid":"gpqa_diamond-ea54154853e4a30a","rd":"2025-02-24","se":0.030088629490217445,"sid":"iv9yfwZ9vTznxQfvJF6J7p","sm":"claude-3-7-sonnet-20250219_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fiv9yfwZ9vTznxQfvJF6J7p.eval","v":0.7676767676767676},{"b":"gpqa_diamond","d":"2025-03-10","evaluated_at":"2025-03-10T20:08:15.034Z","m":"deepseek-r1-distill-llama-70b","oid":"gpqa_diamond-4610f383f5c75502","rd":"2025-01-20","se":0.02973465219179067,"sid":"f6pnAATjxmrFJS353EjQKi","sm":"DeepSeek-R1-Distill-Llama-70B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ff6pnAATjxmrFJS353EjQKi.eval","v":0.5574494949494949},{"b":"gpqa_diamond","d":"2025-02-28","evaluated_at":"2025-02-28T09:17:12.859Z","m":"gpt-4-5-preview-2025-02-27","oid":"gpqa_diamond-f9cfc8b808e885f7","rd":"2025-02-27","se":0.033042050878136546,"sid":"dodibSv7pk8GtqeMy4YLQg","sm":"gpt-4.5-preview-2025-02-27","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdodibSv7pk8GtqeMy4YLQg.eval","v":0.6868686868686869},{"b":"gpqa_diamond","d":"2025-02-27","evaluated_at":"2025-02-27T14:29:45.537Z","m":"gpt-4-turbo-2024-04-09","oid":"gpqa_diamond-2194e074edc42479","rd":"2024-04-09","se":0.026576883332580552,"sid":"TaCoN7eyPQEfDWXqs2MpcF","sm":"gpt-4-turbo-2024-04-09","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTaCoN7eyPQEfDWXqs2MpcF.eval","v":0.4659090909090909},{"b":"gpqa_diamond","d":"2025-02-26","evaluated_at":"2025-02-26T16:56:43.203Z","m":"claude-3-7-sonnet-20250219-16k","oid":"gpqa_diamond-d7b499cddb5c5d5d","rd":"2025-02-24","se":0.030088629490217445,"sid":"Vn3Ug88CRbxEaJhLbRzAWD","sm":"claude-3-7-sonnet-20250219_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVn3Ug88CRbxEaJhLbRzAWD.eval","v":0.7676767676767676},{"b":"gpqa_diamond","d":"2025-02-25","evaluated_at":"2025-02-25T12:31:34.069Z","m":"mistral-large-2411","oid":"gpqa_diamond-3079c534e5995040","rd":"2024-11-18","se":0.026887818295509087,"sid":"n9gpXfUzo8SbWqvrG2q4xM","sm":"mistral-large-2411","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fn9gpXfUzo8SbWqvrG2q4xM.eval","v":0.5132575757575758},{"b":"gpqa_diamond","d":"2025-02-24","evaluated_at":"2025-02-24T19:14:07.847Z","m":"claude-3-7-sonnet-2025-02-19","oid":"gpqa_diamond-15c5b22b7b322cef","rd":"2025-02-24","se":0.026703858684060923,"sid":"PyNEkFCfMEReodeE2oiPhh","sm":"claude-3-7-sonnet-20250219","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPyNEkFCfMEReodeE2oiPhh.eval","v":0.6603535353535354},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:12:11.445Z","m":"o1-2024-12-17","oid":"gpqa_diamond-b91a40c475ac1889","rd":"2024-12-17","se":0.030088629490217445,"sid":"NAkCFiFiDSN3NXMj7HMvMS","sm":"o1-2024-12-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNAkCFiFiDSN3NXMj7HMvMS.eval","v":0.7676767676767676},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:07:50.981Z","m":"o1-mini-2024-09-12","oid":"gpqa_diamond-68c9591af263badc","rd":"2024-09-12","se":0.027427723511230753,"sid":"X2gKewiGPZx5DLFEddUUqW","sm":"o1-mini-2024-09-12_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX2gKewiGPZx5DLFEddUUqW.eval","v":0.6237373737373737},{"b":"gpqa_diamond","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:07:50.336Z","m":"o3-mini-2025-01-31","oid":"gpqa_diamond-1ea1c9efafadd95c","rd":"2025-01-31","se":0.025855332698878485,"sid":"MzsTvCEEfEpzn4WvxYdJkX","sm":"o3-mini-2025-01-31_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMzsTvCEEfEpzn4WvxYdJkX.eval","v":0.7702020202020202},{"b":"gpqa_diamond","d":"2025-02-07","evaluated_at":"2025-02-07T07:28:03.988Z","m":"gemini-2-0-pro-exp-02-05","oid":"gpqa_diamond-e98ed276f3646aa7","rd":"2025-02-05","se":0.03383201223244441,"sid":"g5YE7G6DXb45EH4HxEdiht","sm":"gemini-2.0-pro-exp-02-05","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fg5YE7G6DXb45EH4HxEdiht.eval","v":0.6565656565656566},{"b":"gpqa_diamond","d":"2025-02-06","evaluated_at":"2025-02-06T22:40:04.398Z","m":"gemini-2-0-flash-thinking-exp-01-21","oid":"gpqa_diamond-8e0b372b11d1ec7a","rd":"2025-01-21","se":0.03526552724601199,"sid":"ag9w2w4xa47ghZaWt5oZey","sm":"gemini-2.0-flash-thinking-exp-01-21","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fag9w2w4xa47ghZaWt5oZey.eval","v":0.5707070707070707},{"b":"gpqa_diamond","d":"2025-02-06","evaluated_at":"2025-02-06T15:51:21.852Z","m":"gemini-2-0-flash-001","oid":"gpqa_diamond-6351cf5765642b0b","rd":"2025-02-05","se":0.027395618179027574,"sid":"fbszwTApm3f28z5VA6oXBK","sm":"gemini-2.0-flash-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfbszwTApm3f28z5VA6oXBK.eval","v":0.6414141414141414},{"b":"gpqa_diamond","d":"2025-02-05","evaluated_at":"2025-02-05T08:31:15.914Z","m":"gpt-4o-2024-11-20","oid":"gpqa_diamond-6b50d764beaacd80","rd":"2024-11-20","se":0.026336972178726107,"sid":"gULKGqAe9nG9syxmYMtkhi","sm":"gpt-4o-2024-11-20","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgULKGqAe9nG9syxmYMtkhi.eval","v":0.4788510101010101},{"b":"gpqa_diamond","d":"2025-01-31","evaluated_at":"2025-01-31T23:03:31.781Z","m":"phi-4","oid":"gpqa_diamond-8768672a66add967","rd":"2024-12-12","se":0.02590274664132679,"sid":"RAwuPRwJedy7P2KeQhEmr8","sm":"phi-4","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRAwuPRwJedy7P2KeQhEmr8.eval","v":0.5606060606060606},{"b":"gpqa_diamond","d":"2025-01-31","evaluated_at":"2025-01-31T22:41:25.625Z","m":"phi-3-medium-128k-instruct","oid":"gpqa_diamond-e0be0a3a7e6b6d81","rd":"2024-04-23","se":0.016117685500292727,"sid":"bDS2AGEf3AywbVcdDyAKZK","sm":"Phi-3-medium-128k-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbDS2AGEf3AywbVcdDyAKZK.eval","v":0.2758838383838384},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-01-31","e":"medium","evaluated_at":"2025-01-31T19:21:57.338Z","m":"o3-mini-2025-01-31","oid":"gpqa_diamond-0e5621e51b709de4","rd":"2025-01-31","se":0.025411943362857046,"sid":"ehvA5nisC7GMbgbyy3Z4Et","sm":"o3-mini-2025-01-31_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FehvA5nisC7GMbgbyy3Z4Et.eval","v":0.742739898989899},{"b":"gpqa_diamond","d":"2025-01-30","evaluated_at":"2025-01-30T21:10:36.084Z","m":"mistral-small-2501","oid":"gpqa_diamond-eded4f0031d2782e","rd":"2025-01-25","se":0.024794688677057853,"sid":"oDq358GfyV5Au65scERdVo","sm":"mistral-small-2501","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoDq358GfyV5Au65scERdVo.eval","v":0.4529671717171717},{"b":"gpqa_diamond","d":"2025-01-30","evaluated_at":"2025-01-30T18:31:31.252Z","m":"qwen2-5-32b-instruct","oid":"gpqa_diamond-07507d7e545c4b97","rd":"2024-09-17","se":0.02595783004579065,"sid":"d6JihAFVqUy3u9suNt4P3L","sm":"qwen2.5-32b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fd6JihAFVqUy3u9suNt4P3L.eval","v":0.46085858585858586},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-flash-001","oid":"gpqa_diamond-5c88533865495336","rd":"2024-05-23","se":0.023852475604024478,"sid":"Hc2ycMP6DyCZxCPjKbZFQU","sm":"gemini-1.5-flash-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHc2ycMP6DyCZxCPjKbZFQU.eval","v":0.40372474747474746},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-2-0","oid":"gpqa_diamond-3310ea703a54602e","rd":"2023-07-11","se":0.025076255983551475,"sid":"mTa4f52zsEuxdrPy7WfQvX","sm":"claude-2.0","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmTa4f52zsEuxdrPy7WfQvX.eval","v":0.3465909090909091},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-opus-2024-02-29","oid":"gpqa_diamond-8a252320a583a199","rd":"2024-02-29","se":0.026395393956533515,"sid":"hTnDa4gn8RYduCEYqoNywe","sm":"claude-3-opus-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhTnDa4gn8RYduCEYqoNywe.eval","v":0.4715909090909091},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-01-27","e":"medium","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-mini-2024-09-12","oid":"gpqa_diamond-d2a494bf1e8726f2","rd":"2024-09-12","se":0.02795124238637276,"sid":"4NiUpCrLfz3oFaerkrMeBt","sm":"o1-mini-2024-09-12_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4NiUpCrLfz3oFaerkrMeBt.eval","v":0.5950126262626263},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-2024-08-06","oid":"gpqa_diamond-94698875279161d5","rd":"2024-08-06","se":0.025999270277271707,"sid":"fairJpL9VekDBKxYHG9ZoG","sm":"gpt-4o-2024-08-06","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfairJpL9VekDBKxYHG9ZoG.eval","v":0.49210858585858586},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-preview-2024-09-12","oid":"gpqa_diamond-f2e9b15b7c71d94a","rd":"2024-09-12","se":0.028257821072909393,"sid":"7CkADTsoNbDd88uqYbqfwj","sm":"o1-preview-2024-09-12","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7CkADTsoNbDd88uqYbqfwj.eval","v":0.5031565656565656},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-sonnet-2024-02-29","oid":"gpqa_diamond-6f9ae3cda419cc2f","rd":"2024-02-29","se":0.023920223491748827,"sid":"QLejPZXhjX6SQCQgzjNvCG","sm":"claude-3-sonnet-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQLejPZXhjX6SQCQgzjNvCG.eval","v":0.4059343434343434},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-5-sonnet-2024-10-22","oid":"gpqa_diamond-530b4af72f4a7712","rd":"2024-10-22","se":0.028208090147078847,"sid":"JCCVcVLz3J4TQXSU67hMJZ","sm":"claude-3-5-sonnet-20241022","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJCCVcVLz3J4TQXSU67hMJZ.eval","v":0.553030303030303},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-0125-preview","oid":"gpqa_diamond-9600edbd6e19b9cf","rd":"2024-01-25","se":0.026795949166711065,"sid":"4j3WRhhruXrfaq9Jki3NDc","sm":"gpt-4-0125-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4j3WRhhruXrfaq9Jki3NDc.eval","v":0.42266414141414144},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-2-1","oid":"gpqa_diamond-dc0a0846d0229077","rd":"2023-11-21","se":0.02324885083265289,"sid":"eMAisrJd7fAUHPfcPWXeXo","sm":"claude-2.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeMAisrJd7fAUHPfcPWXeXo.eval","v":0.32954545454545453},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemma-2-27b-it","oid":"gpqa_diamond-3e46631a9504a8f8","rd":"2024-06-24","se":0.023897884428691632,"sid":"ZUJxsJDpHnLXxNiDcvj22W","sm":"gemma-2-27b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZUJxsJDpHnLXxNiDcvj22W.eval","v":0.3648989898989899},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-haiku-2024-03-07","oid":"gpqa_diamond-64b912a05db6c269","rd":"2024-03-07","se":0.024180910672074975,"sid":"T3FWdCCWBryVjxwTcoY5bK","sm":"claude-3-haiku-20240307","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT3FWdCCWBryVjxwTcoY5bK.eval","v":0.3630050505050505},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-5-sonnet-2024-06-20","oid":"gpqa_diamond-f08cc5bf121c0d77","rd":"2024-06-20","se":0.027528555453348653,"sid":"78Wpvp8WDByAcQRBas72cs","sm":"claude-3-5-sonnet-20240620","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F78Wpvp8WDByAcQRBas72cs.eval","v":0.5404040404040404},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-2024-05-13","oid":"gpqa_diamond-c8a9c4ce81dc7541","rd":"2024-05-13","se":0.026254109873371994,"sid":"Y59q8WbEDp2HH8oovyJCj9","sm":"gpt-4o-2024-05-13","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY59q8WbEDp2HH8oovyJCj9.eval","v":0.4889520202020202},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-flash-002","oid":"gpqa_diamond-05332696f93eaf28","rd":"2024-09-24","se":0.027737408277040958,"sid":"PCLvrs4TEBtqQqcYeU8pAG","sm":"gemini-1.5-flash-002","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPCLvrs4TEBtqQqcYeU8pAG.eval","v":0.47316919191919193},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-0613","oid":"gpqa_diamond-7eb2c293c3c9c4e6","rd":"2023-06-13","se":0.02337322948270623,"sid":"fu7pWqbSC4MUBdwv3GFNa5","sm":"gpt-4-0613","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ffu7pWqbSC4MUBdwv3GFNa5.eval","v":0.30650252525252525},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemma-2-9b-it","oid":"gpqa_diamond-c59089df0aef6804","rd":"2024-06-24","se":0.023032467954778795,"sid":"fQjoHKyUEPAuwdrBGfL8SG","sm":"gemma-2-9b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfQjoHKyUEPAuwdrBGfL8SG.eval","v":0.2746212121212121},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-3-5-turbo-1106","oid":"gpqa_diamond-2cfec058a8106acf","rd":"2023-11-06","se":0.018720372561004013,"sid":"enhRcQmfN6RFB2zu7R3Uxr","sm":"gpt-3.5-turbo-1106","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FenhRcQmfN6RFB2zu7R3Uxr.eval","v":0.2803030303030303},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-pro-001","oid":"gpqa_diamond-9f6d2f15cda7b71f","rd":"2024-05-14","se":0.02617295639558329,"sid":"PKdqpKZ9tqDWyWJ7GksBWz","sm":"gemini-1.5-pro-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPKdqpKZ9tqDWyWJ7GksBWz.eval","v":0.4586489898989899},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-1106-preview","oid":"gpqa_diamond-9af0ffb579ccb5fb","rd":"2023-11-06","se":0.024076142618301676,"sid":"AWewipyfSQb8hXtS8d4fNL","sm":"gpt-4-1106-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAWewipyfSQb8hXtS8d4fNL.eval","v":0.4236111111111111},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-mini-2024-07-18","oid":"gpqa_diamond-e4133ba34cec7398","rd":"2024-07-18","se":0.024068925424620136,"sid":"L9yfwZC3snsKZshBXTaXjc","sm":"gpt-4o-mini-2024-07-18","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FL9yfwZC3snsKZshBXTaXjc.eval","v":0.37720959595959597},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-pro-002","oid":"gpqa_diamond-ec0ea2046186c108","rd":"2024-09-24","se":0.027959709636504244,"sid":"9TrLEEw8TvsyugyCjwvUvD","sm":"gemini-1.5-pro-002","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9TrLEEw8TvsyugyCjwvUvD.eval","v":0.5722853535353535},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-3-5-turbo-0125","oid":"gpqa_diamond-9aff1cc5c10e57fb","rd":"2024-01-25","se":0.01953579968788737,"sid":"eZaQhpbhbzbo3sbKGS5Efa","sm":"gpt-3.5-turbo-0125","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeZaQhpbhbzbo3sbKGS5Efa.eval","v":0.27178030303030304},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-70b-instruct","oid":"gpqa_diamond-ca467f690d01d4c9","rd":"2024-07-23","se":0.024522863472149384,"sid":"MEwMcmSV32JHrsNoAq4BFL","sm":"Llama-3.1-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMEwMcmSV32JHrsNoAq4BFL.eval","v":0.44191919191919193},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-405b-instruct","oid":"gpqa_diamond-55791b8f5fed6fd2","rd":"2024-07-23","se":0.025861992702770648,"sid":"S5QYXSvQBRSbUbXSnAGbMm","sm":"Llama-3.1-405B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FS5QYXSvQBRSbUbXSnAGbMm.eval","v":0.5091540404040404},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"yi-1-5-34b-chat","oid":"gpqa_diamond-b8f7fddab14cb3bd","rd":"2024-05-13","se":0.020005291661900883,"sid":"Jwi8JTeeTxJuJY8qidxngw","sm":"Yi-1.5-34B-Chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJwi8JTeeTxJuJY8qidxngw.eval","v":0.319760101010101},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"yi-34b-chat","oid":"gpqa_diamond-2fbbd2f3228044d7","rd":"2023-11-22","se":0.011372823721448395,"sid":"WxMpEBcTf8qPFPz4BrF86B","sm":"Yi-34B-Chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWxMpEBcTf8qPFPz4BrF86B.eval","v":0.14741161616161616},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"grok-2-1212","oid":"gpqa_diamond-adc03080122d48d5","rd":"2024-12-12","se":0.027066342316607196,"sid":"83DRJD4FvjwqU9eStRaNiU","sm":"grok-2-1212","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F83DRJD4FvjwqU9eStRaNiU.eval","v":0.5378787878787878},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen2-72b-instruct","oid":"gpqa_diamond-63beb35dc48ede83","rd":"2024-06-07","se":0.02574288051254005,"sid":"Pex6SpYJX8ZPySncfNNy7H","sm":"qwen2-72b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPex6SpYJX8ZPySncfNNy7H.eval","v":0.4078282828282828},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen1-5-72b-chat","oid":"gpqa_diamond-5687e7fa3ef8a986","rd":"2024-02-04","se":0.01993691228122493,"sid":"NeqeQSxHmdUNLk9Ytpo6qo","sm":"qwen1.5-72b-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNeqeQSxHmdUNLk9Ytpo6qo.eval","v":0.2881944444444444},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen1-5-32b-chat","oid":"gpqa_diamond-35e20b20b5889170","rd":"2024-04-03","se":0.020200517973424446,"sid":"9K6c9b6v9zQrQYtw827BDn","sm":"qwen1.5-32b-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9K6c9b6v9zQrQYtw827BDn.eval","v":0.307449494949495},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"hermes-2-theta-llama-3-70b","oid":"gpqa_diamond-7c0ea871c1f1d01e","rd":"2024-06-20","se":0.024803009993948306,"sid":"DHdeZNynu4VtRp2VGkCGvd","sm":"Hermes-2-Theta-Llama-3-70B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDHdeZNynu4VtRp2VGkCGvd.eval","v":0.3746843434343434},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-2-70b-chat-hf","oid":"gpqa_diamond-796db845cca0eb28","rd":"2023-07-18","se":0.019901808558284103,"sid":"evPjJw9SKqq3TieSdxrSQn","sm":"Llama-2-70b-chat-hf","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FevPjJw9SKqq3TieSdxrSQn.eval","v":0.26325757575757575},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"meta-llama-3-70b-instruct","oid":"gpqa_diamond-21e37f0f979eec8e","rd":"2024-04-18","se":0.026224849492112068,"sid":"jrvutCoFFG4KxYtU7REdpe","sm":"Meta-Llama-3-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjrvutCoFFG4KxYtU7REdpe.eval","v":0.40561868686868685},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"meta-llama-3-8b-instruct","oid":"gpqa_diamond-7312caf339495f72","rd":"2024-04-18","se":0.01703043361773576,"sid":"iT3BBFWjsZk2rKHKeJDsM8","sm":"Meta-Llama-3-8B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiT3BBFWjsZk2rKHKeJDsM8.eval","v":0.26073232323232326},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-7b-instruct-v0-3","oid":"gpqa_diamond-0f59c0dbf5b3a91f","rd":"2024-05-27","se":0.010148891037227303,"sid":"YbQkMPC2GtNLhxqTEhhVsa","sm":"Mistral-7B-Instruct-v0.3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYbQkMPC2GtNLhxqTEhhVsa.eval","v":0.1518308080808081},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"deepseek-llm-67b-chat","oid":"gpqa_diamond-470771197979d036","rd":"2023-11-29","se":0.013623351950761947,"sid":"82RTgqgiC2UuM5M37L3WWM","sm":"deepseek-llm-67b-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F82RTgqgiC2UuM5M37L3WWM.eval","v":0.24621212121212122},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mixtral-8x7b-instruct-v0-1","oid":"gpqa_diamond-5ae17b844b9983c2","rd":"2023-12-11","se":0.01937059603941351,"sid":"Sn2qbB2HiySe7NcG28jnyM","sm":"Mixtral-8x7B-Instruct-v0.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSn2qbB2HiySe7NcG28jnyM.eval","v":0.3058712121212121},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen2-5-72b-instruct","oid":"gpqa_diamond-ed40800f25d75239","rd":"2024-09-19","se":0.026947767404178667,"sid":"WtALgW6VSFdihDZWXRm2AZ","sm":"qwen2.5-72b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWtALgW6VSFdihDZWXRm2AZ.eval","v":0.4914772727272727},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"wizardlm-2-8x22b","oid":"gpqa_diamond-95f362caa3f48894","rd":"2024-04-15","se":0.023232093333444247,"sid":"MQJA3RUjsQzSubGJiukZ7A","sm":"WizardLM-2-8x22B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMQJA3RUjsQzSubGJiukZ7A.eval","v":0.43434343434343436},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"dbrx-instruct","oid":"gpqa_diamond-a5f561f4c9ba2681","rd":"2024-03-27","se":0.03256944333564935,"sid":"TTa3hu7PgRxHgizVreZsdJ","sm":"dbrx-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTTa3hu7PgRxHgizVreZsdJ.eval","v":0.32891414141414144},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-0-pro-001","oid":"gpqa_diamond-c89cd4268e8f138f","rd":"2023-12-13","se":0.020110933405176737,"sid":"ibpS8bs4z3wD3Hb6F3kpdV","sm":"gemini-1.0-pro-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FibpS8bs4z3wD3Hb6F3kpdV.eval","v":0.33964646464646464},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"ministral-3b-2410","oid":"gpqa_diamond-7d4c4a88075bfcfc","rd":"2024-10-16","se":0.019305614722797293,"sid":"bJV4D8SEo9zMJPEYTTXJNZ","sm":"ministral-3b-2410","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbJV4D8SEo9zMJPEYTTXJNZ.eval","v":0.25252525252525254},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-large-2402","oid":"gpqa_diamond-9d9f7f26b78c4b10","rd":"2024-02-26","se":0.024776598535417284,"sid":"dDeREdDJ5bVyci3APZxzWb","sm":"mistral-large-2402","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdDeREdDJ5bVyci3APZxzWb.eval","v":0.38762626262626265},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mixtral-8x22b","oid":"gpqa_diamond-3ef2d2bbd713556e","rd":"2024-04-17","se":0.02068134779158647,"sid":"ifAM3DJBdm5eU9Y2DjaEvW","sm":"open-mixtral-8x22b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FifAM3DJBdm5eU9Y2DjaEvW.eval","v":0.34059343434343436},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"ministral-8b-2410","oid":"gpqa_diamond-a9f8ad70541e5e2e","rd":"2024-10-16","se":0.020863706671553732,"sid":"DGEzZA6nvTne4MEGSuo3Tm","sm":"ministral-8b-2410","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDGEzZA6nvTne4MEGSuo3Tm.eval","v":0.27146464646464646},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-large-2407","oid":"gpqa_diamond-24b5e830f183463a","rd":"2024-07-24","se":0.025918840988656967,"sid":"ZQPNnicTuhuscJvni85PFY","sm":"mistral-large-2407","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZQPNnicTuhuscJvni85PFY.eval","v":0.49021464646464646},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mixtral-8x7b","oid":"gpqa_diamond-66ddde4e176ee463","rd":"2023-12-11","se":0.019146998037923697,"sid":"Y6KPSujFepdwqinKLf9mDF","sm":"open-mixtral-8x7b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY6KPSujFepdwqinKLf9mDF.eval","v":0.29829545454545453},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mistral-7b","oid":"gpqa_diamond-9972df489af04c8a","rd":"2023-09-27","se":0.009888086213040367,"sid":"5XuVuHeUDekHqjUy2P8gww","sm":"open-mistral-7b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5XuVuHeUDekHqjUy2P8gww.eval","v":0.13226010101010102},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mistral-nemo-2407","oid":"gpqa_diamond-d755c12054c57866","rd":"2024-07-18","se":0.02096191098761666,"sid":"XCSWtbDquA7FUiWfd7kh9q","sm":"open-mistral-nemo-2407","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXCSWtbDquA7FUiWfd7kh9q.eval","v":0.2989267676767677},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-2-90b-vision-instruct","oid":"gpqa_diamond-4f27274a3b5c4f56","rd":"2024-09-24","se":0.022948341091025694,"sid":"9SSHEP95FYVZvHDdoHSRy3","sm":"Llama-3.2-90B-Vision-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9SSHEP95FYVZvHDdoHSRy3.eval","v":0.41035353535353536},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-tulu-3-70b-dpo","oid":"gpqa_diamond-d7bb3277e93ec667","rd":"2024-11-21","se":0.02405836362093961,"sid":"5JgivNDZp2D7Lc7cpDhS28","sm":"Llama-3.1-Tulu-3-70B-DPO","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5JgivNDZp2D7Lc7cpDhS28.eval","v":0.46275252525252525},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"eurus-2-7b-prime","oid":"gpqa_diamond-aff28946a95fe485","rd":"2024-12-31","se":0.02107317439394748,"sid":"RJm4u2vXF6PUmThJGzAfHE","sm":"Eurus-2-7B-PRIME","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRJm4u2vXF6PUmThJGzAfHE.eval","v":0.3390151515151515},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-3-70b-instruct","oid":"gpqa_diamond-b82fd430d3269095","rd":"2024-12-06","se":0.028523027562061026,"sid":"TxXS78Wg2pSCDbpmeJuzQn","sm":"Llama-3.3-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTxXS78Wg2pSCDbpmeJuzQn.eval","v":0.4744318181818182},{"b":"gpqa_diamond","cfg":{"effort":"medium"},"d":"2025-01-27","e":"medium","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-2024-12-17","oid":"gpqa_diamond-9df3d8e811c7526e","rd":"2024-12-17","se":0.030532892233932022,"sid":"Uczhz7MKLstjLkSbbGuwdN","sm":"o1-2024-12-17_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUczhz7MKLstjLkSbbGuwdN.eval","v":0.7575757575757576},{"b":"gpqa_diamond","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"deepseek-v3","oid":"gpqa_diamond-5761b83b4b649f5c","rd":"2024-12-26","se":0.02763990029950893,"sid":"Vwo3nMA8g2gGBCyTdKFMQJ","sm":"DeepSeek-V3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVwo3nMA8g2gGBCyTdKFMQJ.eval","v":0.5653409090909091},{"b":"gpqa_diamond","m":"grok-4-0709","oid":"gpqa_diamond-dcfbe2b1c62e6e4f","rd":"2025-07-09","se":0.02,"sid":"A85Zfq2qguE4X9xXBweBHP","sm":"grok-4-0709","v":0.87},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"code-davinci-002","m":"code-davinci-002","oid":"gsm8k-743720efa686620d","rd":"2022-03-15","sid":"recC9RJ4GUYLfhNZC","sm":"code-davinci-002","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.568},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"text-davinci-003","m":"text-davinci-003","oid":"gsm8k-fe88dff45f55d74e","rd":"2022-11-28","sid":"rec8hZWOfPZGc8t7Q","sm":"text-davinci-003","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.506},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Llama 2 (70B)","m":"llama-2-70b-hf","oid":"gsm8k-aa5390c1e1064e6d","rd":"2023-07-18","sid":"recXJZtKCyb4P4SVF","sm":"Llama-2-70b-hf","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.484},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"gpt-3.5-turbo-0613","m":"gpt-3-5-turbo-0613","oid":"gsm8k-c88e48ec066e7e17","rd":"2023-06-13","sid":"recqp1V0mwUdsVRem","sm":"gpt-3.5-turbo-0613","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.469},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"LLaMA (65B)","m":"llama-65b","oid":"gsm8k-32cbb85fa6047e0d","rd":"2023-02-24","sid":"recroBXEiAHrbJ0G6","sm":"LLaMA-65B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.466},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"text-davinci-002","m":"text-davinci-002","oid":"gsm8k-286c0b955fcd54f9","rd":"2022-03-15","sid":"recoDKfXnKDzKIo3R","sm":"text-davinci-002","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.415},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Mistral v0.1 (7B)","m":"mistral-7b-v0-1","oid":"gsm8k-616e4b027dc3a9c3","rd":"2023-09-27","sid":"recHcwXx5qAtSwauC","sm":"Mistral-7B-v0.1","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.381},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"MPT-Instruct (30B)","m":"mpt-30b-instruct","oid":"gsm8k-655d09fdb52100ca","rd":"2023-06-22","sid":"reciGMGxgYhthpF6j","sm":"mpt-30b-instruct","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.344},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon-Instruct (40B)","m":"falcon-40b-instruct","oid":"gsm8k-e40785a44bdd1b7e","rd":"2023-05-25","sid":"rece6CXnlwtcJqjXw","sm":"falcon-40b-instruct","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.338},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"LLaMA (30B)","m":"llama-33b","oid":"gsm8k-e47304294aa9cbb5","rd":"2023-02-24","sid":"recftyEtwpgppPQJY","sm":"LLaMA-33B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.32},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon (40B)","m":"falcon-40b","oid":"gsm8k-5b5382861e1de001","rd":"2023-03-15","sid":"reciN3EJDuJw028OV","sm":"falcon-40b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.25},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Llama 2 (13B)","m":"llama-2-13b","oid":"gsm8k-4c32aee3f827b08f","rd":"2023-07-18","sid":"recpxfs0xcllEJeu8","sm":"Llama-2-13b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.245},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Vicuna v1.3 (13B)","m":"vicuna-13b-v1-3","oid":"gsm8k-050122c01f998eb6","rd":"2023-06-18","sid":"rec3Y7X7VdJ4nw0QX","sm":"vicuna-13b-v1.3","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.226},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"MPT (30B)","m":"mpt-30b","oid":"gsm8k-3714d7d70217b85a","rd":"2023-06-22","sid":"recth8oFuGYbZWmt3","sm":"mpt-30b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.164},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"LLaMA (13B)","m":"llama-13b","oid":"gsm8k-2e6b92435778658a","rd":"2023-02-24","sid":"rec376aACcpELpmU8","sm":"LLaMA-13B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.154},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Llama 2 (7B)","m":"llama-2-70b-hf","oid":"gsm8k-bd1427483ebc52b4","rd":"2023-07-18","sid":"recOJI8uhjSL51VhF","sm":"Llama-2-70b-hf","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.133},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"BLOOM (176B)","m":"bloom","oid":"gsm8k-4d3e53ef09d1e5a6","rd":"2022-07-06","sid":"recAoaesoiJO7uTlT","sm":"bloom","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.095},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"davinci (175B)","m":"davinci","oid":"gsm8k-894f6c733aaa379a","sid":"recN3MvbzHfDJYOx0","sm":"davinci","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.09},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"LLaMA (7B)","m":"llama-7b","oid":"gsm8k-8b502265949da06b","rd":"2023-02-24","sid":"reczTfwmbEvxPqfHI","sm":"LLaMA-7B","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.08},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"OPT (175B)","m":"opt-175b","oid":"gsm8k-1dd24afebe071ebd","rd":"2022-05-02","sid":"rec5Tmjxxco3f4pmV","sm":"opt-175b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.04},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon (7B)","m":"falcon-7b","oid":"gsm8k-e42cc4950c4e6044","rd":"2023-04-24","sid":"recaVUOlS9XK6d01V","sm":"falcon-7b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.04},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"OPT (66B)","m":"opt-66b","oid":"gsm8k-748e4d31b6982902","rd":"2022-05-03","sid":"recNXn5TKfIeDBCX1","sm":"opt-66b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.018},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"curie (6.7B)","m":"curie","oid":"gsm8k-0b6c5e8e45640286","sid":"recG9AtQzNyxmyNcN","sm":"curie","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.016},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"babbage (1.3B)","m":"babbage","oid":"gsm8k-2592aa3aa18b223b","rd":"2020-06-22","sid":"rec2iML006beoMXFn","sm":"babbage","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.007},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"ada (350M)","m":"ada","oid":"gsm8k-6c780735d86bb21b","sid":"recV2EaPfHNBvr0hb","sm":"ada","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.006},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"text-curie-001","m":"text-curie-001","oid":"gsm8k-2556d0dbb639d82e","sid":"rec191CB0bMuoPuGK","sm":"text-curie-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.006},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"text-ada-001","m":"text-ada-001","oid":"gsm8k-2388e39b94db82dd","sid":"rec0V8SZd0uPBlgyp","sm":"text-ada-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0.004},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"text-babbage-001","m":"text-babbage-001","oid":"gsm8k-d7dfff62515122e7","sid":"recPnofjm1vlqkfwp","sm":"text-babbage-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/gsm","v":0},{"b":"gsm8k","m":"llama-2-7b","oid":"gsm8k-0daf9a8c5a1381db","rd":"2023-07-18","sid":"rec1iVL6AnUBdhAMO","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2403.08295","v":0.146},{"b":"gsm8k","m":"llama-2-13b","oid":"gsm8k-a9e46aec2ff45292","rd":"2023-07-18","sid":"recZfU3XHlvMu46sP","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2403.08295","v":0.287},{"b":"gsm8k","dn":"Mistral 7B","m":"mistral-7b-instruct-v0-2","oid":"gsm8k-f35c06d279fa0be8","rd":"2023-12-11","sid":"rec0smApu8Mq2BOsY","sm":"Mistral-7B-Instruct-v0.2","src":"http://arxiv.org/abs/2403.08295","v":0.354},{"b":"gsm8k","m":"gemma-2b","oid":"gsm8k-0f4adbb7c9e3d0f4","rd":"2024-02-21","sid":"recYegDkHsIh69wUi","sm":"gemma-2b","src":"http://arxiv.org/abs/2403.08295","v":0.177},{"b":"gsm8k","m":"gemma-7b","oid":"gsm8k-1a6e57723ffea950","rd":"2024-02-21","sid":"reccmnrR6t9A5tbfZ","sm":"gemma-7b","src":"http://arxiv.org/abs/2403.08295","v":0.464},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"QWEN 14B","m":"qwen-14b","oid":"gsm8k-f4aed6aa18d6fcb1","rd":"2023-09-28","sid":"recAv3zIYd8lTYBBs","sm":"Qwen-14B","src":"http://arxiv.org/abs/2402.16819","v":0.601},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"gsm8k-e0f6dd91f818ba08","rd":"2024-02-26","sid":"recoYBTgs9ooX9x9h","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.46},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"gsm8k-84ff3fd52fff71e9","rd":"2023-07-18","sid":"rec87oNIjkEeALpVG","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.287},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"gsm8k-cc5442a15f73d758","rd":"2023-07-18","sid":"recrojyKXfVp438n3","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.422},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Baichuan-2 13B","m":"baichuan-2-13b-base","oid":"gsm8k-48f307cbff826779","rd":"2023-09-06","sid":"reczmqKWq7JkTfpa6","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2402.16819","v":0.528},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"gsm8k-1d6dd8f1d7481d82","rd":"2023-09-27","sid":"recxjNidfoLOuJkgZ","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.354},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Gemma 7B","m":"gemma-7b","oid":"gsm8k-6a23c1b2b880b5dd","rd":"2024-02-21","sid":"recgxNHJ9ndu3AnUb","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.464},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"GPT-4","m":"gpt-4-0613","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-b2c1ef18938950f2","rd":"2023-06-13","sid":"recUjhbldxzRr4009","sm":"gpt-4-0613","src":"http://arxiv.org/abs/2309.10305","v":0.8999},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"GPT-3.5 Turbo","m":"gpt-3-5-turbo-0613","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-db06682805a9ead5","rd":"2023-06-13","sid":"recVbcJgszljOW7Kk","sm":"gpt-3.5-turbo-0613","src":"http://arxiv.org/abs/2309.10305","v":0.5777},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLAMA-7B","m":"llama-7b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-ba2c9f538ce1829e","rd":"2023-02-24","sid":"recUCtsCjtPJ6tU2S","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.10305","v":0.0978},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLAMA 2-7B","m":"llama-2-7b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-3f37634bcfa1ac13","rd":"2023-07-18","sid":"recQl93FcUulVDpYh","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.10305","v":0.1622},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"MPT-7B","m":"mpt-7b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-1bad9c2e027cd97a","rd":"2023-05-05","sid":"recQY9qdqwrrOh82b","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.10305","v":0.0864},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Falcon-7B","m":"falcon-7b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-80570af5a3feaef0","rd":"2023-04-24","sid":"recaRQgaTBxBau6lP","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.10305","v":0.0546},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"ChatGLM 2-6B","m":"chatglm2-6b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-c365d7ba3362e3e8","rd":"2023-06-24","sid":"recyALBBs1Nq8G6a1","sm":"chatglm2-6b","src":"http://arxiv.org/abs/2309.10305","v":0.289},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Baichuan 1-7B","m":"baichuan-7b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-c7688a53b9c618ee","rd":"2023-06-01","sid":"recXedJgypN9E4urq","sm":"Baichuan-7B","src":"http://arxiv.org/abs/2309.10305","v":0.092},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Baichuan 2-7B-Base","m":"baichuan-2-7b-base","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-eb57958b030f1024","rd":"2023-09-20","sid":"rec7vTJoXibWJVN3I","sm":"Baichuan-2-7B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.245},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLaMA-13B","m":"llama-13b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-f09755a586551243","rd":"2023-02-24","sid":"recLX4SpBZawYMjNC","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2309.10305","v":0.2055},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLAMA 2-13B","m":"llama-2-13b","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-a2a6294736ae2da4","rd":"2023-07-18","sid":"recci7OCQPpd5JeXn","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2309.10305","v":0.2889},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Vicuna-13B","m":"vicuna-13b-v1-1","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-6127b8918fd28fc7","rd":"2023-04-12","sid":"rec7e2U57eUieqNli","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.10305","v":0.2813},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Baichuan 1-13B-Base","m":"baichuan-13b-base","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-5c83f333c29e481c","rd":"2023-07-11","sid":"rec9UJIdmGpZliFSB","sm":"Baichuan-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.268},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Baichuan 2-13B-Base","m":"baichuan-2-13b-base","notes":"\"For GSM8K, we adopt 4-shot testing derived from OpenCompass (OpenCompass, 2023).\"","oid":"gsm8k-c82e4219722d918b","rd":"2023-09-06","sid":"recGqlIrVF6UiDf9Q","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.528},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Phi-3.5-mini","m":"phi-3-5-mini-instruct","notes":"CoT","oid":"gsm8k-284b21e0271cc6fb","rd":"2024-08-16","sid":"reccYIVbofOBOPknJ","sm":"Phi-3.5-mini-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.862},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Phi-3.5-MoE","m":"phi-3-5-moe-instruct","notes":"CoT","oid":"gsm8k-195bfd44c85c766e","rd":"2024-08-17","sid":"recDI1z24XCRLbrny","sm":"Phi-3.5-MoE-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.887},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","notes":"CoT","oid":"gsm8k-ec2d121748dc7343","rd":"2023-09-27","sid":"rec4iro1y3Mse53sB","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.544},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Mistral-Nemo 12B","m":"mistral-nemo-base-2407","notes":"CoT","oid":"gsm8k-732ae9aa26a7a267","rd":"2024-07-18","sid":"recdbE6j0lBrX5byj","sm":"Mistral-Nemo-Base-2407","src":"http://arxiv.org/abs/2404.14219","v":0.842},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Llama-3.1-In 8B","m":"llama-3-1-8b-instruct","notes":"CoT","oid":"gsm8k-a745d6948fb1a518","rd":"2024-07-23","sid":"recOcYLjPO17LCZQP","sm":"Llama-3.1-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.824},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Gemma-2 9B","m":"gemma-2-9b","notes":"CoT","oid":"gsm8k-8f19da72b8edc85b","sid":"rec7skWHWt0sr5KwE","sm":"gemma-2-9b","src":"http://arxiv.org/abs/2404.14219","v":0.849},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Gemini-1.5 Flash","m":"gemini-1-5-flash-001","notes":"CoT","oid":"gsm8k-5c190fb4e60c288d","rd":"2024-05-23","sid":"recx5qxkmLl6QN3sM","sm":"gemini-1.5-flash-001","src":"http://arxiv.org/abs/2404.14219","v":0.824},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"GPT-4o-mini","m":"gpt-4o-mini-2024-07-18","notes":"CoT","oid":"gsm8k-e793b76d1c30f369","rd":"2024-07-18","sid":"recf8nbb0TVMGvbfa","sm":"gpt-4o-mini-2024-07-18","src":"http://arxiv.org/abs/2404.14219","v":0.913},{"b":"gsm8k","dn":"GPT-3.5","m":"text-davinci-003","oid":"gsm8k-77926ead42a418ab","rd":"2022-11-28","sid":"recxA7ihKzVOLgmMY","sm":"text-davinci-003","src":"http://arxiv.org/abs/2403.04652","v":0.571},{"b":"gsm8k","dn":"Falcon 180B","m":"falcon-180b","oid":"gsm8k-348e1ad99ed45765","rd":"2023-09-06","sid":"recifLu8aRHvaMnoy","sm":"falcon-180B","src":"http://arxiv.org/abs/2403.04652","v":0.544},{"b":"gsm8k","dn":"Qwen 7B","m":"qwen-7b","oid":"gsm8k-5176517e58267081","rd":"2023-09-28","sid":"recJXNf2osPIVcVJ1","sm":"Qwen-7B","src":"http://arxiv.org/abs/2403.04652","v":0.517},{"b":"gsm8k","dn":"Qwen 14B","m":"qwen-14b","oid":"gsm8k-ddfc1ed7509ddd58","rd":"2023-09-28","sid":"recHeOf9hsuMTB0Nt","sm":"Qwen-14B","src":"http://arxiv.org/abs/2403.04652","v":0.613},{"b":"gsm8k","dn":"Baichuan 2 7B","m":"baichuan-2-7b-base","oid":"gsm8k-a1164e23157a7d79","rd":"2023-09-20","sid":"rect5qryhROP5cKF1","sm":"Baichuan-2-7B-Base","src":"http://arxiv.org/abs/2403.04652","v":0.245},{"b":"gsm8k","dn":"Baichuan 2 13B","m":"baichuan-2-13b-base","oid":"gsm8k-31adc0148014f1f2","rd":"2023-09-06","sid":"recS8XnSp8vwS08Nq","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2403.04652","v":0.221},{"b":"gsm8k","dn":"LlaMA 2 7B","m":"llama-2-7b","oid":"gsm8k-b380abe107a52928","rd":"2023-07-18","sid":"recalDZW02BuhxxDe","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2403.04652","v":0.167},{"b":"gsm8k","dn":"LlaMA 2 34B","m":"llama-2-34b","oid":"gsm8k-67282de47c610aba","rd":"2023-07-18","sid":"recp9w8gDc4q84LTY","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2403.04652","v":0.422},{"b":"gsm8k","dn":"LlaMA 2 70B","m":"llama-2-70b-hf","oid":"gsm8k-704c8e46b61a40b9","rd":"2023-07-18","sid":"recFypD2efi4otQCy","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2403.04652","v":0.568},{"b":"gsm8k","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"gsm8k-a411a553f6e43547","rd":"2023-09-27","sid":"reclGrtotrnt1MfuQ","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2403.04652","v":0.475},{"b":"gsm8k","dn":"InternLM 20B","m":"internlm-20b","oid":"gsm8k-669d937c254ce14f","rd":"2023-09-18","sid":"reccIpxndujtTs3bV","sm":"internlm-20b","src":"http://arxiv.org/abs/2403.04652","v":0.629},{"b":"gsm8k","dn":"Yi 6B","m":"yi-6b","oid":"gsm8k-a654ad430323c9dd","rd":"2023-11-02","sid":"recA2Ili8KeePjYZs","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.325},{"b":"gsm8k","dn":"Yi 34B","m":"yi-34b","oid":"gsm8k-92b1fa4beb1c7e09","rd":"2023-11-02","sid":"recpRrI1v4ew4AMpx","sm":"Yi-34B","src":"http://arxiv.org/abs/2403.04652","v":0.672},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"MPT 7B","m":"mpt-7b","oid":"gsm8k-ec6dad030a2b4c1c","rd":"2023-05-05","sid":"recwye183nOXecXAC","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.068},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"MPT 30B","m":"mpt-30b","oid":"gsm8k-699053c965fa3cc7","rd":"2023-06-22","sid":"rec6UjNAzgKa4aDoH","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.152},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Falcon 7B","m":"falcon-7b","oid":"gsm8k-d68e8ff04d63a640","rd":"2023-04-24","sid":"recbtjtAzdu0vswek","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.068},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Falcon 40B","m":"falcon-40b","oid":"gsm8k-68980727e305d4b8","rd":"2023-03-15","sid":"reccgFXJBe3qRmPEN","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.196},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"gsm8k-f638e86823ee3074","rd":"2023-02-24","sid":"recMklFp5jpk1l1Dr","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.11},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"gsm8k-6b176b11412c2905","rd":"2023-02-24","sid":"recmVk4n0ii6xMABS","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.178},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"gsm8k-b88f688423f26714","rd":"2023-02-24","sid":"recSODAewQMPlTkes","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.356},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"gsm8k-3086ea27a62823b2","rd":"2023-02-24","sid":"rec4NnfPf2dSIvbCy","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.509},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"gsm8k-62b92073dac0cd5d","rd":"2023-07-18","sid":"recUQV5eq2Y7i9TTm","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.146},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"gsm8k-65a78c195618411a","rd":"2023-07-18","sid":"recjXiUynLVUcfghM","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.287},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"gsm8k-3698618b8ab1fc62","rd":"2023-07-18","sid":"recoM9rGsGnmYfvdg","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.422},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"gsm8k-40348efe4daa86be","rd":"2023-07-18","sid":"rec087bDThmBmUa7C","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.568},{"b":"gsm8k","dn":"PaLM 62B","m":"palm-62b","oid":"gsm8k-c5ccc5b3e2dbf1c0","rd":"2022-04-04","sid":"recdeOIkjqHvhVHBF","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.33},{"b":"gsm8k","dn":"PaLM 540B","m":"palm-540b","oid":"gsm8k-61c77204a5c535a3","rd":"2022-04-04","sid":"rec0taUYKp9iVVfCD","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.565},{"b":"gsm8k","dn":"LLaMA 7B","m":"llama-7b","oid":"gsm8k-abf65befcfd713f5","rd":"2023-02-24","sid":"recTfzfAsGKOt9prx","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.11},{"b":"gsm8k","dn":"LLaMA 13B","m":"llama-13b","oid":"gsm8k-dd67fed466bafb38","rd":"2023-02-24","sid":"rec6vcGiZs6MASzjr","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.178},{"b":"gsm8k","dn":"LLaMA 33B","m":"llama-33b","oid":"gsm8k-222ff087db0416f4","rd":"2023-02-24","sid":"recMypFW0m9QPk4fN","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.356},{"b":"gsm8k","dn":"LLaMA 65B","m":"llama-65b","oid":"gsm8k-510146befa92c7ab","rd":"2023-02-24","sid":"recMComWaZ4F2sglY","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.509},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"gsm8k-0ff05421b35ac762","rd":"2023-07-18","sid":"recRTtTwROe3kxBBb","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2401.04088","v":0.16},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"gsm8k-2ff15a7004b86287","rd":"2023-07-18","sid":"recqEMuNOspyhcIMc","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2401.04088","v":0.343},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 1 33B","m":"llama-33b","oid":"gsm8k-71ed1824bf5dbd5c","rd":"2023-02-24","sid":"rece7m5HNbcs75EDJ","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2401.04088","v":0.441},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"gsm8k-b2d9ab5210881a95","rd":"2023-07-18","sid":"rec9b94BalsIQe0HQ","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2401.04088","v":0.696},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"gsm8k-8f71c3d9659fc4a8","rd":"2023-09-27","sid":"recVlEJvA1rFQeLL3","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.5},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"gsm8k-db53d87753182678","rd":"2023-12-11","sid":"reczD1NT6XqiSTtzQ","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.744},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"GPT-3.5 (8-shot)","m":"text-davinci-003","oid":"gsm8k-660d1d748611a29f","rd":"2022-11-28","sid":"rec2uZfuhxZCO1QSe","sm":"text-davinci-003","src":"https://arxiv.org/abs/2309.16609","v":0.782},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"ChatGLM2 6B (8-shot)","m":"chatglm2-6b","oid":"gsm8k-0fd912da77e1f01d","rd":"2023-06-24","sid":"recblK5kWC7IZTktk","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.288},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"MPT 7B","m":"mpt-7b","oid":"gsm8k-68065701a9902f61","rd":"2023-05-05","sid":"rechR9KOVBLbR6gLp","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.091},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"MPT 30B","m":"mpt-30b","oid":"gsm8k-2a55d80c0d0b7edc","rd":"2023-06-22","sid":"recll5gMAvJtlDI4Z","sm":"mpt-30b","src":"https://arxiv.org/abs/2309.16609","v":0.152},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Falcon 7B","m":"falcon-7b","oid":"gsm8k-b2a6b898af651d7b","rd":"2023-04-24","sid":"recBtj68J3F2ighqz","sm":"falcon-7b","src":"https://arxiv.org/abs/2309.16609","v":0.068},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Falcon 40B","m":"falcon-40b","oid":"gsm8k-c0f5014be689d148","rd":"2023-03-15","sid":"rec0JKU1MVgnyXiaG","sm":"falcon-40b","src":"https://arxiv.org/abs/2309.16609","v":0.196},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"ChatGLM2 6B","m":"chatglm2-6b","oid":"gsm8k-ab4a914b4d1e960f","rd":"2023-06-24","sid":"recCpu5aGJJNQDoZt","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.324},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"InternLM 7B","m":"internlm-7b","oid":"gsm8k-be28996b86bfe092","rd":"2023-07-05","sid":"recWftrwAHnRbFITr","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.312},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"InternLM 20B","m":"internlm-20b","oid":"gsm8k-a55d682757e0cbbc","rd":"2023-09-18","sid":"recNoBGJXXooeIWLz","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.526},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Baichuan2 7B","m":"baichuan-2-7b-base","oid":"gsm8k-78babb9c15d20023","rd":"2023-09-20","sid":"recCrNkpJFgDVJ6Tf","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.246},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"gsm8k-04c2ebdaa305178c","rd":"2023-09-06","sid":"rec0gI6pARuxRMVr5","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.528},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 7B","m":"llama-7b","oid":"gsm8k-fd7666a4a470f3ea","rd":"2023-02-24","sid":"recrWri4a3tjHHdlF","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.11},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 13B","m":"llama-13b","oid":"gsm8k-ca2c1aaeeef077e0","rd":"2023-02-24","sid":"rec9PV4HA7E8YFgMJ","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.203},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 33B","m":"llama-33b","oid":"gsm8k-be1674e02f528355","rd":"2023-02-24","sid":"recxheW0e7qyWPJSI","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.423},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLaMA 65B","m":"llama-65b","oid":"gsm8k-b388011ce6223159","rd":"2023-02-24","sid":"rec49SYWqVvnyux3x","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.544},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"gsm8k-6ebbf85301bd04ce","rd":"2023-07-18","sid":"recNitTxcPAdpzITH","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.167},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"gsm8k-d4269983e97b5526","rd":"2023-07-18","sid":"recnufr3ybafoxuxD","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.296},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"gsm8k-4be2bb71cdbe8969","rd":"2023-07-18","sid":"receU4INAtymlLbDr","sm":"Llama-2-34b","src":"https://arxiv.org/abs/2309.16609","v":0.422},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"gsm8k-a72eaadfd25c4ce0","rd":"2023-07-18","sid":"recETvjYPiBguB7sr","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.633},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"StableBeluga2 70B","m":"stablebeluga2","oid":"gsm8k-7423fcfd73cc71c2","rd":"2023-07-20","sid":"recyd0ZfeN48NpVc5","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.696},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"QWEN 1.8B","m":"qwen-1-8b","oid":"gsm8k-a9640e9e5718d66b","rd":"2023-11-30","sid":"recKJ9LXmZvEkfVBk","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.212},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"QWEN 7B","m":"qwen-7b","oid":"gsm8k-9041d2934dc2555b","rd":"2023-09-28","sid":"recJ2BKu4GfbEnMOX","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.517},{"b":"gsm8k","cfg":{"shots":"8"},"dn":"QWEN 14B","m":"qwen-14b","oid":"gsm8k-b0992d85bfaca379","rd":"2023-09-28","sid":"recLeeSk6utRh8JlP","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.613},{"b":"gsm8k","dn":"DS-Coder-V2-Lite-Instruct","m":"deepseek-coder-v2-lite-instruct","oid":"gsm8k-34047965793818a4","rd":"2024-06-13","sid":"rec0LE2Ld8YxKTtGU","sm":"DeepSeek-Coder-V2-Lite-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.876},{"b":"gsm8k","dn":"DS-Coder-V2-Instruct","m":"deepseek-coder-v2-instruct","oid":"gsm8k-aa65899c71ba7895","rd":"2024-06-17","sid":"recgWLov37oVjOk4Y","sm":"DeepSeek-Coder-V2-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.945},{"b":"gsm8k","dn":"Qwen2.5-Coder-3B-Instruct","m":"qwen2-5-coder-3b-instruct","oid":"gsm8k-155db1e2b9670ac0","rd":"2024-11-06","sid":"recEBiI7Vckg3zy0g","sm":"Qwen2.5-Coder-3B-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.807},{"b":"gsm8k","dn":"Qwen2.5-Coder-7B-Instruct","m":"qwen2-5-coder-7b-instruct","oid":"gsm8k-73586987b0911b27","rd":"2024-09-17","sid":"recrxlcLlOEJo7b5M","sm":"Qwen2.5-Coder-7B-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.867},{"b":"gsm8k","dn":"Qwen2.5-Coder-14B-Instruct","m":"qwen2-5-coder-14b-instruct","oid":"gsm8k-5315f2d296b7a1a1","rd":"2024-11-06","sid":"reciL9E2SFpdpTQ9F","sm":"Qwen2.5-Coder-14B-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.942},{"b":"gsm8k","dn":"Qwen2.5-Coder-32B-Instruct","m":"qwen2-5-coder-32b-instruct","oid":"gsm8k-59cd68521c37649b","rd":"2024-11-21","sid":"reckQ3NF72z9ICZI6","sm":"Qwen2.5-Coder-32B-Instruct","src":"http://arxiv.org/abs/2409.12186","v":0.93},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-0.5B","m":"qwen2-5-coder-0-5b","oid":"gsm8k-2ee920e43fb3f80e","rd":"2024-09-18","sid":"recriWzkW7RWIJpUI","sm":"Qwen2.5-Coder-0.5B","src":"http://arxiv.org/abs/2409.12186","v":0.345},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"DS-Coder-1.3B-Base","m":"deepseek-coder-1-3b-base","oid":"gsm8k-088c70451b1c2021","rd":"2023-11-02","sid":"recfazKRlIPHIqw4y","sm":"deepseek-coder-1.3b-base","src":"http://arxiv.org/abs/2409.12186","v":0.044},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-1.5B","m":"qwen2-5-coder-1-5b","oid":"gsm8k-0e9fcf56cc3929e9","rd":"2024-09-18","sid":"recIikbIv1ZKuCk4E","sm":"Qwen2.5-Coder-1.5B","src":"http://arxiv.org/abs/2409.12186","v":0.658},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"StarCoder2-3B","m":"starcoder2-3b","oid":"gsm8k-4b8a77e379ec4181","rd":"2024-02-22","sid":"recoPSMZ3BwAx0twb","sm":"starcoder2-3b","src":"http://arxiv.org/abs/2409.12186","v":0.216},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-3B","m":"qwen2-5-coder-3b","oid":"gsm8k-4c962a28fc800aaa","rd":"2024-09-18","sid":"reccwWD5uXqA4L3aV","sm":"Qwen2.5-Coder-3B","src":"http://arxiv.org/abs/2409.12186","v":0.757},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"StarCoder2-7B","m":"starcoder2-7b","oid":"gsm8k-ddb8286e0a6c5b05","rd":"2024-02-20","sid":"recAA2soUr8CvLUav","sm":"starcoder2-7b","src":"http://arxiv.org/abs/2409.12186","v":0.327},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"DS-Coder-6.7B-Base","m":"deepseek-coder-6-7b-base","oid":"gsm8k-40cf3954986dbc9f","rd":"2023-11-02","sid":"recBna1XK3H5DgEs2","sm":"deepseek-coder-6.7b-base","src":"http://arxiv.org/abs/2409.12186","v":0.213},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"DS-Coder-V2-Lite-Base","m":"deepseek-coder-v2-lite-base","oid":"gsm8k-fa498e5a4ae4c545","rd":"2024-06-13","sid":"recJu2UPUTDAbeHPS","sm":"DeepSeek-Coder-V2-Lite-Base","src":"http://arxiv.org/abs/2409.12186","v":0.671},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"CodeQwen1.5-7B","m":"codeqwen1-5-7b","oid":"gsm8k-58caac2fb2d14748","rd":"2024-04-15","sid":"recK2XPT7uEmiN3aQ","sm":"CodeQwen1.5-7B","src":"http://arxiv.org/abs/2409.12186","v":0.377},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-7B","m":"qwen2-5-coder-7b","oid":"gsm8k-52dd48934b70185f","rd":"2024-09-18","sid":"recvzqMLC4OK292e5","sm":"Qwen2.5-Coder-7B","src":"http://arxiv.org/abs/2409.12186","v":0.839},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"StarCoder2-15B","m":"starcoder2-15b","oid":"gsm8k-588d5ef5a08495e7","rd":"2024-02-20","sid":"reclAG14Sox6ongVu","sm":"starcoder2-15b","src":"http://arxiv.org/abs/2409.12186","v":0.577},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-14B","m":"qwen2-5-coder-14b","oid":"gsm8k-85518e86fabd9673","rd":"2024-09-18","sid":"recE8WEXVD6mou9Pa","sm":"Qwen2.5-Coder-14B","src":"http://arxiv.org/abs/2409.12186","v":0.887},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"DS-Coder-33B-Base","m":"deepseek-coder-33b-base","oid":"gsm8k-5cb4a2caa014cd7a","rd":"2023-11-02","sid":"recgenJ4jFOj6dzga","sm":"deepseek-coder-33b-base","src":"http://arxiv.org/abs/2409.12186","v":0.354},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"DS-Coder-V2-Base","m":"deepseek-coder-v2-base","oid":"gsm8k-3fd573701767cac1","rd":"2024-06-17","sid":"recwfs7UuYljqIWC4","sm":"DeepSeek-Coder-V2-Base","src":"http://arxiv.org/abs/2409.12186","v":0.858},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen2.5-Coder-32B","m":"qwen2-5-coder-32b","oid":"gsm8k-456d6cc06c54252a","rd":"2024-09-18","sid":"recOpZnC24j98jgDQ","sm":"Qwen2.5-Coder-32B","src":"http://arxiv.org/abs/2409.12186","v":0.911},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Llama-65B","m":"llama-65b","oid":"gsm8k-69efd321c7cdb2f5","rd":"2023-02-24","sid":"recr1p7aw6EXnQ1v3","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2309.05463","v":0.509},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Llama2-7B","m":"llama-2-7b","oid":"gsm8k-3fe8d9036ff76515","rd":"2023-07-18","sid":"rec5XefgMSdMfybiu","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.146},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Llama-7B","m":"llama-7b","oid":"gsm8k-ca991fd04622ce07","rd":"2023-02-24","sid":"recyIqPH1anVacKat","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.11},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"gsm8k-9fb268d68e1b6294","rd":"2023-05-05","sid":"recKiblWkHkeFO7ru","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.068},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"gsm8k-346bcfd7e6828b2a","rd":"2023-04-24","sid":"recgD5lGOM5UcF7h2","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.068},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"gsm8k-e467bcb7ddd8416e","rd":"2023-04-24","sid":"recWnPUygvzfdY7Af","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.0462},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"gsm8k-083db59f3b24ce89","rd":"2023-03-15","sid":"recLm39SndNBNiWRp","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.2146},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"gsm8k-03bbd6150482db0a","rd":"2024-05-09","sid":"recVCko9kWNdH2gNm","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.5383},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"GPT-4","m":"gpt-4-0314","notes":"5-shot CoT","oid":"gsm8k-06016d3b678f709f","rd":"2023-03-14","sid":"rece42bDNIveYuuDP","sm":"gpt-4-0314","src":"http://arxiv.org/abs/2303.08774","v":0.92},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"GPT-3.5","m":"text-davinci-003","oid":"gsm8k-0855a5cd50228e47","rd":"2022-11-28","sid":"recIH7lu93SZpnNtQ","sm":"text-davinci-003","src":"http://arxiv.org/abs/2303.08774","v":0.571},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Claude Instant 1.1","m":"claude-instant-1-1","oid":"gsm8k-cfbfc8acc78e89e4","sid":"rec8k8VQ1JZHytR3M","sm":"claude-instant-1.1","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.809},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Claude Instant 1.2","m":"claude-instant-1-2","oid":"gsm8k-5a4c6e70931454c7","rd":"2023-08-09","sid":"recxB6uTAez5LVB1C","sm":"claude-instant-1.2","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.867},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"gsm8k-00c48b1543898ea0","rd":"2023-07-18","sid":"recQvGZxtMqZBhrzX","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.369},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"gsm8k-cc9b8e2ef96843d9","rd":"2023-07-18","sid":"recvSWvfSUrNLC9jO","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.471},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"gsm8k-1b86df370b9924c0","rd":"2023-09-06","sid":"recRNdsAyvpxPEgNH","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.457},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"gsm8k-93e89584e3d1144b","rd":"2023-09-24","sid":"recWEw3LYrO0FK3nA","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.595},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"InternLM-Chat 20B","m":"internlm-chat-20b","oid":"gsm8k-775c11037973ab95","rd":"2023-09-17","sid":"rec1DzxapEWUb1QLT","sm":"internlm-chat-20b","src":"http://arxiv.org/abs/2403.04652","v":0.157},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"gsm8k-d7ff9a64004354cb","rd":"2023-11-22","sid":"recXDDudal1CkzMWV","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.384},{"b":"gsm8k","cfg":{"shots":"0"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"gsm8k-b3d4fbf3c383d375","rd":"2023-11-22","sid":"recYK13JohCBsAbGm","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.717},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"gsm8k-9c01c0b47d651c9b","rd":"2023-07-18","sid":"receIYDoF0g38AmUG","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.027},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"gsm8k-1a6dae3459b0ef02","rd":"2023-07-18","sid":"recjUxUiNVOU4NPaE","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.587},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"gsm8k-a71379eaf6f88746","rd":"2023-09-06","sid":"recpO2MhyWbmloHzx","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.233},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"gsm8k-65e31e1d6df0a0b0","rd":"2023-09-24","sid":"recZlE4AqZQVZIb8G","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.612},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"InternLM-Chat 20B","m":"internlm-20b","oid":"gsm8k-117ee0694323489d","rd":"2023-09-18","sid":"recch1fcHNEU8B7jN","sm":"internlm-20b","src":"http://arxiv.org/abs/2403.04652","v":0.434},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"gsm8k-6594fba95ad371ef","rd":"2023-11-22","sid":"recNmM3qEyqQo0BeU","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.449},{"b":"gsm8k","cfg":{"shots":"4"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"gsm8k-df1fd8288591644a","rd":"2023-11-22","sid":"recmFCa4UkrZjtfQt","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.76},{"b":"gsm8k","cfg":{"shots":"5"},"dn":"INTELLECT-1-Instruct","m":"intellect-1-instruct","oid":"gsm8k-06e6c46fb9a2d193","rd":"2024-11-29","sid":"rec2Fh0JiOpowgEKq","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.3858},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-09-27","m":"claude-fable-5-1","oid":"gso-df8f0ffbc8a88a2a","rd":"2026-09-01","sid":"recQ2XelNSf1hgTyB","sm":"claude-fable-5-1_unknown","src":"https://gso-bench.github.io/index.html","v":0.8824},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-09-27","m":"gpt-6-astra","oid":"gso-02206059889ec286","rd":"2026-09-03","sid":"reciZmjMQUpEwWLdB","sm":"gpt-6-astra_unknown","src":"https://gso-bench.github.io/index.html","v":0.7941},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-09-27","m":"claude-fable-5","oid":"gso-f5663641f43889a9","rd":"2026-06-09","sid":"recynRgjlNkqf7SRD","sm":"claude-fable-5_unknown","src":"https://gso-bench.github.io/index.html","v":0.7843000000000001},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-09-27","m":"gpt-5-6-sol","oid":"gso-5a8a9302900afda4","rd":"2026-07-09","sid":"recrBBgZqqBgYOpXp","sm":"gpt-5.6-sol_unknown","src":"https://gso-bench.github.io/index.html","v":0.7647},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-07-12","m":"claude-opus-4-8","oid":"gso-9b0f2bc2d41dbbe2","rd":"2026-05-28","sid":"recD2WNrQUHIATHKH","sm":"claude-opus-4-8_unknown","src":"https://gso-bench.github.io/index.html","v":0.4706},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2026-04-27","e":"high","m":"claude-opus-4-7","oid":"gso-376e6bf49bca7886","rd":"2026-04-16","sid":"reccHX8eA2pr1NUgv","sm":"claude-opus-4-7_high","src":"https://gso-bench.github.io/index.html","v":0.4412},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-04-27","m":"claude-opus-4-7","oid":"gso-d81aeadd62a8f2db","rd":"2026-04-16","sid":"recB6g5swe17tSdKR","sm":"claude-opus-4-7","v":0.4412},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2026-04-27","e":"high","m":"claude-opus-4-6","oid":"gso-d2aab8d85cef6855","rd":"2026-02-05","sid":"recyQlfHiilyasE8b","sm":"claude-opus-4-6_high","v":0.412},{"b":"gso","cfg":{"effort":"xhigh","scaffold":"OpenHands"},"d":"2026-04-27","e":"xhigh","m":"gpt-5-5","oid":"gso-77fd8de7c7159bbd","rd":"2026-04-23","sid":"recbfEoFyIqdyNOkl","sm":"gpt-5.5_xhigh","src":"https://gso-bench.github.io/index.html","v":0.402},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-07-12","m":"claude-sonnet-5","oid":"gso-e736b52a6fd8e58c","rd":"2026-06-30","sid":"recWjLztriPDdu5G8","sm":"claude-sonnet-5_unknown","src":"https://gso-bench.github.io/index.html","v":0.3725},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-04-27","m":"claude-opus-4-6","oid":"gso-1c06f42ee74d9f0b","rd":"2026-02-05","sid":"recjWm0x4FyWwetTo","sm":"claude-opus-4-6_unknown","src":"https://gso-bench.github.io/index.html","v":0.3333},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-04-27","m":"claude-opus-4-6","oid":"gso-0154c9ec16ae7270","rd":"2026-02-05","sid":"reco0a5ivM5mr8sai","sm":"claude-opus-4-6","v":0.3333},{"b":"gso","cfg":{"effort":"xhigh","scaffold":"OpenHands"},"d":"2026-03-10","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"gso-04d4f7b1b21727a5","rd":"2026-03-05","sid":"reco2vF4MBsh4TlN3","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://gso-bench.github.io/index.html","v":0.31370000000000003},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2025-12-18","e":"high","m":"gpt-5-2-2025-12-11","oid":"gso-f46b7a8ecb5574cd","rd":"2025-12-11","sid":"reco6scbTaMu1KpKP","sm":"gpt-5.2-2025-12-11_high","src":"https://gso-bench.github.io/index.html","v":0.2745},{"b":"gso","cfg":{"scaffold":"OpenHands"},"m":"claude-opus-4-5-2025-11-01","oid":"gso-dfc23ad1fb81150b","rd":"2025-11-24","sid":"recj4DxItKjOUYsAN","sm":"claude-opus-4-5-20251101","src":"https://gso-bench.github.io/leaderboard.html","v":0.265},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-11-28","m":"claude-opus-4-5-2025-11-01","oid":"gso-9ed56d907a32fcbf","rd":"2025-11-24","sid":"recE6ibGfliJ4tCOo","sm":"claude-opus-4-5-20251101_unknown","src":"https://gso-bench.github.io/index.html","v":0.2647},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2026-03-10","e":"high","m":"gpt-5-4-2026-03-05","oid":"gso-9b83e2f229ea361b","rd":"2026-03-05","sid":"recPK6PeVNd5aK2bT","sm":"gpt-5.4-2026-03-05_high","src":"https://gso-bench.github.io/index.html","v":0.2549},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2026-03-09","m":"gemini-3-1-pro-preview","oid":"gso-c76459e1a176dcd8","rd":"2026-02-19","sid":"recxX8wnn05N8Oysr","sm":"gemini-3.1-pro-preview","src":"https://gso-bench.github.io/index.html","v":0.2255},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-11-28","m":"gemini-3-pro-preview","oid":"gso-cf7a7d14fdd50efd","rd":"2025-11-18","sid":"reckInnupZ6V8D9li","sm":"gemini-3-pro-preview","src":"https://gso-bench.github.io/leaderboard.html","v":0.1863},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-10-10","m":"claude-sonnet-4-5-2025-09-29","oid":"gso-3c31bf911230f874","rd":"2025-09-29","sid":"recHDedYHXs08M0kY","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://gso-bench.github.io/index.html","v":0.1471},{"b":"gso","cfg":{"scaffold":"OpenHands"},"m":"claude-sonnet-4-5-2025-09-29","oid":"gso-ef4e3ed3ed330b00","rd":"2025-09-29","sid":"recLNUhUM75fIc6xP","sm":"claude-sonnet-4-5-20250929","src":"https://gso-bench.github.io/leaderboard.html","v":0.147},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-12-18","m":"gpt-5-1-2025-11-13","oid":"gso-d0cf7ff78c91d249","rd":"2025-11-13","sid":"recgp9r5bBUG2Csxs","sm":"gpt-5.1-2025-11-13_unknown","src":"https://gso-bench.github.io/index.html","v":0.1373},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"e":"high","m":"gpt-5-1-2025-11-13","oid":"gso-f058e63cbef64012","rd":"2025-11-13","sid":"recZJBCgxFNxjLDZs","sm":"gpt-5.1-2025-11-13_high","v":0.137},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-12-19","m":"gemini-3-flash-preview","oid":"gso-7dd7cea4aab1bef2","rd":"2025-12-17","sid":"recjjdkkqSrz8EgCW","sm":"gemini-3-flash-preview","src":"https://gso-bench.github.io/index.html","v":0.098},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2025-06-09","e":"high","m":"o3-2025-04-16","oid":"gso-4f47428746175add","rd":"2025-04-16","sid":"recyrQ4JUWjfKS1xR","sm":"o3-2025-04-16_high","src":"https://gso-bench.github.io/leaderboard.html","v":0.0882},{"b":"gso","cfg":{"scaffold":"OpenHands"},"m":"claude-opus-4-2025-05-14","oid":"gso-146157f53b05fedc","rd":"2025-05-22","sid":"recth4YmbJGhtEeae","sm":"claude-opus-4-20250514","src":"https://gso-bench.github.io/leaderboard.html","v":0.069},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2025-08-10","e":"high","m":"gpt-5-2025-08-07","oid":"gso-e82168ecfcdd5c89","rd":"2025-08-07","sid":"recXbtm7TgBsNVAJQ","sm":"gpt-5-2025-08-07_high","src":"https://gso-bench.github.io/leaderboard.html","v":0.06860000000000001},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-08-10","m":"claude-opus-4-2025-05-14","oid":"gso-4c7974ca72372d28","rd":"2025-05-22","sid":"recRyARCVqUatCNzh","sm":"claude-opus-4-20250514_unknown","src":"https://gso-bench.github.io/index.html","v":0.06860000000000001},{"b":"gso","cfg":{"scaffold":"OpenHands"},"m":"claude-sonnet-4-2025-05-14","oid":"gso-f3325fe936d0154f","rd":"2025-05-22","sid":"recCMHpSUHUw7MhjQ","sm":"claude-sonnet-4-20250514","src":"https://gso-bench.github.io/leaderboard.html","v":0.049},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-08-26","m":"kimi-k2-instruct","oid":"gso-63335b88d2ef7388","rd":"2025-07-12","sid":"recslQxYG0KdpDtzW","sm":"Kimi-K2-Instruct","src":"https://gso-bench.github.io/leaderboard.html","v":0.049},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-09-02","m":"qwen3-coder-480b-a35b-instruct","oid":"gso-ead7cfc2619128df","rd":"2025-07-31","sid":"rechj9ltkUUfx20ya","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://gso-bench.github.io/leaderboard.html","v":0.049},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-05-30","m":"claude-sonnet-4-2025-05-14","oid":"gso-1e94794d56563a57","rd":"2025-05-22","sid":"recOVY5kUnc4Mh8iX","sm":"claude-sonnet-4-20250514_unknown","src":"https://gso-bench.github.io/index.html","v":0.049},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-05-30","m":"claude-3-5-sonnet-2024-10-22","oid":"gso-b632b142f3c27532","rd":"2024-10-22","sid":"recTAzfnTtfeArFqk","sm":"claude-3-5-sonnet-20241022","src":"https://gso-bench.github.io/leaderboard.html","v":0.046},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-10-10","m":"gemini-2-5-pro","oid":"gso-3545f77381b37469","rd":"2025-06-17","sid":"recSwFDzdt4zosJsD","sm":"gemini-2.5-pro","src":"https://gso-bench.github.io/leaderboard.html","v":0.0392},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-10-10","m":"gemini-2-5-pro-preview-06-05","oid":"gso-c00ebce03dc6459f","rd":"2025-06-05","sid":"recEk77DIVqhyoCQd","sm":"gemini-2.5-pro-preview-06-05","v":0.0392},{"b":"gso","cfg":{"scaffold":"OpenHands"},"m":"claude-3-7-sonnet-2025-02-19","oid":"gso-36fbad19b3866bcd","rd":"2025-02-24","sid":"recoExXWLABCrn50A","sm":"claude-3-7-sonnet-20250219","src":"https://gso-bench.github.io/leaderboard.html","v":0.038},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-05-30","m":"claude-3-7-sonnet-2025-02-19","oid":"gso-5bc4dffeb635f334","rd":"2025-02-24","sid":"recaKd4PeC7yCrBMH","sm":"claude-3-7-sonnet-20250219_unknown","src":"https://gso-bench.github.io/index.html","v":0.038},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2025-05-30","e":"high","m":"o4-mini-2025-04-16","oid":"gso-3fd38fae27aabed4","rd":"2025-04-16","sid":"recq2Mzz81PDsAp4v","sm":"o4-mini-2025-04-16_high","src":"https://gso-bench.github.io/leaderboard.html","v":0.036000000000000004},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-08-26","m":"glm-4-5-air","oid":"gso-0b3e779e677f6b34","rd":"2025-07-20","sid":"recEcEvRGxBQTGqxO","sm":"GLM-4.5-Air","src":"https://gso-bench.github.io/index.html","v":0.0294},{"b":"gso","cfg":{"effort":"high","scaffold":"OpenHands"},"d":"2025-05-30","e":"high","m":"o3-mini-2025-01-31","oid":"gso-8d1c7a0d925760fe","rd":"2025-01-31","sid":"recmNBNhajAOofv1Q","sm":"o3-mini-2025-01-31_high","src":"https://gso-bench.github.io/leaderboard.html","v":0.013000000000000001},{"b":"gso","cfg":{"effort":"low","scaffold":"OpenHands"},"d":"2025-05-30","e":"low","m":"o3-mini-2025-01-31","oid":"gso-c7ad7fd8bc3994a9","rd":"2025-01-31","sid":"recPJ2z8G5wYQmWoN","sm":"o3-mini-2025-01-31_low","v":0.013000000000000001},{"b":"gso","cfg":{"scaffold":"OpenHands"},"d":"2025-05-30","m":"gpt-4o-2024-11-20","oid":"gso-11d271be75ac409f","rd":"2024-11-20","sid":"recSbAXRQhrp0eicx","sm":"gpt-4o-2024-11-20","src":"https://gso-bench.github.io/leaderboard.html","v":0},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"ada (350M)","m":"ada","oid":"hella_swag-8a326d2dd992aa5e","sid":"recfeLKZvBng82A20","sm":"ada","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.435},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"babbage (1.3B)","m":"babbage","oid":"hella_swag-f597f9514161fe44","rd":"2020-06-22","sid":"rec9GDlYjK3e9FkXR","sm":"babbage","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.555},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Baichuan2 7B","m":"baichuan-2-7b-base","oid":"hella_swag-c3e4c666ff573c70","rd":"2023-09-20","sid":"rec6q2wSPdc9L6x40","sm":"Baichuan-2-7B-Base","src":"http://arxiv.org/abs/2309.16609","v":0.68},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"hella_swag-95dcce809428a7ff","rd":"2023-09-06","sid":"recKMHyLo7iHhIm9m","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2309.16609","v":0.708},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"BLOOM (176B)","m":"bloom","oid":"hella_swag-0d18cfffee9e8aaa","rd":"2022-07-06","sid":"recQSVqDaut5Qbuj2","sm":"bloom","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.744},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"hella_swag-25a18598a9d7047e","rd":"2023-03-20","sid":"rec2aapgpGAXpxWQ6","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.594},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"ChatGLM2","m":"chatglm2-6b","oid":"hella_swag-91165597ecac16e6","rd":"2023-06-24","sid":"recGt5AcRHqUhevHc","sm":"chatglm2-6b","src":"http://arxiv.org/abs/2309.16609","v":0.57},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"curie (6.7B)","m":"curie","oid":"hella_swag-237bb52759775dd5","sid":"recKp5z9jwr9Zh0Ts","sm":"curie","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.682},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"davinci (175B)","m":"davinci","oid":"hella_swag-9f430763080e7591","sid":"recbOlkritQTwwoUh","sm":"davinci","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.775},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"hella_swag-4455a033f0bd7418","rd":"2024-05-07","sid":"recIDwVCV7xiAPtWR","sm":"DeepSeek-V2","src":"https://arxiv.org/pdf/2412.19437","v":0.871},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"hella_swag-5e166aaac8f2ccb7","rd":"2024-12-26","sid":"recnADYV42QsRB24p","sm":"DeepSeek-V3","src":"https://arxiv.org/pdf/2412.19437","v":0.889},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"hella_swag-f6b530669e53c24c","rd":"2023-04-11","sid":"recvtwJTHfsxN0xBD","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.708},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon 7B","m":"falcon-7b","oid":"hella_swag-bdd1ed37d595a627","rd":"2023-04-24","sid":"recNf0HzKB7GCWezm","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.7631},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon 7B","m":"falcon-7b","oid":"hella_swag-c165e3a6c2825cb2","rd":"2023-04-24","sid":"reclyVPS1rMuSndsk","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.781},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon 7B","m":"falcon-7b","oid":"hella_swag-f8fb1a6ca62868ff","rd":"2023-04-24","sid":"recUsvn4nbRtm8w4V","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.16609","v":0.741},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon 7B","m":"falcon-7b","oid":"hella_swag-96f35b6e8810b847","rd":"2023-04-24","sid":"reczFRmY0w4zEXGr0","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.7631},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon 7B","m":"falcon-7b","oid":"hella_swag-4765cb87042596e4","rd":"2023-04-24","sid":"recls5hl8NYpOc7i3","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.781},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon 40B","m":"falcon-40b","oid":"hella_swag-4c19448a26e331ff","rd":"2023-03-15","sid":"reciBfJ5AKzP8yAl6","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.828},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon 40B","m":"falcon-40b","oid":"hella_swag-e3d0099b5c3ff2e2","rd":"2023-03-15","sid":"recuz33sIq3QXEcxm","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.8528},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon 40B","m":"falcon-40b","oid":"hella_swag-b2501975dc959e8d","rd":"2023-03-15","sid":"recaRUn4YSr3AcqiP","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.8282},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon 40B","m":"falcon-40b","oid":"hella_swag-46cb8fadb087e80b","rd":"2023-03-15","sid":"recya8LRIunACDqxv","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.8528},{"b":"hella_swag","dn":"Falcon-7B","m":"falcon-7b","oid":"hella_swag-0e3190eb58ab83b6","rd":"2023-04-24","sid":"recAFpIJDF6mgmPuv","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.542},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"hella_swag-8e83f5b964e69cac","rd":"2023-04-24","sid":"recmVOgkewjlqKh8h","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.764},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon-180B","m":"falcon-180b","oid":"hella_swag-f0f4dc01db44bf69","rd":"2023-09-06","sid":"recGmteOirEpRk74U","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.89},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"hella_swag-f490696e95a657c0","rd":"2024-05-09","sid":"recIC1QoBZPGnA5tT","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.8207},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"hella_swag-e6b0c70e7a1ac142","rd":"2024-05-09","sid":"recag3Lz0AvhGRmS4","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.8291},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"hella_swag-593a4578df236b96","rd":"2024-05-09","sid":"recsqpRPEKmC1nrEh","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.8207},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"hella_swag-7bd95929f0cfebbb","rd":"2024-05-09","sid":"recQ4BBxuAOaopnCC","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.8291},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Gemma 2B","m":"gemma-2b","oid":"hella_swag-db2facf30503fad8","rd":"2024-02-21","sid":"recBqQ9CYxvNxJKHn","sm":"gemma-2b","src":"https://arxiv.org/pdf/2403.08295","v":0.714},{"b":"hella_swag","dn":"Gemma 7B","m":"gemma-7b","notes":"Run with HuggingFace H6 suite","oid":"hella_swag-f5147a650409849d","rd":"2024-02-21","sid":"rec1YZRNwjqtYEQgE","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.822},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Gemma 7B","m":"gemma-7b","oid":"hella_swag-cf3e57b3e75d8e8b","rd":"2024-02-21","sid":"reczjwjnz8igKpnEi","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.812},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GLaM","m":"glam-moe","oid":"hella_swag-f490c9ec6dfad072","rd":"2021-12-13","sid":"rec2jRUCp1AFzsN31","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.766},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"GLaM","m":"glam-moe","oid":"hella_swag-1553e15b7a0452c8","rd":"2021-12-13","sid":"recSe1QPYCHRdaNts","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.768},{"b":"hella_swag","cfg":{"shots":"8"},"dn":"GLaM","m":"glam-moe","oid":"hella_swag-deda13c1bf60caff","rd":"2021-12-13","sid":"rece76lEt406di379","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.772},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"hella_swag-f4497b75e60f0002","rd":"2021-12-08","sid":"recwxUQUprmMr9I8j","sm":"Gopher (280B)","src":"https://arxiv.org/pdf/2201.11990","v":0.792},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Gopher (280B)","m":"gopher-280b","oid":"hella_swag-c84b696d78c06561","rd":"2021-12-08","sid":"rec1fTXYPgL0mRzBe","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.06905","v":0.792},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"hella_swag-4ec80f06146eafde","rd":"2022-01-27","sid":"recnKx5JEOw8tZlbl","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.789},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"hella_swag-7b055700115ba5e8","rd":"2022-01-27","sid":"recCvPkcxVrRq6YCL","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.781},{"b":"hella_swag","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"hella_swag-74226c9e4bc2c0a7","rd":"2022-01-27","sid":"recsTSf13JcXIzyWp","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.793},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"hella_swag-91ae290c1c25c4a9","rd":"2022-01-27","sid":"recdvddfEJCDhen1x","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.789},{"b":"hella_swag","cfg":{"shots":"20"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"hella_swag-9d1284b543a9dcc0","rd":"2022-01-27","sid":"recc3BHdP90Z1fI20","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.793},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"hella_swag-d5db46b3e5ccdbad","rd":"2022-01-27","sid":"recr3mOusR0K1TilG","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.781},{"b":"hella_swag","cfg":{"shots":"few"},"dn":"GPT-3 Few-Shot","m":"text-davinci-001","oid":"hella_swag-ded7e16ce08ff8d1","rd":"2022-01-27","sid":"rec9hsfppA15xBC1Z","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.793},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"GPT-3 One-Shot","m":"text-davinci-001","oid":"hella_swag-8b91049338e883ce","rd":"2022-01-27","sid":"recLlYcFFQ2fkEaY4","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.781},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GPT-3 Zero-Shot","m":"text-davinci-001","oid":"hella_swag-d70ae1d3326df0c6","rd":"2022-01-27","sid":"recw7hL34oyte0hO4","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.789},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"GPT-3.5","m":"text-davinci-003","oid":"hella_swag-f078fc0bb16c7b22","rd":"2022-11-28","sid":"recDZLIX5tZdFHfZV","sm":"text-davinci-003","src":"http://arxiv.org/abs/2311.16867","v":0.855},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"GPT-3.5","m":"text-davinci-003","oid":"hella_swag-7e42880387f56ced","rd":"2022-11-28","sid":"recSoaSkn6hHNrdDG","sm":"text-davinci-003","src":"http://arxiv.org/abs/2303.08774","v":0.855},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"GPT-4","m":"gpt-4-32k-0314","oid":"hella_swag-4d7c8c1135a020dc","rd":"2023-03-14","sid":"recOyxFNOYBvgCzru","sm":"gpt-4-32k-0314","src":"http://arxiv.org/abs/2311.16867","v":0.953},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"GPT-4","m":"gpt-4-0314","oid":"hella_swag-b894811ccc117bb9","rd":"2023-03-14","sid":"recGg30WxShdoma6K","sm":"gpt-4-0314","src":"http://arxiv.org/abs/2303.08774","v":0.953},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"hella_swag-603a37e1cbf5676e","rd":"2021-08-05","sid":"reckIFCQgW9j4dxIA","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.662},{"b":"hella_swag","dn":"GPT-Neo-2.7B","m":"gpt-neo-2-7b","oid":"hella_swag-ffc2132f84167f70","rd":"2021-03-30","sid":"recdUHL8y6j09NCyI","sm":"gpt-neo-2.7B","src":"http://arxiv.org/abs/2309.05463","v":0.427},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"hella_swag-bd65e6c424835c9d","rd":"2022-04-07","sid":"recKBrB9BpH26k9lE","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.705},{"b":"hella_swag","dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"hella_swag-7939a95131c19d7b","rd":"2019-11-05","sid":"rechzRZkyb0rIGBL4","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.4},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"INTELLECT-1-Instruct","m":"intellect-1-instruct","oid":"hella_swag-9d2eb712949f90cb","rd":"2024-11-29","sid":"rec9T90Eeu3oz0AqY","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.7142},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"InternLM 7B","m":"internlm-7b","oid":"hella_swag-0826bf4e7a2379c6","rd":"2023-07-05","sid":"recystu4Gc11FUp5X","sm":"internlm-7b","src":"http://arxiv.org/abs/2309.16609","v":0.706},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"InternLM 20B","m":"internlm-20b","oid":"hella_swag-28a11000596ff407","rd":"2023-09-18","sid":"recKfpodrVZGjl5CL","sm":"internlm-20b","src":"http://arxiv.org/abs/2309.16609","v":0.781},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"hella_swag-5eb7b4aea05c8926","rd":"2023-07-18","sid":"recX2iDmTM7fmBQcr","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2309.16609","v":0.853},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"hella_swag-ca55ad73d2ff5202","rd":"2023-02-24","sid":"recjYC87LW5FsDXJe","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2302.13971","v":0.761},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"hella_swag-27237383620bea98","rd":"2023-02-24","sid":"recltSGk7UqgYJQGD","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2302.13971","v":0.792},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA 33B","m":"llama-33b","oid":"hella_swag-908465a1b12b27d4","rd":"2023-02-24","sid":"recSk7gas28ZeRmQV","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2302.13971","v":0.828},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA 65B","m":"llama-65b","oid":"hella_swag-68f7caf62f4ac08f","rd":"2023-02-24","sid":"recbhMjETLxddNfTo","sm":"LLaMA-65B","src":"https://arxiv.org/pdf/2302.13971","v":0.842},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"hella_swag-cf2443d489050d8d","rd":"2023-07-18","sid":"recLoCMBOHteyqSFV","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.772},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"hella_swag-c693c0bda624a7cd","rd":"2023-07-18","sid":"rec3jliHk20I9fwvl","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2403.08295","v":0.807},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"hella_swag-1a80b4a855ebcff9","rd":"2024-07-23","sid":"recOisJQ9AhmUw58Z","sm":"Llama-3.1-405B","src":"https://arxiv.org/pdf/2412.19437","v":0.892},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"hella_swag-540d7d4875f603d1","rd":"2023-02-24","sid":"recLOfEzuA2K0aFdy","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.762},{"b":"hella_swag","dn":"Llama-7B","m":"llama-7b","oid":"hella_swag-e76409e81cfce005","rd":"2023-02-24","sid":"recIgJOQCzWeRSNU8","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.562},{"b":"hella_swag","dn":"Llama2-7B","m":"llama-2-7b","oid":"hella_swag-88b95fbb1054c564","rd":"2023-07-18","sid":"rec18ziKAqmhepn9d","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.571},{"b":"hella_swag","dn":"Megatron-NLG (530B)","m":"megatron-turing-nlg-530b","oid":"hella_swag-07e1b86021d7d9ff","rd":"2021-10-11","sid":"recb8yDVZF6TqwUZI","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2112.06905","v":0.824},{"b":"hella_swag","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"hella_swag-43531ff8ed6d402e","rd":"2023-09-27","sid":"recvCEZxPk8kXUffy","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.81},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"hella_swag-68c675ed764d1554","rd":"2023-12-11","sid":"rec5L8vbrflxgqG9Y","sm":"Mixtral-8x7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.867},{"b":"hella_swag","dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"hella_swag-bb3364480f1e1f13","rd":"2023-12-11","sid":"receQE7wX2SUXU5kj","sm":"Mixtral-8x7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.844},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"MPT 7B","m":"mpt-7b","oid":"hella_swag-c26950b0e924aa11","rd":"2023-05-05","sid":"recJnSsj1yJW0DpOz","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.16609","v":0.764},{"b":"hella_swag","dn":"MPT-7B","m":"mpt-7b","oid":"hella_swag-aed920b175e407d4","rd":"2023-05-05","sid":"recTjndtio3wpkQtJ","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.571},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"hella_swag-2bb751a54c005982","rd":"2023-05-05","sid":"recjgraDh1VdRw31V","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.761},{"b":"hella_swag","cfg":{"shots":"few"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"hella_swag-5f4c98ed27072148","rd":"2021-10-11","sid":"recChavgsXvRqIfA4","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.824},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"hella_swag-be74e5296487602b","rd":"2021-10-11","sid":"recOS1vYKc0O63Nm9","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.802},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"hella_swag-da0cea35b4fb4706","rd":"2021-10-11","sid":"reclD9XvCzARYAaJD","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8024},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"hella_swag-9fa48b9d8c391d25","rd":"2024-02-26","sid":"recxJ90xKTkaaA1JI","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.824},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"hella_swag-e5babf10a23af7ce","rd":"2023-06-07","sid":"recqC3Si74hlV1FmN","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.718},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"OPT (66B)","m":"opt-66b","oid":"hella_swag-51de1d59fd1e4670","rd":"2022-05-03","sid":"recevvZp5GvZqnhTY","sm":"opt-66b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.745},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"OPT (175B)","m":"opt-175b","oid":"hella_swag-26b5aef746087de1","rd":"2022-05-02","sid":"recedbGBvF8R6FFVE","sm":"opt-175b","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.791},{"b":"hella_swag","dn":"OPT-1.3B","m":"opt-1-3b","oid":"hella_swag-7c2a09d738527c6d","rd":"2022-05-11","sid":"recX8gW8qBbzDWMF3","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.415},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"hella_swag-e775539441a3f84c","rd":"2022-05-11","sid":"reczB7H8nIbkL9Jpr","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.699},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"PaLM 2-L","m":"palm-2-l","oid":"hella_swag-d7721928c82a9117","rd":"2023-05-17","sid":"recPWo0FVBOyy6UQL","sm":"PaLM 2-L","src":"https://arxiv.org/pdf/2305.10403","v":0.868},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"hella_swag-8b18371419485211","rd":"2023-05-17","sid":"rec1xIDMlT4yARh9N","sm":"PaLM 2-M","src":"https://arxiv.org/pdf/2305.10403","v":0.84},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"hella_swag-a0cb496998e32283","rd":"2023-05-17","sid":"reclJ23a6dlbZmII4","sm":"PaLM 2-S","src":"https://arxiv.org/pdf/2305.10403","v":0.82},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"PaLM-540B","m":"palm-540b","oid":"hella_swag-d3983d090d605b31","rd":"2022-04-04","sid":"recDP6AFPqvGUU7ko","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.834},{"b":"hella_swag","cfg":{"shots":"few"},"dn":"PaLM-540B","m":"palm-540b","oid":"hella_swag-a3cd62bdd7bc2bfc","rd":"2022-04-04","sid":"recMqMvo2ZR5xB7Td","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.838},{"b":"hella_swag","cfg":{"shots":"1"},"dn":"PaLM-540B","m":"palm-540b","oid":"hella_swag-930b6182105888a1","rd":"2022-04-04","sid":"recZj2LOfHy2dxU3Y","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.836},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"phi-1.5","m":"phi-1-5","oid":"hella_swag-ff588541d4dbcf05","rd":"2023-09-11","sid":"recC009XHETCkqHuq","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.476},{"b":"hella_swag","cfg":{"shots":"5"},"dn":"Phi-2","m":"phi-2","oid":"hella_swag-c0d14471c59617cf","rd":"2023-12-12","sid":"rec6Wjhlpz51HwZAb","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.536},{"b":"hella_swag","cfg":{"shots":"5"},"dn":"Phi-3-medium","m":"phi-3-medium-128k-instruct","oid":"hella_swag-a142332075031277","rd":"2024-04-23","sid":"recY158Aob3my3MXx","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.824},{"b":"hella_swag","cfg":{"shots":"5"},"dn":"Phi-3-mini","m":"phi-3-mini-4k-instruct","oid":"hella_swag-f276d08cd5561bb8","rd":"2024-04-23","sid":"recMJYP7H2VFfBaU5","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.767},{"b":"hella_swag","cfg":{"shots":"5"},"dn":"Phi-3-small","m":"phi-3-small-8k-instruct","oid":"hella_swag-dbf69457a834a8e8","rd":"2024-04-23","sid":"rechnntOSjcnmoV5r","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.77},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"hella_swag-3158a086f0371545","rd":"2024-09-19","sid":"recwpbI0o0B1hbdpF","sm":"Qwen2.5-72B","src":"https://arxiv.org/pdf/2412.19437","v":0.848},{"b":"hella_swag","dn":"Qwen2.5-Coder-0.5B","m":"qwen2-5-coder-0-5b","oid":"hella_swag-d33da80a103ad4a8","rd":"2024-09-18","sid":"recKJQ9e6G5eR4LY9","sm":"Qwen2.5-Coder-0.5B","src":"https://arxiv.org/pdf/2409.12186","v":0.484},{"b":"hella_swag","dn":"Qwen2.5-Coder-1.5B","m":"qwen2-5-coder-1-5b","oid":"hella_swag-af5234db3b32a354","rd":"2024-09-18","sid":"recJ6JRT9b7jqkffL","sm":"Qwen2.5-Coder-1.5B","src":"https://arxiv.org/pdf/2409.12186","v":0.618},{"b":"hella_swag","dn":"Qwen2.5-Coder-3B","m":"qwen2-5-coder-3b","oid":"hella_swag-e34834e11235bc1d","rd":"2024-09-18","sid":"recIljwSiqroHuB4D","sm":"Qwen2.5-Coder-3B","src":"https://arxiv.org/pdf/2409.12186","v":0.709},{"b":"hella_swag","dn":"Qwen2.5-Coder-7B","m":"qwen2-5-coder-7b","oid":"hella_swag-9b247d37ee19f2b3","rd":"2024-09-18","sid":"rec7vcmFtc4HEyNvG","sm":"Qwen2.5-Coder-7B","src":"https://arxiv.org/pdf/2409.12186","v":0.768},{"b":"hella_swag","dn":"Qwen2.5-Coder-14B","m":"qwen2-5-coder-14b","oid":"hella_swag-654e713bc370b5e2","rd":"2024-09-18","sid":"recJnaCHTOREy7sKU","sm":"Qwen2.5-Coder-14B","src":"https://arxiv.org/pdf/2409.12186","v":0.802},{"b":"hella_swag","dn":"Qwen2.5-Coder-32B","m":"qwen2-5-coder-32b","oid":"hella_swag-b60c3e3b7931e4ae","rd":"2024-09-18","sid":"recxMCtl8GTjOanHW","sm":"Qwen2.5-Coder-32B","src":"https://arxiv.org/pdf/2409.12186","v":0.83},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"hella_swag-d3c54055f63c037d","rd":"2023-05-04","sid":"recd9h5DECjR1gnE7","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.703},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"StableBeluga2","m":"stablebeluga2","oid":"hella_swag-4a2bdb1f621d48cc","rd":"2023-07-20","sid":"recRGvz0NvjpFvSSH","sm":"StableBeluga2","src":"http://arxiv.org/abs/2309.16609","v":0.841},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"hella_swag-ced3029af58df930","rd":"2023-04-19","sid":"reczrfJheZMsfglky","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.407},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"text-ada-001","m":"text-ada-001","oid":"hella_swag-d16cdfeb77aba659","sid":"recNkQMe0Nqd3deYK","sm":"text-ada-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.429},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"text-babbage-001","m":"text-babbage-001","oid":"hella_swag-ed1d2ec7e80f7da3","sid":"recxrPuCbT16sDZrG","sm":"text-babbage-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.561},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"text-curie-001","m":"text-curie-001","oid":"hella_swag-daeb2173f6df2f8b","sid":"recYhmf2CsNqDxs5a","sm":"text-curie-001","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.676},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"text-davinci-002","m":"text-davinci-002","oid":"hella_swag-cd83a35739edad27","rd":"2022-03-15","sid":"rechG6XtFFAfGaxk8","sm":"text-davinci-002","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.815},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"text-davinci-003","m":"text-davinci-003","oid":"hella_swag-6a9071619084a517","rd":"2022-11-28","sid":"recaAsccv7Ypq42Vf","sm":"text-davinci-003","src":"https://crfm.stanford.edu/helm/classic/latest/#/groups/hellaswag","v":0.822},{"b":"hella_swag","dn":"Vicuna-13B","m":"vicuna-13b-v1-1","oid":"hella_swag-7723fe4569a489d6","rd":"2023-04-12","sid":"recci6ceGcEr7HPnk","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.578},{"b":"hella_swag","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"hella_swag-8512a3a38276c834","rd":"2023-06-27","sid":"recy1P9FZOSqOxfEa","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.742},{"b":"hella_swag","cfg":{"shots":"10"},"dn":"Yi-6B","m":"yi-6b","oid":"hella_swag-e0641f2b5876147e","rd":"2023-11-02","sid":"recrDBkh4IHJOxHX3","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.744},{"b":"hella_swag","dn":"Yi-9B","m":"yi-9b","oid":"hella_swag-230bf35bafe9c55a","rd":"2024-03-01","sid":"recrHT1sajTQ2TT4C","sm":"Yi-9B","src":"http://arxiv.org/abs/2403.04652","v":0.764},{"b":"hle","dn":"GPT 6 Astra","m":"gpt-6-astra","metrics":{"calibration_error":39},"oid":"hle-a85bc68ad8ccaddb","rd":"2026-09-03","se":0.0194,"sid":"recbv6dU38s3cvqyO","sm":"gpt-6-astra_unknown","v":0.5479999999999999},{"b":"hle","cfg":{"effort":"xhigh"},"dn":"Fable 5.1 (xhigh)","e":"xhigh","m":"claude-fable-5-1","metrics":{"calibration_error":20},"oid":"hle-c6ca16357621677c","rd":"2026-09-01","se":0.02,"sid":"rec1P9SLLAtCSdTMk","sm":"claude-fable-5-1_xhigh","v":0.465},{"b":"hle","dn":"gemini-3.1-pro-preview (thinking high)","m":"gemini-3-1-pro-preview","metrics":{"calibration_error":51},"oid":"hle-2affa04bb0ec31b7","rd":"2026-02-19","se":0.0196,"sid":"recuBCQLkHd2mOtrr","sm":"gemini-3.1-pro-preview","v":0.4644},{"b":"hle","cfg":{"effort":"xhigh"},"dn":"Sonnet 5.5 (xhigh)","e":"xhigh","m":"claude-sonnet-5-5","oid":"hle-606a5a05bf2d95ba","rd":"2026-09-28","se":0.0195,"sid":"recxGPs2ikoJ9KUyA","sm":"claude-sonnet-5-5_xhigh","v":0.4612},{"b":"hle","dn":"Gemini 3.8 Flash","m":"gemini-3-8-flash","metrics":{"calibration_error":51},"oid":"hle-805cd9a4667a9362","rd":"2026-09-02","se":0.0196,"sid":"recIqVer2D18rE192","sm":"gemini-3.8-flash_unknown","v":0.44520000000000004},{"b":"hle","dn":"gpt-5.4-pro-2026-03-05","m":"gpt-5-4-pro-2026-03-05","metrics":{"calibration_error":38},"oid":"hle-3994909952aa0c33","rd":"2026-03-05","se":0.0195,"sid":"recreMdBwgtJ8NoJL","sm":"gpt-5.4-pro-2026-03-05_unknown","v":0.44320000000000004},{"b":"hle","dn":"Muse Spark","m":"muse-spark","metrics":{"calibration_error":50},"oid":"hle-a605465830bbde00","rd":"2026-04-08","se":0.0192,"sid":"recvslFSn9YBYB4kI","sm":"muse-spark","v":0.4056},{"b":"hle","dn":"gemini-3-pro-preview","m":"gemini-3-pro-preview","metrics":{"calibration_error":57},"oid":"hle-1d89678dbf73d9ec","rd":"2025-11-18","se":0.019,"sid":"recXvAi7z6kYifmzb","sm":"gemini-3-pro-preview","v":0.37520000000000003},{"b":"hle","cfg":{"effort":"xhigh"},"dn":"gpt-5.4-2026-03-05 (xhigh thinking)","e":"xhigh","m":"gpt-5-4-2026-03-05","metrics":{"calibration_error":42},"oid":"hle-659c5c36c7264f26","rd":"2026-03-05","se":0.0188,"sid":"reczymcIUNR10O9vo","sm":"gpt-5.4-2026-03-05_xhigh","v":0.3624},{"b":"hle","dn":"claude-opus-4-7","m":"claude-opus-4-7","metrics":{"calibration_error":47},"oid":"hle-cf79bfed6d97f413","rd":"2026-04-16","se":0.0188,"sid":"recVgBx1IpXMKMIDN","sm":"claude-opus-4-7_unknown","v":0.36200000000000004},{"b":"hle","cfg":{"effort":"max"},"dn":"claude-opus-4-6-thinking-max","e":"max","m":"claude-opus-4-6","metrics":{"calibration_error":46},"oid":"hle-1f3b37efda1d79ed","rd":"2026-02-05","se":0.018600000000000002,"sid":"recRaT3ySEJuEiBMU","sm":"claude-opus-4-6_max","v":0.3444},{"b":"hle","dn":"gpt-5-pro-2025-10-06","m":"gpt-5-pro-2025-10-06","metrics":{"calibration_error":49},"oid":"hle-8b143e896d7d0eac","rd":"2025-10-07","se":0.0182,"sid":"recsQ0oM0i71F5lGd","sm":"gpt-5-pro-2025-10-06_unknown","v":0.3164},{"b":"hle","dn":"Claude Haiku 5.5","m":"claude-haiku-5-5","oid":"hle-c5bd9a9e61aae142","rd":"2026-10-07","se":0.0183,"sid":"rectuouKismbEFle9","sm":"claude-haiku-5-5_unknown","v":0.3012},{"b":"hle","dn":"gpt-5.2-2025-12-11","m":"gpt-5-2-2025-12-11","metrics":{"calibration_error":45},"oid":"hle-ffd3a829f6ec6e44","rd":"2025-12-11","se":0.0176,"sid":"recQxZu8DefCNl6dq","sm":"gpt-5.2-2025-12-11_unknown","v":0.278},{"b":"hle","cfg":{"effort":"high"},"dn":"gpt-5-2025-08-07","e":"high","m":"gpt-5-2025-08-07","metrics":{"calibration_error":50},"oid":"hle-476b77dcdee1fc6f","rd":"2025-08-07","se":0.017,"sid":"recWhEbJqDmLaXyxu","sm":"gpt-5-2025-08-07_high","v":0.2532},{"b":"hle","dn":"gpt-5-2025-08-07","m":"gpt-5-2025-08-07","metrics":{"calibration_error":50},"oid":"hle-73a9350015b67cf4","rd":"2025-08-07","se":0.017,"sid":"recQ4tGIHl6FByXLl","sm":"gpt-5-2025-08-07_unknown","v":0.2532},{"b":"hle","dn":"claude-opus-4-5-20251101-thinking","m":"claude-opus-4-5-2025-11-01","metrics":{"calibration_error":55},"oid":"hle-6fa20e5c19b88184","rd":"2025-11-24","se":0.017,"sid":"recfMdmXCFFRUCxHi","sm":"claude-opus-4-5-20251101_unknown","v":0.252},{"b":"hle","dn":"kimi-k2.5","m":"kimi-k2-5","metrics":{"calibration_error":67},"oid":"hle-956997649af2ab04","rd":"2026-01-27","se":0.0181,"sid":"recj1DEYXMJzvbCpt","sm":"kimi-k2.5","v":0.24370000000000003},{"b":"hle","dn":"gpt-5.1-thinking","m":"gpt-5-1-2025-11-13","metrics":{"calibration_error":55},"oid":"hle-d5305833aa81e3ef","rd":"2025-11-13","se":0.0167,"sid":"recZGWz9x8AupIcR3","sm":"gpt-5.1-2025-11-13_unknown","v":0.2368},{"b":"hle","dn":"gemini-2.5-pro-preview-06-05","m":"gemini-2-5-pro-preview-06-05","metrics":{"calibration_error":72},"oid":"hle-be38dc893a47b562","rd":"2025-06-05","se":0.0161,"sid":"recuifDMJFbDLs8xq","sm":"gemini-2.5-pro-preview-06-05","v":0.2164},{"b":"hle","cfg":{"effort":"high"},"dn":"o3 (high) (April 2025)","e":"high","m":"o3-2025-04-16","metrics":{"calibration_error":34},"oid":"hle-cee4f61b407998ac","rd":"2025-04-16","se":0.0158,"sid":"recAokfZ8Ah2IzhPy","sm":"o3-2025-04-16_high","v":0.20320000000000002},{"b":"hle","dn":"gpt-5-mini-2025-08-07","m":"gpt-5-mini-2025-08-07","metrics":{"calibration_error":65},"oid":"hle-9fd8fcbd9eb5bfc0","rd":"2025-08-07","se":0.015500000000000002,"sid":"reczvkyjanol6Vn9f","sm":"gpt-5-mini-2025-08-07_unknown","v":0.19440000000000002},{"b":"hle","cfg":{"effort":"medium"},"dn":"o3 (medium) (April 2025)","e":"medium","m":"o3-2025-04-16","metrics":{"calibration_error":39},"oid":"hle-a012146ba38273d2","rd":"2025-04-16","se":0.0154,"sid":"recGb81W0KGwOPFN2","sm":"o3-2025-04-16_medium","v":0.192},{"b":"hle","dn":"claude-opus-4-6 (Non-Thinking)","m":"claude-opus-4-6","metrics":{"calibration_error":44},"oid":"hle-5492d37d2c9db995","rd":"2026-02-05","se":0.0154,"sid":"recYxQkYiHyzMnn5w","sm":"claude-opus-4-6","v":0.19},{"b":"hle","dn":"Gemini 2.5 Pro Experimental (March 2025)","m":"gemini-2-5-pro-exp-03-25","metrics":{"calibration_error":71},"oid":"hle-6721c3df2b43494a","rd":"2025-03-25","se":0.0151,"sid":"recwvcOM1X2e9xinL","sm":"gemini-2.5-pro-exp-03-25","v":0.1816},{"b":"hle","cfg":{"effort":"high"},"dn":"o4-mini (high) (April 2025)","e":"high","m":"o4-mini-2025-04-16","metrics":{"calibration_error":57},"oid":"hle-c5ab9a0a4769cc9f","rd":"2025-04-16","se":0.0151,"sid":"recGUVrntyz0xRSWS","sm":"o4-mini-2025-04-16_high","v":0.1808},{"b":"hle","dn":"Gemini 2.5 Pro Preview (May 06 2025)","m":"gemini-2-5-pro-preview-05-06","metrics":{"calibration_error":70},"oid":"hle-59ad607f35472db5","rd":"2025-05-06","se":0.015,"sid":"recqPy0eZ1dSztvhv","sm":"gemini-2.5-pro-preview-05-06","v":0.17800000000000002},{"b":"hle","cfg":{"effort":"medium"},"dn":"o4-mini (medium) (April 2025)","e":"medium","m":"o4-mini-2025-04-16","metrics":{"calibration_error":59},"oid":"hle-cea6c9a20d36a625","rd":"2025-04-16","se":0.013700000000000002,"sid":"rec9Y7oPa9KGyhXSZ","sm":"o4-mini-2025-04-16_medium","v":0.1428},{"b":"hle","dn":"claude-opus-4-5-20251101","m":"claude-opus-4-5-2025-11-01","metrics":{"calibration_error":56},"oid":"hle-ef24bef9a49340c5","rd":"2025-11-24","se":0.013700000000000002,"sid":"recxYcr9VmKPUVR3Z","sm":"claude-opus-4-5-20251101_unknown","v":0.1416},{"b":"hle","dn":"claude-sonnet-4-5-20250929-thinking","m":"claude-sonnet-4-5-2025-09-29","metrics":{"calibration_error":65},"oid":"hle-9884a72ebcc57c43","rd":"2025-09-29","se":0.013500000000000002,"sid":"receLskUASD1G2QuH","sm":"claude-sonnet-4-5-20250929_unknown","v":0.13720000000000002},{"b":"hle","dn":"Gemini 2.5 Flash (April 2025)","m":"gemini-2-5-flash-preview-04-17","metrics":{"calibration_error":80},"oid":"hle-25797f318b89de35","rd":"2025-04-17","se":0.0128,"sid":"recLRKaQqj6097BJl","sm":"gemini-2.5-flash-preview-04-17","v":0.1208},{"b":"hle","dn":"claude-opus-4-1-20250805-thinking","m":"claude-opus-4-1-2025-08-05","metrics":{"calibration_error":71},"oid":"hle-6b5e34f155c21fb6","rd":"2025-08-05","se":0.0125,"sid":"recjZHPwO8vEk0XnB","sm":"claude-opus-4-1-20250805_unknown","v":0.1152},{"b":"hle","dn":"Gemini 2.5 Flash Preview (May 2025)","m":"gemini-2-5-flash-preview-05-20","metrics":{"calibration_error":82},"oid":"hle-506f469d13b21094","rd":"2025-05-20","se":0.0122,"sid":"recncpy8WpYML12AD","sm":"gemini-2.5-flash-preview-05-20","v":0.10960000000000002},{"b":"hle","dn":"Claude Opus 4 (Thinking)","m":"claude-opus-4-2025-05-14","metrics":{"calibration_error":73},"oid":"hle-ac43b302d75ab7d5","rd":"2025-05-22","se":0.0121,"sid":"recDGGt1nZaGx87Ql","sm":"claude-opus-4-20250514_unknown","v":0.1072},{"b":"hle","dn":"gemini-3.1-flash-lite-preview","m":"gemini-3-1-flash-lite","metrics":{"calibration_error":83},"oid":"hle-ecc76a00d7bf763a","rd":"2026-03-03","se":0.011000000000000001,"sid":"recsvNt37Kyyg9qCI","sm":"gemini-3.1-flash-lite","v":0.0864},{"b":"hle","dn":"glm-4p5","m":"glm-4-5","metrics":{"calibration_error":79},"oid":"hle-1df02ccf0cdef255","rd":"2025-08-03","se":0.0108,"sid":"recLxM4WtI97UUniT","sm":"glm-4.5","v":0.08320000000000001},{"b":"hle","dn":"glm-4p5-air","m":"glm-4-5-air","metrics":{"calibration_error":77},"oid":"hle-150d2f472080c498","rd":"2025-07-20","se":0.010700000000000001,"sid":"recqBBEb2sQdj6vn5","sm":"GLM-4.5-Air","v":0.0812},{"b":"hle","dn":"o1 Pro","m":"o1-pro-2025-03-19","metrics":{"calibration_error":82},"oid":"hle-2ba5da5c8167077c","rd":"2025-03-19","se":0.010700000000000001,"sid":"recxgQqtw3fmiFs3B","sm":"o1-pro-2025-03-19","v":0.0812},{"b":"hle","dn":"Claude 3.7 Sonnet (Thinking)","m":"claude-3-7-sonnet-2025-02-19","metrics":{"calibration_error":80},"oid":"hle-61971e9a56ba1546","rd":"2025-02-24","se":0.010700000000000001,"sid":"recSaAOoaHzSl2qT2","sm":"claude-3-7-sonnet-20250219_unknown","v":0.0804},{"b":"hle","dn":"o1 (December 2024)","m":"o1-2024-12-17","metrics":{"calibration_error":83},"oid":"hle-d9a5ed257cb17214","rd":"2024-12-17","se":0.0106,"sid":"recg1cdjMzYHVdlNz","sm":"o1-2024-12-17_unknown","v":0.0796},{"b":"hle","dn":"claude-opus-4-1-20250805","m":"claude-opus-4-1-2025-08-05","metrics":{"calibration_error":70},"oid":"hle-2e7bd0a70a9c230e","rd":"2025-08-05","se":0.0106,"sid":"recNm82odoKRxfsNj","sm":"claude-opus-4-1-20250805_unknown","v":0.0792},{"b":"hle","dn":"Claude Sonnet 4 (Thinking)","m":"claude-sonnet-4-2025-05-14","metrics":{"calibration_error":75},"oid":"hle-509283149280fe97","rd":"2025-05-22","se":0.0105,"sid":"recBG4cqUawtpE2Av","sm":"claude-sonnet-4-20250514_unknown","v":0.0776},{"b":"hle","dn":"claude-sonnet-4-5-20250929","m":"claude-sonnet-4-5-2025-09-29","metrics":{"calibration_error":70},"oid":"hle-c2c36ec5e1fa3456","rd":"2025-09-29","se":0.0103,"sid":"recs2YfzlLXFgzEAG","sm":"claude-sonnet-4-5-20250929_unknown","v":0.0752},{"b":"hle","cfg":{"effort":"none"},"dn":"gpt-5.1-instant","e":"none","m":"gpt-5-1-2025-11-13","metrics":{"calibration_error":69},"oid":"hle-daed90058afaa808","rd":"2025-11-13","se":0.0099,"sid":"recyAOwkCQoXPsGUC","sm":"gpt-5.1-2025-11-13_none","v":0.068},{"b":"hle","dn":"Claude Opus 4","m":"claude-opus-4-2025-05-14","metrics":{"calibration_error":74},"oid":"hle-3c2698ea4910b568","rd":"2025-05-22","se":0.0098,"sid":"rec7DOdaKkCsIp2zt","sm":"claude-opus-4-20250514_unknown","v":0.0668},{"b":"hle","dn":"Gemini 2.0 Flash Thinking (January 2025)","m":"gemini-2-0-flash-thinking-exp-01-21","metrics":{"calibration_error":82},"oid":"hle-5b14c3f7bdad3f3b","rd":"2025-01-21","se":0.0097,"sid":"recpj5kBIlMjPoiDi","sm":"gemini-2.0-flash-thinking-exp-01-21","v":0.06559999999999999},{"b":"hle","dn":"Llama 4 Maverick","m":"llama-4-maverick-17b-128e-instruct","metrics":{"calibration_error":83},"oid":"hle-2588ad8568fc1c96","rd":"2025-04-05","se":0.0091,"sid":"recS0T6kjQeFZjFzH","sm":"Llama-4-Maverick-17B-128E-Instruct","v":0.056799999999999996},{"b":"hle","dn":"Claude Sonnet 4","m":"claude-sonnet-4-2025-05-14","metrics":{"calibration_error":76},"oid":"hle-856fa4c6bbf012db","rd":"2025-05-22","se":0.009000000000000001,"sid":"recDcYVlY8ya80Qq0","sm":"claude-sonnet-4-20250514_unknown","v":0.0552},{"b":"hle","dn":"GPT 4.5 Preview","m":"gpt-4-5-preview-2025-02-27","metrics":{"calibration_error":85},"oid":"hle-254b694747be2279","rd":"2025-02-27","se":0.0089,"sid":"recH4gsEnQtsYl58v","sm":"gpt-4.5-preview-2025-02-27","v":0.054400000000000004},{"b":"hle","dn":"GPT-4.1","m":"gpt-4-1-2025-04-14","metrics":{"calibration_error":89},"oid":"hle-b3fe6326be3f6ccd","rd":"2025-04-14","se":0.0089,"sid":"recXkRBDMBFxIp3rr","sm":"gpt-4.1-2025-04-14","v":0.054000000000000006},{"b":"hle","dn":"Gemini-1.5-Pro-002","m":"gemini-1-5-pro-002","metrics":{"calibration_error":88},"oid":"hle-56ba0ce2a12dae06","rd":"2024-09-24","se":0.008199999999999999,"sid":"recgO7p5JHoSiiFv3","sm":"gemini-1.5-pro-002","v":0.046},{"b":"hle","dn":"Mistral Medium 3","m":"mistral-medium-2505","metrics":{"calibration_error":77},"oid":"hle-d13a3bee0d3935e9","rd":"2025-05-07","se":0.008100000000000001,"sid":"rec84Eh6Y4FPpn9zg","sm":"mistral-medium-2505","v":0.0452},{"b":"hle","dn":"Nova Pro","m":"amazon-nova-pro-v10","metrics":{"calibration_error":80},"oid":"hle-dd97f32207b6793a","rd":"2024-12-03","se":0.008,"sid":"recPzTH64KyHad3Mb","sm":"amazon.nova-pro-v1:0","v":0.044000000000000004},{"b":"hle","dn":"Claude 3.5 Sonnet (October 2024)","m":"claude-3-5-sonnet-2024-10-22","metrics":{"calibration_error":84},"oid":"hle-2b0ef3c0df5454ab","rd":"2024-10-22","se":0.0078000000000000005,"sid":"recRcZwfwiWXVED57","sm":"claude-3-5-sonnet-20241022","v":0.0408},{"b":"hle","dn":"Nova Lite","m":"amazon-nova-lite-v10","metrics":{"calibration_error":82},"oid":"hle-eb9cb1804a2a2c51","rd":"2024-12-03","se":0.0073,"sid":"recujVA9XFIMx29bE","sm":"amazon.nova-lite-v1:0","v":0.0364},{"b":"hle","dn":"GPT-4o (November 2024)","m":"gpt-4o-2024-11-20","metrics":{"calibration_error":89},"oid":"hle-7a5702c54ff491e1","rd":"2024-11-20","se":0.0064,"sid":"recIkpfM7qdP40Q5g","sm":"gpt-4o-2024-11-20","v":0.027200000000000002},{"b":"lambada","m":"mpt-7b","oid":"lambada-77d46bbbe1285efc","rd":"2023-05-05","sid":"rec2xXtaYsETkAIic","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.7},{"b":"lambada","dn":"ChatGLM2","m":"chatglm2-6b","oid":"lambada-e212225e7f570c97","rd":"2023-06-24","sid":"reczTSlKRTLtNSENB","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.543},{"b":"lambada","m":"internlm-7b","oid":"lambada-b1eb7a482999384c","rd":"2023-07-05","sid":"rec7doikBryvY92fX","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.67},{"b":"lambada","m":"internlm-20b","oid":"lambada-ce59beca7994e967","rd":"2023-09-18","sid":"reclSAZPhymLQvV01","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.718},{"b":"lambada","m":"baichuan-2-7b-base","oid":"lambada-a634b74d79ddb5ce","rd":"2023-09-20","sid":"recYHhnjZgZIXdZsp","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","m":"baichuan-2-13b-base","oid":"lambada-745bfcd63cd45995","rd":"2023-09-06","sid":"recNL96PBikW3geVY","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.74},{"b":"lambada","m":"llama-7b","oid":"lambada-4f8f1b93ad752e98","rd":"2023-02-24","sid":"recXxy43JUGK2jPzL","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","m":"llama-13b","oid":"lambada-590c87fdc7a9e5d1","rd":"2023-02-24","sid":"recWRJ7s4P8mK4b3y","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.752},{"b":"lambada","m":"llama-33b","oid":"lambada-88af5f9983076f6b","rd":"2023-02-24","sid":"rec1P4zL04qfPtmrX","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.772},{"b":"lambada","m":"llama-65b","oid":"lambada-e831223653bf18f8","rd":"2023-02-24","sid":"recfGUja0cWwBFXaA","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.777},{"b":"lambada","m":"llama-2-7b","oid":"lambada-3a564ab2c4531a87","rd":"2023-07-18","sid":"recU0DvIGj0Cx7FUc","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","m":"llama-2-13b","oid":"lambada-b602d73c9e361687","rd":"2023-07-18","sid":"rechMH4HPGuohVfC6","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.765},{"b":"lambada","m":"llama-2-70b-hf","oid":"lambada-80378c379503fef4","rd":"2023-07-18","sid":"recoEvP3kjsU0RhfV","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.789},{"b":"lambada","m":"stablebeluga2","oid":"lambada-b5b8db6eedc96c45","rd":"2023-07-20","sid":"recaCNICEk1UnsEbl","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.713},{"b":"lambada","m":"qwen-1-8b","oid":"lambada-0f02995514458df9","rd":"2023-11-30","sid":"recC7agcItthoBTMw","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.584},{"b":"lambada","m":"qwen-7b","oid":"lambada-1f541be33cf345ed","rd":"2023-09-28","sid":"rechdx8dakBhmNq77","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.679},{"b":"lambada","m":"qwen-14b","oid":"lambada-51ff9879d56f5c8a","rd":"2023-09-28","sid":"recNTDzKb9IjM2007","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.711},{"b":"lambada","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"lambada-4afb5b0cb90b34f7","rd":"2022-01-27","sid":"recJMbsvJvQxBVWGL","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.725},{"b":"lambada","cfg":{"shots":"few"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"lambada-36a93908040bc58c","rd":"2022-01-27","sid":"recKkMcRrA3QYrqcl","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.864},{"b":"lambada","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"lambada-be4d044190b2783f","rd":"2022-01-27","sid":"recam11tENdCcJOOT","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.762},{"b":"lambada","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"lambada-2c6e2a2bdcce5453","rd":"2021-12-08","sid":"recnJL8PNsKurWZQJ","sm":"Gopher (280B)","src":"https://arxiv.org/pdf/2201.11990","v":0.745},{"b":"lambada","cfg":{"shots":"0"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"lambada-6f5ad9ccfb7ebda7","rd":"2021-10-11","sid":"rectiOvDVbt6MvVp1","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.7656},{"b":"lambada","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"lambada-458a7fb3e5ba577c","rd":"2022-01-27","sid":"rec9y9ui3cTNMI7fL","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.725},{"b":"lambada","cfg":{"shots":"1"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"lambada-ad7bd8814d88438a","rd":"2021-10-11","sid":"recsoiPkr7WH9pRDy","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.7306},{"b":"lambada","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"lambada-527b2761ce7df7c6","rd":"2022-01-27","sid":"rec0YBtMZPv6PvP3L","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.864},{"b":"lambada","cfg":{"shots":"few"},"dn":"MT-NLG (ours)","m":"megatron-turing-nlg-530b","oid":"lambada-f3b74ed8bdc1d577","rd":"2021-10-11","sid":"reckRUXCbqhiNITwm","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8715},{"b":"lambada","cfg":{"shots":"0"},"dn":"Gopher (k-Shot)","m":"gopher-280b","oid":"lambada-a399883a71a7c70a","rd":"2021-12-08","sid":"rec6TGPjtqdj0scyj","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.11446","v":0.745},{"b":"lambada","dn":"MPT 7B","m":"mpt-7b","oid":"lambada-aa96d86442d2067c","rd":"2023-05-05","sid":"recu9bbbL7HC6GIN7","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.7},{"b":"lambada","dn":"ChatGLM2 6B","m":"chatglm2-6b","oid":"lambada-2080e6ab3105bf38","rd":"2023-06-24","sid":"rec27gUWwQKDPiwAW","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.543},{"b":"lambada","dn":"InternLM 7B","m":"internlm-7b","oid":"lambada-1409489e1d08a5a4","rd":"2023-07-05","sid":"recHfcdJyV48YnXwf","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.67},{"b":"lambada","dn":"InternLM 20B","m":"internlm-20b","oid":"lambada-93506819390c95c8","rd":"2023-09-18","sid":"recLdEoATQGGlAACE","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.718},{"b":"lambada","dn":"Baichuan2 7B","m":"baichuan-2-7b-base","oid":"lambada-4537378846bca28e","rd":"2023-09-20","sid":"recIy3Bb5HqxiMKfB","sm":"Baichuan-2-7B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"lambada-a2ccb3b4091b980e","rd":"2023-09-06","sid":"recJIddONftE4eWpk","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.74},{"b":"lambada","dn":"LLaMA 7B","m":"llama-7b","oid":"lambada-f1c34043ba90b72e","rd":"2023-02-24","sid":"rec874oZp2QZs6G61","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","dn":"LLaMA 13B","m":"llama-13b","oid":"lambada-9e9b70b8ab165700","rd":"2023-02-24","sid":"recgQDXaNeJ8VWV3x","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.752},{"b":"lambada","dn":"LLaMA 33B","m":"llama-33b","oid":"lambada-2fda22f6e42ba90b","rd":"2023-02-24","sid":"recSSkgQnDpJ9KDZ5","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.772},{"b":"lambada","dn":"LLaMA 65B","m":"llama-65b","oid":"lambada-34485b4f877f8573","rd":"2023-02-24","sid":"recKARMKetCVFKMuN","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.777},{"b":"lambada","dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"lambada-5fbeacd6e0f04858","rd":"2023-07-18","sid":"recrz4vp9TeQDMijg","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"lambada","dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"lambada-565ca13cb91c9824","rd":"2023-07-18","sid":"recVe2Wo6SWeUqwfw","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.765},{"b":"lambada","dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"lambada-9038c9261c3f0d4a","rd":"2023-07-18","sid":"rec0O43m1j1P0bghs","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.789},{"b":"lambada","dn":"StableBeluga2 70B","m":"stablebeluga2","oid":"lambada-4d22fc502f7a1a0d","rd":"2023-07-20","sid":"rec4vNMLVeNultNFD","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.713},{"b":"lambada","dn":"QWEN 1.8B","m":"qwen-1-8b","oid":"lambada-900cde8e3db38020","rd":"2023-11-30","sid":"recgB9JIq6NldBDrd","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.584},{"b":"lambada","dn":"QWEN 7B","m":"qwen-7b","oid":"lambada-47c29190d7a59ad6","rd":"2023-09-28","sid":"rec4HYw9Hi7O5f0C3","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.679},{"b":"lambada","dn":"QWEN 14B","m":"qwen-14b","oid":"lambada-d97bab6d2dcb4d2e","rd":"2023-09-28","sid":"recn5IxI2fxa7AR5f","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.711},{"b":"lambada","dn":"GPT-3","m":"text-davinci-001","oid":"lambada-74703fb4289687d2","rd":"2022-01-27","sid":"recO5w0E8iGLF03SH","sm":"text-davinci-001","src":"http://arxiv.org/abs/2311.16867","v":0.762},{"b":"lambada","dn":"Gopher","m":"gopher-280b","oid":"lambada-17ab730db545984b","rd":"2021-12-08","sid":"reci74pJGLLVqHXJt","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2311.16867","v":0.745},{"b":"lambada","dn":"Chinchilla","m":"chinchilla-70b","oid":"lambada-be071d03892970b3","rd":"2022-03-29","sid":"recs8hCOMtPRefESy","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2311.16867","v":0.774},{"b":"lambada","dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"lambada-f01590ac6b510852","rd":"2021-10-11","sid":"recdQcFMIIooSeC1P","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2311.16867","v":0.766},{"b":"lambada","dn":"PaLM","m":"palm-540b","oid":"lambada-37f680a9e70bb245","rd":"2022-04-04","sid":"recbH5Bhjf8PSH14k","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.779},{"b":"lambada","dn":"Inflection-1","m":"inflection-1","oid":"lambada-be0126c9ed75a4c0","rd":"2023-06-22","sid":"recvjENyQBR54sD6E","sm":"Inflection-1","src":"http://arxiv.org/abs/2311.16867","v":0.785},{"b":"lambada","dn":"Falcon 7B","m":"falcon-7b","oid":"lambada-9bef718387e7e868","rd":"2023-04-24","sid":"recnSUBDS2uY8d8BI","sm":"falcon-7b","src":"http://arxiv.org/abs/2311.16867","v":0.749},{"b":"lambada","dn":"Falcon 40B","m":"falcon-40b","oid":"lambada-92b29b125feab251","rd":"2023-03-15","sid":"rec7WtVRmVKLEnZmN","sm":"falcon-40b","src":"http://arxiv.org/abs/2311.16867","v":0.773},{"b":"lambada","dn":"Falcon 180B","m":"falcon-180b","oid":"lambada-3694401be7340fb6","rd":"2023-09-06","sid":"recr1kUAaOEhwPRGb","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.798},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o3-2025-04-16","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-64ec712794365596","rd":"2025-04-16","sm":"o3-2025-04-16_medium","src":"https://github.com/lechmazur/Writing","v":8.39},{"b":"lech_mazur_writing","m":"qwen3-235b-a22b","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-54dfe74d1cf11967","rd":"2025-04-29","sm":"qwen3-235b-a22b","src":"https://github.com/lechmazur/Writing","v":8.3},{"b":"lech_mazur_writing","m":"deepseek-r1","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-5a2d20f36e908a0c","rd":"2025-01-20","sm":"DeepSeek-R1","src":"https://github.com/lechmazur/Writing","v":8.3},{"b":"lech_mazur_writing","m":"gpt-4o-2024-11-20","notes":"Assuming latest version of gpt-4o as of March 2025, which was released in November 2024.\n\nScore from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-ba3bffa1692fff7a","rd":"2024-11-20","sm":"gpt-4o-2024-11-20","src":"https://github.com/lechmazur/Writing","v":8.18},{"b":"lech_mazur_writing","m":"claude-3-7-sonnet-20250219-16k","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-3dc1a401663fed48","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219_16K","src":"https://github.com/lechmazur/Writing","v":8.11},{"b":"lech_mazur_writing","m":"qwq-32b-16k-thinking","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-74fa3650148bef7c","rd":"2025-03-05","sm":"QwQ-32B (16K thinking)","src":"https://github.com/lechmazur/Writing","v":8.02},{"b":"lech_mazur_writing","m":"gemini-2-5-pro-exp-03-25","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-3566f24c4538b91a","rd":"2025-03-25","sm":"gemini-2.5-pro-exp-03-25","src":"https://github.com/lechmazur/Writing","v":8.05},{"b":"lech_mazur_writing","m":"claude-3-5-sonnet-2024-10-22","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-c1e9966b98b50f52","rd":"2024-10-22","sm":"claude-3-5-sonnet-20241022","src":"https://github.com/lechmazur/Writing","v":8.03},{"b":"lech_mazur_writing","m":"gemma-3-27b-it","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-6b9ef32fc0fa09ee","rd":"2025-03-12","sm":"gemma-3-27b-it","src":"https://github.com/lechmazur/Writing","v":7.99},{"b":"lech_mazur_writing","m":"claude-3-7-sonnet-2025-02-19","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-194d4299236be7e4","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219","src":"https://github.com/lechmazur/Writing","v":7.94},{"b":"lech_mazur_writing","m":"deepseek-v3-0324","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-939ad188989898c3","rd":"2025-03-24","sm":"DeepSeek-V3-0324","src":"https://github.com/lechmazur/Writing","v":7.7},{"b":"lech_mazur_writing","m":"gemini-2-5-flash-preview-04-17-24k-thinking","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-1136318f90565a4e","rd":"2025-04-17","sm":"gemini-2.5-flash-preview-04-17 (24K thinking)","src":"https://github.com/lechmazur/Writing","v":7.65},{"b":"lech_mazur_writing","m":"grok-3-beta","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-bed7330519bb2ccc","rd":"2025-04-09","sm":"grok-3-beta","src":"https://github.com/lechmazur/Writing","v":7.64},{"b":"lech_mazur_writing","m":"gpt-4-5-preview-2025-02-27","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-5cc6551eb85a1c0e","rd":"2025-02-27","sm":"gpt-4.5-preview-2025-02-27","src":"https://github.com/lechmazur/Writing","v":7.56},{"b":"lech_mazur_writing","m":"qwen3-30b-a3b","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-65a3be39f14db10e","rd":"2025-04-29","sm":"Qwen3-30B-A3B","src":"https://github.com/lechmazur/Writing","v":7.53},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o4-mini-2025-04-16","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-f49b3695eaa7811b","rd":"2025-04-16","sm":"o4-mini-2025-04-16_medium","src":"https://github.com/lechmazur/Writing","v":7.5},{"b":"lech_mazur_writing","m":"gemini-2-0-flash-thinking-exp-01-21","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-997bac974d1d1abe","rd":"2025-01-21","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://github.com/lechmazur/Writing","v":7.38},{"b":"lech_mazur_writing","m":"claude-3-5-haiku-2024-10-22","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-efef430a7865cbed","rd":"2024-10-22","sm":"claude-3-5-haiku-20241022","src":"https://github.com/lechmazur/Writing","v":7.35},{"b":"lech_mazur_writing","cfg":{"effort":"low"},"e":"low","m":"grok-3-mini-beta","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-deb08ac7ad85e61b","rd":"2025-04-09","sm":"grok-3-mini-beta_low","src":"https://github.com/lechmazur/Writing","v":7.35},{"b":"lech_mazur_writing","m":"qwen2-5-max","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-068c80822f32b1f7","rd":"2025-01-28","sm":"qwen2.5-max","src":"https://github.com/lechmazur/Writing","v":7.29},{"b":"lech_mazur_writing","m":"gemini-2-0-flash-exp","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-8ae559e11df2183c","rd":"2024-12-11","sm":"gemini-2.0-flash-exp","src":"https://github.com/lechmazur/Writing","v":7.15},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o1-2024-12-17","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-8760e3dd5d3e9d28","rd":"2024-12-17","sm":"o1-2024-12-17_medium","src":"https://github.com/lechmazur/Writing","v":7.02},{"b":"lech_mazur_writing","m":"mistral-large-2407","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-95b776e974a35cc1","rd":"2024-07-24","sm":"mistral-large-2407","src":"https://github.com/lechmazur/Writing","v":6.9},{"b":"lech_mazur_writing","m":"gpt-4o-mini-2024-07-18","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-d7429aee8dfa3ab1","rd":"2024-07-18","sm":"gpt-4o-mini-2024-07-18","src":"https://github.com/lechmazur/Writing","v":6.72},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o1-mini-2024-09-12","notes":"Assuming medium reasoning effort.\n\nScore from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-6a0b57b7e7eef075","rd":"2024-09-12","sm":"o1-mini-2024-09-12_medium","src":"https://github.com/lechmazur/Writing","v":6.49},{"b":"lech_mazur_writing","m":"grok-2-1212","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-8190b579cebe644b","rd":"2024-12-12","sm":"grok-2-1212","src":"https://github.com/lechmazur/Writing","v":6.36},{"b":"lech_mazur_writing","m":"phi-4","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-9520025afafdbf8a","rd":"2024-12-12","sm":"phi-4","src":"https://github.com/lechmazur/Writing","v":6.26},{"b":"lech_mazur_writing","m":"llama-4-maverick-17b-128e-instruct","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-63db367199d22736","rd":"2025-04-05","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://github.com/lechmazur/Writing","v":6.2},{"b":"lech_mazur_writing","cfg":{"effort":"high"},"e":"high","m":"o3-mini-2025-01-31","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-c91987cf2111f70a","rd":"2025-01-31","sm":"o3-mini-2025-01-31_high","src":"https://github.com/lechmazur/Writing","v":6.17},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o3-mini-2025-01-31","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-a1bcbb2e46b1816f","rd":"2025-01-31","sm":"o3-mini-2025-01-31_medium","src":"https://github.com/lechmazur/Writing","v":6.15},{"b":"lech_mazur_writing","m":"amazon-nova-pro-v10","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-0e798d3380502557","rd":"2024-12-03","sm":"amazon.nova-pro-v1:0","src":"https://github.com/lechmazur/Writing","v":6.05},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-2025-08-07","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-9a954b114c2e8340","rd":"2025-08-07","sm":"gpt-5-2025-08-07_medium","src":"https://github.com/lechmazur/Writing","v":8.6},{"b":"lech_mazur_writing","m":"kimi-k2-instruct","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-e8d215e30e321866","rd":"2025-07-12","sm":"Kimi-K2-Instruct","src":"https://github.com/lechmazur/Writing","v":8.56},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"o3-pro-2025-06-10","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-dc021c57d6b95cbe","rd":"2025-06-10","sm":"o3-pro-2025-06-10_medium","src":"https://github.com/lechmazur/Writing","v":8.44},{"b":"lech_mazur_writing","m":"gemini-2-5-pro","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-1844677dc36b9164","rd":"2025-06-17","sm":"gemini-2.5-pro","src":"https://github.com/lechmazur/Writing","v":8.38},{"b":"lech_mazur_writing","m":"claude-opus-4-1-2025-08-05","notes":"No thinking\n\nScore from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-a2479d00c0089e38","rd":"2025-08-05","sm":"claude-opus-4-1-20250805","src":"https://github.com/lechmazur/Writing","v":8.47},{"b":"lech_mazur_writing","m":"qwen3-235b-a22b-thinking-2507","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-8c5c2a6c8fb0abf4","rd":"2025-07-25","sm":"Qwen3-235B-A22B-Thinking-2507","src":"https://github.com/lechmazur/Writing","v":8.24},{"b":"lech_mazur_writing","m":"grok-4-0709","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-f7c9488cd34955e1","rd":"2025-07-09","sm":"grok-4-0709","src":"https://github.com/lechmazur/Writing","v":7.69},{"b":"lech_mazur_writing","m":"glm-4-5","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-a0f24dc1d21b43f1","rd":"2025-08-03","sm":"glm-4.5","src":"https://github.com/lechmazur/Writing","v":7.34},{"b":"lech_mazur_writing","m":"gpt-oss-120b","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-d02d424040f03e20","rd":"2025-08-05","sm":"gpt-oss-120b","src":"https://github.com/lechmazur/Writing","v":7.71},{"b":"lech_mazur_writing","m":"claude-opus-4-1-20250805-16k","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-14e73e6441c6bb46","rd":"2025-08-05","sm":"claude-opus-4-1-20250805_16K","src":"https://github.com/lechmazur/Writing","v":8.45},{"b":"lech_mazur_writing","m":"claude-opus-4-20250514-16k","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-64b02ad2936ccd12","rd":"2025-05-22","sm":"claude-opus-4-20250514_16K","src":"https://github.com/lechmazur/Writing","v":8.36},{"b":"lech_mazur_writing","m":"claude-opus-4-2025-05-14","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-0dd9b0046a1f1fb6","rd":"2025-05-22","sm":"claude-opus-4-20250514","src":"https://github.com/lechmazur/Writing","v":8.31},{"b":"lech_mazur_writing","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-mini-2025-08-07","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-93c00e5c23625da7","rd":"2025-08-07","sm":"gpt-5-mini-2025-08-07_medium","src":"https://github.com/lechmazur/Writing","v":8.31},{"b":"lech_mazur_writing","m":"deepseek-r1-0528","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-2fb3b4360b46d987","rd":"2025-05-28","sm":"DeepSeek-R1-0528","src":"https://github.com/lechmazur/Writing","v":8.19},{"b":"lech_mazur_writing","m":"claude-sonnet-4-20250514-16k","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-c775f2cb09c0cbd2","rd":"2025-05-22","sm":"claude-sonnet-4-20250514_16K","src":"https://github.com/lechmazur/Writing","v":8.14},{"b":"lech_mazur_writing","m":"claude-sonnet-4-2025-05-14","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-211762fcfa786d28","rd":"2025-05-22","sm":"claude-sonnet-4-20250514","src":"https://github.com/lechmazur/Writing","v":8.09},{"b":"lech_mazur_writing","m":"gemini-2-5-pro-preview-05-06","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-4a4ab44fbb7bca24","rd":"2025-05-06","sm":"gemini-2.5-pro-preview-05-06","src":"https://github.com/lechmazur/Writing","v":8.09},{"b":"lech_mazur_writing","m":"mistral-medium-2505","notes":"Score from the final V1 leaderboard (0-10 scale) of lechmazur/Writing, commit 80b7f17 (Aug 8, 2025). Frozen: the benchmark's later versions use a different scoring system that is not comparable to this scale.","oid":"lech_mazur_writing-ac8e8455fd378eb3","rd":"2025-05-07","sm":"mistral-medium-2505","src":"https://github.com/lechmazur/Writing","v":7.73},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-5-pro-exp-03-25","oid":"live_bench-88f48d4d02bf174c","rd":"2025-03-25","sid":"gemini-2.5-pro-exp-03-25","sm":"gemini-2.5-pro-exp-03-25","src":"https://livebench.ai/#/","v":82.35},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"claude-3-7-sonnet-2025-02-19","notes":"Assumed baseline 3.7 Sonnet","oid":"live_bench-b40f92e6fc85c101","rd":"2025-02-24","sid":"claude-3-7-sonnet-thinking","sm":"claude-3-7-sonnet-20250219","src":"https://livebench.ai/#/","v":76.1},{"b":"live_bench","bv":"LiveBench-2024-11-25","cfg":{"effort":"high"},"e":"high","m":"o3-mini-2025-01-31","oid":"live_bench-bfd85bfa05d5bbed","rd":"2025-01-31","sid":"o3-mini-2025-01-31-high","sm":"o3-mini-2025-01-31_high","src":"https://livebench.ai/#/","v":75.88},{"b":"live_bench","bv":"LiveBench-2024-11-25","cfg":{"effort":"high"},"e":"high","m":"o1-2024-12-17","oid":"live_bench-c706c36fbb28cde5","rd":"2024-12-17","sid":"o1-2024-12-17-high","sm":"o1-2024-12-17_high","src":"https://livebench.ai/#/","v":75.67},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"qwq-32b","oid":"live_bench-e9af0309344a284e","rd":"2025-03-05","sid":"qwq-32b","sm":"QwQ-32B","src":"https://livebench.ai/#/","v":71.96},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"deepseek-r1","oid":"live_bench-9352ae74cd7dd838","rd":"2025-01-20","sid":"deepseek-r1","sm":"DeepSeek-R1","src":"https://livebench.ai/#/","v":71.57},{"b":"live_bench","bv":"LiveBench-2024-11-25","cfg":{"effort":"medium"},"e":"medium","m":"o3-mini-2025-01-31","oid":"live_bench-87ec61edfcf7dcde","rd":"2025-01-31","sid":"o3-mini-2025-01-31-medium","sm":"o3-mini-2025-01-31_medium","src":"https://livebench.ai/#/","v":70.01},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gpt-4-5-preview-2025-02-27","oid":"live_bench-8dd3331076bee3b4","rd":"2025-02-27","sid":"gpt-4.5-preview","sm":"gpt-4.5-preview-2025-02-27","src":"https://livebench.ai/#/","v":68.95},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-flash-thinking-exp-01-21","oid":"live_bench-f62d54742242db05","rd":"2025-01-21","sid":"gemini-2.0-flash-thinking-exp-01-21","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://livebench.ai/#/","v":66.92},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"deepseek-v3-0324","oid":"live_bench-4a480ac4ca972955","rd":"2025-03-24","sid":"deepseek-v3-0324","sm":"DeepSeek-V3-0324","src":"https://livebench.ai/#/","v":66.86},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"claude-3-7-sonnet-2025-02-19","oid":"live_bench-bde335b3a4e9a2ed","rd":"2025-02-24","sid":"claude-3-7-sonnet","sm":"claude-3-7-sonnet-20250219","src":"https://livebench.ai/#/","v":65.56},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-pro-exp-02-05","oid":"live_bench-e8df2c14397e002e","rd":"2025-02-05","sid":"gemini-2.0-pro-exp-02-05","sm":"gemini-2.0-pro-exp-02-05","src":"https://livebench.ai/#/","v":65.13},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-exp-1206","oid":"live_bench-9b7118f447b5aa8b","rd":"2024-12-06","sid":"gemini-exp-1206","sm":"gemini-exp-1206","src":"https://livebench.ai/#/","v":64.09},{"b":"live_bench","bv":"LiveBench-2024-11-25","cfg":{"effort":"low"},"e":"low","m":"o3-mini-2025-01-31","oid":"live_bench-c5c9c009c50e9329","rd":"2025-01-31","sid":"o3-mini-2025-01-31-low","sm":"o3-mini-2025-01-31_low","src":"https://livebench.ai/#/","v":62.45},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"qwen2-5-max","oid":"live_bench-0d85ea8c03e4e14f","rd":"2025-01-28","sid":"qwen2.5-max","sm":"qwen2.5-max","src":"https://livebench.ai/#/","v":62.29},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-flash-001","oid":"live_bench-29158fc407d5a6a8","rd":"2025-02-05","sid":"gemini-2.0-flash","sm":"gemini-2.0-flash-001","src":"https://livebench.ai/#/","v":61.47},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"deepseek-v3","oid":"live_bench-555fb1ef7b6a1509","rd":"2024-12-26","sid":"deepseek-v3","sm":"DeepSeek-V3","src":"https://livebench.ai/#/","v":60.45},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-flash-exp","oid":"live_bench-260581fad49b8bcb","rd":"2024-12-11","sid":"gemini-2.0-flash-exp","sm":"gemini-2.0-flash-exp","src":"https://livebench.ai/#/","v":59.26},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"claude-3-5-sonnet-2024-10-22","oid":"live_bench-b749163aed755e32","rd":"2024-10-22","sid":"claude-3-5-sonnet-20241022","sm":"claude-3-5-sonnet-20241022","src":"https://livebench.ai/#/","v":59.03},{"b":"live_bench","bv":"LiveBench-2024-11-25","cfg":{"effort":"medium"},"e":"medium","m":"o1-mini-2024-09-12","notes":"Assuming medium thinking effort for o1-mini","oid":"live_bench-9999a11404e51528","rd":"2024-09-12","sid":"o1-mini-2024-09-12","sm":"o1-mini-2024-09-12_medium","src":"https://livebench.ai/#/","v":57.76},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gpt-4o-2024-08-06","oid":"live_bench-6cf237aa1ae79642","rd":"2024-08-06","sid":"gpt-4o-2024-08-06","sm":"gpt-4o-2024-08-06","src":"https://livebench.ai/#/","v":55.33},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"deepseek-r1-distill-llama-70b","oid":"live_bench-f81be38dbae6c2fa","rd":"2025-01-20","sid":"deepseek-r1-distill-llama-70b","sm":"DeepSeek-R1-Distill-Llama-70B","src":"https://livebench.ai/#/","v":54.5},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"grok-2-1212","oid":"live_bench-6e52542ca68c3526","rd":"2024-12-12","sid":"grok-2-1212","sm":"grok-2-1212","src":"https://livebench.ai/#/","v":54.3},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-flash-lite","oid":"live_bench-028f1954e368e28b","rd":"2025-02-05","sid":"gemini-2.0-flash-lite","sm":"gemini-2.0-flash-lite","src":"https://livebench.ai/#/","v":54.29},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemini-2-0-flash-lite-preview-02-05","oid":"live_bench-65cc5661935d7081","rd":"2025-02-05","sid":"gemini-2.0-flash-lite-preview-02-05","sm":"gemini-2.0-flash-lite-preview-02-05","src":"https://livebench.ai/#/","v":53.24},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"dracarys2-72b-instruct","oid":"live_bench-0b9c4791e9d1410d","rd":"2024-09-30","sid":"dracarys2-72b-instruct","sm":"Dracarys2-72B-Instruct","src":"https://livebench.ai/#/","v":52.64},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gpt-4o-2024-11-20","oid":"live_bench-5c5fc8a9599d06ae","rd":"2024-11-20","sid":"gpt-4o-2024-11-20","sm":"gpt-4o-2024-11-20","src":"https://livebench.ai/#/","v":52.19},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"learnlm-1-5-pro-experimental","oid":"live_bench-52c081b14728c241","sid":"learnlm-1.5-pro-experimental","sm":"learnlm-1.5-pro-experimental","src":"https://livebench.ai/#/","v":52.19},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"llama-3-3-70b-instruct","oid":"live_bench-abe78c640219696c","rd":"2024-12-06","sid":"llama-3.3-70b-instruct-turbo","sm":"Llama-3.3-70B-Instruct","src":"https://livebench.ai/#/","v":50.16},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemma-3-27b-it","oid":"live_bench-8b53784a14de333b","rd":"2025-03-12","sid":"gemma-3-27b-it","sm":"gemma-3-27b-it","src":"https://livebench.ai/#/","v":49.99},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"claude-3-opus-2024-02-29","oid":"live_bench-2e045d173e9314c8","rd":"2024-02-29","sid":"claude-3-opus-20240229","sm":"claude-3-opus-20240229","src":"https://livebench.ai/#/","v":49.16},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"mistral-large-2411","oid":"live_bench-64af859d2343c3c3","rd":"2024-11-18","sid":"mistral-large-2411","sm":"mistral-large-2411","src":"https://livebench.ai/#/","v":48.43},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"sonar","oid":"live_bench-071682d6bb383ac0","sid":"perplexity-sonar","sm":"sonar","src":"https://livebench.ai/#/","v":46.88},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"qwen2-5-coder-32b-instruct","oid":"live_bench-67bdcea2ed5f527f","rd":"2024-11-21","sid":"qwen2.5-coder-32b-instruct","sm":"Qwen2.5-Coder-32B-Instruct","src":"https://livebench.ai/#/","v":46.23},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"dracarys2-llama-3-1-70b-instruct","oid":"live_bench-44477bb9112193b9","rd":"2024-08-14","sid":"dracarys2-llama-3.1-70b-instruct","sm":"Dracarys2-Llama-3.1-70B-Instruct","src":"https://livebench.ai/#/","v":46.21},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"deepseek-r1-distill-qwen-32b","oid":"live_bench-32546c83c4240a82","rd":"2025-01-20","sid":"deepseek-r1-distill-qwen-32b","sm":"DeepSeek-R1-Distill-Qwen-32B","src":"https://livebench.ai/#/","v":45.55},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"mistral-small-2503","oid":"live_bench-d37a7cd5099ec71e","rd":"2025-03-17","sid":"mistral-small-2503","sm":"mistral-small-2503","src":"https://livebench.ai/#/","v":43.96},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"amazon-nova-pro-v10","oid":"live_bench-e8b75e03db1c79a5","rd":"2024-12-03","sid":"amazon.nova-pro-v1:0","sm":"amazon.nova-pro-v1:0","src":"https://livebench.ai/#/","v":43.53},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"claude-3-5-haiku-2024-10-22","oid":"live_bench-914ba63facb02568","rd":"2024-10-22","sid":"claude-3-5-haiku-20241022","sm":"claude-3-5-haiku-20241022","src":"https://livebench.ai/#/","v":43.45},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"mistral-small-2501","oid":"live_bench-67af61be90372605","rd":"2025-01-25","sid":"mistral-small-2501","sm":"mistral-small-2501","src":"https://livebench.ai/#/","v":42.55},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"phi-4","oid":"live_bench-6a15aa5da53b1936","rd":"2024-12-12","sid":"phi-4","sm":"phi-4","src":"https://livebench.ai/#/","v":41.61},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gpt-4o-mini-2024-07-18","oid":"live_bench-dbf3e972a23daeae","rd":"2024-07-18","sid":"gpt-4o-mini-2024-07-18","sm":"gpt-4o-mini-2024-07-18","src":"https://livebench.ai/#/","v":41.26},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"qwq-32b-preview","oid":"live_bench-5fa964ab3c9d1ad9","rd":"2024-11-28","sid":"qwq-32b-preview","sm":"QwQ-32B-Preview","src":"https://livebench.ai/#/","v":40.25},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemma-2-27b-it","oid":"live_bench-c5b47b581de5cee3","rd":"2024-06-24","sid":"gemma-2-27b-it","sm":"gemma-2-27b-it","src":"https://livebench.ai/#/","v":38.18},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"amazon-nova-lite-v10","oid":"live_bench-929d4f9dfb15a9c7","rd":"2024-12-03","sid":"amazon.nova-lite-v1:0","sm":"amazon.nova-lite-v1:0","src":"https://livebench.ai/#/","v":36.35},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"c4ai-command-r-plus-08-2024","oid":"live_bench-9bf314baab0ce512","rd":"2024-08-30","sid":"command-r-plus-08-2024","sm":"c4ai-command-r-plus-08-2024","src":"https://livebench.ai/#/","v":31.76},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"amazon-nova-micro-v10","oid":"live_bench-fb93b372f15c90f4","rd":"2024-12-03","sid":"amazon.nova-micro-v1:0","sm":"amazon.nova-micro-v1:0","src":"https://livebench.ai/#/","v":29.59},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"gemma-2-9b-it","oid":"live_bench-aba5e8337901f307","rd":"2024-06-24","sid":"gemma-2-9b-it","sm":"gemma-2-9b-it","src":"https://livebench.ai/#/","v":28.66},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"c4ai-command-r-08-2024","oid":"live_bench-cc0a7501df3bccae","rd":"2024-08-30","sid":"command-r-08-2024","sm":"c4ai-command-r-08-2024","src":"https://livebench.ai/#/","v":27.48},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"phi-3-small-8k-instruct","oid":"live_bench-57a4963428a12b18","rd":"2024-04-23","sid":"phi-3-small-8k-instruct","sm":"Phi-3-small-8k-instruct","src":"https://livebench.ai/#/","v":24.03},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"phi-3-mini-4k-instruct","oid":"live_bench-5d98e4abffcb4737","rd":"2024-04-23","sid":"phi-3-mini-128k-instruct","sm":"Phi-3-mini-4k-instruct","src":"https://livebench.ai/#/","v":22.36},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"olmo-2-1124-13b-instruct","oid":"live_bench-289fb7e5ec4448c6","rd":"2024-12-31","sid":"olmo-2-1124-13b-instruct","sm":"OLMo-2-1124-13B-Instruct","src":"https://livebench.ai/#/","v":22.12},{"b":"live_bench","bv":"LiveBench-2024-11-25","m":"phi-3-mini-4k-instruct","oid":"live_bench-7ca4e5c89e920d8e","rd":"2024-04-23","sid":"phi-3-mini-4k-instruct","sm":"Phi-3-mini-4k-instruct","src":"https://livebench.ai/#/","v":22.08},{"b":"live_bench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-1-2025-11-13","oid":"live_bench-2b75c7c896a3a889","rd":"2025-11-13","sid":"gpt-5.1_high","sm":"gpt-5.1-2025-11-13_high","src":"https://livebench.ai/#/","v":78.79},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T10:00:02.998Z","m":"gpt-5-mini-2025-08-07","oid":"math_level_5-9ee63dec3f48245f","rd":"2025-08-07","se":0.003369422414145465,"sid":"5duoiNDccsgaW7UVt6feYX","sm":"gpt-5-mini-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5duoiNDccsgaW7UVt6feYX.eval","v":0.9784743202416919},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-10-29","e":"high","evaluated_at":"2025-10-29T22:29:43.092Z","m":"gpt-5-2025-08-07","oid":"math_level_5-f2e869aedad3a652","rd":"2025-08-07","se":0.0031922231523012308,"sid":"4wN6FKe2db9CLQ3NUFfYge","sm":"gpt-5-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4wN6FKe2db9CLQ3NUFfYge.eval","v":0.9813066465256798},{"b":"math_level_5","d":"2025-10-22","evaluated_at":"2025-10-22T13:37:59.284Z","m":"claude-haiku-4-5-20251001-32k","oid":"math_level_5-51520ce8e3085bae","rd":"2025-10-15","se":0.00515811348923114,"sid":"Qwci6JAG4DUWqA9eSHThDJ","sm":"claude-haiku-4-5-20251001_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Qwci6JAG4DUWqA9eSHThDJ.eval","v":0.9636087945413192},{"b":"math_level_5","d":"2025-10-21","evaluated_at":"2025-10-21T16:28:50.811Z","m":"claude-sonnet-4-5-20250929-32k","oid":"math_level_5-ded0a97b7c82d911","rd":"2025-09-29","se":0.004091286174812221,"sid":"FqjAH25VNjgwQTbDJFfqAg","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FqjAH25VNjgwQTbDJFfqAg.eval","v":0.9773413897280967},{"b":"math_level_5","d":"2025-10-16","evaluated_at":"2025-10-16T11:58:16.245Z","m":"claude-haiku-4-5-2025-10-01","oid":"math_level_5-5001d103e89e2598","rd":"2025-10-15","se":0.007747551687214772,"sid":"3w2dD9rcMPHLYsEsJmYsCX","sm":"claude-haiku-4-5-20251001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3w2dD9rcMPHLYsEsJmYsCX.eval","v":0.8691465256797583},{"b":"math_level_5","d":"2025-10-09","evaluated_at":"2025-10-09T15:30:36.002Z","m":"qwen3-max-2025-09-23","oid":"math_level_5-383ff361f17446ab","rd":"2025-09-24","se":0.004200694815744905,"sid":"f2vxirYMpXwkaZxb4ywnYM","sm":"qwen3-max-2025-09-23","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/f2vxirYMpXwkaZxb4ywnYM.eval","v":0.9712990936555891},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-08-20","e":"medium","evaluated_at":"2025-08-20T11:28:53.620Z","m":"gpt-5-2025-08-07","oid":"math_level_5-5c92aafa1d6eb572","rd":"2025-08-07","se":0.003446256264439923,"sid":"eDzsqj2CnVxbkp5JRuULXF","sm":"gpt-5-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eDzsqj2CnVxbkp5JRuULXF.eval","v":0.9792296072507553},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-08-20","e":"medium","evaluated_at":"2025-08-20T11:28:52.351Z","m":"gpt-5-nano-2025-08-07","oid":"math_level_5-14fafa30b6d185c2","rd":"2025-08-07","se":0.004627056893049539,"sid":"SSLfQCm7RvLhSv5aPKaLNk","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SSLfQCm7RvLhSv5aPKaLNk.eval","v":0.952416918429003},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-08-20","e":"high","evaluated_at":"2025-08-20T11:28:52.016Z","m":"gpt-5-nano-2025-08-07","oid":"math_level_5-402e0d34a20dc19a","rd":"2025-08-07","se":0.004848851575113763,"sid":"TYQuXJw9H8NRjSovr5VoBn","sm":"gpt-5-nano-2025-08-07_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/TYQuXJw9H8NRjSovr5VoBn.eval","v":0.9490181268882175},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-08-20","e":"medium","evaluated_at":"2025-08-20T11:28:51.670Z","m":"gpt-5-mini-2025-08-07","oid":"math_level_5-d545fe49ab09d7a5","rd":"2025-08-07","se":0.003853475747353661,"sid":"3NYyCGnM6XhgytjyVGtwh8","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3NYyCGnM6XhgytjyVGtwh8.eval","v":0.9679003021148036},{"b":"math_level_5","d":"2025-06-03","evaluated_at":"2025-06-03T22:33:05.178Z","m":"qwen3-235b-a22b","oid":"math_level_5-699723cd22e90419","rd":"2025-04-29","se":0.009487067376018314,"sid":"BTDXT8mHQu4je5jysTDuXV","sm":"qwen3-235b-a22b","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BTDXT8mHQu4je5jysTDuXV.eval","v":0.6885699899295065},{"b":"math_level_5","d":"2025-05-29","evaluated_at":"2025-05-29T14:09:30.970Z","m":"deepseek-r1-0528","oid":"math_level_5-53d0c02cda47cdb2","rd":"2025-05-28","se":0.0042156161735636515,"sid":"2wd9khVKiBzsAMUquz2R5s","sm":"DeepSeek-R1-0528","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2wd9khVKiBzsAMUquz2R5s.eval","v":0.9663897280966768},{"b":"math_level_5","d":"2025-05-22","evaluated_at":"2025-05-22T17:39:31.972Z","m":"claude-sonnet-4-2025-05-14","oid":"math_level_5-de6fd3a5b88297a0","rd":"2025-05-22","se":0.00998490304339695,"sid":"dZLNhNKQVUgnB5ygAEUCcT","sm":"claude-sonnet-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/dZLNhNKQVUgnB5ygAEUCcT.eval","v":0.8436555891238671},{"b":"math_level_5","d":"2025-05-22","evaluated_at":"2025-05-22T17:39:30.135Z","m":"claude-opus-4-2025-05-14","oid":"math_level_5-20aeb314f2edf206","rd":"2025-05-22","se":0.009804690367660946,"sid":"YbrpPbWuYAXNqbSfU4so7f","sm":"claude-opus-4-20250514","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YbrpPbWuYAXNqbSfU4so7f.eval","v":0.850453172205438},{"b":"math_level_5","d":"2025-05-08","evaluated_at":"2025-05-08T05:28:17.657Z","m":"gemini-2-5-pro-preview-05-06","oid":"math_level_5-7041c50e32f75ddd","rd":"2025-05-06","se":0.004221388011004008,"sid":"Mi8LCxmduKukxazAtmuhqg","sm":"gemini-2.5-pro-preview-05-06","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Mi8LCxmduKukxazAtmuhqg.eval","v":0.9590256797583081},{"b":"math_level_5","d":"2025-05-07","evaluated_at":"2025-05-07T21:59:53.901Z","m":"gemini-2-5-pro-preview-03-25","oid":"math_level_5-6f0ea22231832da3","rd":"2025-03-31","se":0.004475169260814502,"sid":"XRkZCzSoQC5yYiQ3xitNdC","sm":"gemini-2.5-pro-preview-03-25","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XRkZCzSoQC5yYiQ3xitNdC.eval","v":0.9556268882175226},{"b":"math_level_5","d":"2025-05-07","evaluated_at":"2025-05-07T17:32:19.064Z","m":"mistral-medium-2505","oid":"math_level_5-22900d0cef021c15","rd":"2025-05-07","se":0.00899565165100951,"sid":"LNUnvvrDra89Vhdc9bzdCy","sm":"mistral-medium-2505","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/LNUnvvrDra89Vhdc9bzdCy.eval","v":0.8162764350453172},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:08.504Z","m":"o4-mini-2025-04-16","oid":"math_level_5-44d3fcfc0ffc54ef","rd":"2025-04-16","se":0.0033204985518619605,"sid":"727ixGKZ7A7G8LmavE2ScL","sm":"o4-mini-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/727ixGKZ7A7G8LmavE2ScL.eval","v":0.978285498489426},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:06.429Z","m":"o3-2025-04-16","oid":"math_level_5-213e24a075fe6f68","rd":"2025-04-16","se":0.0033230338273894273,"sid":"6oW3GceNQznewQJRwLigth","sm":"o3-2025-04-16_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6oW3GceNQznewQJRwLigth.eval","v":0.9777190332326284},{"b":"math_level_5","d":"2025-04-14","evaluated_at":"2025-04-14T22:50:03.233Z","m":"gpt-4-1-mini-2025-04-14","oid":"math_level_5-cd52f76970c40490","rd":"2025-04-14","se":0.007298484758774334,"sid":"6ybVweWdTfiCuirnS2BKXD","sm":"gpt-4.1-mini-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6ybVweWdTfiCuirnS2BKXD.eval","v":0.8729229607250756},{"b":"math_level_5","d":"2025-04-14","evaluated_at":"2025-04-14T22:49:56.785Z","m":"gpt-4-1-nano-2025-04-14","oid":"math_level_5-5effacb7cc17bbd9","rd":"2025-04-14","se":0.010231591350302844,"sid":"K4TfMXsaPd4Z46zFmiV4Rf","sm":"gpt-4.1-nano-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/K4TfMXsaPd4Z46zFmiV4Rf.eval","v":0.6999622356495468},{"b":"math_level_5","d":"2025-04-14","evaluated_at":"2025-04-14T22:22:31.727Z","m":"gpt-4-1-2025-04-14","oid":"math_level_5-b7c456989a645890","rd":"2025-04-14","se":0.008490146044318322,"sid":"YyJXPVXJjN5EnLfYjfCTrQ","sm":"gpt-4.1-2025-04-14","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YyJXPVXJjN5EnLfYjfCTrQ.eval","v":0.8300604229607251},{"b":"math_level_5","cfg":{"effort":"low"},"d":"2025-04-10","e":"low","evaluated_at":"2025-04-10T20:30:31.009Z","m":"grok-3-mini-beta","oid":"math_level_5-f0e39fcf5f685f13","rd":"2025-04-09","se":0.007892888346623621,"sid":"5QyCm49Lce2PpGVcZLomCz","sm":"grok-3-mini-beta_low","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5QyCm49Lce2PpGVcZLomCz.eval","v":0.9093655589123867},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-04-10","e":"high","evaluated_at":"2025-04-10T18:02:30.275Z","m":"grok-3-mini-beta","oid":"math_level_5-9f6f91255075ae87","rd":"2025-04-09","se":0.008912712785411623,"sid":"HMm8XaDBr4CQdMFddasEWb","sm":"grok-3-mini-beta_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/HMm8XaDBr4CQdMFddasEWb.eval","v":0.8806646525679759},{"b":"math_level_5","d":"2025-04-10","evaluated_at":"2025-04-10T17:55:22.706Z","m":"grok-3-beta","oid":"math_level_5-05d45de92e999032","rd":"2025-04-09","se":0.008688487093324447,"sid":"i9eqMbaVTDDuCxWxzGHxyU","sm":"grok-3-beta","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/i9eqMbaVTDDuCxWxzGHxyU.eval","v":0.8874622356495468},{"b":"math_level_5","d":"2025-04-08","evaluated_at":"2025-04-08T09:46:43.408Z","m":"llama-4-maverick-17b-128e-instruct-fp8","oid":"math_level_5-9591fbf45300dede","rd":"2025-04-05","se":0.01083573699636281,"sid":"HkjCosgMFq5QXB2CyTyU7r","sm":"Llama-4-Maverick-17B-128E-Instruct-FP8","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/HkjCosgMFq5QXB2CyTyU7r.eval","v":0.7301737160120846},{"b":"math_level_5","d":"2025-04-08","evaluated_at":"2025-04-08T09:22:06.630Z","m":"llama-4-scout-17b-16e-instruct","oid":"math_level_5-28b98a6e936869dd","rd":"2025-04-05","se":0.012022405873783983,"sid":"c6rTyTFCeoHMBzC2QwjxFL","sm":"Llama-4-Scout-17B-16E-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/c6rTyTFCeoHMBzC2QwjxFL.eval","v":0.6227341389728097},{"b":"math_level_5","d":"2025-04-07","evaluated_at":"2025-04-07T19:55:58.553Z","m":"qwen-turbo-2024-11-01","oid":"math_level_5-91ea123bbcc0f3b1","rd":"2024-11-01","se":0.011364630947550552,"sid":"jWV936hYKg6rYcPEynzEEt","sm":"qwen-turbo-2024-11-01","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jWV936hYKg6rYcPEynzEEt.eval","v":0.5623111782477341},{"b":"math_level_5","d":"2025-04-07","evaluated_at":"2025-04-07T17:38:39.334Z","m":"qwen-plus-2025-01-25","oid":"math_level_5-53324b25652fab39","rd":"2025-01-25","se":0.011334654774046507,"sid":"J3WoEpLKAJZWGGpyBVCCCn","sm":"qwen-plus-2025-01-25","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/J3WoEpLKAJZWGGpyBVCCCn.eval","v":0.6527567975830816},{"b":"math_level_5","d":"2025-04-01","evaluated_at":"2025-04-01T17:50:34.087Z","m":"qwen-max-2025-01-25","oid":"math_level_5-f18a0bf617750a6a","rd":"2025-01-25","se":0.011143967442882196,"sid":"GFZDNH95TnVegNrEtmUT2Q","sm":"qwen-max-2025-01-25","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GFZDNH95TnVegNrEtmUT2Q.eval","v":0.6718277945619335},{"b":"math_level_5","d":"2025-04-01","evaluated_at":"2025-04-01T14:52:18.473Z","m":"deepseek-v3-0324","oid":"math_level_5-07eb36bb446eef80","rd":"2025-03-24","se":0.008603173106577204,"sid":"YFyn28VJ6m2hfop3KGRkBs","sm":"DeepSeek-V3-0324","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YFyn28VJ6m2hfop3KGRkBs.eval","v":0.75547583081571},{"b":"math_level_5","d":"2025-03-18","evaluated_at":"2025-03-18T16:08:03.120Z","m":"mistral-small-2503","oid":"math_level_5-8a7298136136c17d","rd":"2025-03-17","se":0.011995700377965737,"sid":"Kas7Y5PBsT7Li5Wm6Y8mj8","sm":"mistral-small-2503","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Kas7Y5PBsT7Li5Wm6Y8mj8.eval","v":0.4677114803625378},{"b":"math_level_5","d":"2025-03-13","evaluated_at":"2025-03-13T21:33:10.936Z","m":"claude-3-7-sonnet-20250219-64k","oid":"math_level_5-037727b92d92ab3d","rd":"2025-02-24","se":0.007803306377082018,"sid":"epCyFNTf6vyUi5Dby7GeTq","sm":"claude-3-7-sonnet-20250219_64K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/epCyFNTf6vyUi5Dby7GeTq.eval","v":0.9116314199395771},{"b":"math_level_5","d":"2025-03-13","evaluated_at":"2025-03-13T05:52:35.397Z","m":"gemma-3-27b-it","oid":"math_level_5-b9e71858d6f50bdd","rd":"2025-03-12","se":0.011195857525727288,"sid":"amrtN7UjAtuyEmv5DaGshP","sm":"gemma-3-27b-it","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/amrtN7UjAtuyEmv5DaGshP.eval","v":0.740370090634441},{"b":"math_level_5","d":"2025-03-12","evaluated_at":"2025-03-12T23:51:45.093Z","m":"claude-3-5-haiku-2024-10-22","oid":"math_level_5-c63c8cb7209449a0","rd":"2024-10-22","se":0.011611728105526228,"sid":"GSvi9Zf5aK3t5TfCPqNRHy","sm":"claude-3-5-haiku-20241022","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GSvi9Zf5aK3t5TfCPqNRHy.eval","v":0.46355740181268884},{"b":"math_level_5","d":"2025-03-12","evaluated_at":"2025-03-12T19:45:08.221Z","m":"claude-3-7-sonnet-20250219-32k","oid":"math_level_5-72fad4edeee126fd","rd":"2025-02-24","se":0.008236774684678254,"sid":"WHGMHrneqJs4nefAh4TjZR","sm":"claude-3-7-sonnet-20250219_32K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WHGMHrneqJs4nefAh4TjZR.eval","v":0.9003021148036254},{"b":"math_level_5","d":"2025-03-11","evaluated_at":"2025-03-11T00:50:44.877Z","m":"deepseek-r1-distill-qwen-14b","oid":"math_level_5-26acb6549dac75b7","rd":"2025-01-20","se":0.007486217204155208,"sid":"R8XGt5ScHpmTbD5DnXR59V","sm":"DeepSeek-R1-Distill-Qwen-14B","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/R8XGt5ScHpmTbD5DnXR59V.eval","v":0.8712235649546828},{"b":"math_level_5","d":"2025-03-10","evaluated_at":"2025-03-10T22:53:10.180Z","m":"deepseek-r1-distill-llama-70b","oid":"math_level_5-ecc45822477007d9","rd":"2025-01-20","se":0.006818015200174493,"sid":"oPDoTJFMhMNvU4sybYu2jQ","sm":"DeepSeek-R1-Distill-Llama-70B","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/oPDoTJFMhMNvU4sybYu2jQ.eval","v":0.8989803625377644},{"b":"math_level_5","d":"2025-02-28","evaluated_at":"2025-02-28T16:37:00.565Z","m":"gpt-4-5-preview-2025-02-27","oid":"math_level_5-ce405593bb1148ff","rd":"2025-02-27","se":0.011270694428136297,"sid":"japqZFi3FS7um32EHG6wwW","sm":"gpt-4.5-preview-2025-02-27","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/japqZFi3FS7um32EHG6wwW.eval","v":0.7862537764350453},{"b":"math_level_5","d":"2025-02-27","evaluated_at":"2025-02-27T14:29:46.292Z","m":"gpt-4-turbo-2024-04-09","oid":"math_level_5-3cf1356c2a9aa8ab","rd":"2024-04-09","se":0.01142825893039598,"sid":"AZAJomaSEfZtqYqXUre9fS","sm":"gpt-4-turbo-2024-04-09","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/AZAJomaSEfZtqYqXUre9fS.eval","v":0.46733383685800606},{"b":"math_level_5","d":"2025-02-26","evaluated_at":"2025-02-26T16:56:42.885Z","m":"claude-3-7-sonnet-20250219-16k","oid":"math_level_5-3c33925a75edf5cb","rd":"2025-02-24","se":0.009466748483397175,"sid":"hWeVyRo3hN5H4AspNc9cuf","sm":"claude-3-7-sonnet-20250219_16K","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hWeVyRo3hN5H4AspNc9cuf.eval","v":0.8625377643504532},{"b":"math_level_5","d":"2025-02-25","evaluated_at":"2025-02-25T12:31:34.494Z","m":"mistral-large-2411","oid":"math_level_5-b0c8e6a313e5ffcd","rd":"2024-11-18","se":0.011469451623480956,"sid":"MMqL6fQSwsBKAdBSfxrYpU","sm":"mistral-large-2411","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MMqL6fQSwsBKAdBSfxrYpU.eval","v":0.5028323262839879},{"b":"math_level_5","d":"2025-02-24","evaluated_at":"2025-02-24T19:14:03.027Z","m":"claude-3-7-sonnet-2025-02-19","oid":"math_level_5-a7d858696eb66d5e","rd":"2025-02-24","se":0.010669651861930321,"sid":"6uSKG2QxR8atueXmDWAPXn","sm":"claude-3-7-sonnet-20250219","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6uSKG2QxR8atueXmDWAPXn.eval","v":0.6818353474320241},{"b":"math_level_5","d":"2025-02-17","evaluated_at":"2025-02-17T21:39:16.270Z","m":"grok-2-1212","oid":"math_level_5-72cfb55240592565","rd":"2024-12-12","se":0.01323438169590836,"sid":"Kpck3pLw3veM5Mt7WZAb8G","sm":"grok-2-1212","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Kpck3pLw3veM5Mt7WZAb8G.eval","v":0.6351963746223565},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:12:08.885Z","m":"o1-2024-12-17","oid":"math_level_5-95b234019dae6df6","rd":"2024-12-17","se":0.006152192005965586,"sid":"jVXtRkTBDnqH8p4sjVUbgN","sm":"o1-2024-12-17_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jVXtRkTBDnqH8p4sjVUbgN.eval","v":0.947129909365559},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:07:52.489Z","m":"o1-mini-2024-09-12","oid":"math_level_5-4cba6c7c7fe64ed8","rd":"2024-09-12","se":0.006802820166065648,"sid":"4DWnpJowAfeQDHc2xDNy2C","sm":"o1-mini-2024-09-12_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4DWnpJowAfeQDHc2xDNy2C.eval","v":0.8918051359516617},{"b":"math_level_5","cfg":{"effort":"high"},"d":"2025-02-13","e":"high","evaluated_at":"2025-02-13T17:07:49.918Z","m":"o3-mini-2025-01-31","oid":"math_level_5-639b5c58594e7f70","rd":"2025-01-31","se":0.004265263461984669,"sid":"3VDhPT5WvzjJZkRc36Kuvi","sm":"o3-mini-2025-01-31_high","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3VDhPT5WvzjJZkRc36Kuvi.eval","v":0.9648791540785498},{"b":"math_level_5","d":"2025-02-06","evaluated_at":"2025-02-06T17:07:01.731Z","m":"gemini-2-0-pro-exp-02-05","oid":"math_level_5-dd469a0e2a195988","rd":"2025-02-05","se":0.01021492751504924,"sid":"fvDt6HaHV5d3gSEuavixsv","sm":"gemini-2.0-pro-exp-02-05","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/fvDt6HaHV5d3gSEuavixsv.eval","v":0.8345921450151057},{"b":"math_level_5","d":"2025-02-06","evaluated_at":"2025-02-06T15:51:21.031Z","m":"gemini-2-0-flash-001","oid":"math_level_5-c85e11681709e6e1","rd":"2025-02-05","se":0.008734081505176936,"sid":"SqwGZz2UfTmbvPiw2rq3LH","sm":"gemini-2.0-flash-001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SqwGZz2UfTmbvPiw2rq3LH.eval","v":0.8216578549848943},{"b":"math_level_5","d":"2025-02-05","evaluated_at":"2025-02-05T08:31:16.384Z","m":"gpt-4o-2024-11-20","oid":"math_level_5-85e8c38d8df76cf7","rd":"2024-11-20","se":0.011102819618096572,"sid":"D3G2BKGnh6YSvw4bEYSdiY","sm":"gpt-4o-2024-11-20","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/D3G2BKGnh6YSvw4bEYSdiY.eval","v":0.49773413897280966},{"b":"math_level_5","d":"2025-01-31","evaluated_at":"2025-01-31T23:48:20.023Z","m":"phi-3-medium-128k-instruct","oid":"math_level_5-796de8fde285779c","rd":"2024-04-23","se":0.007295657661098519,"sid":"njA86GC9DzSXyiZWtUxXiD","sm":"Phi-3-medium-128k-instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/njA86GC9DzSXyiZWtUxXiD.eval","v":0.17560422960725075},{"b":"math_level_5","d":"2025-01-31","evaluated_at":"2025-01-31T23:48:19.611Z","m":"deepseek-r1","oid":"math_level_5-3c11704ff629cec0","rd":"2025-01-20","se":0.006990872160031429,"sid":"XVqLs2khGFSNwEbHJTEgRU","sm":"DeepSeek-R1","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XVqLs2khGFSNwEbHJTEgRU.eval","v":0.9305135951661632},{"b":"math_level_5","d":"2025-01-31","evaluated_at":"2025-01-31T23:48:19.147Z","m":"phi-4","oid":"math_level_5-53fbad6e9ba5af5b","rd":"2024-12-12","se":0.010656871384084826,"sid":"PnvmfAHVxH9oXfcdcUeUZ3","sm":"phi-4","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PnvmfAHVxH9oXfcdcUeUZ3.eval","v":0.6493580060422961},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-01-31","e":"medium","evaluated_at":"2025-01-31T19:32:36.412Z","m":"o3-mini-2025-01-31","oid":"math_level_5-3e3d62a0de242689","rd":"2025-01-31","se":0.004651917857621578,"sid":"2RiSB2ii5kf2Q6fDBv2P3E","sm":"o3-mini-2025-01-31_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2RiSB2ii5kf2Q6fDBv2P3E.eval","v":0.9516616314199395},{"b":"math_level_5","d":"2025-01-30","evaluated_at":"2025-01-30T19:02:48.179Z","m":"qwen2-5-32b-instruct","oid":"math_level_5-343d6dacb781799d","rd":"2024-09-17","se":0.01076720798872177,"sid":"Nk4mmCXFH3mEyCYnQPdPx5","sm":"qwen2.5-32b-instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Nk4mmCXFH3mEyCYnQPdPx5.eval","v":0.5607061933534743},{"b":"math_level_5","d":"2025-01-30","evaluated_at":"2025-01-30T17:26:30.440Z","m":"mistral-small-2501","oid":"math_level_5-f2f582313f1e7608","rd":"2025-01-25","se":0.010854927233993669,"sid":"NbsnvBsMoMizozbPZY8LLb","sm":"mistral-small-2501","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NbsnvBsMoMizozbPZY8LLb.eval","v":0.4481684290030212},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-0613","oid":"math_level_5-62f784c5e1b3f0b2","rd":"2023-06-13","se":0.008465534027077483,"sid":"WsSrH8oHBm7m7vq9yTRqSC","sm":"gpt-4-0613","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WsSrH8oHBm7m7vq9yTRqSC.eval","v":0.22970166163141995},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-flash-001","oid":"math_level_5-56689da73c60c0b3","rd":"2024-05-23","se":0.009456393246526343,"sid":"CPpd4hqQHtZ5pUejBELwKb","sm":"gemini-1.5-flash-001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/CPpd4hqQHtZ5pUejBELwKb.eval","v":0.2512273413897281},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-pro-002","oid":"math_level_5-fd86371968121a18","rd":"2024-09-24","se":0.010174893415515739,"sid":"QyqLmgcJpeDMsnQyaaJVkR","sm":"gemini-1.5-pro-002","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QyqLmgcJpeDMsnQyaaJVkR.eval","v":0.7039274924471299},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-3-5-turbo-0125","oid":"math_level_5-3faa767062e614e9","rd":"2024-01-25","se":0.005543945913851348,"sid":"9tSgxGV3nzhTBpq7GcX7yx","sm":"gpt-3.5-turbo-0125","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9tSgxGV3nzhTBpq7GcX7yx.eval","v":0.1163141993957704},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-pro-001","oid":"math_level_5-848843cafae6df20","rd":"2024-05-14","se":0.010574902459145802,"sid":"PxLmyq7PFfARAuqxg5oPtE","sm":"gemini-1.5-pro-001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PxLmyq7PFfARAuqxg5oPtE.eval","v":0.4074773413897281},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-1106-preview","oid":"math_level_5-6a0e8decb061d40c","rd":"2023-11-06","se":0.010538794632397387,"sid":"VBStRVGFi47EVz2pn33xu4","sm":"gpt-4-1106-preview","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VBStRVGFi47EVz2pn33xu4.eval","v":0.40020770392749244},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-3-5-turbo-1106","oid":"math_level_5-086dfaa6b2499c81","rd":"2023-11-06","se":0.006959408516001194,"sid":"cJWh887zSiTtnxm9XAJVkr","sm":"gpt-3.5-turbo-1106","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cJWh887zSiTtnxm9XAJVkr.eval","v":0.15889350453172205},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-5-flash-002","oid":"math_level_5-2b7d6bc1f452aa36","rd":"2024-09-24","se":0.01120122127580226,"sid":"kDnfjYKQpnLGEjuhBc9AvM","sm":"gemini-1.5-flash-002","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kDnfjYKQpnLGEjuhBc9AvM.eval","v":0.6186744712990937},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-2-0","oid":"math_level_5-541f8c3f2cb8b3bc","rd":"2023-07-11","se":0.006595670913969979,"sid":"Gs4vxvDD5wmrSvyTkqx2YV","sm":"claude-2.0","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Gs4vxvDD5wmrSvyTkqx2YV.eval","v":0.1172583081570997},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-5-sonnet-2024-10-22","oid":"math_level_5-03c7bdc32c7e76e3","rd":"2024-10-22","se":0.011091284283577038,"sid":"AjroyuZECXLtMTwnzaMHtX","sm":"claude-3-5-sonnet-20241022","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/AjroyuZECXLtMTwnzaMHtX.eval","v":0.5694864048338368},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-5-sonnet-2024-06-20","oid":"math_level_5-03f924c0b4887384","rd":"2024-06-20","se":0.0116250120233614,"sid":"H42HJG29bwqzaMewTMsmky","sm":"claude-3-5-sonnet-20240620","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/H42HJG29bwqzaMewTMsmky.eval","v":0.5168051359516617},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-opus-2024-02-29","oid":"math_level_5-8d82468361eb7738","rd":"2024-02-29","se":0.01058728426376393,"sid":"6SNzvpd6kXXxqpJoaHsSDd","sm":"claude-3-opus-20240229","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6SNzvpd6kXXxqpJoaHsSDd.eval","v":0.37481117824773413},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-haiku-2024-03-07","oid":"math_level_5-5c3b041cec388dd8","rd":"2024-03-07","se":0.007229965913719499,"sid":"QSnWVKUVj9six4jYj4HJD9","sm":"claude-3-haiku-20240307","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QSnWVKUVj9six4jYj4HJD9.eval","v":0.1487915407854985},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"claude-3-sonnet-2024-02-29","oid":"math_level_5-0953cd09daf74d66","rd":"2024-02-29","se":0.007981817344047291,"sid":"FD2ztfjFUeqGyrtF8yuCUB","sm":"claude-3-sonnet-20240229","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FD2ztfjFUeqGyrtF8yuCUB.eval","v":0.18174093655589124},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-8b-instruct","oid":"math_level_5-d26478c128b46273","rd":"2024-07-23","se":0.008519457631995396,"sid":"gyxR3YTcmh4UcKZkET5pPX","sm":"Llama-3.1-8B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/gyxR3YTcmh4UcKZkET5pPX.eval","v":0.22875755287009064},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-405b-instruct","oid":"math_level_5-29e9fe048467d951","rd":"2024-07-23","se":0.012185275936569536,"sid":"ZR7amdUBRr5Cke8Aoz4snr","sm":"Llama-3.1-405B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZR7amdUBRr5Cke8Aoz4snr.eval","v":0.49773413897280966},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-70b-instruct","oid":"math_level_5-a724228bfeefd85c","rd":"2024-07-23","se":0.010229604859690931,"sid":"6JiXiQ67iyiC7cFFRiogvs","sm":"Llama-3.1-70B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6JiXiQ67iyiC7cFFRiogvs.eval","v":0.36678625377643503},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-2024-05-13","oid":"math_level_5-843a346892ce101f","rd":"2024-05-13","se":0.01120841984218505,"sid":"Uqsvq9TAGoqD2xpKfhHBmS","sm":"gpt-4o-2024-05-13","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Uqsvq9TAGoqD2xpKfhHBmS.eval","v":0.5104796072507553},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-mini-2024-07-18","oid":"math_level_5-48a193b6e9ebf693","rd":"2024-07-18","se":0.011275275696871733,"sid":"5iyKgWVhAmNwrA5rNL7MbW","sm":"gpt-4o-mini-2024-07-18","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5iyKgWVhAmNwrA5rNL7MbW.eval","v":0.5263406344410876},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4o-2024-08-06","oid":"math_level_5-6ab1182d74b7c124","rd":"2024-08-06","se":0.01122129021441016,"sid":"ZQgz8AGm4KHin9MFupLu58","sm":"gpt-4o-2024-08-06","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZQgz8AGm4KHin9MFupLu58.eval","v":0.5327605740181269},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-01-27","e":"medium","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-mini-2024-09-12","oid":"math_level_5-25d56136126857f4","rd":"2024-09-12","se":0.008256715083788129,"sid":"FaRZDKSriUbS3Brt6qprP9","sm":"o1-mini-2024-09-12_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FaRZDKSriUbS3Brt6qprP9.eval","v":0.8425226586102719},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"hermes-2-theta-llama-3-70b","oid":"math_level_5-c9a8b6871368cc25","rd":"2024-06-20","se":0.008127708089237323,"sid":"ZAVTwsDqrvbWbqSgeNdXFd","sm":"Hermes-2-Theta-Llama-3-70B","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZAVTwsDqrvbWbqSgeNdXFd.eval","v":0.22686933534743203},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"meta-llama-3-8b-instruct","oid":"math_level_5-9cd5a5cd98bed31c","rd":"2024-04-18","se":0.0037433164863856527,"sid":"BURdKU7mhUX4YMPUDoFsed","sm":"Meta-Llama-3-8B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BURdKU7mhUX4YMPUDoFsed.eval","v":0.0612726586102719},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"meta-llama-3-70b-instruct","oid":"math_level_5-38eb9fc1da30fd15","rd":"2024-04-18","se":0.008257639939780867,"sid":"dekbZbU3BtiQZNKG8Txaax","sm":"Meta-Llama-3-70B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/dekbZbU3BtiQZNKG8Txaax.eval","v":0.225547583081571},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-2-70b-chat-hf","oid":"math_level_5-1db9a65b874d4a2e","rd":"2023-07-18","se":0.00254222977415661,"sid":"4bkXhqCNoe6RpTe9taJ7kR","sm":"Llama-2-70b-chat-hf","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4bkXhqCNoe6RpTe9taJ7kR.eval","v":0.032854984894259816},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"yi-1-5-34b-chat","oid":"math_level_5-41643ec015a462ce","rd":"2024-05-13","se":0.008772368987739622,"sid":"bH5kVfLE53CxY3qWmB9C5K","sm":"Yi-1.5-34B-Chat","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bH5kVfLE53CxY3qWmB9C5K.eval","v":0.2548149546827795},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"yi-34b-chat","oid":"math_level_5-684628fa3d53d623","rd":"2023-11-22","se":0.0033693214361751756,"sid":"nzy4SGSeKGzQSKJLWExKpV","sm":"Yi-34B-Chat","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nzy4SGSeKGzQSKJLWExKpV.eval","v":0.05145392749244713},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gpt-4-0125-preview","oid":"math_level_5-7cd377123c544c7c","rd":"2024-01-25","se":0.010855237562085883,"sid":"c4V7sc8TUNJ8wQKRhuA2Rv","sm":"gpt-4-0125-preview","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/c4V7sc8TUNJ8wQKRhuA2Rv.eval","v":0.3541351963746224},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemma-2-9b-it","oid":"math_level_5-65a052e94fa5bb3e","rd":"2024-06-24","se":0.008990724846256171,"sid":"Hdcqs42VnPNVBquXttexhq","sm":"gemma-2-9b-it","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Hdcqs42VnPNVBquXttexhq.eval","v":0.2100641993957704},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemma-2-27b-it","oid":"math_level_5-7917cc97391bed27","rd":"2024-06-24","se":0.009939474306804167,"sid":"PM3GCQcVK7t2MujNp4F4kH","sm":"gemma-2-27b-it","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PM3GCQcVK7t2MujNp4F4kH.eval","v":0.2788897280966767},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen2-72b-instruct","oid":"math_level_5-d861970cbdacbb39","rd":"2024-06-07","se":0.010317258861464411,"sid":"9mWoY9fM2ifKvDg3xaug6h","sm":"qwen2-72b-instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9mWoY9fM2ifKvDg3xaug6h.eval","v":0.39067220543806647},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-7b-instruct-v0-3","oid":"math_level_5-059f5f2aa5edd221","rd":"2024-05-27","se":0.002666943446495743,"sid":"6xHApTdQAWM9e5oCvKYS3W","sm":"Mistral-7B-Instruct-v0.3","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/6xHApTdQAWM9e5oCvKYS3W.eval","v":0.03597054380664653},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mixtral-8x7b-instruct-v0-1","oid":"math_level_5-393617318879af97","rd":"2023-12-11","se":0.005102464546317853,"sid":"4PTATufH2ra3guNeLwT9p9","sm":"Mixtral-8x7B-Instruct-v0.1","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4PTATufH2ra3guNeLwT9p9.eval","v":0.09290030211480363},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"deepseek-llm-67b-chat","oid":"math_level_5-28044cba16ac2568","rd":"2023-11-29","se":0.004016844149550666,"sid":"Cm4UkgA5TtmzQegrq3rbvm","sm":"deepseek-llm-67b-chat","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Cm4UkgA5TtmzQegrq3rbvm.eval","v":0.06391616314199396},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"wizardlm-2-8x22b","oid":"math_level_5-b06fb73cfd071982","rd":"2024-04-15","se":0.008719521575190186,"sid":"4bAYQuh4heJoTNhapVZLNB","sm":"WizardLM-2-8x22B","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4bAYQuh4heJoTNhapVZLNB.eval","v":0.25736404833836857},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"dbrx-instruct","oid":"math_level_5-8e0c9a0ea7c285f7","rd":"2024-03-27","se":0.007355853708151387,"sid":"PMbefrjykQmTBpABMa7FJi","sm":"dbrx-instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PMbefrjykQmTBpABMa7FJi.eval","v":0.11650302114803625},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"gemini-1-0-pro-001","oid":"math_level_5-44e01458d5a03d18","rd":"2023-12-13","se":0.005990949270731778,"sid":"iUdecHq9vU5utytzgTXzqY","sm":"gemini-1.0-pro-001","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iUdecHq9vU5utytzgTXzqY.eval","v":0.11244335347432025},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"ministral-3b-2410","oid":"math_level_5-fbd72d7875d283bc","rd":"2024-10-16","se":0.007446287617112978,"sid":"YNhHxRT9Ee2uEWWJh9RXuP","sm":"ministral-3b-2410","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YNhHxRT9Ee2uEWWJh9RXuP.eval","v":0.14444864048338368},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mistral-7b","oid":"math_level_5-476f80010a7a4af4","rd":"2023-09-27","se":0.0027715208474754028,"sid":"9ZQaypWLwLHbQrqTWfoEor","sm":"open-mistral-7b","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9ZQaypWLwLHbQrqTWfoEor.eval","v":0.0368202416918429},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"ministral-8b-2410","oid":"math_level_5-17f2f155a32b9ce4","rd":"2024-10-16","se":0.007454534670584494,"sid":"hqdiKcvEPSRyvenUVB9qxM","sm":"ministral-8b-2410","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hqdiKcvEPSRyvenUVB9qxM.eval","v":0.14935800604229607},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-large-2407","oid":"math_level_5-a01567b6165ec5bc","rd":"2024-07-24","se":0.010703488686415494,"sid":"div9uE9eovqykqx883kSpE","sm":"mistral-large-2407","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/div9uE9eovqykqx883kSpE.eval","v":0.4481684290030212},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mistral-nemo-2407","oid":"math_level_5-46c1057dd8ef0808","rd":"2024-07-18","se":0.006092598712054613,"sid":"9qCrggpoq8FjQWVQhHaGBT","sm":"open-mistral-nemo-2407","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9qCrggpoq8FjQWVQhHaGBT.eval","v":0.1082892749244713},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mixtral-8x7b","oid":"math_level_5-6660044742877437","rd":"2023-12-11","se":0.005303272245515919,"sid":"GN3Hez798byishFw3U6cpp","sm":"open-mixtral-8x7b","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GN3Hez798byishFw3U6cpp.eval","v":0.09950906344410876},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"open-mixtral-8x22b","oid":"math_level_5-77b812215a5130b0","rd":"2024-04-17","se":0.008296884112232158,"sid":"cGMb2vs9NhfkwZJsQXzLmQ","sm":"open-mixtral-8x22b","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cGMb2vs9NhfkwZJsQXzLmQ.eval","v":0.24244712990936557},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"mistral-large-2402","oid":"math_level_5-091c9dda399c07eb","rd":"2024-02-26","se":0.00892334786643176,"sid":"gypQecfFNK2x4pNGDtGMDv","sm":"mistral-large-2402","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/gypQecfFNK2x4pNGDtGMDv.eval","v":0.24461858006042297},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-2-90b-vision-instruct","oid":"math_level_5-8d0c2810b8ec96c7","rd":"2024-09-24","se":0.010293053094701802,"sid":"iHRmwxsh8M3FGun8x5mxkt","sm":"Llama-3.2-90B-Vision-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iHRmwxsh8M3FGun8x5mxkt.eval","v":0.39435422960725075},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-1-tulu-3-70b-dpo","oid":"math_level_5-4fd6918bacedfcac","rd":"2024-11-21","se":0.010405854683575872,"sid":"Hf2ESooSteQK99yFjhFm8t","sm":"Llama-3.1-Tulu-3-70B-DPO","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Hf2ESooSteQK99yFjhFm8t.eval","v":0.426642749244713},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"llama-3-3-70b-instruct","oid":"math_level_5-3fd04259e487b77b","rd":"2024-12-06","se":0.011816670264886683,"sid":"DeaDxFfiQtCTdnZPH7Awaf","sm":"Llama-3.3-70B-Instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DeaDxFfiQtCTdnZPH7Awaf.eval","v":0.41597432024169184},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"qwen2-5-72b-instruct","oid":"math_level_5-0d32cacda71b722b","rd":"2024-09-19","se":0.01071424933114761,"sid":"ni4DJboYYpdRssVeAuro2V","sm":"qwen2.5-72b-instruct","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ni4DJboYYpdRssVeAuro2V.eval","v":0.631703172205438},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-preview-2024-09-12","oid":"math_level_5-ccee22f6fe4cc44f","rd":"2024-09-12","se":0.010642608258161584,"sid":"8Uu6XsSJCKwcwhXDt8vgzq","sm":"o1-preview-2024-09-12","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8Uu6XsSJCKwcwhXDt8vgzq.eval","v":0.8164652567975831},{"b":"math_level_5","cfg":{"effort":"medium"},"d":"2025-01-27","e":"medium","evaluated_at":"2025-01-27T00:00:00.000Z","m":"o1-2024-12-17","oid":"math_level_5-49e6cada89c7cb39","rd":"2024-12-17","se":0.006315430485333487,"sid":"AzanNoyhhVrtQi63Azo6hu","sm":"o1-2024-12-17_medium","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/AzanNoyhhVrtQi63Azo6hu.eval","v":0.9441087613293051},{"b":"math_level_5","d":"2025-01-27","evaluated_at":"2025-01-27T00:00:00.000Z","m":"deepseek-v3","oid":"math_level_5-ab5c205d50ba71be","rd":"2024-12-26","se":0.010485766934209063,"sid":"exvQCLCiga6gE7zvQNp2Z6","sm":"DeepSeek-V3","trace":"https://epoch-benchmarks-production-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/exvQCLCiga6gE7zvQNp2Z6.eval","v":0.6485083081570997},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":3304.261235,"lo":508.876789,"m":"claude-mythos-preview-early","oid":"metr_time_horizons-a7267e28b7f2a727","rd":"2026-04-07","sid":"recW9gDVlpPKxhewo","sm":"claude-mythos-preview-early","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":1044.780145},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":3949.750392,"lo":319.32091,"m":"claude-opus-4-6","oid":"metr_time_horizons-674fb3415b1136a8","rd":"2026-02-05","sid":"recF8cs4o1pSE7d15","sm":"claude-opus-4-6_unknown","v":718.80683},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":694.750898,"lo":233.50073,"m":"gemini-3-1-pro-preview","oid":"metr_time_horizons-4d35224f0397f87f","rd":"2026-02-19","sid":"recVAgCABLb9qKDM9","sm":"gemini-3.1-pro-preview","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":384.147435},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","cfg":{"effort":"high"},"e":"high","hi":862.339204,"lo":191.31908,"m":"gpt-5-2-2025-12-11","oid":"metr_time_horizons-daba8ddba4ab4d85","rd":"2025-12-11","sid":"rect0q40LX3bSJ36F","sm":"gpt-5.2-2025-12-11_high","v":352.249302},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":858.325814,"lo":192.048863,"m":"gpt-5-3-codex","oid":"metr_time_horizons-31c663701c0ad73d","rd":"2026-02-05","sid":"recLuqsMcBzVzYHTQ","sm":"gpt-5.3-codex","v":349.530732},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","cfg":{"effort":"xhigh"},"e":"xhigh","hi":768.779526,"lo":186.581591,"m":"gpt-5-4-2026-03-05","oid":"metr_time_horizons-2948070abf82ef5d","rd":"2026-03-05","sid":"recVtYP9KdMolGtGf","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":341.735276},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":638.619623,"lo":160.539157,"m":"claude-opus-4-5-2025-11-01","oid":"metr_time_horizons-95f65df82de3acf8","rd":"2025-11-24","sid":"recTbrNrOzFe3rE8x","sm":"claude-opus-4-5-20251101","v":292.994594},{"b":"metr_time_horizons","hi":1225.190511,"lo":108.873592,"m":"claude-opus-4-5-20251101-16k","oid":"metr_time_horizons-60721e34dd900f5a","rd":"2025-11-24","sid":"recPjuZumaNjbylqU","sm":"claude-opus-4-5-20251101_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":288.904952},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":387.478199,"lo":136.865815,"m":"gemini-3-pro-preview","oid":"metr_time_horizons-bec12983de85253a","rd":"2025-11-18","sid":"rec9ydnu1B0h4VV1V","sm":"gemini-3-pro-preview","v":224.325884},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":396.206723,"lo":134.31077,"m":"gpt-5-1-codex-max","oid":"metr_time_horizons-bb6a2607a1aac410","rd":"2025-11-19","sid":"recio8mNX4NbLwCsZ","sm":"gpt-5.1-codex-max","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":223.714694},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","cfg":{"effort":"high"},"e":"high","hi":406.743053,"lo":114.211156,"m":"gpt-5-2025-08-07","oid":"metr_time_horizons-f03d5907f76edaa5","rd":"2025-08-07","sid":"recf6lql8VssXVJPx","sm":"gpt-5-2025-08-07_high","v":203.012577},{"b":"metr_time_horizons","cfg":{"effort":"medium"},"e":"medium","hi":271.115115,"lo":66.869835,"m":"gpt-5-2025-08-07","oid":"metr_time_horizons-a61298fe58cc5b03","rd":"2025-08-07","sid":"recNwo7dbEyB1crye","sm":"gpt-5-2025-08-07_medium","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":137.318539},{"b":"metr_time_horizons","hi":254.328258,"lo":56.041225,"m":"claude-sonnet-4-5-20250929-16k","oid":"metr_time_horizons-f340385be68f18d8","rd":"2025-09-29","sid":"recwCLGT9Kr3iDcqu","sm":"claude-sonnet-4-5-20250929_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":121.946588},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":191.583353,"lo":72.982684,"m":"o3-2025-04-16","oid":"metr_time_horizons-48505517213f6839","rd":"2025-04-16","sid":"recJSUHrT5GjbcWs8","sm":"o3-2025-04-16_unknown","v":119.732634},{"b":"metr_time_horizons","hi":209.192356,"lo":57.552638,"m":"claude-opus-4-1-20250805-16k","oid":"metr_time_horizons-39f72c9518bbadc9","rd":"2025-08-05","sid":"recuz9bui4PgiPuGH","sm":"claude-opus-4-1-20250805_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":113.68507},{"b":"metr_time_horizons","hi":231.836562,"lo":48.190843,"m":"grok-4-0709","oid":"metr_time_horizons-0f23704f63a9705c","rd":"2025-07-09","sid":"recBhYPAeAQhBoGGs","sm":"grok-4-0709","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":110.075251},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":158.015405,"lo":60.105845,"m":"claude-opus-4-1-2025-08-05","oid":"metr_time_horizons-ab043e5e1995f40b","rd":"2025-08-05","sid":"recwfbaxTfx9WUyj1","sm":"claude-opus-4-1-20250805","v":100.472004},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":163.28087,"lo":60.36877,"m":"claude-opus-4-2025-05-14","oid":"metr_time_horizons-62b322c5bbb767cf","rd":"2025-05-22","sid":"recQ05iSph25jn01x","sm":"claude-opus-4-20250514_unknown","v":100.366123},{"b":"metr_time_horizons","cfg":{"effort":"medium"},"e":"medium","hi":163.132889,"lo":45.62758,"m":"o3-2025-04-16","oid":"metr_time_horizons-8c1c86773cb71ad9","rd":"2025-04-16","sid":"recFSX9NsirVMms25","sm":"o3-2025-04-16_medium","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":91.271355},{"b":"metr_time_horizons","hi":146.803709,"lo":45.062156,"m":"claude-opus-4-20250514-16k","oid":"metr_time_horizons-839884b9c1735a42","rd":"2025-05-22","sid":"recjchvjch6vUVmzs","sm":"claude-opus-4-20250514_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":85.556627},{"b":"metr_time_horizons","cfg":{"effort":"medium"},"e":"medium","hi":151.478027,"lo":35.213047,"m":"o4-mini-2025-04-16","oid":"metr_time_horizons-8acc7dd518faa885","rd":"2025-04-16","sid":"recTsdWFhES9SchXH","sm":"o4-mini-2025-04-16_medium","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":76.509015},{"b":"metr_time_horizons","hi":133.985642,"lo":37.850721,"m":"claude-sonnet-4-20250514-16k","oid":"metr_time_horizons-adbb94c5f8a6b338","rd":"2025-05-22","sid":"recueBmSBs2QlEWPm","sm":"claude-sonnet-4-20250514_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":74.942015},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":107.302719,"lo":33.385879,"m":"claude-3-7-sonnet-2025-02-19","oid":"metr_time_horizons-f215b8c1d1d74ab9","rd":"2025-02-24","sid":"recT0BccE0hbyZxlz","sm":"claude-3-7-sonnet-20250219","v":60.388937},{"b":"metr_time_horizons","hi":93.424004,"lo":29.33067,"m":"claude-3-7-sonnet-20250219-16k","oid":"metr_time_horizons-a7f53236db13fd0b","rd":"2025-02-24","sid":"recqRojHxRU5ySXWq","sm":"claude-3-7-sonnet-20250219_16K","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":56.09162},{"b":"metr_time_horizons","hi":98.247233,"lo":25.48195,"m":"kimi-k2-thinking","oid":"metr_time_horizons-b098228f596f6360","rd":"2025-11-06","sid":"recmuexxjm6ucxXtH","sm":"kimi-k2-thinking","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":54.194148},{"b":"metr_time_horizons","hi":81.505677,"lo":18.79454,"m":"gpt-oss-120b","oid":"metr_time_horizons-dcc24f52a9894b37","rd":"2025-08-05","sid":"rec2VBZIfOKcr9NeL","sm":"gpt-oss-120b","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":41.983451},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","cfg":{"effort":"medium"},"e":"medium","hi":64.95249,"lo":21.164512,"m":"o1-2024-12-17","oid":"metr_time_horizons-76f4d49b64bc383c","rd":"2024-12-17","sid":"recrH7UpKH2XmiFW8","sm":"o1-2024-12-17_medium","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":38.831588},{"b":"metr_time_horizons","hi":69.556192,"lo":19.317217,"m":"gemini-2-5-pro-preview-06-05","oid":"metr_time_horizons-9a41a5fc1ace65c7","rd":"2025-06-05","sid":"recxHthxDmMulkuth","sm":"gemini-2.5-pro-preview-06-05","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":38.734274},{"b":"metr_time_horizons","hi":64.108861,"lo":12.849837,"m":"deepseek-r1-0528","oid":"metr_time_horizons-304fe9f676af5359","rd":"2025-05-28","sid":"recasKfxLMpd8pTjF","sm":"DeepSeek-R1-0528","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":31.167847},{"b":"metr_time_horizons","hi":50.97746,"lo":13.478936,"m":"deepseek-r1","oid":"metr_time_horizons-4230187f7cada31b","rd":"2025-01-20","sid":"rectWm0PAizmHcDik","sm":"DeepSeek-R1","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":26.932599},{"b":"metr_time_horizons","hi":41.333511,"lo":11.71071,"m":"deepseek-v3-0324","oid":"metr_time_horizons-636e2a33b748220f","rd":"2025-03-24","sid":"rec5tXt66EEIgBcLi","sm":"DeepSeek-V3-0324","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":23.117113},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":40.82028,"lo":10.144026,"m":"claude-3-5-sonnet-2024-10-22","oid":"metr_time_horizons-619d277a4577997c","rd":"2024-10-22","sid":"recM2JJhzA3vJAZoZ","sm":"claude-3-5-sonnet-20241022","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":20.522872},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":33.379877,"lo":11.716193,"m":"o1-preview-2024-09-12","oid":"metr_time_horizons-2487b2abd24798bc","rd":"2024-09-12","sid":"recJhENih6N50V43Q","sm":"o1-preview-2024-09-12","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":20.326586},{"b":"metr_time_horizons","hi":33.772933,"lo":8.967316,"m":"deepseek-v3","oid":"metr_time_horizons-4cff3a5554225501","rd":"2024-12-26","sid":"rec3j0U1Ecav7Q1FA","sm":"DeepSeek-V3","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":18.472532},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":22.384214,"lo":5.489734,"m":"claude-3-5-sonnet-2024-06-20","oid":"metr_time_horizons-44bc3b23945f87d9","rd":"2024-06-20","sid":"rec3I5CcPGjF3Riat","sm":"claude-3-5-sonnet-20240620","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":11.395377},{"b":"metr_time_horizons","hi":17.963859,"lo":4.224241,"m":"gpt-4o-2024-05-13","oid":"metr_time_horizons-bdd3222ecbe8cdfa","rd":"2024-05-13","sid":"recdwVn3QiDENdt6P","sm":"gpt-4o-2024-05-13","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":9.17045},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":12.414331,"lo":3.851599,"m":"gpt-4o-2024-05-13","oid":"metr_time_horizons-b5ec26bcd41bb5bb","rd":"2024-05-13","sid":"recj2X3OPdvSriHwy","sm":"gpt-4o-2024-05-13","v":6.991195},{"b":"metr_time_horizons","hi":12.196509,"lo":3.215761,"m":"gpt-4-turbo-2024-04-09","oid":"metr_time_horizons-4d9e9fba8bdce0c8","rd":"2024-04-09","sid":"recen58457Be84Pud","sm":"gpt-4-turbo-2024-04-09","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":6.567339},{"b":"metr_time_horizons","hi":9.755795,"lo":2.802412,"m":"gpt-4-0125-preview","oid":"metr_time_horizons-97a2f51c122bd3e1","rd":"2024-01-25","sid":"recD86WJin9bt0ueq","sm":"gpt-4-0125-preview","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":5.366063},{"b":"metr_time_horizons","hi":9.734091,"lo":2.453082,"m":"gpt-4-0314","oid":"metr_time_horizons-ea86b8fc28781168","rd":"2023-03-14","sid":"recO1ntpu18mX9GIG","sm":"gpt-4-0314","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":5.364045},{"b":"metr_time_horizons","hi":10.325614,"lo":2.384633,"m":"qwen2-5-72b-instruct","oid":"metr_time_horizons-dcbc574d7046090a","rd":"2024-09-19","sid":"recmXpRGOYoAcTAgQ","sm":"qwen2.5-72b-instruct","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":5.165813},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":8.443226,"lo":1.866859,"m":"gpt-4-1106-preview","oid":"metr_time_horizons-84d7ebbc4cb26d56","rd":"2023-11-06","sid":"rechSoSit7zTyOpq0","sm":"gpt-4-1106-preview","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":4.044959},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":7.906991,"lo":1.959555,"m":"gpt-4-0314","oid":"metr_time_horizons-e999c8d36e98cc6d","rd":"2023-03-14","sid":"recGv2XrraPE8opN2","sm":"gpt-4-0314","v":3.987428},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":8.76484,"lo":1.706313,"m":"claude-3-opus-2024-02-29","oid":"metr_time_horizons-86c31be14863bf17","rd":"2024-02-29","sid":"recyvUC4eTQ5SK6Gr","sm":"claude-3-opus-20240229","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":3.952262},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.1","hi":6.742026,"lo":1.938233,"m":"gpt-4-turbo-2024-04-09","oid":"metr_time_horizons-25a2d35dc8114e86","rd":"2024-04-09","sid":"rectjHTBu5ytWIXJE","sm":"gpt-4-turbo-2024-04-09","v":3.732787},{"b":"metr_time_horizons","hi":4.830984,"lo":0.854468,"m":"qwen2-72b-instruct","oid":"metr_time_horizons-75970ba47822c887","rd":"2024-06-07","sid":"recIqQ9tqJzeP1sHf","sm":"qwen2-72b-instruct","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":2.24432},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.0","hi":1.115905,"lo":0.255148,"m":"gpt-3-5-turbo-instruct","oid":"metr_time_horizons-d18c5f05c746df38","rd":"2023-09-18","sid":"rec7jJ3gfq0bIgHN4","sm":"gpt-3.5-turbo-instruct","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":0.599247},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.0","hi":0.221209,"lo":0.093319,"m":"davinci-002","oid":"metr_time_horizons-44ce77759b59d2e0","sid":"recbmHH5GLQL9jfZn","sm":"davinci-002","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":0.144057},{"b":"metr_time_horizons","bv":"METR-Horizon-v1.0","hi":0.141825,"lo":0.009994,"m":"gpt2-xl","oid":"metr_time_horizons-fcfe259508ab5f37","rd":"2019-11-05","sid":"rec8JOj4dfN5bKICN","sm":"gpt2-xl","src":"https://metr.org/blog/2025-03-19-measuring-ai-ability-to-complete-long-tasks/","v":0.053778},{"b":"mindcube","dn":"Gemma-3-12B-it","m":"gemma-3-12b-it","oid":"mindcube-1e3233773b6a3943","rd":"2025-03-12","sm":"gemma-3-12b-it","src":"https://mind-cube.github.io/","v":0.4667},{"b":"mindcube","dn":"mPLUG-Owl3-7B-241101","m":"mplug-owl3-7b-241101","oid":"mindcube-374affe27bffdc9a","rd":"2024-11-26","sm":"mPLUG-Owl3-7B-241101","src":"https://mind-cube.github.io/","v":0.4485},{"b":"mindcube","dn":"LLaVA-Video-Qwen-7B","m":"llava-video-7b-qwen2","oid":"mindcube-7da872dbb74f26ba","rd":"2024-09-02","sm":"LLaVA-Video-7B-Qwen2","src":"https://mind-cube.github.io/","v":0.41960000000000003},{"b":"mindcube","dn":"LongVA-7B","m":"longva-7b","oid":"mindcube-2e312dbb6fe83958","rd":"2024-06-13","sm":"LongVA-7B","src":"https://mind-cube.github.io/","v":0.29460000000000003},{"b":"mindcube","dn":"Claude-4-Sonnet-20250514","m":"claude-sonnet-4-2025-05-14","oid":"mindcube-82a9645a70c6c72a","rd":"2025-05-22","sm":"claude-sonnet-4-20250514_unknown","src":"https://mind-cube.github.io/","v":0.4475},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Amazon Nova Lite","m":"amazon-nova-lite-v10","oid":"mmlu-6c1f15a155eeb429","rd":"2024-12-03","sid":"reccNX56mRCIhx9ok","sm":"amazon.nova-lite-v1:0","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.77},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Amazon Nova Micro","m":"amazon-nova-micro-v10","oid":"mmlu-c2f8cb1b9ffb4e7a","rd":"2024-12-03","sid":"recMu6jniW4Q1aHhj","sm":"amazon.nova-micro-v1:0","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.708},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Amazon Nova Pro","m":"amazon-nova-pro-v10","oid":"mmlu-32b6977d25944b0e","rd":"2024-12-03","sid":"recIRlQ5EAWn5dxqD","sm":"amazon.nova-pro-v1:0","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.82},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan 1-7B","m":"baichuan-7b","oid":"mmlu-cabe8aca0c6033f8","rd":"2023-06-01","sid":"rechuNXbQCiXvI5bb","sm":"Baichuan-7B","src":"http://arxiv.org/abs/2309.10305","v":0.423},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan 1-13B-Base","m":"baichuan-13b-base","oid":"mmlu-6f6fb42d68912962","rd":"2023-07-11","sid":"recCxbVxrA9fWanQp","sm":"Baichuan-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.516},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan 2-7B-Base","m":"baichuan-2-7b-base","oid":"mmlu-fa6d7b11017b82a6","rd":"2023-09-20","sid":"recL0KmTLb9YnVqAX","sm":"Baichuan-2-7B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.5416},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan 2-13B-Base","m":"baichuan-2-13b-base","oid":"mmlu-a9562dd657586314","rd":"2023-09-06","sid":"rec2lAAR2TzfQNZIn","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2309.10305","v":0.5917},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan-2 13B","m":"baichuan-2-13b-base","oid":"mmlu-251cd919477d737e","rd":"2023-09-06","sid":"recg7CzyJtjHNlw9z","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2402.16819","v":0.592},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"mmlu-87d723a60f110dda","rd":"2023-09-06","sid":"recic0LQTjjN2UHF2","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.551},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Baichuan2-Chat 13B","m":"baichuan2-13b-chat","oid":"mmlu-69f8d0a364720174","rd":"2023-09-06","sid":"recLNxS9WuckNhW0w","sm":"Baichuan2-13B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.501},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"mmlu-d457961b137a3713","rd":"2023-03-20","sid":"recjibIVUpfsNYJiW","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.262},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"mmlu-f1279b65bbf0c845","rd":"2023-03-20","sid":"recxEX8BTSBULeO3F","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.246},{"b":"mmlu","cfg":{"shots":"5"},"dn":"ChatGLM2 6B","m":"chatglm2-6b","oid":"mmlu-7f2f0e0f1bce951c","rd":"2023-06-24","sid":"recJMYoLpkfhVACXE","sm":"chatglm2-6b","src":"https://arxiv.org/pdf/2309.16609","v":0.479},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"mmlu-3b4cf0e8233375e9","rd":"2022-03-29","sid":"recblkzYf2HuW8IIY","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.675},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 1.3","m":"claude-1-3","notes":"5-shot CoT","oid":"mmlu-4e419184353a5c59","rd":"2023-04-18","sid":"recMiCaKbG8RgAL4z","sm":"claude-1.3","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.77},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 2","m":"claude-2-0","notes":"5-shot CoT","oid":"mmlu-54cece4d27bbff16","rd":"2023-07-11","sid":"recrV38I6BcgNumAG","sm":"claude-2.0","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.785},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 2.1","m":"claude-2-1","oid":"mmlu-ad9f346a97924995","rd":"2023-11-21","sid":"rec3x29hlWlm49CQP","sm":"claude-2.1","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.735},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3 Haiku (20240307)","m":"claude-3-haiku-2024-03-07","oid":"mmlu-2c622edd6ee4d3ab","rd":"2024-03-07","sid":"recFv7M6aEx8RaC1z","sm":"claude-3-haiku-20240307","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.738},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3 Opus (20240229)","m":"claude-3-opus-2024-02-29","oid":"mmlu-5149ee8792edbd43","rd":"2024-02-29","sid":"receM7r5loSsHAQw3","sm":"claude-3-opus-20240229","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.846},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3 Sonnet (20240229)","m":"claude-3-sonnet-2024-02-29","oid":"mmlu-fe0326b721559959","rd":"2024-02-29","sid":"recE3DF6ClfScS431","sm":"claude-3-sonnet-20240229","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.759},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3.5 Haiku (20241022)","m":"claude-3-5-haiku-2024-10-22","oid":"mmlu-8da1235cdc961137","rd":"2024-10-22","sid":"recYAn1b7tiMc4vIW","sm":"claude-3-5-haiku-20241022","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.743},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3.5 Sonnet (20240620)","m":"claude-3-5-sonnet-2024-06-20","oid":"mmlu-6cc8bc00e026ad7d","rd":"2024-06-20","sid":"rectxjtMFzEfdNd3Q","sm":"claude-3-5-sonnet-20240620","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.865},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude 3.5 Sonnet (20241022)","m":"claude-3-5-sonnet-2024-10-22","oid":"mmlu-af9fdcd5e14f8bbe","rd":"2024-10-22","sid":"recoTsRA8ILsdFhsM","sm":"claude-3-5-sonnet-20241022","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.873},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude instant","m":"claude-instant-1-1","notes":"5-shot CoT","oid":"mmlu-1495909242a87cf9","sid":"recvrlr1UHnsu2xMG","sm":"claude-instant-1.1","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.734},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude Instant 1.1","m":"claude-instant-1-1","oid":"mmlu-65cfa1fd3c399980","sid":"rec8sbiLCZ655gCFv","sm":"claude-instant-1.1","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.734},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude Instant 1.2","m":"claude-instant-1-2","oid":"mmlu-ff4a78d91958094a","rd":"2023-08-09","sid":"rec5zO5kkq0fxUAov","sm":"claude-instant-1.2","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.688},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Claude Instant 1.2","m":"claude-instant-1-2","oid":"mmlu-62bbe4df0e6e3c0c","rd":"2023-08-09","sid":"rece9mgHXM0QhmiDh","sm":"claude-instant-1.2","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.732},{"b":"mmlu","dn":"CodeQwen1.5-7B","m":"codeqwen1-5-7b","oid":"mmlu-ed8af380263b38b6","rd":"2024-04-15","sid":"recBAF9M2T4QfIHAR","sm":"CodeQwen1.5-7B","src":"http://arxiv.org/abs/2409.12186","v":0.405},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Command R","m":"c4ai-command-r-08-2024","oid":"mmlu-2898501205c9d0e1","rd":"2024-08-30","sid":"recalVfY5uZ17cXy1","sm":"c4ai-command-r-08-2024","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.652},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Command R Plus","m":"c4ai-command-r-plus-08-2024","oid":"mmlu-975aaab451e6bf15","rd":"2024-08-30","sid":"rectFxVzTinEXE5UP","sm":"c4ai-command-r-plus-08-2024","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.694},{"b":"mmlu","cfg":{"shots":"5"},"dn":"DeepSeek v3","m":"deepseek-v3","oid":"mmlu-ef943bb17394581b","rd":"2024-12-26","sid":"recZAYS2AtKUwDAot","sm":"DeepSeek-V3","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.872},{"b":"mmlu","cfg":{"shots":"5"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"mmlu-5b881503d2c74cff","rd":"2024-05-07","sid":"recUe3PPejBJft24I","sm":"DeepSeek-V2","src":"https://arxiv.org/pdf/2412.19437","v":0.784},{"b":"mmlu","cfg":{"shots":"5"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"mmlu-d9cba8ff321280d8","rd":"2024-12-26","sid":"reciHYY3MaHRp7o7O","sm":"DeepSeek-V3","src":"https://arxiv.org/pdf/2412.19437","v":0.871},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"mmlu-ed1eb3aea5f90c3f","rd":"2023-04-11","sid":"rec97rlUByGZ2zt8v","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.262},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"mmlu-1c8472edc0c05fb0","rd":"2023-04-11","sid":"recWcPAtUKdfuuDcu","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.254},{"b":"mmlu","dn":"DS-Coder-1.3B-Base","m":"deepseek-coder-1-3b-base","oid":"mmlu-6756aecfc669d0ec","rd":"2023-11-02","sid":"reca7wIho24nf1lbz","sm":"deepseek-coder-1.3b-base","src":"http://arxiv.org/abs/2409.12186","v":0.258},{"b":"mmlu","dn":"DS-Coder-6.7B-Base","m":"deepseek-coder-6-7b-base","oid":"mmlu-0f9f8c37c11835ed","rd":"2023-11-02","sid":"rec9GGQ2SbDsozdsi","sm":"deepseek-coder-6.7b-base","src":"http://arxiv.org/abs/2409.12186","v":0.364},{"b":"mmlu","dn":"DS-Coder-33B-Base","m":"deepseek-coder-33b-base","oid":"mmlu-f9b020c591135724","rd":"2023-11-02","sid":"recWXbEo1J1SGz2Pc","sm":"deepseek-coder-33b-base","src":"http://arxiv.org/abs/2409.12186","v":0.394},{"b":"mmlu","dn":"DS-Coder-V2-Lite-Base","m":"deepseek-coder-v2-lite-base","oid":"mmlu-e8c5e0ec04e43b6c","rd":"2024-06-13","sid":"recvb9WHxy2ct1iLh","sm":"DeepSeek-Coder-V2-Lite-Base","src":"http://arxiv.org/abs/2409.12186","v":0.605},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"mmlu-e406a06f192c1cea","rd":"2023-04-24","sid":"recZxu6DjE1qtZ1S3","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.35},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"mmlu-911e5cf6b9df51c8","rd":"2023-04-24","sid":"recWf7mxbT8We02eo","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.262},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"mmlu-aaef4ea96b378359","rd":"2023-04-24","sid":"recCqMLHKTiaKiBA0","sm":"falcon-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.262},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"mmlu-e71979270644f1dc","rd":"2023-03-15","sid":"recz02AVGuolhygEY","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.569},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"mmlu-4d4b2ca5ce7032b7","rd":"2023-03-15","sid":"recRH4MTY2hMvXK0H","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.554},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"mmlu-d70213b104426ec6","rd":"2023-03-15","sid":"recsdVJaTPLaSRk6Q","sm":"falcon-40b","src":"https://arxiv.org/pdf/2309.16609","v":0.554},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon-7B","m":"falcon-7b","oid":"mmlu-f96b5e9a039f56e0","rd":"2023-04-24","sid":"reci0A7hEp1NCjbaL","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.10305","v":0.2603},{"b":"mmlu","cfg":{"shots":"2"},"dn":"Falcon-7B","m":"falcon-7b","oid":"mmlu-dafab196c9b89cda","rd":"2023-04-24","sid":"recqBkEih0PUjiHHs","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.269},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon-7B","m":"falcon-7b","oid":"mmlu-433875faa8aeaa93","rd":"2023-04-24","sid":"recT6jUES7TrDPOPE","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.269},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"mmlu-fc78d57f72e47288","rd":"2023-04-24","sid":"recoziexUycUxDIif","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.239},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon-180B","m":"falcon-180b","oid":"mmlu-c4b55caa2f51ac9a","rd":"2023-09-06","sid":"rec47dDuJMqw0UtNo","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.706},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Falcon2-11B","m":"falcon-11b","oid":"mmlu-d5bcfb1e5db13f21","rd":"2024-05-09","sid":"reca5QOZscaMohhwc","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.584},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.0 Pro (001)","m":"gemini-1-0-pro-001","oid":"mmlu-df98aee1d8e764b1","rd":"2023-12-13","sid":"recp3h3zrPXM6K5E6","sm":"gemini-1.0-pro-001","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.7},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Flash (001)","m":"gemini-1-5-flash-001","oid":"mmlu-d197834060ff6eda","rd":"2024-05-23","sid":"recya8zx93N6YFh4J","sm":"gemini-1.5-flash-001","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.779},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Flash (002)","m":"gemini-1-5-flash-002","oid":"mmlu-debcfa5b20c52aec","rd":"2024-09-24","sid":"recIlo0YrdpQ2gmEe","sm":"gemini-1.5-flash-002","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.739},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Flash (0514 preview)","m":"gemini-1-5-flash-0514","oid":"mmlu-ff44bc38e0a7b6cc","rd":"2024-05-14","sid":"recvHsglkezP7vlhw","sm":"gemini-1.5-flash-0514","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.778},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Pro (001)","m":"gemini-1-5-pro-001-feb24","oid":"mmlu-6125f76584f80740","rd":"2024-02-15","sid":"recWbQsttPCe4RW4T","sm":"gemini-1.5-pro-001-feb24","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.827},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Pro (002)","m":"gemini-1-5-pro-002","oid":"mmlu-a6190df8d16ae060","rd":"2024-09-24","sid":"recosbed3WAJ7x0aX","sm":"gemini-1.5-pro-002","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.869},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Pro (Feb 2024)","m":"gemini-1-5-pro-001-feb24","oid":"mmlu-8e3442b1a31e1c97","rd":"2024-02-15","sid":"rectVJ2cudDhy3DPZ","sm":"gemini-1.5-pro-001-feb24","src":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","v":0.819},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 1.5 Pro (May 2024)","m":"gemini-1-5-pro-001","oid":"mmlu-470da0cdc3832e0b","rd":"2024-05-14","sid":"recUVuFaQQIGL9MJk","sm":"gemini-1.5-pro-001","src":"https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf","v":0.859},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemini 2.0 Flash (Experimental)","m":"gemini-2-0-flash-exp","oid":"mmlu-7220b4aff609cdf7","rd":"2024-12-11","sid":"recGFT80UG5ZvEXom","sm":"gemini-2.0-flash-exp","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.797},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma (7B)","m":"gemma-7b","oid":"mmlu-2c0e90e8f52f9d19","rd":"2024-02-21","sid":"rec9XaJ2TmQNbln0Q","sm":"gemma-7b","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.661},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 2 Instruct (9B)","m":"gemma-2-9b-it","oid":"mmlu-8c44521ad4e32148","rd":"2024-06-24","sid":"receSMvaZsWAJGCwz","sm":"gemma-2-9b-it","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.721},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 2 Instruct (27B)","m":"gemma-2-27b-it","oid":"mmlu-9bea17f80bc9f3ac","rd":"2024-06-24","sid":"recvSF9bkUFYJFbvW","sm":"gemma-2-27b-it","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.757},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 2B","m":"gemma-2b","notes":"5-shot top-1","oid":"mmlu-5d631effefe4e8fa","rd":"2024-02-21","sid":"recFxjYqOdtTFaKbb","sm":"gemma-2b","src":"https://arxiv.org/pdf/2403.08295","v":0.423},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 7B","m":"gemma-7b","notes":"5-shot top-1","oid":"mmlu-65fe64d1290700bd","rd":"2024-02-21","sid":"recVGRLvxWiFAKPkj","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.643},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 7B","m":"gemma-7b","oid":"mmlu-275c58485043dfd2","rd":"2024-02-21","sid":"recZ4bWHun8k676qr","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.643},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gemma 7b","m":"gemma-7b","oid":"mmlu-400bbb4914b4da64","rd":"2024-02-21","sid":"recw9AShxMdslSHLM","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.636},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gopher","m":"gopher-280b","oid":"mmlu-da657cdeab5bad99","rd":"2021-12-08","sid":"recNkiYry1lAvuSwK","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.11446","v":0.6},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Gopher 280B","m":"gopher-280b","oid":"mmlu-2a4d31292a73d28d","rd":"2021-12-08","sid":"rec5n8Sdwq7WFYI2x","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.6},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT 4o-mini","m":"gpt-4o-mini-2024-07-18","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-fbe8111aca17c8a2","rd":"2024-07-18","sid":"rec9BQTN7H8M50fyc","sm":"gpt-4o-mini-2024-07-18","src":"http://arxiv.org/abs/2412.08905","v":0.818},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3","m":"text-davinci-001","oid":"mmlu-0d852c96e69b88e3","rd":"2022-01-27","sid":"recuAsFQiyYxqtm21","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.11446","v":0.439},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3 175B","m":"text-davinci-001","oid":"mmlu-af01198db520c4c6","rd":"2022-01-27","sid":"recvqThL1Bym2eWPd","sm":"text-davinci-001","src":"http://arxiv.org/abs/2302.13971","v":0.439},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3.5","m":"text-davinci-002","oid":"mmlu-79e728e32bcc49a0","rd":"2022-03-15","sid":"recuAOzevJhAkVd3J","sm":"text-davinci-002","src":"https://arxiv.org/pdf/2303.08774","v":0.7},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3.5 Turbo (0125)","m":"gpt-3-5-turbo-0125","oid":"mmlu-2d67f146493f2f17","rd":"2024-01-25","sid":"recUxR8bLyjemBFQ2","sm":"gpt-3.5-turbo-0125","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.673},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3.5 Turbo (0613)","m":"gpt-3-5-turbo-0613","oid":"mmlu-a204a09c330687cb","rd":"2023-06-13","sid":"receZjlh6qpQqkcqQ","sm":"gpt-3.5-turbo-0613","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.689},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"mmlu-366d77c23ddb3c05","rd":"2023-11-06","sid":"recGJbQCjDKrqzKXr","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.714},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4 (0613)","m":"gpt-4-0613","oid":"mmlu-8572b74696310c16","rd":"2023-06-13","sid":"reccKxm0VqsklwpFa","sm":"gpt-4-0613","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.824},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4 (initial release)","m":"gpt-4-0314","oid":"mmlu-dbae37ca95185504","rd":"2023-03-14","sid":"recP6zUYu6rVqvynx","sm":"gpt-4-0314","src":"https://arxiv.org/pdf/2303.08774","v":0.864},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4 Turbo (1106 preview)","m":"gpt-4-turbo","oid":"mmlu-f104cea82eea79f3","rd":"2023-11-06","sid":"rec6xx0AwDEsiaUS8","sm":"gpt-4-turbo","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.796},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4 Turbo (2024-04-09)","m":"gpt-4-turbo-2024-04-09","oid":"mmlu-396e12ed52684dd5","rd":"2024-04-09","sid":"recTZoT00f1052qmh","sm":"gpt-4-turbo-2024-04-09","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.813},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4o","m":"gpt-4o-2024-11-20","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-7a5710ad2bbfd449","rd":"2024-11-20","sid":"recTErMEF70Oq8pZU","sm":"gpt-4o-2024-11-20","src":"http://arxiv.org/abs/2412.08905","v":0.881},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4o (2024-05-13)","m":"gpt-4o-2024-05-13","oid":"mmlu-5bddd93a420e51ae","rd":"2024-05-13","sid":"rec7uXJcDwu5vAB0v","sm":"gpt-4o-2024-05-13","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.842},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4o (2024-08-06)","m":"gpt-4o-2024-08-06","oid":"mmlu-1204ca30b83d38e4","rd":"2024-08-06","sid":"recngZX6kgPdpEuvJ","sm":"gpt-4o-2024-08-06","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.843},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-4o mini (2024-07-18)","m":"gpt-4o-mini-2024-07-18","oid":"mmlu-149924bbb2c14750","rd":"2024-07-18","sid":"rec66ukiUXgRuE5uX","sm":"gpt-4o-mini-2024-07-18","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.767},{"b":"mmlu","cfg":{"shots":"5"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"mmlu-537cc456899f45e0","rd":"2021-08-05","sid":"recSU7vpnDawA53bF","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.251},{"b":"mmlu","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"mmlu-531908556a06f79f","rd":"2021-08-05","sid":"recnPxCnkNQT6Bl8j","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.257},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Inflection 1","m":"inflection-1","oid":"mmlu-19390291a3952f86","rd":"2023-06-22","sid":"rechQVncuCixOKIxc","sm":"Inflection-1","src":"https://inflection.ai/blog/inflection-1","v":0.727},{"b":"mmlu","cfg":{"shots":"5"},"dn":"INTELLECT-1-INSTRUCT","m":"intellect-1-instruct","oid":"mmlu-9376a607d64aa1d9","rd":"2024-11-29","sid":"recOaIUIOT9HiTs0w","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.4989},{"b":"mmlu","cfg":{"shots":"5"},"dn":"InternLM 7B","m":"internlm-7b","oid":"mmlu-2e391cd4de911061","rd":"2023-07-05","sid":"recCRfqyDW6e5Cxj0","sm":"internlm-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.51},{"b":"mmlu","cfg":{"shots":"0"},"dn":"InternLM-Chat 20B","m":"internlm-chat-20b","oid":"mmlu-e4dca17dad2bff8e","rd":"2023-09-17","sid":"recdCL1os0x7URkOn","sm":"internlm-chat-20b","src":"http://arxiv.org/abs/2403.04652","v":0.556},{"b":"mmlu","cfg":{"shots":"5"},"dn":"InternLM-Chat 20B","m":"internlm-chat-20b","oid":"mmlu-6082c978e34738c8","rd":"2023-09-17","sid":"recQkCpGZV2zmuda4","sm":"internlm-chat-20b","src":"http://arxiv.org/abs/2403.04652","v":0.574},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"mmlu-89de11063d292c89","rd":"2023-02-24","sid":"reciRq6HujhO9z9nl","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.351},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"mmlu-2c7f822560a8bbff","rd":"2023-02-24","sid":"recN0xjCpJlAJpL3g","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.469},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"mmlu-c9c078fd36d0244d","rd":"2023-02-24","sid":"rec3LN7X83bnNLYmy","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.578},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 1 33B","m":"llama-33b","oid":"mmlu-7674963563c37692","rd":"2023-02-24","sid":"rec8UiAKlK7vPondE","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2401.04088","v":0.568},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"mmlu-3424316beb43fd9d","rd":"2023-02-24","sid":"rec1vmtLgykjCiHqG","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.634},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 2 (7B)","m":"llama-2-7b","oid":"mmlu-c4f370d92b4104d0","rd":"2023-07-18","sid":"recDRfiRYG3HCQq5K","sm":"Llama-2-7b","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.458},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 2 (13B)","m":"llama-2-13b","oid":"mmlu-ecc67963dc1dd4d7","rd":"2023-07-18","sid":"recHeILlp9WyfKuQk","sm":"Llama-2-13b","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.554},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 2 (70B)","m":"llama-2-70b-hf","oid":"mmlu-52e1f198d9b93180","rd":"2023-07-18","sid":"recOpGv66pVIzJc2y","sm":"Llama-2-70b-hf","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.695},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"mmlu-0847e09a0d56551e","rd":"2023-07-18","sid":"rec8cJlYZZHD6DliH","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.453},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"mmlu-9100a548fc080348","rd":"2023-07-18","sid":"recQtVvlLn7Mwe42i","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2401.04088","v":0.444},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"mmlu-4299f0b8d93bb2f5","rd":"2023-07-18","sid":"recAtLiMOQMukjkr4","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.548},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"mmlu-bc684b03f0a0180d","rd":"2023-07-18","sid":"recQabHfvgaByHw50","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2401.04088","v":0.556},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"mmlu-e39ccd0906e179d9","rd":"2023-07-18","sid":"recXGb1eMBkdHVVPt","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.626},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2 34B","m":"llama-2-34b","oid":"mmlu-0b24ae5656f97afc","rd":"2023-07-18","sid":"rec4EA8N1sXX54njV","sm":"Llama-2-34b","src":"https://arxiv.org/pdf/2309.16609","v":0.626},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"mmlu-a3ef6aa499e32d9d","rd":"2023-07-18","sid":"recCIOv8PV47XWiXE","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.689},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"mmlu-9a42a2974061ecd8","rd":"2023-07-18","sid":"recEQPQjoxax64ZSN","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2309.16609","v":0.698},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"mmlu-7d3f49e90a483d26","rd":"2023-07-18","sid":"recvvZSkhoBXMrVwo","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2401.04088","v":0.699},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2-7B","m":"llama-2-7b","oid":"mmlu-115b57ab936c1faa","rd":"2023-07-18","sid":"rech0sV4TGdAsCGBx","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.10305","v":0.4573},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 2-13B","m":"llama-2-13b","oid":"mmlu-89415f9d3799a18b","rd":"2023-07-18","sid":"rec4FrJ61uDhzqdol","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2309.10305","v":0.5509},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3 (8B)","m":"meta-llama-3-8b-instruct","oid":"mmlu-93b7b7a22085832e","rd":"2024-04-18","sid":"recShUHfQj3yvMmET","sm":"Meta-Llama-3-8B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.688},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3 (70B)","m":"meta-llama-3-70b-instruct","oid":"mmlu-c1e0fbf4a0146fd9","rd":"2024-04-18","sid":"recWvSXL72D2hpbVl","sm":"Meta-Llama-3-70B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.793},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.1 Instruct Turbo (8B)","m":"llama-3-1-8b-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-8fd2b7610bb766fb","rd":"2024-07-23","sid":"recaaHZaz730PgETl","sm":"Llama-3.1-8B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.561},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.1 Instruct Turbo (70B)","m":"llama-3-1-70b-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-5d168befacbabddc","rd":"2024-07-23","sid":"recBq5JDLuhxief9r","sm":"Llama-3.1-70B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.801},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.1 Instruct Turbo (405B)","m":"llama-3-1-405b-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-3f6ea811cfe70013","rd":"2024-07-23","sid":"recqvxNx4ASzVf1pr","sm":"Llama-3.1-405B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.845},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.2 Vision Instruct Turbo (11B)","m":"llama-3-2-11b-vision-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-b94ee39f585151ce","rd":"2024-09-24","sid":"rec91E3tiY2SItmiw","sm":"Llama-3.2-11B-Vision-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.565},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.2 Vision Instruct Turbo (90B)","m":"llama-3-2-90b-vision-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-fa3dd1e12125f1e1","rd":"2024-09-24","sid":"recib3s0R87GkOIJ6","sm":"Llama-3.2-90B-Vision-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.803},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama 3.3 Instruct Turbo (70B)","m":"llama-3-3-70b-instruct","notes":"Seems like the Turbo versions are quantized: https://huggingface.co/spaces/AtlaAI/judge-arena/discussions/4 (that's a different","oid":"mmlu-17a1050cc23524c0","rd":"2024-12-06","sid":"recYtSuvfsDMgxpQC","sm":"Llama-3.3-70B-Instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.791},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 7B","m":"llama-7b","oid":"mmlu-f195d8206ba2e240","rd":"2023-02-24","sid":"rec43wZbfZGjTvUxq","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2309.16609","v":0.356},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 7B","m":"llama-7b","oid":"mmlu-e84bf29b60d7260f","rd":"2023-02-24","sid":"recpktJbxIumcFneG","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.351},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 13B","m":"llama-13b","oid":"mmlu-fd9cc90155675d47","rd":"2023-02-24","sid":"recvzXv36THWKaBgb","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2309.16609","v":0.477},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 13B","m":"llama-13b","oid":"mmlu-006725f4ba2ec323","rd":"2023-02-24","sid":"recIiOs7bTAVwyNpa","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.469},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 33B","m":"llama-33b","oid":"mmlu-0b0e35e92fd42fd0","rd":"2023-02-24","sid":"receAb9h5D57B0gYh","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2309.16609","v":0.587},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 33B","m":"llama-33b","oid":"mmlu-4b103479bf92fbea","rd":"2023-02-24","sid":"recvQI3hl09uCHYs8","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.578},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA 65B","m":"llama-65b","oid":"mmlu-031b9ec53ab1c67a","rd":"2023-02-24","sid":"rec9IhkT5TRasgNag","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.634},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-2 7B","m":"llama-2-7b","notes":"5-shot top-1","oid":"mmlu-dac7fe950dde5ea6","rd":"2023-07-18","sid":"receYgtoMItZ5byvT","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.453},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-2 13B","m":"llama-2-13b","notes":"5-shot top-1","oid":"mmlu-e4f1afe980aac338","rd":"2023-07-18","sid":"recm3ckG9rQdlSf3G","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2403.08295","v":0.548},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"mmlu-f370b2b17cae30fd","rd":"2023-07-18","sid":"recQpivyiHncdya7p","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.548},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"mmlu-8f6d99791a90003b","rd":"2023-07-18","sid":"reccZ0kQv849Nx6S0","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.626},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"mmlu-091652d1e5c9df45","rd":"2024-04-18","sid":"recKeTJ0WxkfqaXc6","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.665},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"mmlu-01b960e316ecc15b","rd":"2024-07-23","sid":"rec7Kb0dlps0bAwVq","sm":"Llama-3.1-405B","src":"https://arxiv.org/pdf/2412.19437","v":0.844},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-7B","m":"llama-7b","oid":"mmlu-ad8aa72b5d020186","rd":"2023-02-24","sid":"recOK4kloLRVBYWyC","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.10305","v":0.351},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-7B","m":"llama-7b","oid":"mmlu-06137e3f05b19899","rd":"2023-02-24","sid":"recyV0OffUnewduYa","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.351},{"b":"mmlu","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"mmlu-b07012720f0f5077","rd":"2023-02-24","sid":"reck2fsCW8rwEIBV3","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.32},{"b":"mmlu","cfg":{"shots":"2"},"dn":"Llama-7B","m":"llama-7b","oid":"mmlu-488f3b10bfcd2d84","rd":"2023-02-24","sid":"recXt1b7mmd7pBCEN","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.352},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA-13B","m":"llama-13b","oid":"mmlu-bc54fd0fbf2a78a9","rd":"2023-02-24","sid":"recC5kLFkYdywVLTE","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2309.10305","v":0.463},{"b":"mmlu","cfg":{"shots":"2"},"dn":"Llama2-7B","m":"llama-2-7b","oid":"mmlu-ecfe40ae3c1d7f56","rd":"2023-07-18","sid":"rechfMlLcdL4d6JYV","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.453},{"b":"mmlu","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"mmlu-10e42f36c16a7b80","rd":"2023-07-18","sid":"reccr6kNapl4XhHMh","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.509},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA2-Chat 13B","m":"llama-2-13b-chat","oid":"mmlu-db6750919d56739d","rd":"2023-07-18","sid":"recP7LM6vdowGXldC","sm":"Llama-2-13b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.473},{"b":"mmlu","cfg":{"shots":"0"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"mmlu-364a72028e303885","rd":"2023-07-18","sid":"rechEaZxx7y6EW89C","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.594},{"b":"mmlu","cfg":{"shots":"5"},"dn":"LLaMA2-Chat 70B","m":"llama-2-70b-chat","oid":"mmlu-0e660e4b556fbc6f","rd":"2023-07-18","sid":"recq7vZurZvyzSaF8","sm":"Llama-2-70b-chat","src":"http://arxiv.org/abs/2403.04652","v":0.599},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Llama3.3 70b instruct","m":"llama-3-3-70b-instruct","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-5f5944c076f20977","rd":"2024-12-06","sid":"recFwE99pUPS1RrqR","sm":"Llama-3.3-70B-Instruct","src":"http://arxiv.org/abs/2412.08905","v":0.863},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral 7B","m":"mistral-7b-instruct-v0-2","notes":"5-shot top-1","oid":"mmlu-d7310f45390ec635","rd":"2023-12-11","sid":"rec5aVLAAe2uwnAJ3","sm":"Mistral-7B-Instruct-v0.2","src":"https://arxiv.org/pdf/2403.08295","v":0.625},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"mmlu-64fb20288f264d0e","rd":"2023-09-27","sid":"reclqttZHU3FrMenP","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.625},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"mmlu-b3ca8d413fb7245b","rd":"2023-09-27","sid":"recobovxkli2yXD7Q","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.601},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"mmlu-8fe48dbd6241b433","rd":"2023-09-27","sid":"recrKIaLtBWuoRKy0","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.617},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral Instruct v0.3 (7B)","m":"mistral-7b-instruct-v0-3","oid":"mmlu-dc15ba8370d61412","rd":"2024-05-27","sid":"recxGM6lCcs0CnKWO","sm":"Mistral-7B-Instruct-v0.3","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.599},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral Large (2402)","m":"mistral-large-2402","oid":"mmlu-d0d4004b7455e87a","rd":"2024-02-26","sid":"recMfHwyb3g0GU3WP","sm":"mistral-large-2402","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.688},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral Large 2 (2407)","m":"mistral-large-2407","oid":"mmlu-57b0e43d3fe75560","rd":"2024-07-24","sid":"recKDxkbfoASopZz7","sm":"mistral-large-2407","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.8},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral Small (2402)","m":"mistral-small-2402","oid":"mmlu-fea0fba91705ed20","rd":"2024-02-26","sid":"recKStU0sPWNiLWCM","sm":"mistral-small-2402","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.687},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mistral v0.1 (7B)","m":"mistral-7b-v0-1","oid":"mmlu-db18d451c20f8432","rd":"2023-09-27","sid":"recdbbEpQDnuQx1SM","sm":"Mistral-7B-v0.1","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.566},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mixtral (8x22B)","m":"mixtral-8x22b-v0-1","oid":"mmlu-8e12847bef3ebc09","rd":"2024-04-17","sid":"reczWpbUvk8LZHdwj","sm":"Mixtral-8x22B-v0.1","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.778},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"mmlu-fb6676e053ac14ce","rd":"2023-12-11","sid":"rec2QbPaiM3fs3ojk","sm":"Mixtral-8x7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.706},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"mmlu-35d1639913392b3f","rd":"2023-12-11","sid":"recVbhuiQ3mCmSZWs","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.705},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT 7B","m":"mpt-7b","oid":"mmlu-bc2f7b3ff80d3d4f","rd":"2023-05-05","sid":"recXxnvP42eSSUf1r","sm":"mpt-7b","src":"https://arxiv.org/pdf/2309.16609","v":0.308},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT 7B","m":"mpt-7b","oid":"mmlu-6830602fbc9c7968","rd":"2023-05-05","sid":"rec6fhu1fu0tDpFCM","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.268},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT 30B","m":"mpt-30b","oid":"mmlu-d0e1661c4bbe9a77","rd":"2023-06-22","sid":"recsvME0N7N3lHCNM","sm":"mpt-30b","src":"https://arxiv.org/pdf/2309.16609","v":0.479},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT 30B","m":"mpt-30b","oid":"mmlu-2f2b495b30b30b94","rd":"2023-06-22","sid":"recORhwxKRN3CKT5H","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.469},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT-7B","m":"mpt-7b","oid":"mmlu-f1d613122fb86bf7","rd":"2023-05-05","sid":"recciuLJt00P8UdBH","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.10305","v":0.2793},{"b":"mmlu","cfg":{"shots":"2"},"dn":"MPT-7B","m":"mpt-7b","oid":"mmlu-c5a7cf547b0c9080","rd":"2023-05-05","sid":"recThSGOCYopyGBAn","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.268},{"b":"mmlu","cfg":{"shots":"5"},"dn":"MPT-7B","m":"mpt-7b","oid":"mmlu-a910e4d862afdba0","rd":"2023-05-05","sid":"recnP0IxuIUI5wyAV","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.267},{"b":"mmlu","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"mmlu-8911e8389dc6d680","rd":"2023-05-05","sid":"recjktWCt7fCSUqIu","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.274},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"mmlu-e122b332cb89da1a","rd":"2024-02-26","sid":"recFoOkw8h4blebt7","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.587},{"b":"mmlu","cfg":{"shots":"5"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"mmlu-7c486a7f444b8841","rd":"2023-06-07","sid":"recS3RPFpiAohJDyr","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.299},{"b":"mmlu","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"mmlu-ee42cd1c6baf52b6","rd":"2023-06-07","sid":"recKitarPK5ZkGKHY","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.286},{"b":"mmlu","cfg":{"shots":"5"},"dn":"OPT-13B","m":"opt-13b","oid":"mmlu-431c8a800aaaea88","rd":"2022-05-11","sid":"recnCITHR0D69TaSW","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.251},{"b":"mmlu","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"mmlu-d632a816f4498ba6","rd":"2022-05-11","sid":"recptjItsyCc9FRVH","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.244},{"b":"mmlu","cfg":{"shots":"5"},"dn":"PaLM 62B","m":"palm-62b","oid":"mmlu-b41725427314edbc","rd":"2022-04-04","sid":"rec9nwU5JvWa3KMNP","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.537},{"b":"mmlu","cfg":{"shots":"5"},"dn":"PaLM 540B","m":"palm-540b","oid":"mmlu-4b8b16fb48519092","rd":"2022-04-04","sid":"recq6tXUJ4hdJZHiK","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.693},{"b":"mmlu","cfg":{"shots":"2"},"dn":"phi-1.5","m":"phi-1-5","notes":"We use the harness-eval zero-shot accuracy on PIQA, Hellaswag, OpenbookQA, 2-shot performance on MMLU,","oid":"mmlu-f575e14bc2ad213e","rd":"2023-09-11","sid":"recIRX5Xrwuj0aVsw","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.376},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-2","m":"phi-2","oid":"mmlu-ee85976992d69f9b","rd":"2023-12-12","sid":"recOjVJ9mVKxRJb67","sm":"phi-2","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.584},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-2","m":"phi-2","oid":"mmlu-96df8eebe768d74e","rd":"2023-12-12","sid":"recZMvyAbMp0eIx88","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.563},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-3 (14B)","m":"phi-3-medium-128k-instruct","oid":"mmlu-13e8e89148ae2d58","rd":"2024-04-23","sid":"recIC2lbeIoOMCnU9","sm":"Phi-3-medium-128k-instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.775},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-3 14b","m":"phi-3-medium-128k-instruct","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-dec103b58aa0aebe","rd":"2024-04-23","sid":"recokPFC1wM7clrm7","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2412.08905","v":0.779},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-3-medium","m":"phi-3-medium-128k-instruct","oid":"mmlu-d8e003e491664b87","rd":"2024-04-23","sid":"recCmqAcPDN7z3eOd","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.78},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-3-mini","m":"phi-3-mini-4k-instruct","oid":"mmlu-7c2513dbe17343f2","rd":"2024-04-23","sid":"recBm3x8Mkas41pg2","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.688},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-3-small","m":"phi-3-small-8k-instruct","oid":"mmlu-1a505f9e6b96d015","rd":"2024-04-23","sid":"reclGoqueJP6ji9WZ","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.757},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Phi-4 14b","m":"phi-4","notes":"\"Specifically, we used log-likelihood evaluations for MMLU (5-shot)\"","oid":"mmlu-b9cf9fa897382680","rd":"2024-12-12","sid":"recPr3djloYa7CtTW","sm":"phi-4","src":"http://arxiv.org/abs/2412.08905","v":0.848},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen 1.8B","m":"qwen-1-8b","oid":"mmlu-34c35200edf2c2df","rd":"2023-11-30","sid":"recJEnP48cP1MNsrP","sm":"Qwen-1_8B","src":"https://arxiv.org/pdf/2309.16609","v":0.282},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen 7B","m":"qwen-7b","oid":"mmlu-ddc5a477aaace74f","rd":"2023-09-28","sid":"recZMyuEA1HyQDC9A","sm":"Qwen-7B","src":"https://arxiv.org/pdf/2309.16609","v":0.45},{"b":"mmlu","cfg":{"shots":"5"},"dn":"QWEN 14B","m":"qwen-14b","oid":"mmlu-4b681e926f7b6a7a","rd":"2023-09-28","sid":"recbr9lo3CeOOBkvY","sm":"Qwen-14B","src":"http://arxiv.org/abs/2402.16819","v":0.663},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen 14B","m":"qwen-14b","oid":"mmlu-0cfa2c5ec0337e1c","rd":"2023-09-28","sid":"reci4VBEctqPF5t2B","sm":"Qwen-14B","src":"https://arxiv.org/pdf/2309.16609","v":0.534},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"mmlu-b5922245f28990dc","rd":"2023-09-24","sid":"rechXjvOrdKm234Fr","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.64},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen-Chat 14B","m":"qwen-14b-chat","oid":"mmlu-318f4c59a45af9ac","rd":"2023-09-24","sid":"recfT37OwcoOgwoYl","sm":"Qwen-14B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.65},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen1.5 (7B)","m":"qwen1-5-7b","oid":"mmlu-904b50878488f46c","rd":"2024-02-04","sid":"recFmlmIXPZeKd0uT","sm":"Qwen1.5-7B","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.626},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen1.5 (14B)","m":"qwen1-5-14b","oid":"mmlu-857d3db8629e0ed5","rd":"2024-02-04","sid":"recgNYd9sZSHzqlh3","sm":"qwen1.5-14B","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.686},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen1.5 (32B)","m":"qwen1-5-32b","oid":"mmlu-98bd277f1a01a686","rd":"2024-02-04","sid":"recpbIA4MZ2d0bMHo","sm":"qwen1.5-32B","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.744},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2 Instruct (72B)","m":"qwen2-72b-instruct","oid":"mmlu-dac00291ba9e400f","rd":"2024-06-07","sid":"rec4k5yqvGWnSlnLK","sm":"qwen2-72b-instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.824},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2.5 14b instruct","m":"qwen2-5-14b-instruct","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-d564f95d6edd22dc","rd":"2024-09-19","sid":"recaLqeKOZiJFuyWC","sm":"qwen2.5-14b-instruct","src":"http://arxiv.org/abs/2412.08905","v":0.799},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"mmlu-70d77a8494c63d57","rd":"2024-09-19","sid":"rec1e9rTGH12ziqp2","sm":"Qwen2.5-72B","src":"https://arxiv.org/pdf/2412.19437","v":0.85},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2.5 72b instruct","m":"qwen2-5-72b-instruct","notes":"Specifically, we used log-likelihood evaluations for MMLU (5-shot)","oid":"mmlu-059a8821c7cb8813","rd":"2024-09-19","sid":"recbl49vkZltt7Qu7","sm":"qwen2.5-72b-instruct","src":"http://arxiv.org/abs/2412.08905","v":0.853},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2.5 Instruct Turbo (7B)","m":"qwen2-5-7b-instruct","oid":"mmlu-c0905ffb432b13ce","rd":"2024-09-19","sid":"rec4M7QClQ8t5k7Fi","sm":"qwen2.5-7b-instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.729},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Qwen2.5 Instruct Turbo (72B)","m":"qwen2-5-72b-instruct","oid":"mmlu-6ebe794fd20e6dc9","rd":"2024-09-19","sid":"recUfCycerC91cgsO","sm":"qwen2.5-72b-instruct","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.834},{"b":"mmlu","dn":"Qwen2.5-Coder-0.5B","m":"qwen2-5-coder-0-5b","oid":"mmlu-5f7865db96788191","rd":"2024-09-18","sid":"recA0z9ynBjuFVr0i","sm":"Qwen2.5-Coder-0.5B","src":"http://arxiv.org/abs/2409.12186","v":0.42},{"b":"mmlu","dn":"Qwen2.5-Coder-1.5B","m":"qwen2-5-coder-1-5b","oid":"mmlu-cd326529f51bdc87","rd":"2024-09-18","sid":"recsb2ABruR7vDJyc","sm":"Qwen2.5-Coder-1.5B","src":"http://arxiv.org/abs/2409.12186","v":0.536},{"b":"mmlu","dn":"Qwen2.5-Coder-3B","m":"qwen2-5-coder-32b","oid":"mmlu-0d1f8140ab0e753d","rd":"2024-09-18","sid":"recFedHhPHlgqWxpq","sm":"Qwen2.5-Coder-32B","src":"http://arxiv.org/abs/2409.12186","v":0.612},{"b":"mmlu","dn":"Qwen2.5-Coder-7B","m":"qwen2-5-coder-7b","oid":"mmlu-3faaa49b90fc18d9","rd":"2024-09-18","sid":"recot1Px8adoUoPs3","sm":"Qwen2.5-Coder-7B","src":"http://arxiv.org/abs/2409.12186","v":0.68},{"b":"mmlu","dn":"Qwen2.5-Coder-14B","m":"qwen2-5-coder-14b","oid":"mmlu-1eb49866fb608c0b","rd":"2024-09-18","sid":"recXLZnQpE0oht07R","sm":"Qwen2.5-Coder-14B","src":"http://arxiv.org/abs/2409.12186","v":0.752},{"b":"mmlu","dn":"Qwen2.5-Coder-32B","m":"qwen2-5-coder-32b","oid":"mmlu-513abcfa082f36c0","rd":"2024-09-18","sid":"recgIGiZ5lVnVM7rT","sm":"Qwen2.5-Coder-32B","src":"http://arxiv.org/abs/2409.12186","v":0.791},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"mmlu-79b6fee41379c87f","rd":"2023-05-04","sid":"reclqOVSaOoy7NX2U","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.263},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"mmlu-21d779e27809337f","rd":"2023-05-04","sid":"recQGR1shnVCw79MB","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.258},{"b":"mmlu","cfg":{"shots":"5"},"dn":"StableBeluga2 70B","m":"stablebeluga2","oid":"mmlu-bbccce56c11cd5bd","rd":"2023-07-20","sid":"reca4GRikJuwzhgQj","sm":"StableBeluga2","src":"https://arxiv.org/pdf/2309.16609","v":0.686},{"b":"mmlu","dn":"StarCoder2-3B","m":"starcoder2-3b","oid":"mmlu-ee31b4ec145dafa2","rd":"2024-02-22","sid":"rec7SmDXXVMzS12UD","sm":"starcoder2-3b","src":"http://arxiv.org/abs/2409.12186","v":0.366},{"b":"mmlu","dn":"StarCoder2-7B","m":"starcoder2-7b","oid":"mmlu-14df99545f6fabb2","rd":"2024-02-20","sid":"recbF6EudRQDiZM0w","sm":"starcoder2-7b","src":"http://arxiv.org/abs/2409.12186","v":0.388},{"b":"mmlu","dn":"StarCoder2-15B","m":"starcoder2-15b","oid":"mmlu-11320fd7d08c0c0a","rd":"2024-02-20","sid":"recslUwoLlToAjB27","sm":"starcoder2-15b","src":"http://arxiv.org/abs/2409.12186","v":0.641},{"b":"mmlu","cfg":{"shots":"5"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"mmlu-83f03db846400866","rd":"2023-06-27","sid":"recmtaX4LELTcHv2G","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.363},{"b":"mmlu","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"mmlu-74f45c39b23afb00","rd":"2023-06-27","sid":"recNXliE4Qvt56aRx","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.321},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi (6B)","m":"yi-6b","oid":"mmlu-d2d6667d0ce4e869","rd":"2023-11-02","sid":"recQDuDGOq1wY5Ouj","sm":"Yi-6B","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.64},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi (34B)","m":"yi-34b","oid":"mmlu-b8e006d24dc76ee0","rd":"2023-11-02","sid":"recjx70KTlRekGQ02","sm":"Yi-34B","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.762},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi 6B","m":"yi-6b","notes":"We report the overall results for MMLU[27](5-shot), CMMLU[42] (5-shot), Gaokao-Bench[90] (5-shot), and BigBench[72] Hard (BBH[74]) (3-shot).","oid":"mmlu-720790cea33ff77c","rd":"2023-11-02","sid":"recGm5EDNDIETSYi6","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.632},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi 34B","m":"yi-34b","notes":"We report the overall results for MMLU[27](5-shot), CMMLU[42] (5-shot), Gaokao-Bench[90] (5-shot), and BigBench[72] Hard (BBH[74]) (3-shot).","oid":"mmlu-413b1278873a54d6","rd":"2023-11-02","sid":"recwCEStuvtsY9T89","sm":"Yi-34B","src":"http://arxiv.org/abs/2403.04652","v":0.763},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi Large (Preview)","m":"yi-large","oid":"mmlu-567f41617dd11eb0","rd":"2024-05-13","sid":"recbChoe3AEIXUMHg","sm":"Yi-large","src":"https://crfm.stanford.edu/helm/lite/latest/#/leaderboard/mmlu","v":0.793},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi-9B","m":"yi-9b","oid":"mmlu-a1a9e7d292f34e2f","rd":"2024-03-01","sid":"recrn20mUuMopGxXJ","sm":"Yi-9B","src":"http://arxiv.org/abs/2403.04652","v":0.684},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"mmlu-f9db19d8cebfee76","rd":"2023-11-22","sid":"recS1feH5g4KJrpW8","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.582},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi-Chat 6B","m":"yi-6b-chat","oid":"mmlu-458ba424cf1edf7a","rd":"2023-11-22","sid":"receVD87dnzgVzRts","sm":"Yi-6B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.61},{"b":"mmlu","cfg":{"shots":"0"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"mmlu-3b8a23d76eb03dfb","rd":"2023-11-22","sid":"recswVGACIIFbdPtu","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.676},{"b":"mmlu","cfg":{"shots":"5"},"dn":"Yi-Chat 34B","m":"yi-34b-chat","oid":"mmlu-627280c3bfcf3ff0","rd":"2023-11-22","sid":"reckSm2dCL3rMGbBd","sm":"Yi-34B-Chat","src":"http://arxiv.org/abs/2403.04652","v":0.735},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-11-06","e":"max","evaluated_at":"2026-11-06T00:59:23.000Z","m":"kimi-k3-1-pro-together","oid":"mystery_game_puzzles-cce86484d78a70a4","rd":"2026-10-23","se":0.04960449637488583,"sid":"ba7BQr2N7kTuzakT8fZ3WU","sm":"kimi-k3.1-pro-together_max","v":0.42},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T13:47:17.000Z","m":"mistral-large-4-0-oss","oid":"mystery_game_puzzles-a0aca742a667de3f","rd":"2026-10-29","se":0.04020151261036849,"sid":"bsKLyeEUqxgEFd9ZV6PfSx","sm":"mistral-large-4-0-oss_high","v":0.2},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-10-26","e":"max","evaluated_at":"2026-10-26T20:00:37.000Z","m":"gpt-6-1-terra","oid":"mystery_game_puzzles-c9807c3926d05806","rd":"2026-10-26","se":0.04408440022768081,"sid":"Q4Sr4fBBypw9WQDkuk9RMN","sm":"gpt-6.1-terra_max","v":0.74},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-10-19","e":"max","evaluated_at":"2026-10-19T22:39:00.000Z","m":"deepseek-v4-1-flash-1017","oid":"mystery_game_puzzles-a2b2eedbad33f648","rd":"2026-10-17","se":0.05024183937956914,"sid":"GLNTwoU6njn58DbPF9cbxb","sm":"deepseek-v4.1-flash-1017_max","v":0.49},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-10-14","e":"max","evaluated_at":"2026-10-14T16:17:18.000Z","m":"muse-wonder-1-0","oid":"mystery_game_puzzles-e641ccabb1ae8be3","rd":"2026-10-12","se":0.05024183937956913,"sid":"gmscEuzbtZeUeFUFZMZiAV","sm":"muse-wonder-1.0_max","v":0.51},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T22:01:32.000Z","m":"mistral-large-4-0","oid":"mystery_game_puzzles-0a868e3bff8071d1","rd":"2026-10-06","se":0.03775251680686369,"sid":"LT9xLJnaU4UHsBQrvu3rbU","sm":"mistral-large-4-0_high","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:14:57.000Z","m":"gemini-4-argon","oid":"mystery_game_puzzles-8a03f7894ec55615","rd":"2026-09-30","se":0.048241815132442176,"sid":"9ipFgNR5MDw7ustUsuSviu","sm":"gemini-4-argon_high","v":0.64},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-10-08","e":"max","evaluated_at":"2026-10-08T19:02:32.000Z","m":"deepseek-v4-1-flash","oid":"mystery_game_puzzles-0b8513a0f8ebff71","rd":"2026-09-09","se":0.0497569851956243,"sid":"K3JABAsJCbXU5YrCPH2yVW","sm":"deepseek-v4.1-flash_max","v":0.43},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-10-07","e":"max","evaluated_at":"2026-10-07T20:51:20.000Z","m":"claude-haiku-5-5","oid":"mystery_game_puzzles-1e1a29f611326669","rd":"2026-10-07","se":0.04605661864718382,"sid":"CKm7NWXfSePfcUycRK87cB","sm":"claude-haiku-5-5_max","v":0.3},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T22:00:34.000Z","m":"gpt-6-1-sol","oid":"mystery_game_puzzles-cce821916dd1bfeb","rd":"2026-09-29","se":0.04020151261036849,"sid":"fQzkNRV69sKaJX7o9KKneQ","sm":"gpt-6.1-sol_max","v":0.8},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T17:34:54.000Z","m":"claude-opus-5-5","oid":"mystery_game_puzzles-52c315b005263cdd","rd":"2026-09-22","se":0.04560480215720687,"sid":"fREi6AAmnfqUWkDqUhLLtw","sm":"claude-opus-5-5_max","v":0.71},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T15:15:14.000Z","m":"claude-sonnet-5-5","oid":"mystery_game_puzzles-255e0ad5637fe913","rd":"2026-09-28","se":0.04793724854411022,"sid":"3PVdaSEYFJKgYUgHnF4DBu","sm":"claude-sonnet-5-5_max","v":0.65},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T22:14:05.000Z","m":"gpt-6-luna","oid":"mystery_game_puzzles-e15ef724cef6ffe4","rd":"2026-09-22","se":0.02564323999762428,"sid":"bAERV8wdmoHMLz2K4bZCSK","sm":"gpt-6-luna_max","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-09-22","e":"xhigh","evaluated_at":"2026-09-22T21:49:22.000Z","m":"grok-4-7","oid":"mystery_game_puzzles-9c9de3846bc84527","rd":"2026-09-21","se":0.045604802157206865,"sid":"Hk8bjr4u6KGqVE8SrPFKLq","sm":"grok-4.7_xhigh","v":0.29},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T19:59:24.000Z","m":"gpt-6-sol","oid":"mystery_game_puzzles-5d0beb19dffa1b69","rd":"2026-09-22","se":0.049888765156985884,"sid":"DnFTaE9qNsmBdYNqohhynY","sm":"gpt-6-sol_max","v":0.56},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-18","e":"max","evaluated_at":"2026-09-18T19:12:25.000Z","m":"muse-spark-1-3","oid":"mystery_game_puzzles-cb379f044da398d8","rd":"2026-09-02","se":0.044640484731156636,"sid":"6Rcc2q3USiUNB6U44bpvGH","sm":"muse-spark-1.3_max","v":0.25},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-09-16","e":"xhigh","evaluated_at":"2026-09-16T23:26:24.000Z","m":"muse-spark-1-3","oid":"mystery_game_puzzles-1b0dd39e223912c8","rd":"2026-09-02","se":0.03519858850692775,"sid":"n6X597deBgzpbDSyxvhUAB","sm":"muse-spark-1.3_xhigh","v":0.1414141414141414},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-09-02","e":"high","evaluated_at":"2026-09-02T17:15:36.000Z","m":"gemini-3-8-flash","oid":"mystery_game_puzzles-b39cfbf45ecfe925","rd":"2026-09-02","se":0.05016135580465918,"sid":"5yyuCCi2Dpfh6xLBMxqFyB","sm":"gemini-3.8-flash_high","v":0.47},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-09-01","e":"max","evaluated_at":"2026-09-01T18:45:06.000Z","m":"claude-fable-5-1","oid":"mystery_game_puzzles-6e063f91d1b63b11","rd":"2026-09-01","se":0.04960449637488583,"sid":"dBHxnnTzLGz3nHis5LET6t","sm":"claude-fable-5-1_max","v":0.58},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-08-30","e":"max","evaluated_at":"2026-08-30T23:33:33.000Z","m":"glm-5-3","oid":"mystery_game_puzzles-ca9b4d58e6513d0c","rd":"2026-08-14","se":0.04725815626252608,"sid":"UUq99Qhq3qizcoJo5vdhuK","sm":"glm-5.3_max","v":0.33},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:33:48.000Z","m":"gpt-5-4-2026-03-05","oid":"mystery_game_puzzles-aa0f62431d2bab17","rd":"2026-03-05","se":0.03684529491774706,"sid":"GtW2cWyAXZPCszLi63e4yJ","sm":"gpt-5.4-2026-03-05_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GtW2cWyAXZPCszLi63e4yJ.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:33:31.000Z","m":"gpt-5-6-luna","oid":"mystery_game_puzzles-707f0c5ce1745f7f","rd":"2026-07-09","se":0.04020151261036849,"sid":"PBrQpAgRCXeUSNveEn4aDE","sm":"gpt-5.6-luna_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PBrQpAgRCXeUSNveEn4aDE.eval","v":0.2},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-08-30","e":"max","evaluated_at":"2026-08-30T14:57:43.000Z","m":"gpt-6-astra","oid":"mystery_game_puzzles-d1e4183dbd956049","rd":"2026-09-03","se":0.036845294917747066,"sid":"FA7pM9hKdkgTXmGHptcno7","sm":"gpt-6-astra_max","v":0.84},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-29","e":"low","evaluated_at":"2026-08-29T11:05:21.000Z","m":"kimi-k3","oid":"mystery_game_puzzles-b4da5f501c8e6846","rd":"2026-07-16","se":0.032659863237109045,"sid":"CAoLSNkdR27AmHb84fnr8B","sm":"kimi-k3_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/CAoLSNkdR27AmHb84fnr8B.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-29","e":"none","evaluated_at":"2026-08-29T11:05:11.000Z","m":"kimi-k2-6","oid":"mystery_game_puzzles-0a53f67f30437f26","rd":"2026-04-20","se":0.03265986323710904,"sid":"8iiRQiAoCepDELQfUv3VDq","sm":"kimi-k2.6_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8iiRQiAoCepDELQfUv3VDq.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-08-28","e":"max","evaluated_at":"2026-08-28T14:38:27.000Z","m":"glm-5-3-flash","oid":"mystery_game_puzzles-4d38e10fa781d20d","rd":"2026-08-20","se":0.027265992434429072,"sid":"TMM3rPrAHcpBG5Pvw9kr4L","sm":"glm-5.3-flash_max","v":0.08},{"b":"mystery_game_puzzles","d":"2026-08-28","evaluated_at":"2026-08-28T07:12:24.000Z","m":"qwen3-7-flash","oid":"mystery_game_puzzles-77e93b7b3a1669a6","rd":"2026-07-27","se":0.03588702812826367,"sid":"5e4Jp9dxtNisWx2wbRRZxM","sm":"qwen3.7-flash","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/5e4Jp9dxtNisWx2wbRRZxM.eval","v":0.15},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-28","e":"medium","evaluated_at":"2026-08-28T07:12:23.000Z","m":"gpt-5-6-terra","oid":"mystery_game_puzzles-4b2c1628de9dc846","rd":"2026-07-09","se":0.03015113445777636,"sid":"WcCQQSHTFAhXk6qCWEeSLh","sm":"gpt-5.6-terra_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WcCQQSHTFAhXk6qCWEeSLh.eval","v":0.1},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-28","e":"medium","evaluated_at":"2026-08-28T07:12:01.000Z","m":"gpt-5-2-2025-12-11","oid":"mystery_game_puzzles-d1a768f05e9d1954","rd":"2025-12-11","se":0.041633319989322654,"sid":"KHS8JvzY7n9EFgpcjEkj5v","sm":"gpt-5.2-2025-12-11_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/KHS8JvzY7n9EFgpcjEkj5v.eval","v":0.22},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-28","e":"low","evaluated_at":"2026-08-28T07:11:53.000Z","m":"gpt-5-5","oid":"mystery_game_puzzles-e170e16b6d68a646","rd":"2026-04-23","se":0.045126085985421296,"sid":"QHfAMMDjjzQF6xitN2vBbA","sm":"gpt-5.5_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/QHfAMMDjjzQF6xitN2vBbA.eval","v":0.28},{"b":"mystery_game_puzzles","d":"2026-08-28","evaluated_at":"2026-08-28T07:11:35.000Z","m":"qwen3-7-plus","oid":"mystery_game_puzzles-d9a49726b077c7d6","rd":"2026-06-02","se":0.03775251680686369,"sid":"DmptTSPXx5uM5TYZoV59JT","sm":"qwen3.7-plus","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DmptTSPXx5uM5TYZoV59JT.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-28","e":"medium","evaluated_at":"2026-08-28T07:11:31.000Z","m":"gpt-5-4-2026-03-05","oid":"mystery_game_puzzles-fe68a486588a94f1","rd":"2026-03-05","se":0.04512608598542129,"sid":"ereDhnap4G3Q6ydsNQW97W","sm":"gpt-5.4-2026-03-05_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ereDhnap4G3Q6ydsNQW97W.eval","v":0.28},{"b":"mystery_game_puzzles","d":"2026-08-28","evaluated_at":"2026-08-28T07:08:14.000Z","m":"gpt-4-0613","oid":"mystery_game_puzzles-649d8e146d54c2a9","rd":"2023-06-13","se":0.032659863237109045,"sid":"CvujZuqfj5r7qsa52Smyx9","sm":"gpt-4-0613","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/CvujZuqfj5r7qsa52Smyx9.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:27:05.000Z","m":"o3-2025-04-16","oid":"mystery_game_puzzles-214c35a7bfd64512","rd":"2025-04-16","se":0.04560480215720687,"sid":"FmLdgAHVM7MPn3JebBvxTb","sm":"o3-2025-04-16_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/FmLdgAHVM7MPn3JebBvxTb.eval","v":0.29},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:26:53.000Z","m":"o4-mini-2025-04-16","oid":"mystery_game_puzzles-952c70e69f893bee","rd":"2025-04-16","se":0.02190429135575903,"sid":"eJbFTXaxZjnW7HNR93u3WH","sm":"o4-mini-2025-04-16_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eJbFTXaxZjnW7HNR93u3WH.eval","v":0.05},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T19:22:47.000Z","m":"o3-2025-04-16","oid":"mystery_game_puzzles-f1bff4ebd31e8df5","rd":"2025-04-16","se":0.04229525846816507,"sid":"cxniYPW74ux4L6syqvPkaD","sm":"o3-2025-04-16_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/cxniYPW74ux4L6syqvPkaD.eval","v":0.23},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T19:18:34.000Z","m":"qwen3-6-flash","oid":"mystery_game_puzzles-259b177267ac58bf","rd":"2026-04-27","se":0.03265986323710904,"sid":"2y5jXmHGjyEyZvGjNDtRZe","sm":"qwen3.6-flash","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2y5jXmHGjyEyZvGjNDtRZe.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:18:00.000Z","m":"gpt-5-4-2026-03-05","oid":"mystery_game_puzzles-0d1332c53cd55839","rd":"2026-03-05","se":0.03775251680686369,"sid":"kWUUr3aHFZfKNARqAFxYZa","sm":"gpt-5.4-2026-03-05_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kWUUr3aHFZfKNARqAFxYZa.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:17:45.000Z","m":"gpt-5-6-sol","oid":"mystery_game_puzzles-d2d58cf57e88863b","rd":"2026-07-09","se":0.04408440022768081,"sid":"RMeUVa4tq3fURqCMtQWkfe","sm":"gpt-5.6-sol_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RMeUVa4tq3fURqCMtQWkfe.eval","v":0.26},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T19:13:41.000Z","m":"qwen3-6-max-preview","oid":"mystery_game_puzzles-aa5928fbd5d66394","rd":"2026-04-20","se":0.039427724440366255,"sid":"mbySBWyYrKdtd7joxHyBdq","sm":"qwen3.6-max-preview_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/mbySBWyYrKdtd7joxHyBdq.eval","v":0.19},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T19:13:33.000Z","m":"minimax-m3","oid":"mystery_game_puzzles-f72fd2c847d6bc95","rd":"2026-06-01","se":0.02564323999762428,"sid":"GGpgwGRFVATVJdQr8TgRc9","sm":"MiniMax-M3","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GGpgwGRFVATVJdQr8TgRc9.eval","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:13:32.000Z","m":"gpt-5-6-terra","oid":"mystery_game_puzzles-7e597f78a3a8e767","rd":"2026-07-09","se":0.039427724440366255,"sid":"iE9NNTJWbGpRia2fUVGkPf","sm":"gpt-5.6-terra_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iE9NNTJWbGpRia2fUVGkPf.eval","v":0.19},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T19:08:52.000Z","m":"qwen3-7-max","oid":"mystery_game_puzzles-117477bc703feadb","rd":"2026-05-19","se":0.0440844002276808,"sid":"ENh7ZPQj7ihBePjsC8a7HV","sm":"qwen3.7-max_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ENh7ZPQj7ihBePjsC8a7HV.eval","v":0.26},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:08:51.000Z","m":"o3-mini-2025-01-31","oid":"mystery_game_puzzles-893e03fadeeb3be9","rd":"2025-01-31","se":0.02564323999762428,"sid":"JSmdEXGULNBJ7PfSoVoa7W","sm":"o3-mini-2025-01-31_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JSmdEXGULNBJ7PfSoVoa7W.eval","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T19:08:40.000Z","m":"gpt-5-1-2025-11-13","oid":"mystery_game_puzzles-951f8d092eadb979","rd":"2025-11-13","se":0.03684529491774706,"sid":"8dvpS2RJKYLwpVh6M7AJkA","sm":"gpt-5.1-2025-11-13_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8dvpS2RJKYLwpVh6M7AJkA.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T19:08:19.000Z","m":"gpt-5-2025-08-07","oid":"mystery_game_puzzles-bfaad2c13a6528cf","rd":"2025-08-07","se":0.03588702812826367,"sid":"3ZzqFaVH7RgvRyxCt6RseN","sm":"gpt-5-2025-08-07_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3ZzqFaVH7RgvRyxCt6RseN.eval","v":0.15},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:03:44.000Z","m":"gpt-5-2-2025-12-11","oid":"mystery_game_puzzles-111117b187d35224","rd":"2025-12-11","se":0.03015113445777636,"sid":"bLxTUZHmq3YkRejnh6baqZ","sm":"gpt-5.2-2025-12-11_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bLxTUZHmq3YkRejnh6baqZ.eval","v":0.1},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:03:18.000Z","m":"gpt-5-2025-08-07","oid":"mystery_game_puzzles-ea424a270e6af083","rd":"2025-08-07","se":0.03684529491774706,"sid":"262FWwUwrjPpxVN8G8sDZT","sm":"gpt-5-2025-08-07_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/262FWwUwrjPpxVN8G8sDZT.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T18:59:15.000Z","m":"gpt-5-mini-2025-08-07","oid":"mystery_game_puzzles-1fd37c1afb2efea8","rd":"2025-08-07","se":0.021904291355759033,"sid":"THodvrqzCUw7fjpxvKMMWT","sm":"gpt-5-mini-2025-08-07_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/THodvrqzCUw7fjpxvKMMWT.eval","v":0.05},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:48:42.000Z","m":"gpt-5-6-luna","oid":"mystery_game_puzzles-6a083a153706ba78","rd":"2026-07-09","se":0.032659863237109045,"sid":"BvShPsULJMkwxxf8vBSNfu","sm":"gpt-5.6-luna_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BvShPsULJMkwxxf8vBSNfu.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T18:48:17.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"mystery_game_puzzles-618d73b637d543ae","rd":"2026-03-17","se":0.021904291355759033,"sid":"2e2NePZeU3iB4wvhwCaTci","sm":"gpt-5.4-nano-2026-03-17_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2e2NePZeU3iB4wvhwCaTci.eval","v":0.05},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:48:08.000Z","m":"qwen3-6-27b","oid":"mystery_game_puzzles-4565637a00098237","rd":"2026-04-22","se":0.02564323999762428,"sid":"dKZDn279PKuc2Kxm68D4r2","sm":"qwen3.6-27b_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/dKZDn279PKuc2Kxm68D4r2.eval","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:47:58.000Z","m":"qwen3-5-397b-a17b","oid":"mystery_game_puzzles-f57e006cef30cb7b","rd":"2026-02-13","se":0.03861229196653691,"sid":"D7WDyCr24AVdzr4eKAreJu","sm":"qwen3.5-397b-a17b_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/D7WDyCr24AVdzr4eKAreJu.eval","v":0.18},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T18:36:07.000Z","m":"gpt-5-1-2025-11-13","oid":"mystery_game_puzzles-840d483d9bb94715","rd":"2025-11-13","se":0.039427724440366255,"sid":"iV2bSueom3C2SKw72ogiYr","sm":"gpt-5.1-2025-11-13_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iV2bSueom3C2SKw72ogiYr.eval","v":0.19},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:36:04.000Z","m":"glm-5-2","oid":"mystery_game_puzzles-759afb9328be531d","rd":"2026-06-16","se":0.03588702812826367,"sid":"NfN7VwMPjSVqViqrBQPWk2","sm":"glm-5.2_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NfN7VwMPjSVqViqrBQPWk2.eval","v":0.15},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:35:48.000Z","m":"qwen3-5-122b-a10b","oid":"mystery_game_puzzles-f0f38fc0d08867ea","rd":"2026-02-24","se":0.03775251680686369,"sid":"X5FL4gkJQYjzJ7axz23rfd","sm":"qwen3.5-122b-a10b_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/X5FL4gkJQYjzJ7axz23rfd.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:31:20.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"mystery_game_puzzles-aa1338da39896e47","rd":"2026-03-17","se":0.02564323999762428,"sid":"4CzMJvWvuiLMCZhe2JGmSb","sm":"gpt-5.4-mini-2026-03-17_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4CzMJvWvuiLMCZhe2JGmSb.eval","v":0.07},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T18:27:45.000Z","m":"qwen3-5-flash","oid":"mystery_game_puzzles-96eba5b10c80ad51","rd":"2026-02-25","se":0.04020151261036849,"sid":"b9csPZ2c9CAsRXmzzfVoEW","sm":"qwen3.5-flash","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/b9csPZ2c9CAsRXmzzfVoEW.eval","v":0.2},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:27:14.000Z","m":"qwen3-6-plus","oid":"mystery_game_puzzles-bbd065eefe7c503b","rd":"2026-03-31","se":0.03265986323710904,"sid":"jGgkDRHbodcaTzQ3Mp6SxS","sm":"qwen3.6-plus_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jGgkDRHbodcaTzQ3Mp6SxS.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:23:25.000Z","m":"qwen3-5-plus","oid":"mystery_game_puzzles-68a2874cc5a8e23d","rd":"2026-02-16","se":0.03775251680686369,"sid":"XbtWdvRz65dbPPnHcSBvmF","sm":"qwen3.5-plus_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/XbtWdvRz65dbPPnHcSBvmF.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:22:49.000Z","m":"qwen3-6-35b-a3b","oid":"mystery_game_puzzles-ff4c511abe91cfa5","rd":"2026-04-14","se":0.041633319989322654,"sid":"Ua5RK6JRxdyrHRa3hVFQwq","sm":"qwen3.6-35b-a3b_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Ua5RK6JRxdyrHRa3hVFQwq.eval","v":0.22},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T18:22:45.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"mystery_game_puzzles-f3cc49498c44d745","rd":"2026-03-17","se":0.027265992434429072,"sid":"9zTyRhqbSKn7bbruvwDP4L","sm":"gpt-5.4-mini-2026-03-17_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/9zTyRhqbSKn7bbruvwDP4L.eval","v":0.08},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T18:18:35.000Z","m":"gpt-5-6-luna","oid":"mystery_game_puzzles-88277dfda0a44e7b","rd":"2026-07-09","se":0.03775251680686369,"sid":"iTY94tKhVEcZsUZivMwLJa","sm":"gpt-5.6-luna_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/iTY94tKhVEcZsUZivMwLJa.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:18:28.000Z","m":"qwen3-7-plus","oid":"mystery_game_puzzles-33a24e814bfc8009","rd":"2026-06-02","se":0.03775251680686369,"sid":"a5qR5FNnftfR8pk9HJ3JUz","sm":"qwen3.7-plus_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/a5qR5FNnftfR8pk9HJ3JUz.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:18:13.000Z","m":"qwen3-7-flash","oid":"mystery_game_puzzles-895019bb91586c11","rd":"2026-07-27","se":0.0348735088019777,"sid":"i7MVHsH5pikQKjYn6LBqHS","sm":"qwen3.7-flash_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/i7MVHsH5pikQKjYn6LBqHS.eval","v":0.14},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:14:08.000Z","m":"qwen3-6-flash","oid":"mystery_game_puzzles-0d35f0a6f7c0768e","rd":"2026-04-27","se":0.03861229196653691,"sid":"3QtkFWzSLzd4hgYGsNd5cL","sm":"qwen3.6-flash_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3QtkFWzSLzd4hgYGsNd5cL.eval","v":0.18},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T18:13:46.000Z","m":"glm-5-2","oid":"mystery_game_puzzles-86746fef8a4ef5b0","rd":"2026-06-16","se":0.039427724440366255,"sid":"7DQNxGG5q6i7VtnzF7vNxA","sm":"glm-5.2_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/7DQNxGG5q6i7VtnzF7vNxA.eval","v":0.19},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:13:19.000Z","m":"gpt-5-mini-2025-08-07","oid":"mystery_game_puzzles-2f925c155d03140a","rd":"2025-08-07","se":0.019694638556693237,"sid":"8RmBWoBLhvciTpz2oPSbDs","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8RmBWoBLhvciTpz2oPSbDs.eval","v":0.04},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:09:34.000Z","m":"minimax-m3","oid":"mystery_game_puzzles-327e1301c9816d03","rd":"2026-06-01","se":0.027265992434429075,"sid":"jWHk7BTYHQRXk2rXCbnpcS","sm":"MiniMax-M3_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/jWHk7BTYHQRXk2rXCbnpcS.eval","v":0.08},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T18:09:14.000Z","m":"gpt-5-nano-2025-08-07","oid":"mystery_game_puzzles-416a283fb2670d33","rd":"2025-08-07","se":0.027265992434429075,"sid":"aW4ppDktWqoy5qVWMUws6z","sm":"gpt-5-nano-2025-08-07_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/aW4ppDktWqoy5qVWMUws6z.eval","v":0.08},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:08:44.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"mystery_game_puzzles-bf8d8258e5ab5d07","rd":"2026-03-17","se":0.0238683256575942,"sid":"hRMMiGMPvHSGh3ygSjgrs2","sm":"gpt-5.4-nano-2026-03-17_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/hRMMiGMPvHSGh3ygSjgrs2.eval","v":0.06},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:07:54.000Z","m":"gpt-5-6-sol","oid":"mystery_game_puzzles-a3c440002f171265","rd":"2026-07-09","se":0.04725815626252608,"sid":"4uxDyWmq5f5azVYfWBGUHh","sm":"gpt-5.6-sol_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4uxDyWmq5f5azVYfWBGUHh.eval","v":0.33},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:07:51.000Z","m":"gpt-5-5","oid":"mystery_game_puzzles-2bd6c06eee3461b7","rd":"2026-04-23","se":0.03861229196653691,"sid":"MHAMdn52f7wLMZ6MSCs5E2","sm":"gpt-5.5_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MHAMdn52f7wLMZ6MSCs5E2.eval","v":0.18},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T18:07:41.000Z","m":"qwen3-max-2025-09-23","oid":"mystery_game_puzzles-281688ba098fe5ef","rd":"2025-09-24","se":0.021904291355759033,"sid":"SfzmmHYZjuvWnvoYWit58g","sm":"qwen3-max-2025-09-23","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/SfzmmHYZjuvWnvoYWit58g.eval","v":0.05},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T18:02:21.000Z","m":"gpt-oss-120b","oid":"mystery_game_puzzles-73b649adf17389bf","rd":"2025-08-05","se":0,"sid":"GbQrTQyihBUsb9sxbqT8pT","sm":"gpt-oss-120b_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/GbQrTQyihBUsb9sxbqT8pT.eval","v":0},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:02:02.000Z","m":"gpt-5-2-2025-12-11","oid":"mystery_game_puzzles-16c9ec3374ef8c18","rd":"2025-12-11","se":0.0348735088019777,"sid":"eSfCVMMp2HXYqvpVpyNjjr","sm":"gpt-5.2-2025-12-11_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/eSfCVMMp2HXYqvpVpyNjjr.eval","v":0.14},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-27","e":"minimal","evaluated_at":"2026-08-27T18:01:49.000Z","m":"glm-5-2","oid":"mystery_game_puzzles-f2f3f9de2096c8f9","rd":"2026-06-16","se":0.03861229196653691,"sid":"aekViWbdqVVeXpEG5hrSbQ","sm":"glm-5.2_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/aekViWbdqVVeXpEG5hrSbQ.eval","v":0.18},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:01:12.000Z","m":"qwen3-5-flash","oid":"mystery_game_puzzles-3dd489721af59639","rd":"2026-02-25","se":0.036845294917747066,"sid":"DjFkVt62QwqQ4xXwzYR5Ng","sm":"qwen3.5-flash_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/DjFkVt62QwqQ4xXwzYR5Ng.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:01:08.000Z","m":"deepseek-v4-pro","oid":"mystery_game_puzzles-e550abf4d198724e","rd":"2026-04-24","se":0.03775251680686369,"sid":"BmvJaHMCnZzb24ErpAsozU","sm":"deepseek-v4-pro_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BmvJaHMCnZzb24ErpAsozU.eval","v":0.17},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T18:01:04.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"mystery_game_puzzles-5fc3f0cf5768eccf","rd":"2026-03-17","se":0.01714466079977653,"sid":"aAAtrj4DqLUuEY6P7W5yLe","sm":"gpt-5.4-nano-2026-03-17_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/aAAtrj4DqLUuEY6P7W5yLe.eval","v":0.03},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T18:00:58.000Z","m":"nemotron-3-ultra","oid":"mystery_game_puzzles-2187c000a1609462","rd":"2026-06-04","se":0.04020151261036849,"sid":"U8H8juE3Ko83jsR6TRntbh","sm":"nemotron-3-ultra","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/U8H8juE3Ko83jsR6TRntbh.eval","v":0.2},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T18:00:41.000Z","m":"gpt-5-6-terra","oid":"mystery_game_puzzles-cdbfed3b9013fd1b","rd":"2026-07-09","se":0.0348735088019777,"sid":"8grv57UmR89LXkEYhyssr9","sm":"gpt-5.6-terra_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/8grv57UmR89LXkEYhyssr9.eval","v":0.14},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T18:00:36.000Z","m":"gpt-5-nano-2025-08-07","oid":"mystery_game_puzzles-88be9ed8b7dc6e1b","rd":"2025-08-07","se":0.028762349126466143,"sid":"ZwQdGcba6Hez4ztMzmZP9j","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZwQdGcba6Hez4ztMzmZP9j.eval","v":0.09},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-27","e":"minimal","evaluated_at":"2026-08-27T17:58:55.000Z","m":"gpt-5-2025-08-07","oid":"mystery_game_puzzles-51c15011e6d21632","rd":"2025-08-07","se":0.0348735088019777,"sid":"G3tHLT8qE9ncseZazqTXRX","sm":"gpt-5-2025-08-07_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/G3tHLT8qE9ncseZazqTXRX.eval","v":0.14},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T17:58:50.000Z","m":"gpt-5-1-2025-11-13","oid":"mystery_game_puzzles-2ac62b4f05fff24c","rd":"2025-11-13","se":0.03588702812826368,"sid":"2a2uhidAfksbnAJC8yUrZK","sm":"gpt-5.1-2025-11-13_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/2a2uhidAfksbnAJC8yUrZK.eval","v":0.15},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T17:58:41.000Z","m":"gpt-4-1-mini-2025-04-14","oid":"mystery_game_puzzles-0a66a4391b712c93","rd":"2025-04-14","se":0.02564323999762428,"sid":"giKk2ut3JVFVQUvbykxvSu","sm":"gpt-4.1-mini-2025-04-14","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/giKk2ut3JVFVQUvbykxvSu.eval","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-27","e":"minimal","evaluated_at":"2026-08-27T17:58:37.000Z","m":"gpt-5-mini-2025-08-07","oid":"mystery_game_puzzles-58b2bbdcaa16ee78","rd":"2025-08-07","se":0.03015113445777636,"sid":"kW6hSbkCwQczqbazduGXPR","sm":"gpt-5-mini-2025-08-07_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kW6hSbkCwQczqbazduGXPR.eval","v":0.1},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T17:58:06.000Z","m":"glm-5-2","oid":"mystery_game_puzzles-2806d3e53effdf18","rd":"2026-06-16","se":0.03861229196653691,"sid":"BLjXudxuEyLbZiKBSZz326","sm":"glm-5.2_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BLjXudxuEyLbZiKBSZz326.eval","v":0.18},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T17:58:05.000Z","m":"qwen3-235b-a22b-thinking-2507","oid":"mystery_game_puzzles-be4fdae50b375d1f","rd":"2025-07-25","se":0.028762349126466143,"sid":"BP3KYD8rYyFg3LviNGtjLL","sm":"Qwen3-235B-A22B-Thinking-2507","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BP3KYD8rYyFg3LviNGtjLL.eval","v":0.09},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T17:57:53.000Z","m":"gpt-3-5-turbo-0125","oid":"mystery_game_puzzles-fbc56150bca8b9ea","rd":"2024-01-25","se":0.01714466079977653,"sid":"UfmijCsx59sZ8xpcbcWDms","sm":"gpt-3.5-turbo-0125","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/UfmijCsx59sZ8xpcbcWDms.eval","v":0.03},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T17:57:44.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"mystery_game_puzzles-a4692222d7a232b2","rd":"2026-03-17","se":0.03144660377352201,"sid":"NEysw7ag6KJxUahem2PHQv","sm":"gpt-5.4-mini-2026-03-17_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/NEysw7ag6KJxUahem2PHQv.eval","v":0.11},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T17:57:43.000Z","m":"gpt-oss-120b","oid":"mystery_game_puzzles-8b7a7fa7f5e7d861","rd":"2025-08-05","se":0.014070529413628966,"sid":"58MQQiJ7vvgvP9e8LcG6eV","sm":"gpt-oss-120b_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/58MQQiJ7vvgvP9e8LcG6eV.eval","v":0.02},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T17:57:33.000Z","m":"gpt-5-nano-2025-08-07","oid":"mystery_game_puzzles-0e7d329efdd88d42","rd":"2025-08-07","se":0.028762349126466143,"sid":"PLbKjMLohFB9epj3qHPcuS","sm":"gpt-5-nano-2025-08-07_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/PLbKjMLohFB9epj3qHPcuS.eval","v":0.09},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-27","e":"minimal","evaluated_at":"2026-08-27T17:56:54.000Z","m":"gpt-5-nano-2025-08-07","oid":"mystery_game_puzzles-3048fa6f5e176f00","rd":"2025-08-07","se":0.021904291355759033,"sid":"RVRYWkwXwrvbCyZW8AhqEm","sm":"gpt-5-nano-2025-08-07_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/RVRYWkwXwrvbCyZW8AhqEm.eval","v":0.05},{"b":"mystery_game_puzzles","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T17:56:52.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"mystery_game_puzzles-b5cbfc3da5febb07","rd":"2026-03-17","se":0.028762349126466143,"sid":"fxNDgLaScEoHo2canW89yv","sm":"gpt-5.4-nano-2026-03-17_none","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/fxNDgLaScEoHo2canW89yv.eval","v":0.09},{"b":"mystery_game_puzzles","d":"2026-08-27","evaluated_at":"2026-08-27T17:56:46.000Z","m":"gpt-4o-mini-2024-07-18","oid":"mystery_game_puzzles-d0e2ff658156241e","rd":"2024-07-18","se":0.03265986323710904,"sid":"mbviVfQ7Bh7wHSCDDmrt7u","sm":"gpt-4o-mini-2024-07-18","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/mbviVfQ7Bh7wHSCDDmrt7u.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-08-19","e":"max","evaluated_at":"2026-08-19T18:14:49.000Z","m":"deepseek-v4-pro-0813","oid":"mystery_game_puzzles-0dfbc3354366ba8d","rd":"2026-08-13","se":0.0497569851956243,"sid":"c2KVkMCWnpo6BbnMWgdaDH","sm":"deepseek-v4-pro-0813_max","v":0.43},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-08-15","e":"xhigh","evaluated_at":"2026-08-15T01:02:40.000Z","m":"inkling-small","oid":"mystery_game_puzzles-e9a56a90b3bc23d7","rd":"2026-07-15","se":0.023868325657594204,"sid":"nPFA9ufjfVqg3YNmDRqWeC","sm":"inkling-small_xhigh","v":0.06},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-14","e":"high","evaluated_at":"2026-08-14T21:35:50.000Z","m":"gemini-3-7-flash","oid":"mystery_game_puzzles-45a2bba93f5359cc","rd":"2026-08-13","se":0.048523658709390974,"sid":"ZavSMMq9jk5s9WSbARZr6X","sm":"gemini-3.7-flash_high","v":0.37},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-08-14","e":"xhigh","evaluated_at":"2026-08-14T20:27:46.000Z","m":"grok-4-6","oid":"mystery_game_puzzles-81a66551a6a880b1","rd":"2026-08-12","se":0.04760952285695234,"sid":"9cXtChGrowgGDwa9AcAecB","sm":"grok-4.6_xhigh","v":0.34},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T14:07:21.000Z","m":"gpt-5-2-2025-12-11","oid":"mystery_game_puzzles-6e6c980c30e48d4e","rd":"2025-12-11","se":0.04229525846816508,"sid":"AsFSc8AQu4F46Tzp4xVwYu","sm":"gpt-5.2-2025-12-11_high","v":0.23},{"b":"mystery_game_puzzles","d":"2026-08-06","evaluated_at":"2026-08-06T00:18:43.000Z","m":"claude-sonnet-4-6","oid":"mystery_game_puzzles-e1a55f56f36f56ab","rd":"2026-02-17","se":0.0348735088019777,"sid":"VjfWxdhdNerrh4huSHT8PT","sm":"claude-sonnet-4-6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/VjfWxdhdNerrh4huSHT8PT.eval","v":0.14},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T00:18:14.000Z","m":"claude-sonnet-4-6","oid":"mystery_game_puzzles-8ae907dc1e2e2b88","rd":"2026-02-17","se":0.03684529491774706,"sid":"YdyvoMQ4mA5V8Ymg6Tksxu","sm":"claude-sonnet-4-6_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/YdyvoMQ4mA5V8Ymg6Tksxu.eval","v":0.16},{"b":"mystery_game_puzzles","d":"2026-08-06","evaluated_at":"2026-08-06T00:14:11.000Z","m":"claude-opus-5","oid":"mystery_game_puzzles-728b8d7687c8b925","rd":"2026-07-24","se":0.048523658709390974,"sid":"bWnggCv6to2QKSjngrtuos","sm":"claude-opus-5","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/bWnggCv6to2QKSjngrtuos.eval","v":0.37},{"b":"mystery_game_puzzles","d":"2026-08-06","evaluated_at":"2026-08-06T00:13:47.000Z","m":"claude-opus-4-7","oid":"mystery_game_puzzles-1a160a46cd775548","rd":"2026-04-16","se":0.033799766898963114,"sid":"LsnPrukDADRTP8fsn8zD9S","sm":"claude-opus-4-7","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/LsnPrukDADRTP8fsn8zD9S.eval","v":0.13},{"b":"mystery_game_puzzles","d":"2026-08-06","evaluated_at":"2026-08-06T00:13:31.000Z","m":"claude-opus-4-6","oid":"mystery_game_puzzles-6f064c2937cf03ee","rd":"2026-02-05","se":0.03588702812826367,"sid":"BbM4xbJcS5sjrzZABGg8K7","sm":"claude-opus-4-6","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/BbM4xbJcS5sjrzZABGg8K7.eval","v":0.15},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T00:13:22.000Z","m":"claude-opus-4-6","oid":"mystery_game_puzzles-613cbdc9bc561f6e","rd":"2026-02-05","se":0.02564323999762428,"sid":"JarZHkwUMZ6HuA4ArMNT45","sm":"claude-opus-4-6_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/JarZHkwUMZ6HuA4ArMNT45.eval","v":0.07},{"b":"mystery_game_puzzles","cfg":{"effort":"medium"},"d":"2026-08-06","e":"medium","evaluated_at":"2026-08-06T00:12:06.000Z","m":"gemini-3-1-pro-preview","oid":"mystery_game_puzzles-2e1d7fa4e117e82c","rd":"2026-02-19","se":0.04688261722621506,"sid":"WueZbBM3NwMUZPveuagrZo","sm":"gemini-3.1-pro-preview_medium","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/WueZbBM3NwMUZPveuagrZo.eval","v":0.32},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T00:11:58.000Z","m":"gemini-3-1-pro-preview","oid":"mystery_game_puzzles-9040bdb19013ca6f","rd":"2026-02-19","se":0.04560480215720687,"sid":"4a3M8rRWUoPphCSr2dguXH","sm":"gemini-3.1-pro-preview_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/4a3M8rRWUoPphCSr2dguXH.eval","v":0.29},{"b":"mystery_game_puzzles","d":"2026-08-06","evaluated_at":"2026-08-06T00:09:48.000Z","m":"claude-sonnet-5","oid":"mystery_game_puzzles-0ab127fe74121edf","rd":"2026-06-30","se":0.036845294917747066,"sid":"AZHoMLTNQEcKLwtv7QzBLZ","sm":"claude-sonnet-5","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/AZHoMLTNQEcKLwtv7QzBLZ.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-05","e":"low","evaluated_at":"2026-08-05T23:51:58.000Z","m":"gemini-3-5-flash","oid":"mystery_game_puzzles-1c45b16c56eae92e","rd":"2026-05-19","se":0.045126085985421296,"sid":"nrh3xa3Bn765Q6DXWoVfbS","sm":"gemini-3.5-flash_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/nrh3xa3Bn765Q6DXWoVfbS.eval","v":0.28},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-05","e":"minimal","evaluated_at":"2026-08-05T23:51:57.000Z","m":"gemini-3-6-flash","oid":"mystery_game_puzzles-6ad52929a4151380","rd":"2026-07-21","se":0.04351941398892446,"sid":"3L22tfAid5ZM42p92tqC73","sm":"gemini-3.6-flash_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/3L22tfAid5ZM42p92tqC73.eval","v":0.25},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-05","e":"low","evaluated_at":"2026-08-05T23:51:40.000Z","m":"gemini-3-6-flash","oid":"mystery_game_puzzles-29ab26bb25a38134","rd":"2026-07-21","se":0.04229525846816508,"sid":"M8sY8vHnvSLxQUg3HTMTCp","sm":"gemini-3.6-flash_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/M8sY8vHnvSLxQUg3HTMTCp.eval","v":0.23},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-05","e":"minimal","evaluated_at":"2026-08-05T23:47:51.000Z","m":"gemini-3-flash-preview","oid":"mystery_game_puzzles-2e170b7a8e51fb7a","rd":"2025-12-17","se":0.04351941398892446,"sid":"njpkLAWZTvept6tnkraYrR","sm":"gemini-3-flash-preview_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/njpkLAWZTvept6tnkraYrR.eval","v":0.25},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-05","e":"high","evaluated_at":"2026-08-05T23:47:26.000Z","m":"gemini-3-flash-preview","oid":"mystery_game_puzzles-6a9b547e305fa433","rd":"2025-12-17","se":0.04020151261036849,"sid":"MvdunHkVURna3JuyDjhzEc","sm":"gemini-3-flash-preview_high","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/MvdunHkVURna3JuyDjhzEc.eval","v":0.2},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-05","e":"low","evaluated_at":"2026-08-05T23:47:24.000Z","m":"gemini-3-flash-preview","oid":"mystery_game_puzzles-c382933f5dea2cbf","rd":"2025-12-17","se":0.0440844002276808,"sid":"ZQMcTQjHcDSevCN2dYYHKY","sm":"gemini-3-flash-preview_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/ZQMcTQjHcDSevCN2dYYHKY.eval","v":0.26},{"b":"mystery_game_puzzles","cfg":{"effort":"minimal"},"d":"2026-08-05","e":"minimal","evaluated_at":"2026-08-05T23:43:12.000Z","m":"gemini-3-5-flash-lite","oid":"mystery_game_puzzles-271cfd12fcf3df3f","rd":"2026-07-21","se":0.03265986323710904,"sid":"fJKhFRm5dn9wzbXojee7uj","sm":"gemini-3.5-flash-lite_minimal","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/fJKhFRm5dn9wzbXojee7uj.eval","v":0.12},{"b":"mystery_game_puzzles","cfg":{"effort":"low"},"d":"2026-08-05","e":"low","evaluated_at":"2026-08-05T23:42:56.000Z","m":"gemini-3-5-flash-lite","oid":"mystery_game_puzzles-7213d9cb1e883d7d","rd":"2026-07-21","se":0.039427724440366255,"sid":"Mg2yb9Tp3yc2H3ApJX6JBK","sm":"gemini-3.5-flash-lite_low","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/Mg2yb9Tp3yc2H3ApJX6JBK.eval","v":0.19},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-08-05","e":"max","evaluated_at":"2026-08-05T23:40:26.000Z","m":"deepseek-v4-flash-0731","oid":"mystery_game_puzzles-f5165a179b2b4f93","rd":"2026-07-31","se":0.04760952285695233,"sid":"4S4GXbkRhLz8wWP97jYG57","sm":"deepseek-v4-flash-0731_max","v":0.34},{"b":"mystery_game_puzzles","d":"2026-08-05","evaluated_at":"2026-08-05T23:33:02.000Z","m":"qwen3-5-plus","oid":"mystery_game_puzzles-f21e20f402f14615","rd":"2026-02-16","se":0.036845294917747066,"sid":"kCQr6ARwJyUfkwWqKAfuDX","sm":"qwen3.5-plus","trace":"https://epoch-benchmarks-staging-private.s3.us-east-2.amazonaws.com/inspect_ai_logs/kCQr6ARwJyUfkwWqKAfuDX.eval","v":0.16},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-05","e":"high","evaluated_at":"2026-08-05T19:32:49.000Z","m":"gpt-5-2025-08-07","oid":"mystery_game_puzzles-b2fd7d091bec8236","rd":"2025-08-07","se":0.04229525846816508,"sid":"NKdPCHXg9sphwKW2UEnmkD","sm":"gpt-5-2025-08-07_high","v":0.23},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-08-05","e":"high","evaluated_at":"2026-08-05T01:49:56.000Z","m":"gemini-3-6-flash","oid":"mystery_game_puzzles-718fd786db7574fb","rd":"2026-07-21","se":0.04605661864718382,"sid":"YBqiCXPjMUSAgAjDUoz5sy","sm":"gemini-3.6-flash_high","v":0.3},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-08-05","e":"xhigh","evaluated_at":"2026-08-05T01:42:38.000Z","m":"qwen3-8-max","oid":"mystery_game_puzzles-64a143042b9d6cbf","rd":"2026-08-02","se":0.048783173121456344,"sid":"c2YLifDU4zYCX5Jbi9HLL7","sm":"qwen3.8-max_xhigh","v":0.38},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-31","e":"max","evaluated_at":"2026-07-31T19:05:04.000Z","m":"claude-fable-5","oid":"mystery_game_puzzles-a3190d7b35cbf7fe","rd":"2026-06-09","se":0.05021167315686782,"sid":"X29V6KG3aVe5Nv5r3tsLHX","sm":"claude-fable-5_max","v":0.52},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-29","e":"max","evaluated_at":"2026-07-29T03:26:08.000Z","m":"kimi-k3","oid":"mystery_game_puzzles-a678ce4f2e39dc11","rd":"2026-07-16","se":0.0440844002276808,"sid":"eCq63CuekohAswYftWXJVj","sm":"kimi-k3_max","v":0.26},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-28","e":"max","evaluated_at":"2026-07-28T21:09:50.000Z","m":"claude-sonnet-5","oid":"mystery_game_puzzles-35a8d90f9a7bbb25","rd":"2026-06-30","se":0.04793724854411023,"sid":"YdcyYXLzqfPUSSgGv4MTED","sm":"claude-sonnet-5_max","v":0.35},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-28","e":"max","evaluated_at":"2026-07-28T20:59:07.000Z","m":"gpt-5-6-luna","oid":"mystery_game_puzzles-f716bf43a03bef85","rd":"2026-07-09","se":0.040936018074033236,"sid":"Epg7eNLwwGPXtX3sSAwPf8","sm":"gpt-5.6-luna_max","v":0.21},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-28","e":"max","evaluated_at":"2026-07-28T20:58:46.000Z","m":"gpt-5-6-terra","oid":"mystery_game_puzzles-fe3e0bd5c55fb6b8","rd":"2026-07-09","se":0.047937248544110224,"sid":"MFTd4BUbtzbfcCky6d79d4","sm":"gpt-5.6-terra_max","v":0.35},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-28","e":"max","evaluated_at":"2026-07-28T20:58:29.000Z","m":"gpt-5-6-sol","oid":"mystery_game_puzzles-afff75359514985b","rd":"2026-07-09","se":0.04960449637488583,"sid":"5kMRXeasRvtghXpDotCdgV","sm":"gpt-5.6-sol_max","v":0.58},{"b":"mystery_game_puzzles","d":"2026-07-28","evaluated_at":"2026-07-28T19:23:45.000Z","m":"qwen3-7-max","oid":"mystery_game_puzzles-a78de7898a39bb8f","rd":"2026-05-19","se":0.04688261722621506,"sid":"f7dotY9NheKNKK2zz6bfKZ","sm":"qwen3.7-max","v":0.32},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-07-27","e":"high","evaluated_at":"2026-07-27T19:10:39.000Z","m":"gpt-5-5","oid":"mystery_game_puzzles-5cecd6e403246e62","rd":"2026-04-23","se":0.05021167315686782,"sid":"Knij32Jp6bQJ2V9NHpg5VS","sm":"gpt-5.5_high","v":0.52},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-07-27","e":"high","evaluated_at":"2026-07-27T18:29:16.000Z","m":"gemini-3-1-pro-preview","oid":"mystery_game_puzzles-9a57be7c4c0a87c1","rd":"2026-02-19","se":0.04760952285695234,"sid":"g4bbCaiFDqw2YYNDAcmn2a","sm":"gemini-3.1-pro-preview_high","v":0.34},{"b":"mystery_game_puzzles","cfg":{"effort":"high"},"d":"2026-07-27","e":"high","evaluated_at":"2026-07-27T18:28:52.000Z","m":"gemini-3-5-flash","oid":"mystery_game_puzzles-464ab24f0a554fa4","rd":"2026-05-19","se":0.04688261722621506,"sid":"GQKsxFbdBWB546zE8ci49f","sm":"gemini-3.5-flash_high","v":0.32},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-07-26","e":"xhigh","evaluated_at":"2026-07-26T17:43:35.000Z","m":"claude-opus-4-8","oid":"mystery_game_puzzles-ba78ca29cf71188c","rd":"2026-05-28","se":0.04648231987117317,"sid":"3ZnPtGRJkvqHk8wvWWRgCe","sm":"claude-opus-4-8_xhigh","v":0.31},{"b":"mystery_game_puzzles","d":"2026-07-25","evaluated_at":"2026-07-25T07:06:00.000Z","m":"claude-sonnet-4-5-20250929-48k","oid":"mystery_game_puzzles-c53c47aa72b9bc56","rd":"2025-09-29","se":0.03775251680686369,"sid":"4oSbA2RKFsfuAfVayAMAww","sm":"claude-sonnet-4-5-20250929_48K","v":0.17},{"b":"mystery_game_puzzles","d":"2026-07-25","evaluated_at":"2026-07-25T07:05:43.000Z","m":"claude-opus-4-1-20250805-24k","oid":"mystery_game_puzzles-4d0dbe17f9437253","rd":"2025-08-05","se":0.040936018074033236,"sid":"2AbeCjy9w4KJWKhbKjtFTJ","sm":"claude-opus-4-1-20250805_24K","v":0.21},{"b":"mystery_game_puzzles","d":"2026-07-25","evaluated_at":"2026-07-25T07:05:27.000Z","m":"claude-opus-4-5-20251101-48k","oid":"mystery_game_puzzles-4127b81196ffb554","rd":"2025-11-24","se":0.04163331998932266,"sid":"k4zGHaXESxFRTRMkyDLZrz","sm":"claude-opus-4-5-20251101_48K","v":0.22},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-25","e":"max","evaluated_at":"2026-07-25T04:37:20.000Z","m":"claude-opus-4-8","oid":"mystery_game_puzzles-98d0107acdc11348","rd":"2026-05-28","se":0.04824181513244218,"sid":"Q3WpHdurYNvawj6ycXyj6e","sm":"claude-opus-4-8_max","v":0.36},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-25","e":"max","evaluated_at":"2026-07-25T04:37:04.000Z","m":"claude-opus-5","oid":"mystery_game_puzzles-d37d19349b4198a0","rd":"2026-07-24","se":0.04943110704237103,"sid":"FfAvNS5H9bMDSBF4JNNiDz","sm":"claude-opus-5_max","v":0.59},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-25","e":"max","evaluated_at":"2026-07-25T01:44:09.000Z","m":"claude-opus-4-6","oid":"mystery_game_puzzles-922968f715cff7ab","rd":"2026-02-05","se":0.04351941398892446,"sid":"64rokFyL2L8Vd4SZVdayCy","sm":"claude-opus-4-6_max","v":0.25},{"b":"mystery_game_puzzles","cfg":{"effort":"max"},"d":"2026-07-25","e":"max","evaluated_at":"2026-07-25T01:43:24.000Z","m":"claude-opus-4-7","oid":"mystery_game_puzzles-586429be8e728cdc","rd":"2026-04-16","se":0.04512608598542129,"sid":"VFYhE54autgdYcMJNXicwU","sm":"claude-opus-4-7_max","v":0.28},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-07-24","e":"xhigh","evaluated_at":"2026-07-24T18:20:01.000Z","m":"gpt-5-4-2026-03-05","oid":"mystery_game_puzzles-f40d83b86a906082","rd":"2026-03-05","se":0.048523658709390974,"sid":"T6DyawEuhVmDAaDTzDRNNx","sm":"gpt-5.4-2026-03-05_xhigh","v":0.37},{"b":"mystery_game_puzzles","cfg":{"effort":"xhigh"},"d":"2026-07-24","e":"xhigh","evaluated_at":"2026-07-24T18:19:51.000Z","m":"gpt-5-5","oid":"mystery_game_puzzles-0ffcf0e9dc37eae4","rd":"2026-04-23","se":0.049888765156985884,"sid":"EatRQnSaUpUwKzdtjYnQHo","sm":"gpt-5.5_xhigh","v":0.56},{"b":"mystery_game_puzzles","d":"2026-07-17","evaluated_at":"2026-07-17T23:54:57.000Z","m":"kimi-k2-6","oid":"mystery_game_puzzles-f27fd924637de4af","rd":"2026-04-20","se":0.03861229196653691,"sid":"RA5MRbmWJpFPR29BQxfvZt","sm":"kimi-k2.6","v":0.18},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"open_book_qa-c2afa2f2be676d5e","rd":"2023-03-20","sid":"rec3bpxvIXHWi7lr8","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.358},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"open_book_qa-efee2553d289d7d4","rd":"2023-04-11","sid":"recRv6XQk2g8nv2LK","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.392},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Falcon 7B","m":"falcon-7b","oid":"open_book_qa-565b365f8794936a","rd":"2023-04-24","sid":"recOQTvbSFFnxLVK3","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.516},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"open_book_qa-885a4c08913a2f12","rd":"2023-04-24","sid":"reckejrbahCqvjrU6","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.44},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"open_book_qa-992dce8f9743cdd3","rd":"2023-04-24","sid":"recqHoMWEmZsrbHoX","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.32},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Falcon 40B","m":"falcon-40b","oid":"open_book_qa-5ca7bba29012e312","rd":"2023-03-15","sid":"recIIgm5W90RnDpyw","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.566},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"Falcon-180B","m":"falcon-180b","oid":"open_book_qa-476b8b64f7a97a18","rd":"2023-09-06","sid":"reccJipEIOFHnc7eW","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.642},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Gemma 7b","m":"gemma-7b","oid":"open_book_qa-06656b73172ca7c7","rd":"2024-02-21","sid":"recDm6xAaA2hOyE18","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.786},{"b":"open_book_qa","cfg":{"shots":"32"},"dn":"GLaM","m":"glam-moe","oid":"open_book_qa-8a3d7200de5cc702","rd":"2021-12-13","sid":"recKeJAcYdE1fqMp0","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.63},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"GLaM","m":"glam-moe","oid":"open_book_qa-c82bd646584fe442","rd":"2021-12-13","sid":"recLCaTp5a71NVAdg","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.552},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GLaM","m":"glam-moe","oid":"open_book_qa-254411dd8d849d34","rd":"2021-12-13","sid":"recEoJmc12bQ9cfIJ","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.534},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"open_book_qa-17268495578ae058","rd":"2023-11-06","sid":"rec39keFceUY6rwsC","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.86},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"open_book_qa-caec141b6f7d1379","rd":"2021-08-05","sid":"rec9RxSC1PV3tdeZJ","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.382},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-Neo-2.7B","m":"gpt-neo-2-7b","oid":"open_book_qa-f3759b87c6dd5564","rd":"2021-03-30","sid":"recPK1fpfiUrqmnGQ","sm":"gpt-neo-2.7B","src":"http://arxiv.org/abs/2309.05463","v":0.232},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"open_book_qa-b4bcf5ed3d253e1d","rd":"2022-04-07","sid":"recPTpLjU81ZjRoxs","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.388},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"open_book_qa-172e5758f719f5bd","rd":"2019-11-05","sid":"recsEQtJDmLV1Xcch","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.224},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"open_book_qa-de7411921ecad359","rd":"2023-07-18","sid":"rechy2nFTpVEj0Qno","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.586},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Llama2-7B","m":"llama-2-7b","oid":"open_book_qa-708060b374d4e0e8","rd":"2023-07-18","sid":"rec9Y2UBqoSRQvHDn","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.314},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"open_book_qa-f5c229bacc448811","rd":"2023-07-18","sid":"recCV5qTv8V9ymqeW","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.57},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"open_book_qa-94eee1009e0f0414","rd":"2023-07-18","sid":"recUq34yNFmNh5B5G","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.582},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"open_book_qa-af98edee2857c094","rd":"2023-07-18","sid":"rec8fmtGNMDa9AbnI","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.602},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"open_book_qa-0819038fb1ebaecf","rd":"2023-02-24","sid":"recrGbCP8SmW1H1fQ","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.572},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"open_book_qa-54bac2cf17d54e16","rd":"2023-02-24","sid":"reclLod1NhMDGa7DH","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.572},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"open_book_qa-660df6b2f1f7c5cb","rd":"2023-02-24","sid":"recK8s6dic5FAPA4W","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.442},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Llama-7B","m":"llama-7b","oid":"open_book_qa-6676d7704ea2ecc3","rd":"2023-02-24","sid":"rec9Kg010w6BBE51L","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.284},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"open_book_qa-65cf3b2040bbb791","rd":"2023-02-24","sid":"recNZ8YEViePx57jx","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.564},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"open_book_qa-295919a9092df560","rd":"2023-02-24","sid":"reconY997ubXRO6oG","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.564},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"open_book_qa-c0f6ac5303100806","rd":"2023-02-24","sid":"recBLnNexGcpsQ3Sq","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.586},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLaMA 33B","m":"llama-33b","oid":"open_book_qa-ae163eabc395b8fa","rd":"2023-02-24","sid":"rec5Gi0O64VhoK4OT","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.586},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"open_book_qa-edbac752cc84f03d","rd":"2023-02-24","sid":"reczH9kpD4IpqjPk1","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.602},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"LLaMA 65B","m":"llama-65b","oid":"open_book_qa-715e69c23e825a6a","rd":"2023-02-24","sid":"recg3MS1UPXaX6Gdw","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.602},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"open_book_qa-670c8357e97f0770","rd":"2024-04-18","sid":"recE5eGspU6lhyKQV","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.826},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Llama 3 8B","m":"meta-llama-3-8b-instruct","notes":"\"For pre-trained models, we use a 0-shot config and report average accuracy. We run these as choice task.\"","oid":"open_book_qa-81a286288a582355","rd":"2024-04-18","sid":"rec3L2OKijkkbrBp1","sm":"Meta-Llama-3-8B-Instruct","src":"https://arxiv.org/pdf/2407.21783","v":0.45},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"open_book_qa-c4c3815f519387f3","rd":"2023-09-27","sid":"recpUSIUFvbjMUSDA","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.798},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"open_book_qa-593b2ee4dfd6dd84","rd":"2023-12-11","sid":"recSGILLdlIug5JEV","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.858},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"MPT 7B","m":"mpt-7b","oid":"open_book_qa-d7bdbe182b49df99","rd":"2023-05-05","sid":"recf1oPTFEEiXW0lC","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.514},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"open_book_qa-574b4430c1e82f5b","rd":"2023-05-05","sid":"recwkbIifsqHWBylq","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.418},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"open_book_qa-221fd8ba85b7afd8","rd":"2023-05-05","sid":"rec8wCqr8Hi1klVLR","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.314},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"MPT 30B","m":"mpt-30b","oid":"open_book_qa-a825c5c1b319c2bd","rd":"2023-06-22","sid":"rec8lDHmcxKwxjWvp","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.52},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"open_book_qa-d93e3280d7841b8c","rd":"2023-06-07","sid":"reckybuDIUweWoaXN","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.39},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"OPT-1.3B","m":"opt-1-3b","oid":"open_book_qa-ab6fbf65ecc51a0d","rd":"2022-05-11","sid":"recyCxsBCiq0Dgqb2","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.24},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"open_book_qa-2f074e57ba193be0","rd":"2022-05-11","sid":"rec4xtdDLXW9kjtYL","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.398},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"open_book_qa-856899d573c7e9d7","rd":"2023-05-17","sid":"recBNf9DNfr8q6ppe","sm":"PaLM 2-M","src":"https://arxiv.org/pdf/2305.10403","v":0.574},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"PaLM 2-L","m":"palm-2-s","oid":"open_book_qa-42de8d33fb2d8e8c","rd":"2023-05-17","sid":"recGMiq3GPLHIV4OI","sm":"PaLM 2-S","src":"https://arxiv.org/pdf/2305.10403","v":0.562},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"open_book_qa-8a197128d8587b91","rd":"2023-05-17","sid":"recBqXIB4reE9vndJ","sm":"PaLM 2-S","src":"https://arxiv.org/pdf/2305.10403","v":0.536},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"PaLM 62B","m":"palm-62b","oid":"open_book_qa-1bceb1a53ad72bbf","rd":"2022-04-04","sid":"rechEbt7N3ckRpwzN","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.504},{"b":"open_book_qa","cfg":{"shots":"few"},"dn":"PaLM 540B","m":"palm-540b","oid":"open_book_qa-47f1751181699a4e","rd":"2022-04-04","sid":"recUDUft5bxQOJL2u","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.68},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"PaLM 540B","m":"palm-540b","oid":"open_book_qa-c84a3ee87702b658","rd":"2022-04-04","sid":"reczsCmGsOF2cuiTX","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.536},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"PaLM 540B","m":"palm-540b","oid":"open_book_qa-9feb246cfbf81e4e","rd":"2022-04-04","sid":"recvoaYztvUBvsqKw","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.534},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"PaLM 540B","m":"palm-540b","oid":"open_book_qa-fcd918ec78b1e777","rd":"2022-04-04","sid":"rectKDDJrAzsuBGK9","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.534},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"phi-1.5","m":"phi-1-5","oid":"open_book_qa-32f3a534bee959e8","rd":"2023-09-11","sid":"recqLehIM25p3OUAM","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.372},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-2","m":"phi-2","oid":"open_book_qa-bb568ad2d051b43f","rd":"2023-12-12","sid":"rec8R0aZuAAgy9GXj","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.736},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-2 2.7b","m":"phi-2","oid":"open_book_qa-c027f91321a96173","rd":"2023-12-12","sid":"recyxDPo86hq7OszB","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.736},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-medium","m":"phi-3-medium-128k-instruct","oid":"open_book_qa-1b970628b863533b","rd":"2024-04-23","sid":"recVdd6z2Oqgmy2eF","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.874},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-medium 14b","m":"phi-3-medium-128k-instruct","oid":"open_book_qa-08cf228f190086fa","rd":"2024-04-23","sid":"recc9groORxv5t0rq","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.874},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-small","m":"phi-3-mini-4k-instruct","oid":"open_book_qa-3738abcc364aa78e","rd":"2024-04-23","sid":"recwHz5iXxJOt3fJZ","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.88},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-mini","m":"phi-3-mini-4k-instruct","oid":"open_book_qa-dcbb64784e980eb0","rd":"2024-04-23","sid":"recGJPRGYdYCQ5VHo","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.832},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-mini 3.8b","m":"phi-3-mini-4k-instruct","oid":"open_book_qa-be29625191e73321","rd":"2024-04-23","sid":"receyJ3VCrGAW8rJz","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.832},{"b":"open_book_qa","cfg":{"shots":"10"},"dn":"Phi-3-small 7b","m":"phi-3-small-8k-instruct","oid":"open_book_qa-cbb58ff59997760f","rd":"2024-04-23","sid":"recXqivQrRpdZQZOr","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.88},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"open_book_qa-126282faccf9c1ad","rd":"2023-05-04","sid":"recggVvziNm48zqR3","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.4},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"open_book_qa-9de7d46c19bd9a6a","rd":"2023-04-19","sid":"rec1pDbHRlNgb6yqD","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.324},{"b":"open_book_qa","cfg":{"shots":"100"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"open_book_qa-982abc436b18985e","rd":"2022-01-27","sid":"recSDCmLFvBeiQOqG","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.654},{"b":"open_book_qa","cfg":{"shots":"few"},"dn":"GPT-3 Few-Shot","m":"text-davinci-001","oid":"open_book_qa-51d6aa804eaeaada","rd":"2022-01-27","sid":"recV8gA88czCjdZG5","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.654},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"open_book_qa-e3c56b5d92229e88","rd":"2022-01-27","sid":"rec1lRzMumzZON196","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.588},{"b":"open_book_qa","cfg":{"shots":"1"},"dn":"GPT-3 One-Shot","m":"text-davinci-001","oid":"open_book_qa-cc8c359692d47a95","rd":"2022-01-27","sid":"rec6lQxIK5zT1TvsP","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.588},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"open_book_qa-ff914c9abfadb11f","rd":"2022-01-27","sid":"recGiuywzkfGkjfa9","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.576},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-3 Zero-Shot","m":"text-davinci-001","oid":"open_book_qa-8c86d4ff0e9cf293","rd":"2022-01-27","sid":"recFibgTqlVSOR9VX","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.576},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"GPT-3 175B","m":"text-davinci-001","oid":"open_book_qa-6838cd5a1b94b326","rd":"2022-01-27","sid":"recGI43E7k7qQP84S","sm":"text-davinci-001","src":"http://arxiv.org/abs/2302.13971","v":0.576},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"Vicuna-13B","m":"vicuna-13b-v1-1","oid":"open_book_qa-25c04eeed6dad486","rd":"2023-04-12","sid":"recpohf85rUyjWO3J","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.33},{"b":"open_book_qa","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"open_book_qa-4d43ac5729e3b4a7","rd":"2023-06-27","sid":"recM76F7J2TQ2TO7S","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.402},{"b":"os_world","cfg":{"agent":"claude-sonnet-4-6 (100 steps)"},"m":"claude-sonnet-4-6","oid":"os_world-449d2061172afb1e","rd":"2026-02-17","sm":"claude-sonnet-4-6","src":"https://os-world.github.io/","v":72.1},{"b":"os_world","cfg":{"agent":"claude-sonnet-4-5-20250929 (100 steps)"},"m":"claude-sonnet-4-5-2025-09-29","oid":"os_world-193d34098e8c2b95","rd":"2025-09-29","sm":"claude-sonnet-4-5-20250929","src":"https://os-world.github.io/","v":62.9},{"b":"os_world","cfg":{"agent":"claude-sonnet-4-5-20250929 (50 steps)"},"m":"claude-sonnet-4-5-2025-09-29","oid":"os_world-dfaec8404eaeb66f","rd":"2025-09-29","sm":"claude-sonnet-4-5-20250929","src":"https://os-world.github.io/","v":58.1},{"b":"os_world","cfg":{"agent":"claude-4-sonnet-20250514 (50 steps)"},"m":"claude-sonnet-4-2025-05-14","oid":"os_world-26fcee6e5b1f2ca7","rd":"2025-05-22","sm":"claude-sonnet-4-20250514","src":"https://os-world.github.io/","v":43.9},{"b":"os_world","cfg":{"agent":"claude-sonnet-4-5-20250929 (15 steps)"},"m":"claude-sonnet-4-5-2025-09-29","oid":"os_world-1df0cb15c1ec686a","rd":"2025-09-29","sm":"claude-sonnet-4-5-20250929","src":"https://os-world.github.io/","v":42.9},{"b":"os_world","cfg":{"agent":"claude-4-sonnet-20250514 (100 steps)"},"m":"claude-sonnet-4-2025-05-14","oid":"os_world-fcd12e90d3b94b0c","rd":"2025-05-22","sm":"claude-sonnet-4-20250514","src":"https://os-world.github.io/","v":41.4},{"b":"os_world","cfg":{"agent":"claude-3-7-sonnet-20250219 (50 steps)"},"m":"claude-3-7-sonnet-2025-02-19","oid":"os_world-00a005e81c671b6f","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219","src":"https://os-world.github.io/","v":35.8},{"b":"os_world","cfg":{"agent":"claude-3-7-sonnet-20250219 (100 steps)"},"m":"claude-3-7-sonnet-2025-02-19","oid":"os_world-75f84b08f2e981c8","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219","src":"https://os-world.github.io/","v":35.6},{"b":"os_world","cfg":{"agent":"computer-use-preview (50 steps)"},"m":"computer-use-preview-2025-03-11","oid":"os_world-78da3770b82d0d20","rd":"2025-03-11","sm":"computer-use-preview-2025-03-11","src":"https://os-world.github.io/","v":31.3},{"b":"os_world","cfg":{"agent":"claude-4-sonnet-20250514 (15 steps)"},"m":"claude-sonnet-4-2025-05-14","oid":"os_world-446a05e426c1a9e1","rd":"2025-05-22","sm":"claude-sonnet-4-20250514","src":"https://os-world.github.io/","v":31.2},{"b":"os_world","cfg":{"agent":"computer-use-preview (100 steps)"},"m":"computer-use-preview-2025-03-11","oid":"os_world-d8f4d9ab22034864","rd":"2025-03-11","sm":"computer-use-preview-2025-03-11","src":"https://os-world.github.io/","v":30.5},{"b":"os_world","cfg":{"agent":"claude-3-7-sonnet-20250219 (15 steps)"},"m":"claude-3-7-sonnet-2025-02-19","oid":"os_world-f079c072162b4df5","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219","src":"https://os-world.github.io/","v":27.1},{"b":"os_world","cfg":{"agent":"computer-use-preview (15 steps)"},"m":"computer-use-preview-2025-03-11","oid":"os_world-537e9457be0f224c","rd":"2025-03-11","sm":"computer-use-preview-2025-03-11","src":"https://os-world.github.io/","v":26},{"b":"os_world","cfg":{"agent":"o3 (100 steps)","effort":"medium"},"e":"medium","m":"o3-2025-04-16","oid":"os_world-db873da3a4d33ff7","rd":"2025-04-16","sm":"o3-2025-04-16_medium","src":"https://os-world.github.io/","v":23},{"b":"os_world","cfg":{"agent":"o3 (50 steps)","effort":"medium"},"e":"medium","m":"o3-2025-04-16","oid":"os_world-07bc4e77459fbf43","rd":"2025-04-16","sm":"o3-2025-04-16_medium","src":"https://os-world.github.io/","v":17.2},{"b":"os_world","cfg":{"agent":"o3 (15 steps)","effort":"medium"},"e":"medium","m":"o3-2025-04-16","oid":"os_world-33749e5f10958ffa","rd":"2025-04-16","sm":"o3-2025-04-16_medium","src":"https://os-world.github.io/","v":9.1},{"b":"os_world","cfg":{"agent":"qwen2.5-vl-72b-instruct (100 steps)"},"m":"qwen2-5-vl-72b-instruct","oid":"os_world-336faba021f7a3f7","rd":"2024-09-19","sm":"Qwen2.5-VL-72B-Instruct","src":"https://os-world.github.io/","v":5},{"b":"os_world","cfg":{"agent":"qwen2.5-vl-72b-instruct (15 steps)"},"m":"qwen2-5-vl-72b-instruct","oid":"os_world-fa6417368bb96022","rd":"2024-09-19","sm":"Qwen2.5-VL-72B-Instruct","src":"https://os-world.github.io/","v":4.4},{"b":"os_world","cfg":{"agent":"Claude Opus 4.5"},"m":"claude-opus-4-5-2025-11-01","oid":"os_world-c478cb67f0ee9a85","rd":"2025-11-24","sm":"claude-opus-4-5-20251101","src":"https://www.anthropic.com/news/claude-opus-4-5","v":66.3},{"b":"os_world","cfg":{"agent":"Kimi-K2.5"},"m":"kimi-k2-5","oid":"os_world-04e9fb4348b9bfbe","rd":"2026-01-27","sm":"kimi-k2.5","src":"https://os-world.github.io/","v":63.3},{"b":"osworld_2","cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"batch tool"},"dn":"Claude Opus 5 (max)","e":"max","m":"claude-opus-5","oid":"osworld_2-78ffbb3a71bc5132","rd":"2026-07-24","sm":"claude-opus-5_max","src":"https://osworld-v2.xlang.ai/","v":0.3143},{"b":"osworld_2","cfg":{"budget":"500","effort":"xhigh","reasoning":"xhigh","tools":"batch tool"},"dn":"Claude Opus 5 (xhigh)","e":"xhigh","m":"claude-opus-5","oid":"osworld_2-a70714abad8b02c1","rd":"2026-07-24","sm":"claude-opus-5_xhigh","src":"https://osworld-v2.xlang.ai/","v":0.30210000000000004},{"b":"osworld_2","cfg":{"budget":"500","effort":"high","reasoning":"high","tools":"batch tool"},"dn":"Claude Opus 5 (high)","e":"high","m":"claude-opus-5","oid":"osworld_2-a6eb42e5a6482bf4","rd":"2026-07-24","sm":"claude-opus-5_high","src":"https://osworld-v2.xlang.ai/","v":0.2898},{"b":"osworld_2","cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"batch tool"},"dn":"GPT-5.6 Sol (max)","e":"max","m":"gpt-5-6-sol","oid":"osworld_2-b19f452ec7179aa6","rd":"2026-07-09","sm":"gpt-5.6-sol_max","src":"https://osworld-v2.xlang.ai/","v":0.27340000000000003},{"b":"osworld_2","cfg":{"budget":"500","effort":"medium","reasoning":"medium","tools":"batch tool"},"dn":"Claude Opus 5 (medium)","e":"medium","m":"claude-opus-5","oid":"osworld_2-6b4a4a9c3e82fa3a","rd":"2026-07-24","sm":"claude-opus-5_medium","src":"https://osworld-v2.xlang.ai/","v":0.25329999999999997},{"b":"osworld_2","cfg":{"budget":"500","effort":"low","reasoning":"low","tools":"batch tool"},"dn":"Claude Opus 5 (low)","e":"low","m":"claude-opus-5","oid":"osworld_2-6d4995df99d2b119","rd":"2026-07-24","sm":"claude-opus-5_low","src":"https://osworld-v2.xlang.ai/","v":0.2232},{"b":"osworld_2","cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"batched tool"},"dn":"Claude Opus 4.8 (max)","e":"max","m":"claude-opus-4-8","oid":"osworld_2-c9cf60376a82c8c0","rd":"2026-05-28","sm":"claude-opus-4-8_max","src":"https://osworld-v2.xlang.ai/","v":0.20600000000000002},{"b":"osworld_2","cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"standard"},"dn":"Claude Opus 4.8 (max)","e":"max","m":"claude-opus-4-8","oid":"osworld_2-9c02a3b4eff6e41a","rd":"2026-05-28","sm":"claude-opus-4-8_max","src":"https://osworld-v2.xlang.ai/","v":0.1852},{"b":"osworld_2","cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"batched tool"},"dn":"Claude Opus 4.7 (max)","e":"max","m":"claude-opus-4-7","oid":"osworld_2-4265399df87bdb06","rd":"2026-04-16","sm":"claude-opus-4-7_max","src":"https://osworld-v2.xlang.ai/","v":0.182},{"b":"osworld_2","c":3870,"cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"standard"},"dn":"Claude Opus 4.7 (max)","e":"max","m":"claude-opus-4-7","oid":"osworld_2-113d97d5d8b61736","rd":"2026-04-16","sm":"claude-opus-4-7_max","src":"https://osworld-v2.xlang.ai/","v":0.139},{"b":"osworld_2","c":2750,"cfg":{"budget":"500","effort":"xhigh","reasoning":"xhigh","tools":"batch tool"},"dn":"GPT-5.5 (xhigh)","e":"xhigh","m":"gpt-5-5","oid":"osworld_2-f6c1eb7d00b95c44","rd":"2026-04-23","sm":"gpt-5.5_xhigh","src":"https://osworld-v2.xlang.ai/","v":0.13},{"b":"osworld_2","c":1550,"cfg":{"budget":"500","effort":"medium","reasoning":"medium","tools":"standard"},"dn":"Claude Sonnet 4.6 (medium)","e":"medium","m":"claude-sonnet-4-6","oid":"osworld_2-f9f7e01fa0a79fb4","rd":"2026-02-17","sm":"claude-sonnet-4-6_medium","src":"https://osworld-v2.xlang.ai/","v":0.09300000000000001},{"b":"osworld_2","c":2410,"cfg":{"budget":"500","effort":"max","reasoning":"max","tools":"standard"},"dn":"Claude Sonnet 4.6 (max)","e":"max","m":"claude-sonnet-4-6","oid":"osworld_2-658c31f085461359","rd":"2026-02-17","sm":"claude-sonnet-4-6_max","src":"https://osworld-v2.xlang.ai/","v":0.083},{"b":"osworld_2","c":258.78,"cfg":{"budget":"500","reasoning":"enabled","tools":"standard"},"dn":"MiniMax M3 (enabled)","m":"minimax-m3","oid":"osworld_2-a6d9c7522be0e7c9","rd":"2026-06-01","sm":"MiniMax-M3","src":"https://osworld-v2.xlang.ai/","v":0.046},{"b":"osworld_2","c":708,"cfg":{"budget":"500","reasoning":"enabled","tools":"standard"},"dn":"Kimi 2.6 (enabled)","m":"kimi-k2-6","oid":"osworld_2-c1eef5acfa38dfdd","rd":"2026-04-20","sm":"kimi-k2.6","src":"https://osworld-v2.xlang.ai/","v":0.046},{"b":"osworld_2","c":411.56,"cfg":{"budget":"500","effort":"thinking","reasoning":"thinking","tools":"standard"},"dn":"Qwen 3.7-Plus (thinking)","e":"thinking","m":"qwen3-7-plus","oid":"osworld_2-774889d8d03b7233","rd":"2026-06-02","sm":"qwen3.7-plus","src":"https://osworld-v2.xlang.ai/","v":0.027999999999999997},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-11-06","e":"max","evaluated_at":"2026-11-06T00:59:23.000Z","m":"kimi-k3-1-pro-together","oid":"otis_mock_aime_2024_2025-50f6cd0085d49d12","rd":"2026-10-23","se":0,"sid":"8Qe2qf4Eg77PMfp2Eh7V6K","sm":"kimi-k3.1-pro-together_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8Qe2qf4Eg77PMfp2Eh7V6K.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T20:57:35.000Z","m":"doubao-seed-evolving","oid":"otis_mock_aime_2024_2025-9964603ca5fc26e2","rd":"2026-11-02","se":0.004700633956814717,"sid":"RG3zEJdwgXxoYZjFkHc6pa","sm":"doubao-seed-evolving_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRG3zEJdwgXxoYZjFkHc6pa.eval","v":0.9916666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T13:47:17.000Z","m":"mistral-large-4-0-oss","oid":"otis_mock_aime_2024_2025-4d0f190bc735a62e","rd":"2026-10-29","se":0.003883473863441843,"sid":"gmJrht9jW92sRXiYyfSqEP","sm":"mistral-large-4-0-oss_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgmJrht9jW92sRXiYyfSqEP.eval","v":0.9944444444444445},{"b":"otis_mock_aime_2024_2025","d":"2026-10-30","evaluated_at":"2026-10-30T18:28:17.000Z","m":"seed-1-6","oid":"otis_mock_aime_2024_2025-9e22b749d6f206b2","rd":"2025-06-11","se":0.06123151591268345,"sid":"WNAkAqgqwujVtjUKxkjQVu","sm":"seed-1.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWNAkAqgqwujVtjUKxkjQVu.eval","v":0.6944444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T12:30:09.000Z","m":"doubao-seed-2-1-turbo-260628","oid":"otis_mock_aime_2024_2025-b78680f461d9340b","rd":"2026-06-23","se":0.024753271842145104,"sid":"mXHskR6CPxcMBRzmWaPvcq","sm":"doubao-seed-2.1-turbo-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmXHskR6CPxcMBRzmWaPvcq.eval","v":0.9611111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T11:38:36.000Z","m":"doubao-seed-2-0-pro-260215","oid":"otis_mock_aime_2024_2025-93bc59c2ac319be7","rd":"2026-02-14","se":0.02741556776322574,"sid":"GVKTrNdpeZhFRevQFSFkRY","sm":"doubao-seed-2.0-pro-260215_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGVKTrNdpeZhFRevQFSFkRY.eval","v":0.9388888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T11:38:33.000Z","m":"doubao-seed-2-0-lite-260428","oid":"otis_mock_aime_2024_2025-7acdc3c12e3d28a3","rd":"2026-04-28","se":0.036111111111111115,"sid":"RpVeo88Yf93kMhHgwz6jcj","sm":"doubao-seed-2.0-lite-260428_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRpVeo88Yf93kMhHgwz6jcj.eval","v":0.9111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T11:38:23.000Z","m":"doubao-seed-2-1-pro-260628","oid":"otis_mock_aime_2024_2025-a14b5ab53c3dd308","rd":"2026-06-23","se":0.029235728721796906,"sid":"jL9Sf6dcoTTRSSUFXTG7py","sm":"doubao-seed-2.1-pro-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjL9Sf6dcoTTRSSUFXTG7py.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-10-26","e":"max","evaluated_at":"2026-10-26T20:00:37.000Z","m":"gpt-6-1-terra","oid":"otis_mock_aime_2024_2025-aa70ccfe4c0187c1","rd":"2026-10-26","se":0,"sid":"YnwukT2gyJHGKzPNPU9hry","sm":"gpt-6.1-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYnwukT2gyJHGKzPNPU9hry.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-10-19","e":"max","evaluated_at":"2026-10-19T22:39:00.000Z","m":"deepseek-v4-1-flash-1017","oid":"otis_mock_aime_2024_2025-a02403c3b0864d24","rd":"2026-10-17","se":0.003883473863441843,"sid":"chayEvHvpt2Wcif7pg64hF","sm":"deepseek-v4.1-flash-1017_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FchayEvHvpt2Wcif7pg64hF.eval","v":0.9944444444444445},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-10-15","e":"xhigh","evaluated_at":"2026-10-15T18:52:04.000Z","m":"muse-spark-open-816b-a42b","oid":"otis_mock_aime_2024_2025-b532813b9d080d59","rd":"2026-10-15","se":0.01253502388483925,"sid":"hB4XzBdWkdpG3WJVYqeRM8","sm":"Muse-Spark-Open-816B-A42B_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhB4XzBdWkdpG3WJVYqeRM8.eval","v":0.9805555555555555},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:14:57.000Z","m":"gemini-4-argon","oid":"otis_mock_aime_2024_2025-09f26afffda4e0b3","rd":"2026-09-30","se":0,"sid":"43f36RTDoJGxXWQDVvTLJj","sm":"gemini-4-argon_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F43f36RTDoJGxXWQDVvTLJj.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-12","e":"high","evaluated_at":"2026-10-12T21:38:35.000Z","m":"mistral-large-4-0","oid":"otis_mock_aime_2024_2025-41bab3e21159ad90","rd":"2026-10-06","se":0.006405884289881728,"sid":"Pby5WuP8Muu6Jzs2v5gB7F","sm":"mistral-large-4-0_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPby5WuP8Muu6Jzs2v5gB7F.eval","v":0.9833333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-10-12","e":"max","evaluated_at":"2026-10-12T19:00:20.000Z","m":"muse-wonder-1-0","oid":"otis_mock_aime_2024_2025-db014dc22b4d2b93","rd":"2026-10-12","se":0.003883473863441843,"sid":"AFigcTGzpgZgMi6j2fpboa","sm":"muse-wonder-1.0_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAFigcTGzpgZgMi6j2fpboa.eval","v":0.9944444444444445},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-10-08","e":"max","evaluated_at":"2026-10-08T19:02:32.000Z","m":"deepseek-v4-1-flash","oid":"otis_mock_aime_2024_2025-075cbe0e20456028","rd":"2026-09-09","se":0.006405884289881728,"sid":"7eHNiVQCzLcAyj5LxmeeSU","sm":"deepseek-v4.1-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7eHNiVQCzLcAyj5LxmeeSU.eval","v":0.9833333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-10-07","e":"xhigh","evaluated_at":"2026-10-07T21:16:36.000Z","m":"claude-haiku-5-5","oid":"otis_mock_aime_2024_2025-e8cabbdfd24d748c","rd":"2026-10-07","se":0.005362818328685681,"sid":"iiEPrSjWXHH6Zq9CK4Zmcr","sm":"claude-haiku-5-5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiiEPrSjWXHH6Zq9CK4Zmcr.eval","v":0.9888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-10-07","e":"high","evaluated_at":"2026-10-07T21:16:25.000Z","m":"claude-haiku-5-5","oid":"otis_mock_aime_2024_2025-9d49b6a13b9eacb0","rd":"2026-10-07","se":0.013109441770118595,"sid":"LycbwGumF6Q7ySpmGY94XW","sm":"claude-haiku-5-5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLycbwGumF6Q7ySpmGY94XW.eval","v":0.9722222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T22:00:34.000Z","m":"gpt-6-1-sol","oid":"otis_mock_aime_2024_2025-3ead1bc8e523ce57","rd":"2026-09-29","se":0,"sid":"Di7y6sES4PGQbXNkXYhdqM","sm":"gpt-6.1-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDi7y6sES4PGQbXNkXYhdqM.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T15:15:14.000Z","m":"claude-sonnet-5-5","oid":"otis_mock_aime_2024_2025-2b295f6479963acb","rd":"2026-09-28","se":0,"sid":"bSEaZLPehiNVDQL7HP9EJS","sm":"claude-sonnet-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbSEaZLPehiNVDQL7HP9EJS.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T22:14:05.000Z","m":"gpt-6-luna","oid":"otis_mock_aime_2024_2025-647112943d51ff49","rd":"2026-09-22","se":0.006674027922425511,"sid":"4uDz9b6SosxDSZf7swMD9D","sm":"gpt-6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4uDz9b6SosxDSZf7swMD9D.eval","v":0.9888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-09-22","e":"xhigh","evaluated_at":"2026-09-22T21:49:22.000Z","m":"grok-4-7","oid":"otis_mock_aime_2024_2025-6791bd33a3bf32d2","rd":"2026-09-21","se":0.014299486306926387,"sid":"Jhgwm7QpspGcvm7DUVqUDg","sm":"grok-4.7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJhgwm7QpspGcvm7DUVqUDg.eval","v":0.9805555555555555},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T19:59:24.000Z","m":"gpt-6-sol","oid":"otis_mock_aime_2024_2025-cde79cd8d14f24a4","rd":"2026-09-22","se":0,"sid":"VZAUcNkzeMSMm9paxBhj6B","sm":"gpt-6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVZAUcNkzeMSMm9paxBhj6B.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T18:27:39.000Z","m":"claude-opus-5-5","oid":"otis_mock_aime_2024_2025-1428d8b612ff473e","rd":"2026-09-22","se":0,"sid":"LSfona7P3jCAnZFXJ2YA7F","sm":"claude-opus-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLSfona7P3jCAnZFXJ2YA7F.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-09-17","e":"xhigh","evaluated_at":"2026-09-17T13:30:27.000Z","m":"muse-spark-1-3","oid":"otis_mock_aime_2024_2025-1bac2b1d83a15905","rd":"2026-09-02","se":0.006154574548966636,"sid":"Wmxkx5m3qaQuwzem6gzZmg","sm":"muse-spark-1.3_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWmxkx5m3qaQuwzem6gzZmg.eval","v":0.9916666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-09-02","e":"xhigh","evaluated_at":"2026-09-02T18:04:58.000Z","m":"qwen3-8-max-0902","oid":"otis_mock_aime_2024_2025-0f3882be8455b5e4","rd":"2026-09-01","se":0,"sid":"2f8Vn5QBdJnc9i5y9jzXS7","sm":"qwen3.8-max-0902_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2f8Vn5QBdJnc9i5y9jzXS7.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-09-02","e":"high","evaluated_at":"2026-09-02T17:15:36.000Z","m":"gemini-3-8-flash","oid":"otis_mock_aime_2024_2025-29d41d2d2d80b902","rd":"2026-09-02","se":0.008724007409153888,"sid":"NeYuqgQyWtFWAHEjQYPm5Z","sm":"gemini-3.8-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNeYuqgQyWtFWAHEjQYPm5Z.eval","v":0.9888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-09-01","e":"max","evaluated_at":"2026-09-01T18:32:00.000Z","m":"claude-fable-5-1","oid":"otis_mock_aime_2024_2025-475c438ddcf83263","rd":"2026-09-01","se":0,"sid":"mRAASXsUuoJmc2oR5G89zR","sm":"claude-fable-5-1_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmRAASXsUuoJmc2oR5G89zR.eval","v":1},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:47:13.000Z","m":"granite-4-0-micro","oid":"otis_mock_aime_2024_2025-d4b4521a577a0e7f","rd":"2025-10-02","se":0.014262647723679736,"sid":"gVnaLSupMCiWmiScxfMLx3","sm":"granite-4.0-micro","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgVnaLSupMCiWmiScxfMLx3.eval","v":0.027777777777777776},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:46:30.000Z","m":"granite-4-0-1b","oid":"otis_mock_aime_2024_2025-1c05f61fe3646b32","rd":"2025-10-28","se":0.004700633956814717,"sid":"PiFfhBoFbnaoqX5z2gVvra","sm":"granite-4.0-1b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPiFfhBoFbnaoqX5z2gVvra.eval","v":0.008333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:45:56.000Z","m":"qwen3-5-35b-a3b","oid":"otis_mock_aime_2024_2025-6b2bdf5bce9af3c9","rd":"2026-02-24","se":0.06003996985174109,"sid":"h9aoDxyWEhybvARHpyuoco","sm":"qwen3.5-35b-a3b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fh9aoDxyWEhybvARHpyuoco.eval","v":0.7},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:45:36.000Z","m":"qwen3-5-35b-a3b","oid":"otis_mock_aime_2024_2025-1d1c83f023a37272","rd":"2026-02-24","se":0.061949030495634445,"sid":"9WS6kHXT8XFtJWtE2wWLAo","sm":"qwen3.5-35b-a3b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9WS6kHXT8XFtJWtE2wWLAo.eval","v":0.5444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:45:19.000Z","m":"qwen3-32b","oid":"otis_mock_aime_2024_2025-a97bcc16dc8502aa","rd":"2025-04-29","se":0.05148605337836994,"sid":"CfmvbqzjpF6cCJQfzqdD4w","sm":"Qwen3-32B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCfmvbqzjpF6cCJQfzqdD4w.eval","v":0.23055555555555557},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:44:49.000Z","m":"qwen3-32b","oid":"otis_mock_aime_2024_2025-1c6e58a1361d798e","rd":"2025-04-29","se":0.06040106323530329,"sid":"BAi4jidcT3F6reBkSXDJKv","sm":"Qwen3-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBAi4jidcT3F6reBkSXDJKv.eval","v":0.6694444444444444},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:44:45.000Z","m":"qwen3-30b-a3b-thinking-2507","oid":"otis_mock_aime_2024_2025-1f14e54eb361c0d2","rd":"2025-07-30","se":0.060418480757624834,"sid":"6rZFxNR3GjfhWey6DzEiYz","sm":"qwen3-30b-a3b-thinking-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6rZFxNR3GjfhWey6DzEiYz.eval","v":0.7027777777777777},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:44:11.000Z","m":"qwen3-30b-a3b-instruct-2507","oid":"otis_mock_aime_2024_2025-337e53c687b51dc3","rd":"2025-07-29","se":0.06509619058885623,"sid":"Eo5cAQJK28CHpKDfCSDYG4","sm":"qwen3-30b-a3b-instruct-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEo5cAQJK28CHpKDfCSDYG4.eval","v":0.6222222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:43:50.000Z","m":"qwen3-30b-a3b","oid":"otis_mock_aime_2024_2025-3a4526e9251237e8","rd":"2025-04-29","se":0.052095537290747576,"sid":"ZzbWv2QGVfg4o2DTX45nqV","sm":"Qwen3-30B-A3B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZzbWv2QGVfg4o2DTX45nqV.eval","v":0.25555555555555554},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-30","e":"none","evaluated_at":"2026-08-30T22:42:57.000Z","m":"qwen3-14b","oid":"otis_mock_aime_2024_2025-e4951c040e3ec5d0","rd":"2025-04-29","se":0.048232653761625936,"sid":"RMcys99fv45vvgTgakXXWn","sm":"qwen3-14b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRMcys99fv45vvgTgakXXWn.eval","v":0.25833333333333336},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:42:37.000Z","m":"qwen2-5-7b-instruct","oid":"otis_mock_aime_2024_2025-022fbb2dfffa85f7","rd":"2024-09-19","se":0.009401267913629434,"sid":"U8HHNLLLVHJ7sHCfwdiDBX","sm":"qwen2.5-7b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FU8HHNLLLVHJ7sHCfwdiDBX.eval","v":0.025},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:41:17.000Z","m":"mistral-small-3-2-2506","oid":"otis_mock_aime_2024_2025-79dec5e5150e3e49","rd":"2025-06-20","se":0.057774499712758075,"sid":"HyKzRUkV42yTmGrbYQvZxQ","sm":"mistral-small-3.2-2506","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHyKzRUkV42yTmGrbYQvZxQ.eval","v":0.30277777777777776},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:41:08.000Z","m":"mistral-small-3-1-2503","oid":"otis_mock_aime_2024_2025-ba110d58b94457fa","rd":"2025-03-17","se":0.01529498571765449,"sid":"nNZHFNyHgfv3uzPAxspaPT","sm":"mistral-small-3.1-2503","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnNZHFNyHgfv3uzPAxspaPT.eval","v":0.03888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:40:34.000Z","m":"mistral-small-3-2501","oid":"otis_mock_aime_2024_2025-f304d66f8151769a","rd":"2025-01-30","se":0.02682716849943299,"sid":"LEBmfabfasGXFwRwhN9j35","sm":"mistral-small-3-2501","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLEBmfabfasGXFwRwhN9j35.eval","v":0.06666666666666667},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:40:14.000Z","m":"mistral-7b-instruct-v0-3","oid":"otis_mock_aime_2024_2025-ca8b256a11bdb840","rd":"2024-05-27","se":0.002777777777777778,"sid":"j6Mo25oV9Y9jdGxCMHZfpU","sm":"Mistral-7B-Instruct-v0.3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fj6Mo25oV9Y9jdGxCMHZfpU.eval","v":0.002777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:39:41.000Z","m":"magistral-small-2509","oid":"otis_mock_aime_2024_2025-976f69d435b5fab7","rd":"2025-09-18","se":0.0556407175551872,"sid":"cSTYZ3FKp5Mbs3xopn68NK","sm":"magistral-small-2509","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcSTYZ3FKp5Mbs3xopn68NK.eval","v":0.28055555555555556},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:39:12.000Z","m":"llama-3-2-1b-instruct","oid":"otis_mock_aime_2024_2025-e0297d25df6ff9dc","rd":"2024-09-24","se":0.003883473863441843,"sid":"8qwT2CF9U5zV99PFirnVsp","sm":"Llama-3.2-1B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8qwT2CF9U5zV99PFirnVsp.eval","v":0.005555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:38:53.000Z","m":"meta-llama-3-8b-instruct","oid":"otis_mock_aime_2024_2025-9248968baad02aeb","rd":"2024-04-18","se":0.010477515876628662,"sid":"CAcNPzYBEu39HFWPmuUEeP","sm":"Meta-Llama-3-8B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCAcNPzYBEu39HFWPmuUEeP.eval","v":0.019444444444444445},{"b":"otis_mock_aime_2024_2025","d":"2026-08-30","evaluated_at":"2026-08-30T22:37:43.000Z","m":"glm-4-7-flash","oid":"otis_mock_aime_2024_2025-8649f2955083062d","se":0.06564078093085213,"sid":"9ekki5tJhyjmA7ZMLoUxzo","sm":"glm-4.7-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9ekki5tJhyjmA7ZMLoUxzo.eval","v":0.5833333333333334},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-30","e":"max","evaluated_at":"2026-08-30T14:57:43.000Z","m":"gpt-6-astra","oid":"otis_mock_aime_2024_2025-6bc23d074b225acc","rd":"2026-09-03","se":0,"sid":"L7kW8yxFKCjVXhxNnt3VTq","sm":"gpt-6-astra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FL7kW8yxFKCjVXhxNnt3VTq.eval","v":1},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T18:57:38.000Z","m":"deepseek-llm-67b-chat","oid":"otis_mock_aime_2024_2025-b2ccafd6a9c30fdd","rd":"2023-11-29","se":0.004700633956814719,"sid":"Y5rC5REnP83Q7mAoMtjiRq","sm":"deepseek-llm-67b-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY5rC5REnP83Q7mAoMtjiRq.eval","v":0.008333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T18:40:22.000Z","m":"seed-oss-36b-instruct","oid":"otis_mock_aime_2024_2025-a734097e00e8b0c0","rd":"2025-08-20","se":0.06324056203789373,"sid":"eJvV6RezECZwh3b4ekg4pm","sm":"seed-oss-36b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeJvV6RezECZwh3b4ekg4pm.eval","v":0.675},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T18:33:35.000Z","m":"qwen3-5-4b","oid":"otis_mock_aime_2024_2025-8cda84397bd4683a","rd":"2026-02-24","se":0.0616226296489243,"sid":"SAZy8rk2HgyPjA8XZJFUmR","sm":"qwen3.5-4b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSAZy8rk2HgyPjA8XZJFUmR.eval","v":0.5583333333333333},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T17:48:22.000Z","m":"qwen3-4b-instruct-2507","oid":"otis_mock_aime_2024_2025-3b3f0a7b689e8f98","rd":"2025-08-06","se":0.06591537955214949,"sid":"KH7KfZvFc9kZbqiJFdfYTc","sm":"qwen3-4b-instruct-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKH7KfZvFc9kZbqiJFdfYTc.eval","v":0.5222222222222223},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T17:03:13.000Z","m":"qwen3-30b-a3b","oid":"otis_mock_aime_2024_2025-5af9c0b9a84e8728","rd":"2025-04-29","se":0.062118645754659016,"sid":"XfjRkSyP4ub4gDsWcgS7KY","sm":"Qwen3-30B-A3B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXfjRkSyP4ub4gDsWcgS7KY.eval","v":0.6277777777777778},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T16:54:14.000Z","m":"qwen3-14b","oid":"otis_mock_aime_2024_2025-9dc8ea337b2811e5","rd":"2025-04-29","se":0.06190655406811223,"sid":"gRRQHECkYhz5UaYsRKDh2F","sm":"qwen3-14b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgRRQHECkYhz5UaYsRKDh2F.eval","v":0.6638888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T16:47:54.000Z","m":"qwen3-1-7b","oid":"otis_mock_aime_2024_2025-c2a3eefbc8ab2b48","rd":"2025-04-29","se":0.0281353251405273,"sid":"bbqokudairnKtbyjBjuWCx","sm":"qwen3-1.7b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbbqokudairnKtbyjBjuWCx.eval","v":0.08055555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-28","e":"minimal","evaluated_at":"2026-08-28T12:55:56.000Z","m":"gemma-4-26b-a4b-it","oid":"otis_mock_aime_2024_2025-12c84875e59b2fb9","rd":"2026-04-02","se":0.05763774795025092,"sid":"mstwSH7noJfvhGcd6Lz5kd","sm":"gemma-4-26b-a4b-it_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmstwSH7noJfvhGcd6Lz5kd.eval","v":0.8222222222222222},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T12:37:51.000Z","m":"granite-4-0-350m","oid":"otis_mock_aime_2024_2025-e24c2e4085c849c1","rd":"2025-10-28","se":0.002777777777777778,"sid":"aQYiADNH3qVzZkWUmQn6nZ","sm":"granite-4.0-350m","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaQYiADNH3qVzZkWUmQn6nZ.eval","v":0.002777777777777778},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-28","e":"none","evaluated_at":"2026-08-28T11:17:11.000Z","m":"glm-4-7-flash","oid":"otis_mock_aime_2024_2025-59f363f48306c1ac","se":0.049460472943941645,"sid":"fPE4Zajhq5WfccSxEYXXR7","sm":"glm-4.7-flash_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfPE4Zajhq5WfccSxEYXXR7.eval","v":0.25},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T08:35:58.000Z","m":"gemma-3-4b-it","oid":"otis_mock_aime_2024_2025-a1fb4ab83e0d0d24","rd":"2025-03-12","se":0.02763853991962833,"sid":"KAPGpH2CLMafAt8MnAQJT3","sm":"gemma-3-4b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKAPGpH2CLMafAt8MnAQJT3.eval","v":0.075},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T08:28:18.000Z","m":"gemma-3-1b-it","oid":"otis_mock_aime_2024_2025-3b23cc30307fc888","rd":"2025-03-12","se":0.005362818328685681,"sid":"3CGChL5vNDM6Df5C8Z8owN","sm":"gemma-3-1b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3CGChL5vNDM6Df5C8Z8owN.eval","v":0.011111111111111112},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T08:22:55.000Z","m":"gemma-3-12b-it","oid":"otis_mock_aime_2024_2025-5ac1ec18d2e1e7c4","rd":"2025-03-12","se":0.0424174778156649,"sid":"CAyLyhDHZiMQBZDB3qd8tG","sm":"gemma-3-12b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCAyLyhDHZiMQBZDB3qd8tG.eval","v":0.16666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T08:15:43.000Z","m":"deepseek-r1-distill-qwen-32b","oid":"otis_mock_aime_2024_2025-6dad28ded042a48e","rd":"2025-01-20","se":0.061999963795652153,"sid":"KbUNQf8nnwPa8ixg6AnEwB","sm":"DeepSeek-R1-Distill-Qwen-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKbUNQf8nnwPa8ixg6AnEwB.eval","v":0.5555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T08:02:29.000Z","m":"deepseek-r1-distill-qwen-14b","oid":"otis_mock_aime_2024_2025-d5effcdcb41acd00","rd":"2025-01-20","se":0.05984981671790839,"sid":"EceR3NeHzSxacdmAmfRxgT","sm":"DeepSeek-R1-Distill-Qwen-14B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEceR3NeHzSxacdmAmfRxgT.eval","v":0.5055555555555555},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T07:53:04.000Z","m":"deepseek-r1-distill-qwen-1-5b","oid":"otis_mock_aime_2024_2025-15f2ecb9cfff8994","rd":"2025-01-20","se":0.05279771021800918,"sid":"Gs234P3jzGiLm68KjyzYZL","sm":"DeepSeek-R1-Distill-Qwen-1.5B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGs234P3jzGiLm68KjyzYZL.eval","v":0.21388888888888888},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T07:49:02.000Z","m":"deepseek-r1-0528-qwen3-8b","oid":"otis_mock_aime_2024_2025-72c507e11ba59d7e","rd":"2025-05-28","se":0.0677262181766212,"sid":"LWbCXNGMXtjQYxyM2uoFMV","sm":"deepseek-r1-0528-qwen3-8b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLWbCXNGMXtjQYxyM2uoFMV.eval","v":0.4388888888888889},{"b":"otis_mock_aime_2024_2025","d":"2026-08-28","evaluated_at":"2026-08-28T07:20:55.000Z","m":"qwq-32b","oid":"otis_mock_aime_2024_2025-2ea28ca1f0cd7a52","rd":"2025-03-05","se":0.06348963965431165,"sid":"2xMn6dbY9mw4pmkh8Tz9Gn","sm":"QwQ-32B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2xMn6dbY9mw4pmkh8Tz9Gn.eval","v":0.5916666666666667},{"b":"otis_mock_aime_2024_2025","d":"2026-08-27","evaluated_at":"2026-08-27T16:29:11.000Z","m":"llama-3-1-8b-instruct","oid":"otis_mock_aime_2024_2025-0b91663db7faf836","rd":"2024-07-23","se":0.009401267913629434,"sid":"6YAzFq89fwWiJ3uJdptCMr","sm":"Llama-3.1-8B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6YAzFq89fwWiJ3uJdptCMr.eval","v":0.016666666666666666},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T14:31:08.000Z","m":"qwen3-5-9b","oid":"otis_mock_aime_2024_2025-13090a70b9ebc2e6","rd":"2026-02-24","se":0.06003996985174109,"sid":"YBMTDnW5LmCN6tBp2QYHzi","sm":"qwen3.5-9B_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYBMTDnW5LmCN6tBp2QYHzi.eval","v":0.6166666666666667},{"b":"otis_mock_aime_2024_2025","d":"2026-08-27","evaluated_at":"2026-08-27T14:30:39.000Z","m":"qwen3-5-9b","oid":"otis_mock_aime_2024_2025-06e2772f76f86eb8","rd":"2026-02-24","se":0.06533819469304575,"sid":"5FVmQUSS4uRd23f6nNgeRe","sm":"qwen3.5-9B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5FVmQUSS4uRd23f6nNgeRe.eval","v":0.42777777777777776},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-27","e":"none","evaluated_at":"2026-08-27T14:22:01.000Z","m":"qwen3-8b","oid":"otis_mock_aime_2024_2025-f8485b13fe448f17","rd":"2025-04-28","se":0.0452770806168054,"sid":"ASLdzByzxxMxFeLc7dwTPi","sm":"qwen3-8b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FASLdzByzxxMxFeLc7dwTPi.eval","v":0.2222222222222222},{"b":"otis_mock_aime_2024_2025","d":"2026-08-27","evaluated_at":"2026-08-27T14:21:53.000Z","m":"qwen3-8b","oid":"otis_mock_aime_2024_2025-53605705dadc7ea7","rd":"2025-04-28","se":0.0645035213369854,"sid":"gG2B6fLUDBy5jRraxL2pz8","sm":"qwen3-8b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgG2B6fLUDBy5jRraxL2pz8.eval","v":0.5611111111111111},{"b":"otis_mock_aime_2024_2025","d":"2026-08-27","evaluated_at":"2026-08-27T14:17:59.000Z","m":"gemma-3-27b-it","oid":"otis_mock_aime_2024_2025-0b7b076b89cc7077","rd":"2025-03-12","se":0.052794388667238824,"sid":"DhJ248BX3ZPBHXvkRG4Qsz","sm":"gemma-3-27b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDhJ248BX3ZPBHXvkRG4Qsz.eval","v":0.225},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T14:13:32.000Z","m":"gpt-oss-20b","oid":"otis_mock_aime_2024_2025-444d0401da41541c","rd":"2025-08-05","se":0.06711496995394688,"sid":"GMngscbSmqoPbGZWNv2Jvw","sm":"gpt-oss-20b_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGMngscbSmqoPbGZWNv2Jvw.eval","v":0.5083333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T14:12:40.000Z","m":"gpt-oss-20b","oid":"otis_mock_aime_2024_2025-d336fc861b129914","rd":"2025-08-05","se":0.0593111883162336,"sid":"DB8ja6pEZWGocQbdciyisL","sm":"gpt-oss-20b_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDB8ja6pEZWGocQbdciyisL.eval","v":0.6527777777777778},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T14:12:25.000Z","m":"gpt-oss-20b","oid":"otis_mock_aime_2024_2025-bd599d9e5ae22212","rd":"2025-08-05","se":0.06317675132490254,"sid":"cMwETYmC6b4yKupyJfEHv2","sm":"gpt-oss-20b_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcMwETYmC6b4yKupyJfEHv2.eval","v":0.4027777777777778},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-26","e":"max","evaluated_at":"2026-08-26T22:26:55.000Z","m":"glm-5-3-flash","oid":"otis_mock_aime_2024_2025-fa8e7850a360527b","rd":"2026-08-20","se":0.021622267380853318,"sid":"NCikvVWc5SEyMEbCqvVNb2","sm":"glm-5.3-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNCikvVWc5SEyMEbCqvVNb2.eval","v":0.9388888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-24","e":"max","evaluated_at":"2026-08-24T19:44:18.000Z","m":"glm-5-3","oid":"otis_mock_aime_2024_2025-2cdf5fbbe8f3ef9a","rd":"2026-08-14","se":0.03454754040013868,"sid":"goZR5yGxZic9TZ23ezCL3w","sm":"glm-5.3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgoZR5yGxZic9TZ23ezCL3w.eval","v":0.9111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-18","e":"max","evaluated_at":"2026-08-18T21:03:23.000Z","m":"deepseek-v4-pro-0813","oid":"otis_mock_aime_2024_2025-c89a107b2c6a4945","rd":"2026-08-13","se":0.00990980650017605,"sid":"4HVvM9ZWKKPLQPcC8WYYMT","sm":"deepseek-v4-pro-0813_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4HVvM9ZWKKPLQPcC8WYYMT.eval","v":0.9861111111111112},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-14","e":"high","evaluated_at":"2026-08-14T18:29:12.000Z","m":"gemini-3-7-flash","oid":"otis_mock_aime_2024_2025-7630a6d7c5e78ed1","rd":"2026-08-13","se":0.018586784946171872,"sid":"kQjrYCaa7yYyMXofeR23Dd","sm":"gemini-3.7-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkQjrYCaa7yYyMXofeR23Dd.eval","v":0.9722222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-08-14","e":"xhigh","evaluated_at":"2026-08-14T14:19:47.000Z","m":"inkling-small","oid":"otis_mock_aime_2024_2025-2a7b8cdecb5aca66","rd":"2026-07-15","se":0.03964807305493795,"sid":"53BDFebxiT9KGeP5FRt6bJ","sm":"inkling-small_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F53BDFebxiT9KGeP5FRt6bJ.eval","v":0.9},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-08-14","e":"xhigh","evaluated_at":"2026-08-14T13:47:42.000Z","m":"grok-4-6","oid":"otis_mock_aime_2024_2025-62694e5f7e6ad046","rd":"2026-08-12","se":0.004700633956814717,"sid":"7crLvnfQc987dC6mr3iGbf","sm":"grok-4.6_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7crLvnfQc987dC6mr3iGbf.eval","v":0.9916666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-12","e":"high","evaluated_at":"2026-08-12T19:39:52.000Z","m":"grok-4-6","oid":"otis_mock_aime_2024_2025-ef6cba7d052f4745","rd":"2026-08-12","se":0.00996275330534286,"sid":"jtfdCURnwfZzak4BSiBKzx","sm":"grok-4.6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjtfdCURnwfZzak4BSiBKzx.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2026-08-10","evaluated_at":"2026-08-10T17:47:10.000Z","m":"nemotron-3-ultra","oid":"otis_mock_aime_2024_2025-5fe9304f1deae35d","rd":"2026-06-04","se":0.051247074319053836,"sid":"WYTWr3Xk4qxymhMsm7vRQ2","sm":"nemotron-3-ultra","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWYTWr3Xk4qxymhMsm7vRQ2.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-10","e":"none","evaluated_at":"2026-08-10T17:46:50.000Z","m":"minimax-m3","oid":"otis_mock_aime_2024_2025-8edd23696d31f090","rd":"2026-06-01","se":0.06666666666666668,"sid":"97t77hRrNae6Va8fBqBhPy","sm":"MiniMax-M3_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F97t77hRrNae6Va8fBqBhPy.eval","v":0.26666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2026-08-10","evaluated_at":"2026-08-10T17:46:09.000Z","m":"minimax-m3","oid":"otis_mock_aime_2024_2025-c483500d79192fd5","rd":"2026-06-01","se":0.0683294324254051,"sid":"79ZP6iwoBaF5t9a5VRqPnw","sm":"MiniMax-M3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F79ZP6iwoBaF5t9a5VRqPnw.eval","v":0.7111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-10","e":"none","evaluated_at":"2026-08-10T17:45:48.000Z","m":"glm-5-2","oid":"otis_mock_aime_2024_2025-9832a6250ae6a673","rd":"2026-06-16","se":0.06832943242540508,"sid":"J8hYKbXiGhetQRwu5bUSp6","sm":"glm-5.2_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJ8hYKbXiGhetQRwu5bUSp6.eval","v":0.28888888888888886},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-10","e":"low","evaluated_at":"2026-08-10T17:45:24.000Z","m":"glm-5-2","oid":"otis_mock_aime_2024_2025-351055080c7d6dcd","rd":"2026-06-16","se":0.06478835438717,"sid":"9uKmMPn6eiRPsfcLKod7M7","sm":"glm-5.2_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9uKmMPn6eiRPsfcLKod7M7.eval","v":0.7555555555555555},{"b":"otis_mock_aime_2024_2025","d":"2026-08-10","evaluated_at":"2026-08-10T17:44:52.000Z","m":"glm-5-1","oid":"otis_mock_aime_2024_2025-92c366cc53be74de","rd":"2026-04-07","se":0.03760507165451774,"sid":"b7Lw9ExxAhVY3hdExoh4MP","sm":"glm-5.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fb7Lw9ExxAhVY3hdExoh4MP.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-08-07","e":"xhigh","evaluated_at":"2026-08-07T00:56:59.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"otis_mock_aime_2024_2025-35dc4b44266b078c","rd":"2026-03-17","se":0.047377936967913425,"sid":"QGMk6JNCYuUkFMF4zdVCsM","sm":"gpt-5.4-mini-2026-03-17_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQGMk6JNCYuUkFMF4zdVCsM.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:54:12.000Z","m":"gpt-5-6-sol","oid":"otis_mock_aime_2024_2025-5d2f72fc36979a76","rd":"2026-07-09","se":0.0697920592732311,"sid":"g7czyLHudHegzP9yRD939X","sm":"gpt-5.6-sol_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fg7czyLHudHegzP9yRD939X.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:53:58.000Z","m":"gpt-5-5","oid":"otis_mock_aime_2024_2025-268320dfbdd44aa9","rd":"2026-04-23","se":0.07446027270295805,"sid":"QuqPmMUTbiEsUvoFYZx5Mk","sm":"gpt-5.5_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQuqPmMUTbiEsUvoFYZx5Mk.eval","v":0.5777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:46:06.000Z","m":"kimi-k3","oid":"otis_mock_aime_2024_2025-61524da4d1212f9e","rd":"2026-07-16","se":0.06979205927323111,"sid":"frszScNxpHXXjSHawCt9Zb","sm":"kimi-k3_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfrszScNxpHXXjSHawCt9Zb.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-07","e":"high","evaluated_at":"2026-08-07T00:46:05.000Z","m":"kimi-k3","oid":"otis_mock_aime_2024_2025-30bcb0da7838ef36","rd":"2026-07-16","se":0.03760507165451775,"sid":"4E5xGsuRMwMBhAfC4F2G4N","sm":"kimi-k3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4E5xGsuRMwMBhAfC4F2G4N.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:42:38.000Z","m":"gpt-5-6-terra","oid":"otis_mock_aime_2024_2025-1a6f7c5829475035","rd":"2026-07-09","se":0.07521014330903548,"sid":"cjUVqF2qQJ4qhFKBU5BQpW","sm":"gpt-5.6-terra_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcjUVqF2qQJ4qhFKBU5BQpW.eval","v":0.5333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:42:28.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"otis_mock_aime_2024_2025-29511392280a7ba6","rd":"2026-03-17","se":0.06666666666666668,"sid":"R4NrJwtfdSQkGCWyKnrxgh","sm":"gpt-5.4-mini-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR4NrJwtfdSQkGCWyKnrxgh.eval","v":0.26666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:42:24.000Z","m":"qwen3-6-max-preview","oid":"otis_mock_aime_2024_2025-b95d7b47a805476d","rd":"2026-04-20","se":0.04290254662948546,"sid":"EfJpx5HztMmL5EQSwsyqMe","sm":"qwen3.6-max-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEfJpx5HztMmL5EQSwsyqMe.eval","v":0.9111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:42:21.000Z","m":"gpt-5-6-sol","oid":"otis_mock_aime_2024_2025-92944ac58ca60977","rd":"2026-07-09","se":0.031067790907534736,"sid":"Mifz96Jm442bn43Qnxj5rT","sm":"gpt-5.6-sol_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMifz96Jm442bn43Qnxj5rT.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:39:05.000Z","m":"gpt-5-5","oid":"otis_mock_aime_2024_2025-0e64e25d253053c9","rd":"2026-04-23","se":0.05463890236888291,"sid":"CdmtwZ8NR8YeTXWiLVTRDJ","sm":"gpt-5.5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCdmtwZ8NR8YeTXWiLVTRDJ.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:39:02.000Z","m":"qwen3-7-max","oid":"otis_mock_aime_2024_2025-0a8b7defd5241481","rd":"2026-05-19","se":0.031067790907534736,"sid":"D3YDUZSfEUDsS4frLbEKDu","sm":"qwen3.7-max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FD3YDUZSfEUDsS4frLbEKDu.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2026-08-07","e":"medium","evaluated_at":"2026-08-07T00:28:27.000Z","m":"o3-2025-04-16","oid":"otis_mock_aime_2024_2025-466a98b05aa2846e","rd":"2025-04-16","se":0.05463890236888291,"sid":"8GjRH3T4wVQMnBKX4fv4ub","sm":"o3-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8GjRH3T4wVQMnBKX4fv4ub.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:24:39.000Z","m":"gpt-5-1-2025-11-13","oid":"otis_mock_aime_2024_2025-e61a29467a0b6c39","rd":"2025-11-13","se":0.07309112127323451,"sid":"H5NgPXUD57sbusTLp3yKQE","sm":"gpt-5.1-2025-11-13_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH5NgPXUD57sbusTLp3yKQE.eval","v":0.37777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:20:43.000Z","m":"kimi-k2-7-code","oid":"otis_mock_aime_2024_2025-76adb6f3f96b080e","rd":"2026-06-12","se":0.031067790907534736,"sid":"ENc5ZM5TpBYNKkTWN4xyQM","sm":"kimi-k2.7-code","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FENc5ZM5TpBYNKkTWN4xyQM.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:20:21.000Z","m":"gpt-5-6-terra","oid":"otis_mock_aime_2024_2025-8348e9c084b5887a","rd":"2026-07-09","se":0.04737793696791343,"sid":"Z6S3nMC2tnkpKUixCPLiDj","sm":"gpt-5.6-terra_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ6S3nMC2tnkpKUixCPLiDj.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:20:15.000Z","m":"gpt-5-6-luna","oid":"otis_mock_aime_2024_2025-22e40cf283f7bd12","rd":"2026-07-09","se":0.07385489458759964,"sid":"QYZ6wRuT7WMkaYPKc9dRPy","sm":"gpt-5.6-luna_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQYZ6wRuT7WMkaYPKc9dRPy.eval","v":0.4},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:16:48.000Z","m":"qwen3-6-plus","oid":"otis_mock_aime_2024_2025-941081ab7677fcb8","rd":"2026-03-31","se":0.03760507165451774,"sid":"jYiGAFfUQKJuRStPzU3PX3","sm":"qwen3.6-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjYiGAFfUQKJuRStPzU3PX3.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2026-08-07","e":"medium","evaluated_at":"2026-08-07T00:16:13.000Z","m":"o4-mini-2025-04-16","oid":"otis_mock_aime_2024_2025-d757c9d9c58b76c4","rd":"2025-04-16","se":0.06666666666666667,"sid":"TWJtwC5bzXa9HorTH8DNJy","sm":"o4-mini-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTWJtwC5bzXa9HorTH8DNJy.eval","v":0.7333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:12:36.000Z","m":"qwen3-6-27b","oid":"otis_mock_aime_2024_2025-f355eb8691012a19","rd":"2026-04-22","se":0.04290254662948546,"sid":"452pB5w5ea6uy92e2q2NMv","sm":"qwen3.6-27b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F452pB5w5ea6uy92e2q2NMv.eval","v":0.9111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:12:23.000Z","m":"qwen3-6-27b","oid":"otis_mock_aime_2024_2025-8cd4170769beb1ab","rd":"2026-04-22","se":0.07106690545187015,"sid":"o6VtA8qruLpoShhPy2JdLg","sm":"qwen3.6-27b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo6VtA8qruLpoShhPy2JdLg.eval","v":0.6666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:12:18.000Z","m":"qwen3-5-397b-a17b","oid":"otis_mock_aime_2024_2025-d2ec2f8ad1658dfc","rd":"2026-02-13","se":0.04737793696791343,"sid":"JYuFdyZZSiMvmSZNzMTywb","sm":"qwen3.5-397b-a17b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJYuFdyZZSiMvmSZNzMTywb.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:12:04.000Z","m":"qwen3-5-397b-a17b","oid":"otis_mock_aime_2024_2025-6a6f107044e93f45","rd":"2026-02-13","se":0.057637747950250916,"sid":"i2ZUgMREKiFAnE2M54w3Py","sm":"qwen3.5-397b-a17b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi2ZUgMREKiFAnE2M54w3Py.eval","v":0.8222222222222222},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:11:58.000Z","m":"qwen3-5-plus","oid":"otis_mock_aime_2024_2025-ffb915a93ad5b0cd","rd":"2026-02-16","se":0.051247074319053836,"sid":"cFbntXuxUtfreULvt7vEt7","sm":"qwen3.5-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcFbntXuxUtfreULvt7vEt7.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-07","e":"minimal","evaluated_at":"2026-08-07T00:07:32.000Z","m":"gemini-3-6-flash","oid":"otis_mock_aime_2024_2025-74c9c29133e48035","rd":"2026-07-21","se":0.06030226891555273,"sid":"cF97JqHbFxxp7ePjYVK9HV","sm":"gemini-3.6-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcF97JqHbFxxp7ePjYVK9HV.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:07:20.000Z","m":"gemini-3-6-flash","oid":"otis_mock_aime_2024_2025-71adc69ffe9301a1","rd":"2026-07-21","se":0.057637747950250916,"sid":"6iEkMa88YjLDgcde4bLm8B","sm":"gemini-3.6-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6iEkMa88YjLDgcde4bLm8B.eval","v":0.8222222222222222},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:03:16.000Z","m":"qwen3-6-35b-a3b","oid":"otis_mock_aime_2024_2025-244739e5aa4abb96","rd":"2026-04-14","se":0.051247074319053836,"sid":"3n5KDcBaKqJh3WTg7qU2rs","sm":"qwen3.6-35b-a3b","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3n5KDcBaKqJh3WTg7qU2rs.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:03:06.000Z","m":"qwen3-6-35b-a3b","oid":"otis_mock_aime_2024_2025-b281488bf02de625","rd":"2026-04-14","se":0.06979205927323111,"sid":"7GiYWM7UagUsHpNsdymd4z","sm":"qwen3.6-35b-a3b_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7GiYWM7UagUsHpNsdymd4z.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:02:57.000Z","m":"qwen3-7-plus","oid":"otis_mock_aime_2024_2025-f51d5418d64392f5","rd":"2026-06-02","se":0.03760507165451775,"sid":"Jvdw6GAXiS6T79bkpbk6Ra","sm":"qwen3.7-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJvdw6GAXiS6T79bkpbk6Ra.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:02:56.000Z","m":"qwen3-7-plus","oid":"otis_mock_aime_2024_2025-633e21cc5083bcbf","rd":"2026-06-02","se":0.06030226891555273,"sid":"o5th8Rb9WiaDbv4oEMDoFa","sm":"qwen3.7-plus_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fo5th8Rb9WiaDbv4oEMDoFa.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:02:46.000Z","m":"qwen3-7-flash","oid":"otis_mock_aime_2024_2025-374c204c7f4e756f","rd":"2026-07-27","se":0.051247074319053836,"sid":"TjoS7sNuqVTc4LVyGRo4wa","sm":"qwen3.7-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTjoS7sNuqVTc4LVyGRo4wa.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:02:33.000Z","m":"qwen3-7-flash","oid":"otis_mock_aime_2024_2025-8b70a949ecc2c0b2","rd":"2026-07-27","se":0.06267511942419625,"sid":"SZDgstFQJAGeGQwMXgNx6w","sm":"qwen3.7-flash_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSZDgstFQJAGeGQwMXgNx6w.eval","v":0.7777777777777778},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:02:15.000Z","m":"gpt-5-6-luna","oid":"otis_mock_aime_2024_2025-e6dfa257e7722823","rd":"2026-07-09","se":0.07106690545187014,"sid":"ZgU8BagFmaj7BE94y3JDZA","sm":"gpt-5.6-luna_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZgU8BagFmaj7BE94y3JDZA.eval","v":0.6666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:02:08.000Z","m":"qwen3-6-flash","oid":"otis_mock_aime_2024_2025-b32e5c71b933b733","rd":"2026-04-27","se":0.05463890236888291,"sid":"9zx2EXPTvPmxsL9qFCjP3x","sm":"qwen3.6-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9zx2EXPTvPmxsL9qFCjP3x.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-07","e":"none","evaluated_at":"2026-08-07T00:01:03.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"otis_mock_aime_2024_2025-f19d7a65192f0e3c","rd":"2026-03-17","se":0.07521014330903548,"sid":"eUR9o3bvU2Yu3KY5w5uk4j","sm":"gpt-5.4-nano-2026-03-17_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeUR9o3bvU2Yu3KY5w5uk4j.eval","v":0.4666666666666667},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:00:50.000Z","m":"gpt-3-5-turbo-0125","oid":"otis_mock_aime_2024_2025-2e4d63336d112c42","rd":"2024-01-25","se":0.022222222222222223,"sid":"7ZVhkukwaRXU2Sx2ttibPF","sm":"gpt-3.5-turbo-0125","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7ZVhkukwaRXU2Sx2ttibPF.eval","v":0.022222222222222223},{"b":"otis_mock_aime_2024_2025","d":"2026-08-07","evaluated_at":"2026-08-07T00:00:21.000Z","m":"qwen3-5-flash","oid":"otis_mock_aime_2024_2025-0d546c42f2ae348c","rd":"2026-02-25","se":0.05463890236888291,"sid":"awDKyKbPXRzBVZzfdQucef","sm":"qwen3.5-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FawDKyKbPXRzBVZzfdQucef.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-07","e":"minimal","evaluated_at":"2026-08-07T00:00:04.000Z","m":"gpt-5-mini-2025-08-07","oid":"otis_mock_aime_2024_2025-03c4464fb2a300ef","rd":"2025-08-07","se":0.07491109582924914,"sid":"7VLdaSiAanBwM9TJUB5BhU","sm":"gpt-5-mini-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7VLdaSiAanBwM9TJUB5BhU.eval","v":0.5555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-07","e":"low","evaluated_at":"2026-08-07T00:00:00.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"otis_mock_aime_2024_2025-5651b1dbde1df189","rd":"2026-03-17","se":0.06979205927323111,"sid":"bxvsuZ4Xpi3gZ4rsA8T7kV","sm":"gpt-5.4-nano-2026-03-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbxvsuZ4Xpi3gZ4rsA8T7kV.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T23:59:45.000Z","m":"gemma-4-31b-it","oid":"otis_mock_aime_2024_2025-b9926b7e5d0f2377","rd":"2026-04-02","se":0.06666666666666667,"sid":"jFJucVunfLgoQqoFvPVQWP","sm":"gemma-4-31b-it_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjFJucVunfLgoQqoFvPVQWP.eval","v":0.7333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T23:59:11.000Z","m":"gemini-3-1-pro-preview","oid":"otis_mock_aime_2024_2025-ec69ca4697c19cd8","rd":"2026-02-19","se":0.031067790907534736,"sid":"nhim2QCRgpir5cZdqrmUz3","sm":"gemini-3.1-pro-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fnhim2QCRgpir5cZdqrmUz3.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-06","e":"none","evaluated_at":"2026-08-06T23:59:05.000Z","m":"deepseek-v4-pro","oid":"otis_mock_aime_2024_2025-c667b0c362e2505e","rd":"2026-04-24","se":0.07521014330903548,"sid":"7bwsMdahVHrkaiN5x49Dk7","sm":"deepseek-v4-pro_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7bwsMdahVHrkaiN5x49Dk7.eval","v":0.4666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T23:58:58.000Z","m":"deepseek-v4-pro","oid":"otis_mock_aime_2024_2025-df21435244adcf70","rd":"2026-04-24","se":0.031067790907534743,"sid":"GLZkYFDFDENRiHyL68Nq43","sm":"deepseek-v4-pro_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGLZkYFDFDENRiHyL68Nq43.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T23:58:17.000Z","m":"claude-opus-4-6","oid":"otis_mock_aime_2024_2025-243a204068230ac4","rd":"2026-02-05","se":0.04290254662948546,"sid":"coxYk5vK5cBhQ7iZohMtA5","sm":"claude-opus-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcoxYk5vK5cBhQ7iZohMtA5.eval","v":0.9111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:36:02.000Z","m":"gemini-3-5-flash-lite","oid":"otis_mock_aime_2024_2025-daf9dc09b03be241","rd":"2026-07-21","se":0.07385489458759963,"sid":"B3Kno3buYWRYbZxU33STiX","sm":"gemini-3.5-flash-lite_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FB3Kno3buYWRYbZxU33STiX.eval","v":0.6},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T22:35:57.000Z","m":"gemini-3-5-flash-lite","oid":"otis_mock_aime_2024_2025-234c1cefb17ba5ac","rd":"2026-07-21","se":0.07535922203472521,"sid":"ivr2LfmdZoCMfLME7jh2L8","sm":"gemini-3.5-flash-lite_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fivr2LfmdZoCMfLME7jh2L8.eval","v":0.5111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:35:36.000Z","m":"gemini-3-5-flash-lite","oid":"otis_mock_aime_2024_2025-b0b46adaff4a9194","rd":"2026-07-21","se":0.0683294324254051,"sid":"mqsc4vFuKtAHXxbYyX9WwK","sm":"gemini-3.5-flash-lite_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fmqsc4vFuKtAHXxbYyX9WwK.eval","v":0.7111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:34:49.000Z","m":"gemini-3-5-flash","oid":"otis_mock_aime_2024_2025-2cffb36b69bbf92e","rd":"2026-05-19","se":0.047377936967913425,"sid":"ig7gQ23WRms9SQLLNPtbTi","sm":"gemini-3.5-flash_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fig7gQ23WRms9SQLLNPtbTi.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-08-06","e":"minimal","evaluated_at":"2026-08-06T22:30:17.000Z","m":"gemini-3-1-flash-lite","oid":"otis_mock_aime_2024_2025-c7c5719472c46a57","rd":"2026-03-03","se":0.07309112127323451,"sid":"EoSFpgFhJB7iDPLcMNuLH4","sm":"gemini-3.1-flash-lite_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEoSFpgFhJB7iDPLcMNuLH4.eval","v":0.37777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:30:07.000Z","m":"gemini-3-1-flash-lite","oid":"otis_mock_aime_2024_2025-f847ffef176e3e62","rd":"2026-03-03","se":0.07491109582924914,"sid":"CFBCC2QrkCAFBAvFj5pqPF","sm":"gemini-3.1-flash-lite_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCFBCC2QrkCAFBAvFj5pqPF.eval","v":0.4444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:29:57.000Z","m":"gemini-3-1-flash-lite","oid":"otis_mock_aime_2024_2025-5af67516b5c81640","rd":"2026-03-03","se":0.06030226891555273,"sid":"NMdaAXyqqPRmWfuxZUfHFF","sm":"gemini-3.1-flash-lite_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNMdaAXyqqPRmWfuxZUfHFF.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:25:57.000Z","m":"gemini-3-flash-preview","oid":"otis_mock_aime_2024_2025-974eb9aaa30cdddc","rd":"2025-12-17","se":0.031067790907534743,"sid":"a4uE6KgMhGSUjcnemRnbkA","sm":"gemini-3-flash-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fa4uE6KgMhGSUjcnemRnbkA.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:21:48.000Z","m":"claude-sonnet-5","oid":"otis_mock_aime_2024_2025-00abfa60a5655716","rd":"2026-06-30","se":0.06030226891555273,"sid":"XRutLYUVUePe69APTSTw8S","sm":"claude-sonnet-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXRutLYUVUePe69APTSTw8S.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:21:20.000Z","m":"claude-opus-5","oid":"otis_mock_aime_2024_2025-9e51748588d4fda4","rd":"2026-07-24","se":0.03760507165451774,"sid":"Z5FAHm8aqmk8NUUm7tojng","sm":"claude-opus-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ5FAHm8aqmk8NUUm7tojng.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:21:16.000Z","m":"claude-sonnet-4-6","oid":"otis_mock_aime_2024_2025-53d64746eb884f1b","rd":"2026-02-17","se":0.06832943242540508,"sid":"XnQVH5qSfxVjpzoPwUEmnh","sm":"claude-sonnet-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXnQVH5qSfxVjpzoPwUEmnh.eval","v":0.7111111111111111},{"b":"otis_mock_aime_2024_2025","d":"2026-08-06","evaluated_at":"2026-08-06T22:21:15.000Z","m":"claude-opus-5","oid":"otis_mock_aime_2024_2025-d2f3afc22620f3d8","rd":"2026-07-24","se":0.022222222222222223,"sid":"gsBidnvEcmCfLnn5sxePtk","sm":"claude-opus-5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgsBidnvEcmCfLnn5sxePtk.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-08-06","e":"none","evaluated_at":"2026-08-06T22:21:06.000Z","m":"claude-opus-4-8","oid":"otis_mock_aime_2024_2025-7c11f21f84d03cc5","rd":"2026-05-28","se":0.05463890236888291,"sid":"adGE8rXdNuUFs96ioppbc5","sm":"claude-opus-4-8_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FadGE8rXdNuUFs96ioppbc5.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:20:24.000Z","m":"claude-opus-4-8","oid":"otis_mock_aime_2024_2025-606d5d7b92cc3c10","rd":"2026-05-28","se":0.022222222222222223,"sid":"MEhV5sh6nbM8cKJGQ6HzcS","sm":"claude-opus-4-8_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMEhV5sh6nbM8cKJGQ6HzcS.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-06","e":"max","evaluated_at":"2026-08-06T22:19:46.000Z","m":"claude-opus-4-7","oid":"otis_mock_aime_2024_2025-71f078422cd13979","rd":"2026-04-16","se":0.051247074319053836,"sid":"T4y4iGHMqA8tXCdBKr5GjS","sm":"claude-opus-4-7_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT4y4iGHMqA8tXCdBKr5GjS.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-08-06","e":"low","evaluated_at":"2026-08-06T22:18:24.000Z","m":"claude-fable-5","oid":"otis_mock_aime_2024_2025-43b102fb2bb4bcc1","rd":"2026-06-09","se":0.022222222222222223,"sid":"6E989Xxeuw925T7cCV3XGB","sm":"claude-fable-5_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6E989Xxeuw925T7cCV3XGB.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-06","e":"high","evaluated_at":"2026-08-06T22:18:19.000Z","m":"claude-fable-5","oid":"otis_mock_aime_2024_2025-237313e18f36c12a","rd":"2026-06-09","se":0,"sid":"hwrnvtEbDAocFtjcsHqsQh","sm":"claude-fable-5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhwrnvtEbDAocFtjcsHqsQh.eval","v":1},{"b":"otis_mock_aime_2024_2025","d":"2026-08-06","evaluated_at":"2026-08-06T22:17:05.000Z","m":"magistral-small-2506","oid":"otis_mock_aime_2024_2025-a67a535aac33d5ce","rd":"2025-06-10","se":0.05924610544247621,"sid":"hmZPoiHBpqufv9r7ujiK8V","sm":"magistral-small-2506","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhmZPoiHBpqufv9r7ujiK8V.eval","v":0.3},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-08-05","e":"xhigh","evaluated_at":"2026-08-05T18:55:39.000Z","m":"inkling","oid":"otis_mock_aime_2024_2025-18eef6f4ecf97b3c","rd":"2026-07-15","se":0.039171949640122654,"sid":"j2fELMf364qfh2cpaVDL6W","sm":"Inkling_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fj2fELMf364qfh2cpaVDL6W.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-08-04","e":"xhigh","evaluated_at":"2026-08-04T16:28:46.000Z","m":"qwen3-8-max","oid":"otis_mock_aime_2024_2025-96249ef6d086dc24","rd":"2026-08-02","se":0.0038834738634418433,"sid":"aGRKHaq9uSawfYVpC565zU","sm":"qwen3.8-max_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaGRKHaq9uSawfYVpC565zU.eval","v":0.9944444444444445},{"b":"otis_mock_aime_2024_2025","d":"2026-08-02","evaluated_at":"2026-08-02T02:46:37.000Z","m":"gpt-5-5-instant","oid":"otis_mock_aime_2024_2025-5f0d0c880b9fb12c","rd":"2026-05-05","se":0.062127114361967634,"sid":"hGzLQWbxPdgQiiMbaDqF8k","sm":"gpt-5.5-instant","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhGzLQWbxPdgQiiMbaDqF8k.eval","v":0.6805555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-08-02","e":"max","evaluated_at":"2026-08-02T00:22:31.000Z","m":"deepseek-v4-flash-0731","oid":"otis_mock_aime_2024_2025-d948c91c2d69f6b9","rd":"2026-07-31","se":0.020854366486931897,"sid":"FQ3Suh2vKacz7UNqJybTWk","sm":"deepseek-v4-flash-0731_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFQ3Suh2vKacz7UNqJybTWk.eval","v":0.9444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-08-02","e":"high","evaluated_at":"2026-08-02T00:21:27.000Z","m":"gemini-3-6-flash","oid":"otis_mock_aime_2024_2025-baa0a84060c7063b","rd":"2026-07-21","se":0.030926354359015452,"sid":"ajQFZGMjvyMTH2ULDZhwtz","sm":"gemini-3.6-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FajQFZGMjvyMTH2ULDZhwtz.eval","v":0.9416666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-07-24","e":"max","evaluated_at":"2026-07-24T18:40:39.000Z","m":"claude-opus-5","oid":"otis_mock_aime_2024_2025-2fe2628cb6df4f05","rd":"2026-07-24","se":0.011111111111111112,"sid":"KxvHAT9SV2dTDn7jBnTk6f","sm":"claude-opus-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKxvHAT9SV2dTDn7jBnTk6f.eval","v":0.9888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-20","e":"low","evaluated_at":"2026-07-20T18:28:54.000Z","m":"o3-mini-2025-01-31","oid":"otis_mock_aime_2024_2025-cac3e5a626da9bec","rd":"2025-01-31","se":0.07491109582924914,"sid":"DvMvnsvXgBB38PEnLGbTPH","sm":"o3-mini-2025-01-31_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDvMvnsvXgBB38PEnLGbTPH.eval","v":0.4444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-07-20","e":"minimal","evaluated_at":"2026-07-20T18:28:50.000Z","m":"gpt-5-2025-08-07","oid":"otis_mock_aime_2024_2025-4650d5136a3c72ca","rd":"2025-08-07","se":0.07521014330903548,"sid":"RMsr7t8ZwJmox4m863sJGm","sm":"gpt-5-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRMsr7t8ZwJmox4m863sJGm.eval","v":0.4666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-07-16","e":"max","evaluated_at":"2026-07-16T21:59:40.000Z","m":"kimi-k3","oid":"otis_mock_aime_2024_2025-7c3304a75a499669","rd":"2026-07-16","se":0.011158359139033182,"sid":"A33om9MTYMC7xYeLv9X7Ha","sm":"kimi-k3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FA33om9MTYMC7xYeLv9X7Ha.eval","v":0.9722222222222222},{"b":"otis_mock_aime_2024_2025","d":"2026-07-16","evaluated_at":"2026-07-16T00:10:43.000Z","m":"deepseek-chat","oid":"otis_mock_aime_2024_2025-3634c96df162756d","rd":"2025-12-01","se":0.0753592220347252,"sid":"DzXSCtsyMBfoBAhBDMxhsQ","sm":"deepseek-chat","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDzXSCtsyMBfoBAhBDMxhsQ.eval","v":0.4888888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-15","e":"low","evaluated_at":"2026-07-15T06:14:17.000Z","m":"o1-2024-12-17","oid":"otis_mock_aime_2024_2025-6e14d244c6a794fb","rd":"2024-12-17","se":0.07521014330903548,"sid":"JsAVnfoWmxCaChyiNU9VFi","sm":"o1-2024-12-17_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJsAVnfoWmxCaChyiNU9VFi.eval","v":0.5333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-07-15","e":"high","evaluated_at":"2026-07-15T06:14:12.000Z","m":"o1-2024-12-17","oid":"otis_mock_aime_2024_2025-9754418ebb3a2004","rd":"2024-12-17","se":0.06666666666666667,"sid":"AEQDjKHZzrVUubU42ZUZwA","sm":"o1-2024-12-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAEQDjKHZzrVUubU42ZUZwA.eval","v":0.7333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-15","e":"low","evaluated_at":"2026-07-15T03:15:08.000Z","m":"o3-2025-04-16","oid":"otis_mock_aime_2024_2025-7cdf5d16ce5251e8","rd":"2025-04-16","se":0.07385489458759963,"sid":"8ygEN7GrcTGeBLbVWT9JQ8","sm":"o3-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8ygEN7GrcTGeBLbVWT9JQ8.eval","v":0.6},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2026-07-15","e":"medium","evaluated_at":"2026-07-15T03:14:58.000Z","m":"gpt-5-4-2026-03-05","oid":"otis_mock_aime_2024_2025-931b4b5deda3b512","rd":"2026-03-05","se":0.031067790907534736,"sid":"eUJNMJSdLnJWNgRY8D2rYw","sm":"gpt-5.4-2026-03-05_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeUJNMJSdLnJWNgRY8D2rYw.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-07-15","e":"none","evaluated_at":"2026-07-15T03:14:58.000Z","m":"gpt-5-4-2026-03-05","oid":"otis_mock_aime_2024_2025-8ec0035e1edf4981","rd":"2026-03-05","se":0.07446027270295805,"sid":"fjxs2EWts9GjpuBtFD5BN3","sm":"gpt-5.4-2026-03-05_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ffjxs2EWts9GjpuBtFD5BN3.eval","v":0.5777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-07-15","e":"high","evaluated_at":"2026-07-15T03:14:50.000Z","m":"gpt-5-4-2026-03-05","oid":"otis_mock_aime_2024_2025-41f5eb2bd658c024","rd":"2026-03-05","se":0.022222222222222223,"sid":"DpPahiEawokzgidE3cNZbS","sm":"gpt-5.4-2026-03-05_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDpPahiEawokzgidE3cNZbS.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-15","e":"low","evaluated_at":"2026-07-15T03:14:49.000Z","m":"gpt-5-4-2026-03-05","oid":"otis_mock_aime_2024_2025-476f82e7ccf6c22a","rd":"2026-03-05","se":0.05463890236888291,"sid":"bezRyWjXxqHnQeiL4MkSfZ","sm":"gpt-5.4-2026-03-05_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbezRyWjXxqHnQeiL4MkSfZ.eval","v":0.8444444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-07-15","e":"minimal","evaluated_at":"2026-07-15T03:14:20.000Z","m":"gemini-3-5-flash","oid":"otis_mock_aime_2024_2025-31ad4e24c0ec6b87","rd":"2026-05-19","se":0.06030226891555273,"sid":"JuM7FArkmhLqqKBqudySsC","sm":"gemini-3.5-flash_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJuM7FArkmhLqqKBqudySsC.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-13","e":"low","evaluated_at":"2026-07-13T17:28:21.000Z","m":"gpt-5-nano-2025-08-07","oid":"otis_mock_aime_2024_2025-7c085033f83ab0d6","rd":"2025-08-07","se":0.07521014330903548,"sid":"298stsbTPpZp7toZsS5cp8","sm":"gpt-5-nano-2025-08-07_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F298stsbTPpZp7toZsS5cp8.eval","v":0.4666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-07-13","e":"high","evaluated_at":"2026-07-13T17:28:15.000Z","m":"claude-sonnet-4-6","oid":"otis_mock_aime_2024_2025-0301f0fad06f5b90","rd":"2026-02-17","se":0.06478835438717,"sid":"oXZ7ZVZtADrwFPKAe6zTvL","sm":"claude-sonnet-4-6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FoXZ7ZVZtADrwFPKAe6zTvL.eval","v":0.7555555555555555},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"minimal"},"d":"2026-07-13","e":"minimal","evaluated_at":"2026-07-13T17:28:09.000Z","m":"gpt-5-nano-2025-08-07","oid":"otis_mock_aime_2024_2025-06c8bdad7cac4d79","rd":"2025-08-07","se":0.0721639236343101,"sid":"kJHhuWKGEEunTq9FxwKx9E","sm":"gpt-5-nano-2025-08-07_minimal","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkJHhuWKGEEunTq9FxwKx9E.eval","v":0.35555555555555557},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"none"},"d":"2026-07-13","e":"none","evaluated_at":"2026-07-13T17:28:07.000Z","m":"gpt-5-2-2025-12-11","oid":"otis_mock_aime_2024_2025-6d77db44651de398","rd":"2025-12-11","se":0.07309112127323451,"sid":"UBKiF8QVUmHwqgx4rF7CDA","sm":"gpt-5.2-2025-12-11_none","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUBKiF8QVUmHwqgx4rF7CDA.eval","v":0.6222222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2026-07-13","e":"medium","evaluated_at":"2026-07-13T17:28:00.000Z","m":"claude-sonnet-4-6","oid":"otis_mock_aime_2024_2025-b25129ad88570175","rd":"2026-02-17","se":0.057637747950250916,"sid":"kDNNbUYXhrAH5uDBGykFaf","sm":"claude-sonnet-4-6_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkDNNbUYXhrAH5uDBGykFaf.eval","v":0.8222222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2026-07-13","e":"low","evaluated_at":"2026-07-13T16:50:13.000Z","m":"o4-mini-2025-04-16","oid":"otis_mock_aime_2024_2025-c9a220d6a9cb319a","rd":"2025-04-16","se":0.07446027270295805,"sid":"WNYdVF9tnK7Nf6We9NEhJS","sm":"o4-mini-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWNYdVF9tnK7Nf6We9NEhJS.eval","v":0.5777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2026-07-13","evaluated_at":"2026-07-13T16:45:48.000Z","m":"grok-4-20-0309-reasoning","oid":"otis_mock_aime_2024_2025-319a32c36759d71f","rd":"2026-02-17","se":0.032829739672413996,"sid":"mM6vdJpgjLY9prnetXzZJS","sm":"grok-4.20-0309-reasoning","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmM6vdJpgjLY9prnetXzZJS.eval","v":0.9222222222222223},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T17:07:54.000Z","m":"gpt-5-6-terra","oid":"otis_mock_aime_2024_2025-6f007ebad4523aab","rd":"2026-07-09","se":0.002777777777777778,"sid":"jUmmCNezoW7jWiK72SVCaP","sm":"gpt-5.6-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjUmmCNezoW7jWiK72SVCaP.eval","v":0.9972222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T13:36:49.000Z","m":"gpt-5-6-luna","oid":"otis_mock_aime_2024_2025-c358a80e9fa869ce","rd":"2026-07-09","se":0.011650421590325524,"sid":"AhtQ5ggs4pTfLDqHFjueGy","sm":"gpt-5.6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAhtQ5ggs4pTfLDqHFjueGy.eval","v":0.9833333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-07-09","e":"max","evaluated_at":"2026-07-09T02:07:13.000Z","m":"gpt-5-6-sol","oid":"otis_mock_aime_2024_2025-d7e3c753a0f02cbf","rd":"2026-07-09","se":0,"sid":"7RjsANKwwsePSstcZeSMhX","sm":"gpt-5.6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7RjsANKwwsePSstcZeSMhX.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-07-08","e":"high","evaluated_at":"2026-07-08T23:54:22.000Z","m":"grok-4-5","oid":"otis_mock_aime_2024_2025-1e1c9be36d452acd","rd":"2026-07-08","se":0.012743145865697103,"sid":"f8Pac6hqib4t7x2s4HCRoT","sm":"grok-4.5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Ff8Pac6hqib4t7x2s4HCRoT.eval","v":0.9777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-07-01","e":"xhigh","evaluated_at":"2026-07-01T16:06:23.000Z","m":"claude-sonnet-5","oid":"otis_mock_aime_2024_2025-19cf92e77f2d7d78","rd":"2026-06-30","se":0.022644349282618412,"sid":"T2wFfuSkUdyjpyqurxNmNK","sm":"claude-sonnet-5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT2wFfuSkUdyjpyqurxNmNK.eval","v":0.9472222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-06-25","e":"max","evaluated_at":"2026-06-25T00:15:08.000Z","m":"glm-5-2","oid":"otis_mock_aime_2024_2025-0e9c8551b01dda3d","rd":"2026-06-16","se":0.04219779330510574,"sid":"XjFqou6MyT2XLpkUsCrgxM","sm":"glm-5.2_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXjFqou6MyT2XLpkUsCrgxM.eval","v":0.8638888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-06-17","e":"high","evaluated_at":"2026-06-17T17:51:54.000Z","m":"grok-4-3","oid":"otis_mock_aime_2024_2025-9a42ae54f640da78","rd":"2026-04-17","se":0.03041173683586219,"sid":"fzoUFmy77NpEDtXyA6w6ap","sm":"grok-4.3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfzoUFmy77NpEDtXyA6w6ap.eval","v":0.9333333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-06-17","e":"max","evaluated_at":"2026-06-17T00:34:52.000Z","m":"deepseek-v4-pro","oid":"otis_mock_aime_2024_2025-0151b7397266bb24","rd":"2026-04-24","se":0.02002208376727653,"sid":"HjTDENSpy9sancDzr7pccY","sm":"deepseek-v4-pro_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHjTDENSpy9sancDzr7pccY.eval","v":0.9666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-06-10","e":"max","evaluated_at":"2026-06-10T20:03:17.000Z","m":"claude-fable-5","oid":"otis_mock_aime_2024_2025-7f6c1f55c764380d","rd":"2026-06-09","se":0.002777777777777778,"sid":"eiLEpRUvK48wUKDh4sYkzV","sm":"claude-fable-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeiLEpRUvK48wUKDh4sYkzV.eval","v":0.9972222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"max"},"d":"2026-06-07","e":"max","evaluated_at":"2026-06-07T23:44:54.000Z","m":"claude-opus-4-8","oid":"otis_mock_aime_2024_2025-2db295a2ad38dac7","rd":"2026-05-28","se":0.014101901870444154,"sid":"37pJqPpXTE6ia9ikmmGPJ4","sm":"claude-opus-4-8_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F37pJqPpXTE6ia9ikmmGPJ4.eval","v":0.9833333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-05-25","e":"high","evaluated_at":"2026-05-25T13:10:18.000Z","m":"gemini-3-5-flash","oid":"otis_mock_aime_2024_2025-6e5a973d98d4004d","rd":"2026-05-19","se":0.026696111689702044,"sid":"eYmgjYJfM7Utm46nfoRfZ6","sm":"gemini-3.5-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeYmgjYJfM7Utm46nfoRfZ6.eval","v":0.9555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2026-05-02","evaluated_at":"2026-05-02T09:08:49.000Z","m":"kimi-k2-6","oid":"otis_mock_aime_2024_2025-c368e40f141d855f","rd":"2026-04-20","se":0.023777636174410793,"sid":"BTeRdhgt6TFBj6QkGX2j6P","sm":"kimi-k2.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBTeRdhgt6TFBj6QkGX2j6P.eval","v":0.9611111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-04-24","e":"xhigh","evaluated_at":"2026-04-24T23:38:07.000Z","m":"gpt-5-5-pro-pre-release","oid":"otis_mock_aime_2024_2025-6051aeb9eda07cfe","rd":"2026-04-23","se":0,"sid":"YxbseUDNgFUjokfYXui62r","sm":"gpt-5.5-pro-pre-release_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYxbseUDNgFUjokfYXui62r.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-04-24","e":"xhigh","evaluated_at":"2026-04-24T14:06:46.000Z","m":"gpt-5-5-pre-release","oid":"otis_mock_aime_2024_2025-3bad32d6076c0c51","rd":"2026-04-23","se":0,"sid":"dP6L7ViZSdcTiWxooLCTUh","sm":"gpt-5.5-pre-release_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdP6L7ViZSdcTiWxooLCTUh.eval","v":1},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-04-17","e":"xhigh","evaluated_at":"2026-04-17T18:26:45.000Z","m":"claude-opus-4-7","oid":"otis_mock_aime_2024_2025-1e9bb48c888c3989","rd":"2026-04-16","se":0.022,"sid":"jyCZnpXG2bh9G43X8pT75m","sm":"claude-opus-4-7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjyCZnpXG2bh9G43X8pT75m.eval","v":0.978},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-04-15","e":"high","evaluated_at":"2026-04-15T14:17:38.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"otis_mock_aime_2024_2025-a88c5015b22e941b","rd":"2026-03-17","se":0.042508334166668306,"sid":"Z5Sp2AKYzTBNdMEyTfTRuN","sm":"gpt-5.4-mini-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ5Sp2AKYzTBNdMEyTfTRuN.eval","v":0.8722222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2026-04-14","e":"high","evaluated_at":"2026-04-14T15:08:33.525Z","m":"gpt-5-4-nano-2026-03-17","oid":"otis_mock_aime_2024_2025-691e3b2656118b6b","rd":"2026-03-17","se":0.0394,"sid":"jVv9m8DjkdpZaLPdUzAnG9","sm":"gpt-5.4-nano-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjVv9m8DjkdpZaLPdUzAnG9.eval","v":0.8778},{"b":"otis_mock_aime_2024_2025","d":"2026-04-08","evaluated_at":"2026-04-08T16:30:04.184Z","m":"muse-spark","oid":"otis_mock_aime_2024_2025-c4aff408ac4f54eb","rd":"2026-04-08","se":0.047,"sid":"muse-spark-pre-release-mock-aime","sm":"muse-spark","v":0.889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2026-03-06","e":"xhigh","evaluated_at":"2026-03-06T03:24:03.099Z","m":"gpt-5-4-2026-03-05","oid":"otis_mock_aime_2024_2025-cdc9b5990fbb8342","rd":"2026-03-05","se":0.032,"sid":"HVsPchr34pWYHNuv5sKYyK","sm":"gpt-5.4-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHVsPchr34pWYHNuv5sKYyK.eval","v":0.953},{"b":"otis_mock_aime_2024_2025","d":"2026-02-20","evaluated_at":"2026-02-20T16:18:28.678Z","m":"claude-sonnet-4-6-32k","oid":"otis_mock_aime_2024_2025-4be428edbc6178ab","rd":"2026-02-17","se":0.04539927553202047,"sid":"9LWdzVN5w4ihNvncsaCWMu","sm":"claude-sonnet-4-6_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9LWdzVN5w4ihNvncsaCWMu.eval","v":0.857936507936508},{"b":"otis_mock_aime_2024_2025","d":"2026-02-20","evaluated_at":"2026-02-20T04:39:36.998Z","m":"gemini-3-1-pro-preview","oid":"otis_mock_aime_2024_2025-a1513f79784218cc","rd":"2026-02-19","se":0.031,"sid":"bXgZZgzRE3wXoSRUgrbsVf","sm":"gemini-3.1-pro-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbXgZZgzRE3wXoSRUgrbsVf.eval","v":0.956},{"b":"otis_mock_aime_2024_2025","d":"2026-02-12","evaluated_at":"2026-02-12T16:08:22.578Z","m":"glm-5","oid":"otis_mock_aime_2024_2025-dca50bb16d6492a0","rd":"2026-02-11","se":0.06,"sid":"ccbx7KQY9bTD563vzBXwSg","sm":"glm-5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fccbx7KQY9bTD563vzBXwSg.eval","v":0.8},{"b":"otis_mock_aime_2024_2025","d":"2026-02-06","evaluated_at":"2026-02-06T04:32:30.354Z","m":"claude-opus-4-6-64k","oid":"otis_mock_aime_2024_2025-63b49105808994f0","rd":"2026-02-05","se":0.028247294597471075,"sid":"TUPC8VdX4jApFa4MGfNP4t","sm":"claude-opus-4-6_64K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTUPC8VdX4jApFa4MGfNP4t.eval","v":0.9444444444444444},{"b":"otis_mock_aime_2024_2025","d":"2026-02-06","evaluated_at":"2026-02-06T04:31:07.514Z","m":"claude-opus-4-6-32k","oid":"otis_mock_aime_2024_2025-f2042de1442d74f4","rd":"2026-02-05","se":0.03337013961212755,"sid":"kZ3VgrQJxfKQ5msjpmSaAz","sm":"claude-opus-4-6_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkZ3VgrQJxfKQ5msjpmSaAz.eval","v":0.9305555555555556},{"b":"otis_mock_aime_2024_2025","d":"2026-02-02","evaluated_at":"2026-02-02T10:53:14.986Z","m":"fireworkskimi-k2p5","oid":"otis_mock_aime_2024_2025-42d8abf21befde14","rd":"2026-01-27","se":0.039,"sid":"mbfZvrSNoXSJqCyD4VKqLe","sm":"fireworks/kimi-k2p5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmbfZvrSNoXSJqCyD4VKqLe.eval","v":0.922},{"b":"otis_mock_aime_2024_2025","d":"2026-01-29","evaluated_at":"2026-01-29T16:46:50.435Z","m":"glm-4-7","oid":"otis_mock_aime_2024_2025-54edb0cdac76051d","rd":"2025-12-22","se":0.04767312946227961,"sid":"GYzmrzMfeSjNjm2J6L9PeT","sm":"glm-4.7","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGYzmrzMfeSjNjm2J6L9PeT.eval","v":0.8333333333333334},{"b":"otis_mock_aime_2024_2025","d":"2025-12-17","evaluated_at":"2025-12-17T22:40:30.896Z","m":"gemini-3-flash-preview","oid":"otis_mock_aime_2024_2025-2f0cf155eebb21fb","rd":"2025-12-17","se":0.035196259111492545,"sid":"H3E7EjjXfGCPnm9ooLdmAo","sm":"gemini-3-flash-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FH3E7EjjXfGCPnm9ooLdmAo.eval","v":0.9277777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-12-16","evaluated_at":"2025-12-16T14:32:17.922Z","m":"deepseek-reasoner","oid":"otis_mock_aime_2024_2025-fb7e4c1c44a0dceb","rd":"2025-12-01","se":0.041306766974863386,"sid":"8szQqvTsGG8gJ2TiFReRn9","sm":"deepseek-reasoner","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8szQqvTsGG8gJ2TiFReRn9.eval","v":0.878174603174603},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"xhigh"},"d":"2025-12-13","e":"xhigh","evaluated_at":"2025-12-13T15:56:00.000Z","m":"gpt-5-2-2025-12-11","oid":"otis_mock_aime_2024_2025-54e892913f3e3036","rd":"2025-12-11","se":0.027,"sid":"R393BZjXtkEYju29gs5R8B","sm":"gpt-5.2-2025-12-11_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR393BZjXtkEYju29gs5R8B.eval","v":0.961},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T22:37:41.702Z","m":"openaigpt-oss-120b","oid":"otis_mock_aime_2024_2025-730df1bb0484c44c","rd":"2025-08-05","se":0.04463343655613273,"sid":"3QjP5HPi8tKUzHtNJktfv4","sm":"openai/gpt-oss-120b_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3QjP5HPi8tKUzHtNJktfv4.eval","v":0.8888888888888888},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-12-11","e":"medium","evaluated_at":"2025-12-11T20:49:22.447Z","m":"gpt-5-2-2025-12-11","oid":"otis_mock_aime_2024_2025-9dd8e10da409baf0","rd":"2025-12-11","se":0.030675842208062715,"sid":"Z2ZwwqLDDrBrEYhtKtFLCQ","sm":"gpt-5.2-2025-12-11_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ2ZwwqLDDrBrEYhtKtFLCQ.eval","v":0.9388888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-12-11","e":"high","evaluated_at":"2025-12-11T20:49:15.912Z","m":"gpt-5-2-2025-12-11","oid":"otis_mock_aime_2024_2025-9e66437661236742","rd":"2025-12-11","se":0.025997226390952678,"sid":"LiUMZyNSM8q93KDubWgYCh","sm":"gpt-5.2-2025-12-11_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLiUMZyNSM8q93KDubWgYCh.eval","v":0.9611111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2025-12-11","e":"low","evaluated_at":"2025-12-11T20:24:25.981Z","m":"gpt-5-2-2025-12-11","oid":"otis_mock_aime_2024_2025-fcf420d18063240e","rd":"2025-12-11","se":0.05006659426035015,"sid":"5Hcw3Td96qF66quWrDMKJB","sm":"gpt-5.2-2025-12-11_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5Hcw3Td96qF66quWrDMKJB.eval","v":0.7888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-12-10","evaluated_at":"2025-12-10T18:03:27.753Z","m":"qwen3-235b-a22b-thinking-2507","oid":"otis_mock_aime_2024_2025-51e3cb53c43891e0","rd":"2025-07-25","se":0.051247074319053836,"sid":"XWpHpVPCaVjcwPpdpXuuGH","sm":"Qwen3-235B-A22B-Thinking-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXWpHpVPCaVjcwPpdpXuuGH.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2025-11-25","e":"low","evaluated_at":"2025-11-25T15:18:46.210Z","m":"gpt-5-1-2025-11-13","oid":"otis_mock_aime_2024_2025-5622f5e9f84027e5","rd":"2025-11-13","se":0.058821311521534046,"sid":"6J9aPwieMfq2eSse8ZknBW","sm":"gpt-5.1-2025-11-13_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6J9aPwieMfq2eSse8ZknBW.eval","v":0.6388888888888888},{"b":"otis_mock_aime_2024_2025","d":"2025-11-24","evaluated_at":"2025-11-24T20:47:49.364Z","m":"claude-opus-4-5-2025-11-01","oid":"otis_mock_aime_2024_2025-ea8264938eab48db","rd":"2025-11-24","se":0.06405336755004033,"sid":"nSb2focDRMyPJXErk3MV94","sm":"claude-opus-4-5-20251101","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnSb2focDRMyPJXErk3MV94.eval","v":0.48055555555555557},{"b":"otis_mock_aime_2024_2025","d":"2025-11-24","evaluated_at":"2025-11-24T20:14:50.875Z","m":"claude-opus-4-5-20251101-32k","oid":"otis_mock_aime_2024_2025-ce216c51baa06f64","rd":"2025-11-24","se":0.042644255740946095,"sid":"ikUWdu6HJ2qRgZynfzDCFe","sm":"claude-opus-4-5-20251101_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FikUWdu6HJ2qRgZynfzDCFe.eval","v":0.8611111111111112},{"b":"otis_mock_aime_2024_2025","d":"2025-11-24","evaluated_at":"2025-11-24T20:02:12.729Z","m":"claude-opus-4-5-20251101-16k","oid":"otis_mock_aime_2024_2025-360e539786a85c24","rd":"2025-11-24","se":0.05018903659106632,"sid":"SH5jr76ni3RnpXVMETMy2H","sm":"claude-opus-4-5-20251101_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSH5jr76ni3RnpXVMETMy2H.eval","v":0.8166666666666667},{"b":"otis_mock_aime_2024_2025","d":"2025-11-19","evaluated_at":"2025-11-19T15:16:13.231Z","m":"gemini-3-pro-preview","oid":"otis_mock_aime_2024_2025-781391e4019650eb","rd":"2025-11-18","se":0.03658872186755414,"sid":"4QM4h7ndQiDRL2anjKeddK","sm":"gemini-3-pro-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4QM4h7ndQiDRL2anjKeddK.eval","v":0.9138888888888889},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-11-17","e":"medium","evaluated_at":"2025-11-17T21:28:27.888Z","m":"gpt-5-1-2025-11-13","oid":"otis_mock_aime_2024_2025-f84d8dec38e84692","rd":"2025-11-13","se":0.04387258359693737,"sid":"k7inKYrGDT2LFcCiQhGfTP","sm":"gpt-5.1-2025-11-13_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fk7inKYrGDT2LFcCiQhGfTP.eval","v":0.8555555555555555},{"b":"otis_mock_aime_2024_2025","d":"2025-11-16","evaluated_at":"2025-11-16T19:00:06.794Z","m":"gemini-2-5-pro","oid":"otis_mock_aime_2024_2025-70c9e2980c2359e0","rd":"2025-06-17","se":0.048143196054711346,"sid":"iJp7PgYZa3PiXAV8UWd7MC","sm":"gemini-2.5-pro","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiJp7PgYZa3PiXAV8UWd7MC.eval","v":0.8416666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-11-13","e":"high","evaluated_at":"2025-11-13T19:52:15.595Z","m":"gpt-5-1-2025-11-13","oid":"otis_mock_aime_2024_2025-d56441de8a7a40d3","rd":"2025-11-13","se":0.040479727089927024,"sid":"mYKKTdV27TRJv9FpUg7rkn","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmYKKTdV27TRJv9FpUg7rkn.eval","v":0.8861111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-11-13","evaluated_at":"2025-11-13T14:21:18.345Z","m":"kimi-k2-thinking-turbo","oid":"otis_mock_aime_2024_2025-c90aba859aa220bd","rd":"2025-11-06","se":0.048192643258048534,"sid":"8BM7dCHaUZgcYJwZWiBhiQ","sm":"kimi-k2-thinking-turbo","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8BM7dCHaUZgcYJwZWiBhiQ.eval","v":0.8305555555555556},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-10-31","e":"high","evaluated_at":"2025-10-31T09:55:38.159Z","m":"gpt-5-nano-2025-08-07","oid":"otis_mock_aime_2024_2025-8f87fc9d3f630945","rd":"2025-08-07","se":0.04626556218905063,"sid":"Y9YnaJQWpPFAPhe6jtKMhH","sm":"gpt-5-nano-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY9YnaJQWpPFAPhe6jtKMhH.eval","v":0.8111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-10-30","e":"high","evaluated_at":"2025-10-30T09:59:54.926Z","m":"gpt-5-mini-2025-08-07","oid":"otis_mock_aime_2024_2025-50a7e38ba76dfcfe","rd":"2025-08-07","se":0.04286165199944133,"sid":"R82gycJfm5jaCdh32tFkH4","sm":"gpt-5-mini-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR82gycJfm5jaCdh32tFkH4.eval","v":0.8666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-10-29","e":"high","evaluated_at":"2025-10-29T22:29:41.753Z","m":"gpt-5-2025-08-07","oid":"otis_mock_aime_2024_2025-354d1eea253014b7","rd":"2025-08-07","se":0.03658872186755414,"sid":"3D8aaFBLZwQcnWHYKwvnmq","sm":"gpt-5-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3D8aaFBLZwQcnWHYKwvnmq.eval","v":0.9138888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-10-28","evaluated_at":"2025-10-28T10:21:02.293Z","m":"claude-sonnet-4-5-20250929-59k","oid":"otis_mock_aime_2024_2025-30419030121508f5","rd":"2025-09-29","se":0.06267511942419625,"sid":"k5PDL6MpCC6P5eLAhodJiS","sm":"claude-sonnet-4-5-20250929_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fk5PDL6MpCC6P5eLAhodJiS.eval","v":0.7777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-10-28","evaluated_at":"2025-10-28T09:57:34.696Z","m":"claude-sonnet-4-5-20250929-16k","oid":"otis_mock_aime_2024_2025-73a52e9bb207c77f","rd":"2025-09-29","se":0.0683294324254051,"sid":"7QkPRbRPpRx8nJT2ZcqSx6","sm":"claude-sonnet-4-5-20250929_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7QkPRbRPpRx8nJT2ZcqSx6.eval","v":0.7111111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-10-23","evaluated_at":"2025-10-23T16:09:26.069Z","m":"gpt-4-0613","oid":"otis_mock_aime_2024_2025-64d9f3198fd9df23","rd":"2023-06-13","se":0.005362818328685681,"sid":"hGVxg4EhSSJSCdH88m9soq","sm":"gpt-4-0613","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhGVxg4EhSSJSCdH88m9soq.eval","v":0.011111111111111112},{"b":"otis_mock_aime_2024_2025","d":"2025-10-22","evaluated_at":"2025-10-22T16:49:17.118Z","m":"gpt-4-0314","oid":"otis_mock_aime_2024_2025-2975baa8d330d009","rd":"2023-03-14","se":0.003883473863441843,"sid":"cLTgetxaaTCZoA9vwY4SdL","sm":"gpt-4-0314","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcLTgetxaaTCZoA9vwY4SdL.eval","v":0.005555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2025-10-22","evaluated_at":"2025-10-22T10:31:07.212Z","m":"claude-haiku-4-5-20251001-32k","oid":"otis_mock_aime_2024_2025-8b7bf133681a1c7c","rd":"2025-10-15","se":0.07106690545187014,"sid":"neZvEzLi6yeL9z5KjGef7H","sm":"claude-haiku-4-5-20251001_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FneZvEzLi6yeL9z5KjGef7H.eval","v":0.6666666666666666},{"b":"otis_mock_aime_2024_2025","d":"2025-10-21","evaluated_at":"2025-10-21T12:33:20.414Z","m":"claude-sonnet-4-5-20250929-32k","oid":"otis_mock_aime_2024_2025-1fa16447bd8ca7d5","rd":"2025-09-29","se":0.06267511942419625,"sid":"kqTyhmHyT9irTAWk3xBt2x","sm":"claude-sonnet-4-5-20250929_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FkqTyhmHyT9irTAWk3xBt2x.eval","v":0.7777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-10-16","evaluated_at":"2025-10-16T11:58:20.630Z","m":"claude-haiku-4-5-2025-10-01","oid":"otis_mock_aime_2024_2025-701e2692f7494a96","rd":"2025-10-15","se":0.06238626014232183,"sid":"Wf2kwcXu8YXNgzDCxChtvj","sm":"claude-haiku-4-5-20251001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWf2kwcXu8YXNgzDCxChtvj.eval","v":0.35833333333333334},{"b":"otis_mock_aime_2024_2025","d":"2025-10-06","evaluated_at":"2025-10-06T14:32:01.826Z","m":"qwen3-max-2025-09-23","oid":"otis_mock_aime_2024_2025-666b79a589ff0ec4","rd":"2025-09-24","se":0.059139451700829515,"sid":"ULeuPtwp3ZoyuXQYuuQ82i","sm":"qwen3-max-2025-09-23","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FULeuPtwp3ZoyuXQYuuQ82i.eval","v":0.7333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-09-29","evaluated_at":"2025-09-29T18:39:45.629Z","m":"claude-sonnet-4-5-2025-09-29","oid":"otis_mock_aime_2024_2025-960f5f7840e4ea18","rd":"2025-09-29","se":0.0721639236343101,"sid":"dEVgttSw8u6Tt2RsPWTc3C","sm":"claude-sonnet-4-5-20250929","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdEVgttSw8u6Tt2RsPWTc3C.eval","v":0.35555555555555557},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T21:58:31.032Z","m":"gpt-5-2025-08-07","oid":"otis_mock_aime_2024_2025-ea16db7132d1ddc1","rd":"2025-08-07","se":0.03882118926889545,"sid":"ctsGxm6WkzCGEqiAbJR3kk","sm":"gpt-5-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FctsGxm6WkzCGEqiAbJR3kk.eval","v":0.8722222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T19:34:33.118Z","m":"gpt-5-mini-2025-08-07","oid":"otis_mock_aime_2024_2025-0a4f3d412d425f24","rd":"2025-08-07","se":0.04790431329560243,"sid":"65DJZN2XzPw4RGpa4TCfA5","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F65DJZN2XzPw4RGpa4TCfA5.eval","v":0.7833333333333333},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-08-07","e":"medium","evaluated_at":"2025-08-07T19:34:30.191Z","m":"gpt-5-nano-2025-08-07","oid":"otis_mock_aime_2024_2025-a4ee765f51c1cb47","rd":"2025-08-07","se":0.05231388366621641,"sid":"XRFS8R4WV7q6RMv5keEAdM","sm":"gpt-5-nano-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXRFS8R4WV7q6RMv5keEAdM.eval","v":0.7416666666666667},{"b":"otis_mock_aime_2024_2025","d":"2025-08-05","evaluated_at":"2025-08-05T16:53:30.846Z","m":"claude-opus-4-1-20250805-27k","oid":"otis_mock_aime_2024_2025-f8105298ae7c27c5","rd":"2025-08-05","se":0.06979205927323111,"sid":"UH9RY8MCAotDgFocHKA4j6","sm":"claude-opus-4-1-20250805_27K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUH9RY8MCAotDgFocHKA4j6.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-08-05","evaluated_at":"2025-08-05T16:51:45.842Z","m":"claude-opus-4-1-20250805-16k","oid":"otis_mock_aime_2024_2025-12347b3d68f195fc","rd":"2025-08-05","se":0.0721639236343101,"sid":"kf9vy9sMjMBBswrX6NMzce","sm":"claude-opus-4-1-20250805_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fkf9vy9sMjMBBswrX6NMzce.eval","v":0.6444444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-08-05","evaluated_at":"2025-08-05T16:45:26.245Z","m":"claude-opus-4-1-2025-08-05","oid":"otis_mock_aime_2024_2025-ca845cfc9675c7e9","rd":"2025-08-05","se":0.07385489458759964,"sid":"RaiKuDSzUJYDVMxn4jfzNC","sm":"claude-opus-4-1-20250805","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRaiKuDSzUJYDVMxn4jfzNC.eval","v":0.4},{"b":"otis_mock_aime_2024_2025","d":"2025-07-30","evaluated_at":"2025-07-30T15:47:06.669Z","m":"gpt-4o-mini-2024-07-18","oid":"otis_mock_aime_2024_2025-3205f3948edb43c1","rd":"2024-07-18","se":0.0273963714244758,"sid":"MPJkYPEMsQrowqtecK3mmz","sm":"gpt-4o-mini-2024-07-18","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMPJkYPEMsQrowqtecK3mmz.eval","v":0.06944444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-05-29","evaluated_at":"2025-05-29T10:59:10.779Z","m":"deepseek-r1-0528","oid":"otis_mock_aime_2024_2025-d18037486f7d5ece","rd":"2025-05-28","se":0.04911177955575991,"sid":"bBV5wCrrZtihrgxEsg9jPD","sm":"DeepSeek-R1-0528","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbBV5wCrrZtihrgxEsg9jPD.eval","v":0.6638888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-05-28","evaluated_at":"2025-05-28T18:42:14.809Z","m":"claude-opus-4-20250514-27k","oid":"otis_mock_aime_2024_2025-7b07e1f0abdaa0cb","rd":"2025-05-22","se":0.0721639236343101,"sid":"VrRccga4uRVuCjNp637MeY","sm":"claude-opus-4-20250514_27K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVrRccga4uRVuCjNp637MeY.eval","v":0.6444444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-05-23","evaluated_at":"2025-05-23T21:01:53.668Z","m":"claude-sonnet-4-20250514-59k","oid":"otis_mock_aime_2024_2025-4cf1333fab41e1e1","rd":"2025-05-22","se":0.06979205927323111,"sid":"g8qaq3s2TJqfvrskHYmapM","sm":"claude-sonnet-4-20250514_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fg8qaq3s2TJqfvrskHYmapM.eval","v":0.6888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-05-22","evaluated_at":"2025-05-22T23:06:11.220Z","m":"claude-sonnet-4-20250514-32k","oid":"otis_mock_aime_2024_2025-a673a131af660208","rd":"2025-05-22","se":0.06832943242540508,"sid":"FZHzYjwAQexVykNQTnRtwT","sm":"claude-sonnet-4-20250514_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFZHzYjwAQexVykNQTnRtwT.eval","v":0.7111111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-05-22","evaluated_at":"2025-05-22T22:54:06.990Z","m":"claude-opus-4-20250514-16k","oid":"otis_mock_aime_2024_2025-1d41e52c2d27737a","rd":"2025-05-22","se":0.07385489458759964,"sid":"Xz2F28L5k8HFNfY7Lo8Ggw","sm":"claude-opus-4-20250514_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXz2F28L5k8HFNfY7Lo8Ggw.eval","v":0.6},{"b":"otis_mock_aime_2024_2025","d":"2025-05-22","evaluated_at":"2025-05-22T22:54:05.950Z","m":"claude-sonnet-4-20250514-16k","oid":"otis_mock_aime_2024_2025-95ac8c6d7372afdc","rd":"2025-05-22","se":0.07521014330903548,"sid":"nDrnRwsADSyj8U6BD4in4r","sm":"claude-sonnet-4-20250514_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnDrnRwsADSyj8U6BD4in4r.eval","v":0.5333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-05-22","evaluated_at":"2025-05-22T17:26:48.979Z","m":"claude-opus-4-2025-05-14","oid":"otis_mock_aime_2024_2025-e08ed4115736c095","rd":"2025-05-22","se":0.07446027270295805,"sid":"aCbfcxGpLpPc9XkZNcuF4o","sm":"claude-opus-4-20250514","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaCbfcxGpLpPc9XkZNcuF4o.eval","v":0.4222222222222222},{"b":"otis_mock_aime_2024_2025","d":"2025-05-22","evaluated_at":"2025-05-22T17:26:46.358Z","m":"claude-sonnet-4-2025-05-14","oid":"otis_mock_aime_2024_2025-0a7573eb6836793c","rd":"2025-05-22","se":0.06832943242540508,"sid":"Ua8sCkk3R3mBLWeSnnPG7a","sm":"claude-sonnet-4-20250514","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUa8sCkk3R3mBLWeSnnPG7a.eval","v":0.28888888888888886},{"b":"otis_mock_aime_2024_2025","d":"2025-05-20","evaluated_at":"2025-05-20T21:22:01.376Z","m":"gemini-2-5-flash-preview-05-20","oid":"otis_mock_aime_2024_2025-3e40dcc58b7c086f","rd":"2025-05-20","se":0.05715807427254223,"sid":"9e62UjoHQbcZD2xjgfRkdK","sm":"gemini-2.5-flash-preview-05-20","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9e62UjoHQbcZD2xjgfRkdK.eval","v":0.7083333333333334},{"b":"otis_mock_aime_2024_2025","d":"2025-05-07","evaluated_at":"2025-05-07T17:32:21.604Z","m":"mistral-medium-2505","oid":"otis_mock_aime_2024_2025-bbdf1c74f52d7d76","rd":"2025-05-07","se":0.05965611798482264,"sid":"QWw65xeTn8x42SymMsVREN","sm":"mistral-medium-2505","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQWw65xeTn8x42SymMsVREN.eval","v":0.32222222222222224},{"b":"otis_mock_aime_2024_2025","d":"2025-04-22","evaluated_at":"2025-04-22T10:03:18.492Z","m":"gemini-2-5-flash-preview-04-17","oid":"otis_mock_aime_2024_2025-559447431cc2aa96","rd":"2025-04-17","se":0.05532781604070319,"sid":"KpZYG5nvdJ6QYweLz6dn5a","sm":"gemini-2.5-flash-preview-04-17","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKpZYG5nvdJ6QYweLz6dn5a.eval","v":0.7305555555555555},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:08.217Z","m":"o4-mini-2025-04-16","oid":"otis_mock_aime_2024_2025-5d519fdb45c6edb3","rd":"2025-04-16","se":0.046939139620180814,"sid":"TVsxkJ8ZyfMVmVx88Qq3KN","sm":"o4-mini-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTVsxkJ8ZyfMVmVx88Qq3KN.eval","v":0.8166666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-04-16","e":"high","evaluated_at":"2025-04-16T18:17:05.608Z","m":"o3-2025-04-16","oid":"otis_mock_aime_2024_2025-cf80cd70bc337013","rd":"2025-04-16","se":0.04381258550587166,"sid":"6anLU9CfBBNQkWyJG2GDCa","sm":"o3-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6anLU9CfBBNQkWyJG2GDCa.eval","v":0.8388888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-04-14","evaluated_at":"2025-04-14T22:50:00.937Z","m":"gpt-4-1-nano-2025-04-14","oid":"otis_mock_aime_2024_2025-61a6796d1aefc9a3","rd":"2025-04-14","se":0.05970019560922746,"sid":"3qLHwibDhgtWEbKGwxUYVa","sm":"gpt-4.1-nano-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3qLHwibDhgtWEbKGwxUYVa.eval","v":0.28888888888888886},{"b":"otis_mock_aime_2024_2025","d":"2025-04-14","evaluated_at":"2025-04-14T22:49:56.953Z","m":"gpt-4-1-mini-2025-04-14","oid":"otis_mock_aime_2024_2025-da7a222d9705f21c","rd":"2025-04-14","se":0.06148017573965347,"sid":"cXbbdQDW2jeYassmRuZYpU","sm":"gpt-4.1-mini-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcXbbdQDW2jeYassmRuZYpU.eval","v":0.44722222222222224},{"b":"otis_mock_aime_2024_2025","d":"2025-04-14","evaluated_at":"2025-04-14T18:34:29.241Z","m":"gpt-4-1-2025-04-14","oid":"otis_mock_aime_2024_2025-5230e17b2a22e89e","rd":"2025-04-14","se":0.06324056203789373,"sid":"VAQtARXUPTpdxGKMekUSbX","sm":"gpt-4.1-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVAQtARXUPTpdxGKMekUSbX.eval","v":0.38333333333333336},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"low"},"d":"2025-04-10","e":"low","evaluated_at":"2025-04-10T20:30:32.715Z","m":"grok-3-mini-beta","oid":"otis_mock_aime_2024_2025-760d08f2807e3505","rd":"2025-04-09","se":0.07309112127323451,"sid":"TTmhagb2ovbi7GJnDxEfrj","sm":"grok-3-mini-beta_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTTmhagb2ovbi7GJnDxEfrj.eval","v":0.6222222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-04-10","e":"high","evaluated_at":"2025-04-10T18:02:28.291Z","m":"grok-3-mini-beta","oid":"otis_mock_aime_2024_2025-572a306110a01816","rd":"2025-04-09","se":0.06267511942419625,"sid":"fPtize44vDPcDAzMFuAurw","sm":"grok-3-mini-beta_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfPtize44vDPcDAzMFuAurw.eval","v":0.7777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-04-10","evaluated_at":"2025-04-10T17:55:19.398Z","m":"grok-3-beta","oid":"otis_mock_aime_2024_2025-a547ac5d2b799bf1","rd":"2025-04-09","se":0.07491109582924914,"sid":"DKTzh9kSoipZGkDsC4NGf3","sm":"grok-3-beta","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDKTzh9kSoipZGkDsC4NGf3.eval","v":0.5555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2025-04-08","evaluated_at":"2025-04-08T09:58:55.652Z","m":"llama-4-maverick-17b-128e-instruct-fp8","oid":"otis_mock_aime_2024_2025-410ca85f42631495","rd":"2025-04-05","se":0.04948528559541873,"sid":"GJEpPThqUxXJBSbvkuXVnw","sm":"Llama-4-Maverick-17B-128E-Instruct-FP8","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGJEpPThqUxXJBSbvkuXVnw.eval","v":0.20555555555555555},{"b":"otis_mock_aime_2024_2025","d":"2025-04-08","evaluated_at":"2025-04-08T09:43:28.090Z","m":"llama-4-scout-17b-16e-instruct","oid":"otis_mock_aime_2024_2025-a02f395453f6081f","rd":"2025-04-05","se":0.030069597533844094,"sid":"T3X5DeC47iqCaoRYXu5k8L","sm":"Llama-4-Scout-17B-16E-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FT3X5DeC47iqCaoRYXu5k8L.eval","v":0.07777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-04-07","evaluated_at":"2025-04-07T19:18:53.147Z","m":"qwen-turbo-2024-11-01","oid":"otis_mock_aime_2024_2025-fb24bdc31af86b36","rd":"2024-11-01","se":0.022340279838327228,"sid":"G5gxbTH33S7LXA29o5fgmv","sm":"qwen-turbo-2024-11-01","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FG5gxbTH33S7LXA29o5fgmv.eval","v":0.06111111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-04-07","evaluated_at":"2025-04-07T19:09:43.970Z","m":"qwen-plus-2025-01-25","oid":"otis_mock_aime_2024_2025-7d62e69c1ed3aaff","rd":"2025-01-25","se":0.04360797453817489,"sid":"AULNJG2C75JBP6uaiAgj3P","sm":"qwen-plus-2025-01-25","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAULNJG2C75JBP6uaiAgj3P.eval","v":0.17777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-04-01","evaluated_at":"2025-04-01T17:11:08.937Z","m":"qwen-max-2025-01-25","oid":"otis_mock_aime_2024_2025-dbd86db5760571df","rd":"2025-01-25","se":0.04102194808229969,"sid":"PZt4Yb9bpXVPDoPi4uTxwK","sm":"qwen-max-2025-01-25","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPZt4Yb9bpXVPDoPi4uTxwK.eval","v":0.16111111111111112},{"b":"otis_mock_aime_2024_2025","d":"2025-04-01","evaluated_at":"2025-04-01T15:03:10.644Z","m":"hermes-2-theta-llama-3-70b","oid":"otis_mock_aime_2024_2025-d947e83daded0998","rd":"2024-06-20","se":0.012309149097933271,"sid":"63MnE4e4AXvbQ585MaWTFv","sm":"Hermes-2-Theta-Llama-3-70B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F63MnE4e4AXvbQ585MaWTFv.eval","v":0.025},{"b":"otis_mock_aime_2024_2025","d":"2025-04-01","evaluated_at":"2025-04-01T14:33:01.704Z","m":"deepseek-v3-0324","oid":"otis_mock_aime_2024_2025-87c35668070cf086","rd":"2025-03-24","se":0.062372203800719374,"sid":"VpsMexBEhXkPYJBe7v6Fk9","sm":"DeepSeek-V3-0324","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVpsMexBEhXkPYJBe7v6Fk9.eval","v":0.37777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2025-03-18","evaluated_at":"2025-03-18T16:39:25.325Z","m":"mistral-small-2501","oid":"otis_mock_aime_2024_2025-5795c9d2bcc80d52","rd":"2025-01-25","se":0.02589584594604903,"sid":"b4vEwq8UVmGmAKh4eqp6G3","sm":"mistral-small-2501","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fb4vEwq8UVmGmAKh4eqp6G3.eval","v":0.05277777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-03-18","evaluated_at":"2025-03-18T16:08:02.049Z","m":"mistral-small-2503","oid":"otis_mock_aime_2024_2025-c8fa04f4d7887dee","rd":"2025-03-17","se":0.0226830377599695,"sid":"JxqtHWrwCtYoQYBB3jVqB9","sm":"mistral-small-2503","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJxqtHWrwCtYoQYBB3jVqB9.eval","v":0.058333333333333334},{"b":"otis_mock_aime_2024_2025","d":"2025-03-13","evaluated_at":"2025-03-13T21:20:00.444Z","m":"claude-3-7-sonnet-20250219-64k","oid":"otis_mock_aime_2024_2025-087d6f2af4e442e6","rd":"2025-02-24","se":0.07446027270295805,"sid":"QW3FUBn5qtAFEX4oQuzEAV","sm":"claude-3-7-sonnet-20250219_64K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQW3FUBn5qtAFEX4oQuzEAV.eval","v":0.5777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2025-03-12","evaluated_at":"2025-03-12T19:36:44.645Z","m":"claude-3-7-sonnet-20250219-32k","oid":"otis_mock_aime_2024_2025-dff87ff696ef7516","rd":"2025-02-24","se":0.07521014330903548,"sid":"dSuGgEbnQErengjEboPn62","sm":"claude-3-7-sonnet-20250219_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdSuGgEbnQErengjEboPn62.eval","v":0.5333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T23:13:12.814Z","m":"llama-3-1-tulu-3-70b-dpo","oid":"otis_mock_aime_2024_2025-d6f3ea5b67cbf712","rd":"2024-11-21","se":0.01827278031830863,"sid":"FGcofc566CradiNmHUGqVB","sm":"Llama-3.1-Tulu-3-70B-DPO","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFGcofc566CradiNmHUGqVB.eval","v":0.044444444444444446},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T23:11:28.104Z","m":"gemma-2-27b-it","oid":"otis_mock_aime_2024_2025-2e6335c24adba700","rd":"2024-06-24","se":0.00907862525871088,"sid":"jD6rJdvfNPErKEz9qkpssj","sm":"gemma-2-27b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjD6rJdvfNPErKEz9qkpssj.eval","v":0.013888888888888888},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T23:11:27.610Z","m":"gemma-2-9b-it","oid":"otis_mock_aime_2024_2025-ad7f3fd467e57cab","rd":"2024-06-24","se":0.003883473863441843,"sid":"WNE7cveVN8rsi7DmsFsFVy","sm":"gemma-2-9b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWNE7cveVN8rsi7DmsFsFVy.eval","v":0.005555555555555556},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T23:03:59.002Z","m":"deepseek-r1-distill-llama-70b","oid":"otis_mock_aime_2024_2025-7efbf593cba3ed86","rd":"2025-01-20","se":0.060540263104731575,"sid":"89zcekqEYnE8zhwG4pR5Nu","sm":"DeepSeek-R1-Distill-Llama-70B","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F89zcekqEYnE8zhwG4pR5Nu.eval","v":0.5138888888888888},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T22:18:53.415Z","m":"claude-2-1","oid":"otis_mock_aime_2024_2025-11cfc082a0bea4f2","rd":"2023-11-21","se":0.010477515876628662,"sid":"2vg9qw9pd8zSyRMu9YNiVm","sm":"claude-2.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2vg9qw9pd8zSyRMu9YNiVm.eval","v":0.019444444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T22:18:52.824Z","m":"claude-2-0","oid":"otis_mock_aime_2024_2025-084d44ecad04dcb5","rd":"2023-07-11","se":0.015179897658722631,"sid":"7LLTmqs4LC5GLVfJEqJWSC","sm":"claude-2.0","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7LLTmqs4LC5GLVfJEqJWSC.eval","v":0.025},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T22:18:52.419Z","m":"gemini-2-0-flash-thinking-exp-01-21","oid":"otis_mock_aime_2024_2025-bdf3019958e17e10","rd":"2025-01-21","se":0.07446027270295805,"sid":"cXz2gsCmGYSvQ6Ey4LDiN7","sm":"gemini-2.0-flash-thinking-exp-01-21","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcXz2gsCmGYSvQ6Ey4LDiN7.eval","v":0.5777777777777777},{"b":"otis_mock_aime_2024_2025","d":"2025-03-07","evaluated_at":"2025-03-07T22:14:04.083Z","m":"o1-preview-2024-09-12","oid":"otis_mock_aime_2024_2025-b8f69e11d507fc28","rd":"2024-09-12","se":0.0697920592732311,"sid":"Cprjaj9JQ9MY5NCXqeQR5H","sm":"o1-preview-2024-09-12","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCprjaj9JQ9MY5NCXqeQR5H.eval","v":0.3111111111111111},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-03-06","e":"high","evaluated_at":"2025-03-06T21:47:29.432Z","m":"o1-mini-2024-09-12","oid":"otis_mock_aime_2024_2025-898d9dd214115893","rd":"2024-09-12","se":0.06456873681038247,"sid":"9zhPZLvNeUPQ7JBSpTwKPp","sm":"o1-mini-2024-09-12_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9zhPZLvNeUPQ7JBSpTwKPp.eval","v":0.46944444444444444},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-03-06","e":"medium","evaluated_at":"2025-03-06T21:47:28.091Z","m":"o1-mini-2024-09-12","oid":"otis_mock_aime_2024_2025-6a10dfcec1c50992","rd":"2024-09-12","se":0.06312676751311953,"sid":"QUTfpiKjphedrKg8H6fGzF","sm":"o1-mini-2024-09-12_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQUTfpiKjphedrKg8H6fGzF.eval","v":0.44722222222222224},{"b":"otis_mock_aime_2024_2025","d":"2025-02-28","evaluated_at":"2025-02-28T09:17:11.571Z","m":"gpt-4-5-preview-2025-02-27","oid":"otis_mock_aime_2024_2025-c8a5c221db48d4e6","rd":"2025-02-27","se":0.07309112127323451,"sid":"9DCJMt7YsTMDenTdX4Db6V","sm":"gpt-4.5-preview-2025-02-27","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9DCJMt7YsTMDenTdX4Db6V.eval","v":0.37777777777777777},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"high"},"d":"2025-02-27","e":"high","evaluated_at":"2025-02-27T14:29:45.288Z","m":"o3-mini-2025-01-31","oid":"otis_mock_aime_2024_2025-f4dd00fe8da13fd7","rd":"2025-01-31","se":0.05239762076828802,"sid":"Mjx6vYbgAb2DHXK8xcupX8","sm":"o3-mini-2025-01-31_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMjx6vYbgAb2DHXK8xcupX8.eval","v":0.7694444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-02-27","evaluated_at":"2025-02-27T14:29:44.767Z","m":"gpt-4-turbo-2024-04-09","oid":"otis_mock_aime_2024_2025-9f1cf18dcefa9ab2","rd":"2024-04-09","se":0.024034381938205007,"sid":"MZABVPEY8VCPAyCZ4iir68","sm":"gpt-4-turbo-2024-04-09","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FMZABVPEY8VCPAyCZ4iir68.eval","v":0.06666666666666667},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-02-27","e":"medium","evaluated_at":"2025-02-27T14:16:13.444Z","m":"o1-2024-12-17","oid":"otis_mock_aime_2024_2025-7f66540eeee77905","rd":"2024-12-17","se":0.06666666666666667,"sid":"KvcqycaJpCQMBigiEosyzC","sm":"o1-2024-12-17_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKvcqycaJpCQMBigiEosyzC.eval","v":0.7333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-02-26","evaluated_at":"2025-02-26T16:56:46.931Z","m":"claude-3-7-sonnet-20250219-16k","oid":"otis_mock_aime_2024_2025-878c5281412cdc68","rd":"2025-02-24","se":0.07521014330903548,"sid":"64LAJymM6eXDkRZb92Mdg8","sm":"claude-3-7-sonnet-20250219_16K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F64LAJymM6eXDkRZb92Mdg8.eval","v":0.4666666666666667},{"b":"otis_mock_aime_2024_2025","d":"2025-02-26","evaluated_at":"2025-02-26T16:56:45.594Z","m":"deepseek-r1","oid":"otis_mock_aime_2024_2025-5492bfeb0186ead9","rd":"2025-01-20","se":0.07521014330903548,"sid":"bBY5VZzpDkmTBdSSvu5Ups","sm":"DeepSeek-R1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbBY5VZzpDkmTBdSSvu5Ups.eval","v":0.5333333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:38:33.050Z","m":"meta-llama-3-70b-instruct","oid":"otis_mock_aime_2024_2025-a28afd9c564cb292","rd":"2024-04-18","se":0.02192429710598427,"sid":"6RD3A6J432TZzxrKYRxph2","sm":"Meta-Llama-3-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6RD3A6J432TZzxrKYRxph2.eval","v":0.043055555555555555},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:38:32.379Z","m":"gemini-1-5-pro-001","oid":"otis_mock_aime_2024_2025-aa19ed12e702b432","rd":"2024-05-14","se":0.01893034295514878,"sid":"3S6fjGAMKWKK3zcdxkmh9H","sm":"gemini-1.5-pro-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3S6fjGAMKWKK3zcdxkmh9H.eval","v":0.06805555555555555},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:38:30.971Z","m":"grok-2-1212","oid":"otis_mock_aime_2024_2025-447bd434f8df8b21","rd":"2024-12-12","se":0.03382424700930314,"sid":"PpSVDb67Ucbxpcd8JQv5ii","sm":"grok-2-1212","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPpSVDb67Ucbxpcd8JQv5ii.eval","v":0.11527777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:10.812Z","m":"llama-3-3-70b-instruct","oid":"otis_mock_aime_2024_2025-2ab1cb0858bf4130","rd":"2024-12-06","se":0.023611111111111107,"sid":"ZjsXQpZ7SsejXDo52aetZZ","sm":"Llama-3.3-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZjsXQpZ7SsejXDo52aetZZ.eval","v":0.05138888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:10.587Z","m":"llama-3-2-90b-vision-instruct","oid":"otis_mock_aime_2024_2025-87a44a750ef01bf7","rd":"2024-09-24","se":0.00941990275434065,"sid":"LHRL6nQJZXGJmYppR7kbGw","sm":"Llama-3.2-90B-Vision-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLHRL6nQJZXGJmYppR7kbGw.eval","v":0.02638888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:09.775Z","m":"llama-2-70b-chat-hf","oid":"otis_mock_aime_2024_2025-e0fc4a6730fece2a","rd":"2023-07-18","se":0,"sid":"NLk2oE4rzSFihPzF95peHr","sm":"Llama-2-70b-chat-hf","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNLk2oE4rzSFihPzF95peHr.eval","v":0},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:09.497Z","m":"claude-3-5-sonnet-2024-06-20","oid":"otis_mock_aime_2024_2025-5e4babdef91b86af","rd":"2024-06-20","se":0.022293131815660016,"sid":"2xNPhKLsiWLHQBLLDGU4VJ","sm":"claude-3-5-sonnet-20240620","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2xNPhKLsiWLHQBLLDGU4VJ.eval","v":0.06527777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:08.808Z","m":"gemini-1-5-pro-002","oid":"otis_mock_aime_2024_2025-9eb985cd0c9f1ea4","rd":"2024-09-24","se":0.05186782011216038,"sid":"fhCdyX9uUS4WPtjLiVdBdK","sm":"gemini-1.5-pro-002","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfhCdyX9uUS4WPtjLiVdBdK.eval","v":0.23055555555555557},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:08.205Z","m":"gpt-4o-2024-08-06","oid":"otis_mock_aime_2024_2025-81f560be061f1541","rd":"2024-08-06","se":0.025630380139261548,"sid":"mBUKqiVQsqXd3qmdtKP5UP","sm":"gpt-4o-2024-08-06","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmBUKqiVQsqXd3qmdtKP5UP.eval","v":0.06388888888888888},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:07.974Z","m":"gemini-1-5-flash-002","oid":"otis_mock_aime_2024_2025-e2dedf94d0753a8f","rd":"2024-09-24","se":0.04111375571619442,"sid":"QdGZ2m52PWt3EkLPbcoRwz","sm":"gemini-1.5-flash-002","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQdGZ2m52PWt3EkLPbcoRwz.eval","v":0.1625},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:07.648Z","m":"qwen2-5-32b-instruct","oid":"otis_mock_aime_2024_2025-82b9afd46c5ae1aa","rd":"2024-09-17","se":0.021420595022864277,"sid":"NRCNnCCL5aQKsZf6cKC69Z","sm":"qwen2.5-32b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNRCNnCCL5aQKsZf6cKC69Z.eval","v":0.07361111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:07.542Z","m":"mistral-large-2402","oid":"otis_mock_aime_2024_2025-d5a66d2d8f15633c","rd":"2024-02-26","se":0.00948945749721607,"sid":"ZVHvk3GTPUNTgjc5w3vkhW","sm":"mistral-large-2402","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZVHvk3GTPUNTgjc5w3vkhW.eval","v":0.019444444444444445},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:06.918Z","m":"gemini-1-5-flash-8b-001","oid":"otis_mock_aime_2024_2025-ccff441c98c94f32","rd":"2024-10-03","se":0.020100756305184247,"sid":"jGY5Ee5P9dyStA4Fw5Pa6q","sm":"gemini-1.5-flash-8b-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjGY5Ee5P9dyStA4Fw5Pa6q.eval","v":0.04583333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:06.293Z","m":"mistral-large-2411","oid":"otis_mock_aime_2024_2025-f5e6da5c8f71d858","rd":"2024-11-18","se":0.024662778950157508,"sid":"beRGgJXTJggTi8PPEZNoZ5","sm":"mistral-large-2411","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbeRGgJXTJggTi8PPEZNoZ5.eval","v":0.07777777777777778},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:05.872Z","m":"claude-3-sonnet-2024-02-29","oid":"otis_mock_aime_2024_2025-4b466c527180df3b","rd":"2024-02-29","se":0.01950297827553939,"sid":"AeRCEiG87bMFpxbxbmjCyn","sm":"claude-3-sonnet-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAeRCEiG87bMFpxbxbmjCyn.eval","v":0.025},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:05.855Z","m":"deepseek-v3","oid":"otis_mock_aime_2024_2025-3f7f1e28b48e389b","rd":"2024-12-26","se":0.0433650088193118,"sid":"7saSEhfVvXuz4yTcGy5u2y","sm":"DeepSeek-V3","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7saSEhfVvXuz4yTcGy5u2y.eval","v":0.15833333333333333},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:05.526Z","m":"gemini-1-0-pro-001","oid":"otis_mock_aime_2024_2025-d85b042eb9da5d49","rd":"2023-12-13","se":0.006054026310473157,"sid":"NpJvYQDcmwbGgS5MzZUPUg","sm":"gemini-1.0-pro-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNpJvYQDcmwbGgS5MzZUPUg.eval","v":0.011111111111111112},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:05.407Z","m":"llama-3-1-405b-instruct","oid":"otis_mock_aime_2024_2025-55b033ba3e47fc86","rd":"2024-07-23","se":0.03218644609162339,"sid":"2fQJP3brRApfBdQq7Zy3bm","sm":"Llama-3.1-405B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2fQJP3brRApfBdQq7Zy3bm.eval","v":0.09722222222222222},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:04.923Z","m":"claude-3-7-sonnet-2025-02-19","oid":"otis_mock_aime_2024_2025-89e0e69d0bb843f2","rd":"2025-02-24","se":0.04815168903711295,"sid":"N2NcmgmHyGJ9HNmQe5yvcS","sm":"claude-3-7-sonnet-20250219","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FN2NcmgmHyGJ9HNmQe5yvcS.eval","v":0.21944444444444444},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:03.625Z","m":"claude-3-5-sonnet-2024-10-22","oid":"otis_mock_aime_2024_2025-6e8d73b28b58acf2","rd":"2024-10-22","se":0.026363956855902478,"sid":"geVVaRWxuprKUqnW2VzvCa","sm":"claude-3-5-sonnet-20241022","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgeVVaRWxuprKUqnW2VzvCa.eval","v":0.08472222222222223},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:03.625Z","m":"qwen2-5-72b-instruct","oid":"otis_mock_aime_2024_2025-e29f217d4f0159cd","rd":"2024-09-19","se":0.026098213018967212,"sid":"67YbVQfJh7sd77KH3zFgTn","sm":"qwen2.5-72b-instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F67YbVQfJh7sd77KH3zFgTn.eval","v":0.08055555555555556},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:03.623Z","m":"gpt-4o-2024-05-13","oid":"otis_mock_aime_2024_2025-411a0d51938c6f5d","rd":"2024-05-13","se":0.023249986423369555,"sid":"9cCn5aW522cTRxJSE69crm","sm":"gpt-4o-2024-05-13","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9cCn5aW522cTRxJSE69crm.eval","v":0.0625},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:03.270Z","m":"llama-3-1-70b-instruct","oid":"otis_mock_aime_2024_2025-3245aade4a371385","rd":"2024-07-23","se":0.01691212350465157,"sid":"X2tQXGELXtEbc2VX6gARd7","sm":"Llama-3.1-70B-Instruct","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FX2tQXGELXtEbc2VX6gARd7.eval","v":0.03611111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:03.257Z","m":"claude-3-5-haiku-2024-10-22","oid":"otis_mock_aime_2024_2025-9dbec392595fb9a7","rd":"2024-10-22","se":0.016237638827815495,"sid":"cqhrR5KnwpcuAVpHkZGQFd","sm":"claude-3-5-haiku-20241022","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcqhrR5KnwpcuAVpHkZGQFd.eval","v":0.043055555555555555},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:02.246Z","m":"gemini-1-5-flash-001","oid":"otis_mock_aime_2024_2025-303964f7d20ec9fe","rd":"2024-05-23","se":0.014363726487520755,"sid":"U9GVBhYjRzU55YVjna5mMT","sm":"gemini-1.5-flash-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FU9GVBhYjRzU55YVjna5mMT.eval","v":0.03888888888888889},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:02.017Z","m":"claude-3-haiku-2024-03-07","oid":"otis_mock_aime_2024_2025-8bd0892ff47f7fbf","rd":"2024-03-07","se":0.008098544298396251,"sid":"ThMd6DNDTtyWxhPcb9Kq4y","sm":"claude-3-haiku-20240307","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FThMd6DNDTtyWxhPcb9Kq4y.eval","v":0.018055555555555554},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T14:05:00.461Z","m":"claude-3-opus-2024-02-29","oid":"otis_mock_aime_2024_2025-4280fa823d6ffcbb","rd":"2024-02-29","se":0.019579258477123922,"sid":"mYb9MGQD39RS8VdDyMqujr","sm":"claude-3-opus-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmYb9MGQD39RS8VdDyMqujr.eval","v":0.04722222222222222},{"b":"otis_mock_aime_2024_2025","cfg":{"effort":"medium"},"d":"2025-02-25","e":"medium","evaluated_at":"2025-02-25T13:33:25.508Z","m":"o3-mini-2025-01-31","oid":"otis_mock_aime_2024_2025-10882ea4999abd28","rd":"2025-01-31","se":0.060377105937901555,"sid":"ajytUBY9bDPNNxuBfi5UBS","sm":"o3-mini-2025-01-31_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FajytUBY9bDPNNxuBfi5UBS.eval","v":0.6388888888888888},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T13:19:42.221Z","m":"phi-4","oid":"otis_mock_aime_2024_2025-d72448ebe9eac714","rd":"2024-12-12","se":0.03698083382925635,"sid":"5mmrBSQb7hVSJ9VYtNm46i","sm":"phi-4","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5mmrBSQb7hVSJ9VYtNm46i.eval","v":0.1375},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T13:10:02.200Z","m":"gpt-4o-2024-11-20","oid":"otis_mock_aime_2024_2025-1dc3e23f97b235ba","rd":"2024-11-20","se":0.024165360466267546,"sid":"buNVL2TWcdVgxAcMfVE8xJ","sm":"gpt-4o-2024-11-20","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbuNVL2TWcdVgxAcMfVE8xJ.eval","v":0.0625},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T12:25:50.002Z","m":"gemini-2-0-flash-001","oid":"otis_mock_aime_2024_2025-2d2716b94e8aa087","rd":"2025-02-05","se":0.06311218144454454,"sid":"ZnGSRrKbiEnuju4kk4sgVb","sm":"gemini-2.0-flash-001","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZnGSRrKbiEnuju4kk4sgVb.eval","v":0.3111111111111111},{"b":"otis_mock_aime_2024_2025","d":"2025-02-25","evaluated_at":"2025-02-25T11:59:42.758Z","m":"mistral-large-2407","oid":"otis_mock_aime_2024_2025-b4b6688ea083ab01","rd":"2024-07-24","se":0.024742642797709363,"sid":"bHLfPfZj8HgD9Gef75NtrM","sm":"mistral-large-2407","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbHLfPfZj8HgD9Gef75NtrM.eval","v":0.08472222222222223},{"b":"otis_mock_aime_2024_2025","m":"grok-4-0709","oid":"otis_mock_aime_2024_2025-593436812fed1c9e","rd":"2025-07-09","se":0.05,"sid":"cvTPRDCM38zSTn9Y3MUb9d","sm":"grok-4-0709","v":0.84},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-2-7b","oid":"piqa-6f22821eeac7ced5","rd":"2023-07-18","sid":"rec1xnVLr7TTueuek","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2403.08295","v":0.788},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-2-13b","oid":"piqa-ede0418c396a91c0","rd":"2023-07-18","sid":"recIN14fg5Z6cb0Dy","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2403.08295","v":0.805},{"b":"piqa","cfg":{"shots":"0"},"dn":"Mistral 7B","m":"mistral-7b-instruct-v0-2","oid":"piqa-dadcf87681ebcfd4","rd":"2023-12-11","sid":"recwGYCRlmPQfK60l","sm":"Mistral-7B-Instruct-v0.2","src":"http://arxiv.org/abs/2403.08295","v":0.822},{"b":"piqa","cfg":{"shots":"0"},"m":"gemma-2b","oid":"piqa-bc8312f79639f8cc","rd":"2024-02-21","sid":"recJxRvElA0WHGRBp","sm":"gemma-2b","src":"http://arxiv.org/abs/2403.08295","v":0.773},{"b":"piqa","cfg":{"shots":"0"},"m":"gemma-7b","oid":"piqa-157b350888d3b4f5","rd":"2024-02-21","sid":"recjJz03ZH5MPEkVX","sm":"gemma-7b","src":"http://arxiv.org/abs/2403.08295","v":0.812},{"b":"piqa","cfg":{"shots":"0"},"m":"mpt-7b","oid":"piqa-46f8870f6d699eef","rd":"2023-05-05","sid":"reczbCPMBK72FaE5G","sm":"mpt-7b","src":"https://arxiv.org/abs/2309.16609","v":0.806},{"b":"piqa","cfg":{"shots":"0"},"m":"falcon-7b","oid":"piqa-dcb87e5785cb07eb","rd":"2023-04-24","sid":"recOqznsYbiFQfhfo","sm":"falcon-7b","src":"https://arxiv.org/abs/2309.16609","v":0.767},{"b":"piqa","cfg":{"shots":"0"},"dn":"ChatGLM2","m":"chatglm2-6b","oid":"piqa-15ba42bcc0fbd209","rd":"2023-06-24","sid":"recMuGIrZwsS5fFvR","sm":"chatglm2-6b","src":"https://arxiv.org/abs/2309.16609","v":0.696},{"b":"piqa","cfg":{"shots":"0"},"m":"internlm-7b","oid":"piqa-805e42b7da332bf7","rd":"2023-07-05","sid":"rec8tnNTcy4oGb1kc","sm":"internlm-7b","src":"https://arxiv.org/abs/2309.16609","v":0.779},{"b":"piqa","cfg":{"shots":"0"},"m":"internlm-20b","oid":"piqa-8a926597bae58e09","rd":"2023-09-18","sid":"reccHVW9au1hhKQdJ","sm":"internlm-20b","src":"https://arxiv.org/abs/2309.16609","v":0.803},{"b":"piqa","cfg":{"shots":"0"},"m":"baichuan-7b","oid":"piqa-d132053d5f877567","rd":"2023-06-01","sid":"recZ5llwYoXPLlK4z","sm":"Baichuan-7B","src":"https://arxiv.org/abs/2309.16609","v":0.762},{"b":"piqa","cfg":{"shots":"0"},"dn":"Baichuan2 13B","m":"baichuan-2-13b-base","oid":"piqa-7aaedf269d6c8b0e","rd":"2023-09-06","sid":"recPOD8pH0jFEUrRY","sm":"Baichuan-2-13B-Base","src":"https://arxiv.org/abs/2309.16609","v":0.781},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-7b","oid":"piqa-d9908aacb4700720","rd":"2023-02-24","sid":"recHMsC136QYPMTA3","sm":"LLaMA-7B","src":"https://arxiv.org/abs/2309.16609","v":0.798},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-13b","oid":"piqa-9d6e75a20b4d028b","rd":"2023-02-24","sid":"recXKa35srTYRSqea","sm":"LLaMA-13B","src":"https://arxiv.org/abs/2309.16609","v":0.801},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-33b","oid":"piqa-c11ebd2f77e35e25","rd":"2023-02-24","sid":"recXN4J1Jsj6CbzV2","sm":"LLaMA-33B","src":"https://arxiv.org/abs/2309.16609","v":0.823},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-65b","oid":"piqa-d22e2425fc9da1a3","rd":"2023-02-24","sid":"recwOATQe7kYtlwDp","sm":"LLaMA-65B","src":"https://arxiv.org/abs/2309.16609","v":0.828},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-2-7b","oid":"piqa-69f766c03cb9c56a","rd":"2023-07-18","sid":"rect8NgxFMui2UBT9","sm":"Llama-2-7b","src":"https://arxiv.org/abs/2309.16609","v":0.788},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-2-13b","oid":"piqa-2efa353d4a8b9fcd","rd":"2023-07-18","sid":"recK6uOl6du0fRDlX","sm":"Llama-2-13b","src":"https://arxiv.org/abs/2309.16609","v":0.805},{"b":"piqa","cfg":{"shots":"0"},"m":"llama-2-70b-hf","oid":"piqa-5036224130983496","rd":"2023-07-18","sid":"recoepOMA4acXl4mG","sm":"Llama-2-70b-hf","src":"https://arxiv.org/abs/2309.16609","v":0.828},{"b":"piqa","cfg":{"shots":"0"},"m":"stablebeluga2","oid":"piqa-de1f3d3ab24f3a34","rd":"2023-07-20","sid":"recmTF7ULiFtmEP58","sm":"StableBeluga2","src":"https://arxiv.org/abs/2309.16609","v":0.833},{"b":"piqa","cfg":{"shots":"0"},"m":"qwen-1-8b","oid":"piqa-f3ae2477a20efa84","rd":"2023-11-30","sid":"recRKZzksugdKpDPC","sm":"Qwen-1_8B","src":"https://arxiv.org/abs/2309.16609","v":0.733},{"b":"piqa","cfg":{"shots":"0"},"m":"qwen-7b","oid":"piqa-a179a28258c5a70a","rd":"2023-09-28","sid":"recGbe9gweaWdaQek","sm":"Qwen-7B","src":"https://arxiv.org/abs/2309.16609","v":0.779},{"b":"piqa","cfg":{"shots":"0"},"m":"qwen-14b","oid":"piqa-fd7dfe4559c96c35","rd":"2023-09-28","sid":"recrb6Z8mgHAvK4WU","sm":"Qwen-14B","src":"https://arxiv.org/abs/2309.16609","v":0.799},{"b":"piqa","cfg":{"shots":"0"},"dn":"GPT3 175B","m":"text-davinci-001","oid":"piqa-f1e6d9fa18c1f006","rd":"2022-01-27","sid":"recsYI4pESAvZyS6M","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.804},{"b":"piqa","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"piqa-9a8fc0945dfa6833","rd":"2022-01-27","sid":"recH9xsP3Rbt6iIYp","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.805},{"b":"piqa","cfg":{"shots":"few"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"piqa-9dc285a4d376692b","rd":"2022-01-27","sid":"recmMgB5NbBUgeVIA","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.823},{"b":"piqa","cfg":{"shots":"5"},"dn":"Phi-3.5-mini","m":"phi-3-5-mini-instruct","oid":"piqa-df9909a4bbac2f8c","rd":"2024-08-16","sid":"recPqTKLkju5qZgJl","sm":"Phi-3.5-mini-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.81},{"b":"piqa","cfg":{"shots":"5"},"dn":"Phi-3.5-MoE","m":"phi-3-5-moe-instruct","oid":"piqa-a4f5b7f09ef3e182","rd":"2024-08-17","sid":"rec6rmWvma2wXHWfl","sm":"Phi-3.5-MoE-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.886},{"b":"piqa","cfg":{"shots":"5"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"piqa-a140b8cd58aed65c","rd":"2023-09-27","sid":"reclUJMIrRD6JaeFJ","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.734},{"b":"piqa","cfg":{"shots":"5"},"dn":"Mistral-Nemo 12B","m":"mistral-nemo-base-2407","oid":"piqa-756200f42b7c4478","rd":"2024-07-18","sid":"recg6Csx6Pm68gfKA","sm":"Mistral-Nemo-Base-2407","src":"http://arxiv.org/abs/2404.14219","v":0.835},{"b":"piqa","cfg":{"shots":"5"},"dn":"Llama-3.1-In 8B","m":"llama-3-1-8b-instruct","oid":"piqa-993a65cbeb21eebc","rd":"2024-07-23","sid":"recMRkMxBSeChUEeH","sm":"Llama-3.1-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.812},{"b":"piqa","cfg":{"shots":"5"},"dn":"Gemma-2 9B","m":"gemma-2-9b","oid":"piqa-9ff0aaeef579ea22","sid":"recwF8TNC8B3AT1J3","sm":"gemma-2-9b","src":"http://arxiv.org/abs/2404.14219","v":0.837},{"b":"piqa","cfg":{"shots":"5"},"dn":"Gemini-1.5 Flash","m":"gemini-1-5-flash-002","oid":"piqa-8b46edd632add50f","rd":"2024-09-24","sid":"recgEUyXX8JGSCqol","sm":"gemini-1.5-flash-002","src":"http://arxiv.org/abs/2404.14219","v":0.875},{"b":"piqa","cfg":{"shots":"5"},"dn":"GPT-4o-mini","m":"gpt-4o-mini-2024-07-18","oid":"piqa-07b88a781f56110e","rd":"2024-07-18","sid":"recGz1DuA5ylHICeB","sm":"gpt-4o-mini-2024-07-18","src":"http://arxiv.org/abs/2404.14219","v":0.887},{"b":"piqa","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"piqa-805dbd3683a1ce59","rd":"2022-01-27","sid":"rechyypf7vUAiL6zd","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.81},{"b":"piqa","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"piqa-2b5ff16c1bb0746a","rd":"2022-01-27","sid":"recNGhvAQW5IgDxpV","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.805},{"b":"piqa","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"piqa-4b25d85d33699a60","rd":"2022-01-27","sid":"recjUybr3sNM10801","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.823},{"b":"piqa","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"piqa-ac36015b02093f56","rd":"2021-12-08","sid":"recHKitk77SUFNClZ","sm":"Gopher (280B)","src":"https://arxiv.org/pdf/2201.11990","v":0.818},{"b":"piqa","cfg":{"shots":"0"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"piqa-1dd8ce4a465aa384","rd":"2021-10-11","sid":"recC47h8xcapcI419","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8199},{"b":"piqa","cfg":{"shots":"1"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"piqa-40fbae1a10955586","rd":"2021-10-11","sid":"reccY8xsB8E7wx6VM","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8096},{"b":"piqa","cfg":{"shots":"few"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"piqa-91300ad28256b379","rd":"2021-10-11","sid":"recQ4y5CR1XQQvNwM","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.8319},{"b":"piqa","cfg":{"shots":"0"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"piqa-79adf1aaa96e0fa2","rd":"2024-05-07","sid":"recCDyTiNQ3ihLhEw","sm":"DeepSeek-V2","src":"http://arxiv.org/abs/2412.19437","v":0.839},{"b":"piqa","cfg":{"shots":"0"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"piqa-7e6efecfdde34aa8","rd":"2024-09-19","sid":"recjoWxRpfZ7AQ7yl","sm":"Qwen2.5-72B","src":"http://arxiv.org/abs/2412.19437","v":0.826},{"b":"piqa","cfg":{"shots":"0"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"piqa-34066f4a7a4fe502","rd":"2024-07-23","sid":"recuGgadfFBqsMZOZ","sm":"Llama-3.1-405B","src":"http://arxiv.org/abs/2412.19437","v":0.859},{"b":"piqa","cfg":{"shots":"0"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3-base","oid":"piqa-b3a01998a4f7ee00","rd":"2024-12-26","sid":"recrLJI80qifq9QOn","sm":"DeepSeek-V3-Base","src":"http://arxiv.org/abs/2412.19437","v":0.847},{"b":"piqa","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"piqa-c8afaa20e66e96fb","rd":"2021-12-08","sid":"recMGhvVkVNFWIxCR","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.11446","v":0.818},{"b":"piqa","dn":"MPT 7B","m":"mpt-7b","oid":"piqa-1d27f0fc0008a2aa","rd":"2023-05-05","sid":"recSmuagktvqatZ8Z","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.806},{"b":"piqa","dn":"MPT 30B","m":"mpt-30b","oid":"piqa-f8fd9004a12327be","rd":"2023-06-22","sid":"recr2NM4gHz5DrPHK","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.819},{"b":"piqa","dn":"Falcon 7B","m":"falcon-7b","oid":"piqa-f227ba137e843851","rd":"2023-04-24","sid":"recy3uYhorC17XHuA","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.767},{"b":"piqa","dn":"Falcon 40B","m":"falcon-40b","oid":"piqa-9cd08ce4e902bb18","rd":"2023-03-15","sid":"recRvbbjcqu4tHZUj","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.824},{"b":"piqa","dn":"LLAMA 1 7B","m":"llama-7b","oid":"piqa-cbdefca5a6e5a26f","rd":"2023-02-24","sid":"recqCeWBr36yASU0m","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.798},{"b":"piqa","dn":"LLAMA 1 13B","m":"llama-13b","oid":"piqa-a31fca0029aeae06","rd":"2023-02-24","sid":"rec55TMC1fSIbgbpL","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.801},{"b":"piqa","dn":"LLAMA 1 33B","m":"llama-33b","oid":"piqa-d7b4823803f8a9b8","rd":"2023-02-24","sid":"recN2M5oOIzl0ODKF","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.823},{"b":"piqa","dn":"LLAMA 1 65B","m":"llama-65b","oid":"piqa-10bd71f356afbe16","rd":"2023-02-24","sid":"recHgNAhYhVSOtBCE","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.828},{"b":"piqa","dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"piqa-f8598671aeb5e0c7","rd":"2023-07-18","sid":"recneSbTW5OeVjC1z","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.788},{"b":"piqa","dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"piqa-fbd67f5fa935c459","rd":"2023-07-18","sid":"recgjPq9v01M3wvTo","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.805},{"b":"piqa","dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"piqa-b1e54b3385734e8b","rd":"2023-07-18","sid":"recwvOiQUvfWl8wKI","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.819},{"b":"piqa","dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"piqa-96d140f3b05bd5ee","rd":"2023-07-18","sid":"rechTvbEmR1HjwNVJ","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.828},{"b":"piqa","dn":"GPT-3 175B","m":"text-davinci-001","oid":"piqa-00662cb21b507b51","rd":"2022-01-27","sid":"recsVbigtLpMbL6Qh","sm":"text-davinci-001","src":"http://arxiv.org/abs/2302.13971","v":0.81},{"b":"piqa","dn":"Gopher 280B","m":"gopher-280b","oid":"piqa-989076191a0848c4","rd":"2021-12-08","sid":"rec9o0a54dAKBr38y","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.818},{"b":"piqa","dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"piqa-88936c50f6d49220","rd":"2022-03-29","sid":"recpuR9za4WmoSvTQ","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.818},{"b":"piqa","dn":"PaLM 62B","m":"palm-62b","oid":"piqa-869c269998e68603","rd":"2022-04-04","sid":"recFicEkz7OwkVWpx","sm":"PaLM 62B","src":"http://arxiv.org/abs/2302.13971","v":0.805},{"b":"piqa","dn":"PaLM 540B","m":"palm-540b","oid":"piqa-1b7743ba85bf60df","rd":"2022-04-04","sid":"reciBD1ZNJdnqq65W","sm":"PaLM 540B","src":"http://arxiv.org/abs/2302.13971","v":0.823},{"b":"piqa","dn":"LLaMA 7B","m":"llama-7b","oid":"piqa-054c8f455abd75b4","rd":"2023-02-24","sid":"rec9VBfbQaSSc1u22","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.798},{"b":"piqa","dn":"LLaMA 13B","m":"llama-13b","oid":"piqa-53d47254d9767503","rd":"2023-02-24","sid":"recl6sqNICNc9xkt9","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.801},{"b":"piqa","dn":"LLaMA 33B","m":"llama-33b","oid":"piqa-4708a91a16b7cd49","rd":"2023-02-24","sid":"recJX0keNAJ7aeMtF","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.823},{"b":"piqa","dn":"LLaMA 65B","m":"llama-65b","oid":"piqa-3017200437720a80","rd":"2023-02-24","sid":"reczKzFKydRHC3Y6G","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.828},{"b":"piqa","dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"piqa-88a3b831996bef79","rd":"2023-07-18","sid":"rec9pWZJyyC5cl7TQ","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2401.04088","v":0.779},{"b":"piqa","dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"piqa-f6aaeb3aaa98ce81","rd":"2023-07-18","sid":"recCgq5kWGQ8RBlfe","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2401.04088","v":0.808},{"b":"piqa","dn":"LLaMA 1 33B","m":"llama-33b","oid":"piqa-0875d4f361637422","rd":"2023-02-24","sid":"recysRWV09WKs2EW7","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2401.04088","v":0.822},{"b":"piqa","dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"piqa-c6e843de71d885c4","rd":"2023-07-18","sid":"recF6goVanGsXhTuS","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2401.04088","v":0.826},{"b":"piqa","dn":"Mistral 7B","m":"mistral-7b-instruct-v0-1","oid":"piqa-7fffec761571a69f","rd":"2023-09-27","sid":"recUMmAunynmFGxLQ","sm":"Mistral-7B-Instruct-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.822},{"b":"piqa","dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"piqa-c8f3ab6c49ebe292","rd":"2023-12-11","sid":"rec2IWUndAuaG0h2n","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2401.04088","v":0.836},{"b":"piqa","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"piqa-8fbe3737c304dbc6","rd":"2023-07-18","sid":"recahE13EYTo3Wo1s","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.798},{"b":"piqa","dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"piqa-9990290f4986f5d1","rd":"2023-07-18","sid":"rectIlFXJFhTHN5i7","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.819},{"b":"piqa","dn":"Baichuan-2 13B","m":"baichuan-2-13b-base","oid":"piqa-31d5f788e3934627","rd":"2023-09-06","sid":"recmlOtDLYkLVNRjp","sm":"Baichuan-2-13B-Base","src":"http://arxiv.org/abs/2402.16819","v":0.781},{"b":"piqa","dn":"QWEN 14B","m":"qwen-14b","oid":"piqa-750fdb3c499f76d7","rd":"2023-09-28","sid":"recBuu3qDHZsvO1PL","sm":"Qwen-14B","src":"http://arxiv.org/abs/2402.16819","v":0.799},{"b":"piqa","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"piqa-33a5fbd38080625c","rd":"2023-09-27","sid":"recuKX9gOnTp5HRYM","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.83},{"b":"piqa","dn":"Gemma 7B","m":"gemma-7b","oid":"piqa-f7ecc6b1c0cfd454","rd":"2024-02-21","sid":"recQvIlDdbnoPnv0d","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.812},{"b":"piqa","dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"piqa-355270afbcae5ac4","rd":"2024-02-26","sid":"rectxE1O7JQfuz9EJ","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.824},{"b":"piqa","dn":"Vicuna-13B","m":"vicuna-13b-v1-1","oid":"piqa-77a1eab52b6ef023","rd":"2023-04-12","sid":"recHgkx12MmzkekLN","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.774},{"b":"piqa","dn":"Llama2-7B","m":"llama-2-7b","oid":"piqa-d43f27f94645b576","rd":"2023-07-18","sid":"recaUCCpOTwgqm0Qm","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.781},{"b":"piqa","dn":"Llama-7B","m":"llama-7b","oid":"piqa-052c58344949e745","rd":"2023-02-24","sid":"recsdhbPxzYLRRtjg","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.779},{"b":"piqa","dn":"MPT-7B","m":"mpt-7b","oid":"piqa-91f7131cef10c908","rd":"2023-05-05","sid":"recwVckykBEQi1xoV","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.789},{"b":"piqa","dn":"Falcon-7B","m":"falcon-7b","oid":"piqa-d8d3a9be66da6d09","rd":"2023-04-24","sid":"recb2fIZ2HfaPbGiN","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.794},{"b":"piqa","dn":"OPT-1.3B","m":"opt-1-3b","oid":"piqa-d665ad5635630bdb","rd":"2022-05-11","sid":"recj4wPV6UBqzmQPj","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.69},{"b":"piqa","dn":"GPT-Neo-2.7B","m":"gpt-neo-2-7b","oid":"piqa-b63c6883744ba087","rd":"2021-03-30","sid":"rec1GQU0OQPWk0ZHw","sm":"gpt-neo-2.7B","src":"http://arxiv.org/abs/2309.05463","v":0.729},{"b":"piqa","dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"piqa-4c18884e6401e0ee","rd":"2019-11-05","sid":"recUOcFFT7hhkGKpZ","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.705},{"b":"piqa","dn":"GPT-3","m":"text-davinci-001","oid":"piqa-67cb03d4b9493499","rd":"2022-01-27","sid":"rec25UCR5WZroWYCB","sm":"text-davinci-001","src":"http://arxiv.org/abs/2311.16867","v":0.81},{"b":"piqa","dn":"Gopher","m":"gopher-280b","oid":"piqa-02dc48845b5327b3","rd":"2021-12-08","sid":"rec5VmQW9uyUN1SdY","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2311.16867","v":0.818},{"b":"piqa","dn":"Chinchilla","m":"chinchilla-70b","oid":"piqa-b88df86cb84cdd3c","rd":"2022-03-29","sid":"recqTziBWCcXeSuRY","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2311.16867","v":0.818},{"b":"piqa","dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"piqa-8c16cc0bb8c213d6","rd":"2021-10-11","sid":"recqMqP0JxMAF9zPY","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2311.16867","v":0.82},{"b":"piqa","dn":"PaLM","m":"palm-540b","oid":"piqa-5627001abd0abe3c","rd":"2022-04-04","sid":"recyNe5nwU6obKot5","sm":"PaLM 540B","src":"http://arxiv.org/abs/2311.16867","v":0.823},{"b":"piqa","dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"piqa-6cee8b6a7bb025bc","rd":"2023-07-18","sid":"recGIEisHrwAWU2xV","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2311.16867","v":0.788},{"b":"piqa","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"piqa-ecc487b832ed4bd1","rd":"2023-07-18","sid":"recnLKoW7dq4Du1kv","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2311.16867","v":0.805},{"b":"piqa","dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"piqa-6ba8bc459590648e","rd":"2023-07-18","sid":"recyvBe2DxXox2gja","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2311.16867","v":0.819},{"b":"piqa","dn":"LLaMA-2 70B","m":"llama-2-70b-hf","oid":"piqa-2f983d0ea9b59861","rd":"2023-07-18","sid":"rec4hpg6dNmNaM2ao","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2311.16867","v":0.828},{"b":"piqa","dn":"Inflection-1","m":"inflection-1","oid":"piqa-1b23e1e0e3c29bac","rd":"2023-06-22","sid":"rec82aLV2vuRBCF2f","sm":"Inflection-1","src":"http://arxiv.org/abs/2311.16867","v":0.842},{"b":"piqa","dn":"Falcon 7B","m":"falcon-7b","oid":"piqa-1e47c0f38c926a3a","rd":"2023-04-24","sid":"rec079w7XJrd01GOg","sm":"falcon-7b","src":"http://arxiv.org/abs/2311.16867","v":0.803},{"b":"piqa","dn":"Falcon 40B","m":"falcon-40b","oid":"piqa-eed322726cddfc69","rd":"2023-03-15","sid":"recQxJmvpV3eZ20Lc","sm":"falcon-40b","src":"http://arxiv.org/abs/2311.16867","v":0.83},{"b":"piqa","dn":"Falcon 180B","m":"falcon-180b","oid":"piqa-84e0fa88b6e56f00","rd":"2023-09-06","sid":"recksRN6FGoGt2lU1","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.849},{"b":"piqa","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"piqa-66a5ee070ed27fe2","rd":"2023-06-27","sid":"rectQ5UXB5ekumh3q","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.755},{"b":"piqa","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"piqa-81c2a83d86e252fd","rd":"2023-02-24","sid":"recp7DP6kLNhpXBvf","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.787},{"b":"piqa","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"piqa-b540850f44b38470","rd":"2023-04-24","sid":"recA3hoQGJKtLCEcS","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.794},{"b":"piqa","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"piqa-a9e9d89648baf656","rd":"2023-05-05","sid":"recZRYbGGqawjK4fq","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.791},{"b":"piqa","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"piqa-9a4092d6c523ffac","rd":"2023-06-07","sid":"rec0RNZG538uttbyz","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.76},{"b":"piqa","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"piqa-23d4ce926c4c7f3a","rd":"2023-05-04","sid":"recfbMlV6oYreReFy","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.769},{"b":"piqa","cfg":{"shots":"0"},"dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"piqa-b308c8e43d92f793","rd":"2022-04-07","sid":"rec8Sh8crarIde2aF","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.767},{"b":"piqa","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"piqa-f48ff982efd8f129","rd":"2022-05-11","sid":"rec34MBtTjoOBqvsy","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.757},{"b":"piqa","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"piqa-266bee8d8ad385b9","rd":"2021-08-05","sid":"recieMBRnl774Kp8o","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.754},{"b":"piqa","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"piqa-53b8ab06882bdeb6","rd":"2023-04-11","sid":"recmpn9dENCquDkxR","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.754},{"b":"piqa","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"piqa-f94573b6119eadf0","rd":"2023-03-20","sid":"recNaQ21yNcdLdcSw","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.735},{"b":"piqa","cfg":{"shots":"0"},"dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"piqa-a5b2a15137ed017a","rd":"2023-04-19","sid":"recSBbD5ut1zFgBlR","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.658},{"b":"posttrainbench","cfg":{"effort":"max","scaffold":"Claude Code"},"dn":"Fable 5 (Max)","e":"max","m":"claude-fable-5","oid":"posttrainbench-c12f6d40ae38af81","rd":"2026-06-09","sid":"recP6ZrDsWrdqbEmt","sm":"claude-fable-5_max","v":0.4179},{"b":"posttrainbench","cfg":{"effort":"max","scaffold":"Codex CLI"},"dn":"GPT 5.6 (Sol) (Max)","e":"max","m":"gpt-5-6-sol","oid":"posttrainbench-b25d29bc3af0d615","rd":"2026-07-09","sid":"rec563GDCdzdGWynR","sm":"gpt-5.6-sol_max","v":0.36229999999999996},{"b":"posttrainbench","cfg":{"scaffold":"Claude Code"},"dn":"Opus 5","m":"claude-opus-5","oid":"posttrainbench-d112e8f37655f07c","rd":"2026-07-24","sid":"recxqR2bWuWATTQi2","sm":"claude-opus-5_unknown","v":0.3504},{"b":"posttrainbench","cfg":{"effort":"high","scaffold":"Claude Code"},"dn":"Opus 4.8 (High)","e":"high","m":"claude-opus-4-8","oid":"posttrainbench-e4c998c39be94a34","rd":"2026-05-28","sid":"recXnLB0qsZWf1z1O","sm":"claude-opus-4-8_high","v":0.33840000000000003},{"b":"posttrainbench","cfg":{"effort":"max","scaffold":"Claude Code"},"dn":"Opus 4.8 (Max)","e":"max","m":"claude-opus-4-8","oid":"posttrainbench-62ba2905c5a26604","rd":"2026-05-28","sid":"recxwUdcUZfUJ0Ajy","sm":"claude-opus-4-8_max","v":0.329},{"b":"posttrainbench","cfg":{"scaffold":"Claude Code"},"dn":"Kimi K3","m":"kimi-k3","oid":"posttrainbench-0e5dbc730466a071","rd":"2026-07-16","sid":"reczFkgvLx23FFjG9","sm":"kimi-k3_unknown","v":0.3196},{"b":"posttrainbench","cfg":{"effort":"max","scaffold":"Claude Code"},"dn":"GLM 5.2 (Max)","e":"max","m":"glm-5-2","oid":"posttrainbench-4e9e661e9183dabc","rd":"2026-06-16","sid":"recFx69MKlTBBGigF","sm":"glm-5.2_max","v":0.317},{"b":"posttrainbench","cfg":{"effort":"xhigh","scaffold":"Claude Code"},"dn":"Opus 4.7 (xHigh)","e":"xhigh","m":"claude-opus-4-7","oid":"posttrainbench-ce5179729f43ac5c","rd":"2026-04-16","sid":"recbaD2sTNbt5xVFL","sm":"claude-opus-4-7_xhigh","v":0.2856},{"b":"posttrainbench","cfg":{"effort":"xhigh","scaffold":"Codex CLI"},"dn":"GPT 5.5 (xHigh)","e":"xhigh","m":"gpt-5-5","oid":"posttrainbench-d0510e86e5cca890","rd":"2026-04-23","sid":"recMSOXhMGW3VztFi","sm":"gpt-5.5_xhigh","v":0.2723},{"b":"posttrainbench","cfg":{"effort":"high","scaffold":"Cursor CLI"},"dn":"Grok 4.5 (High)","e":"high","m":"grok-4-5","oid":"posttrainbench-33e8901f485d96b1","rd":"2026-07-08","sid":"rec4HQ0qLNJVhRyw9","sm":"grok-4.5_high","v":0.2345},{"b":"posttrainbench","cfg":{"scaffold":"OpenCode"},"dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"posttrainbench-23909c98d5791e5c","rd":"2026-02-19","sid":"recZWNED3JvA1VUDZ","sm":"gemini-3.1-pro-preview","v":0.21989999999999998},{"b":"posttrainbench","cfg":{"effort":"high","scaffold":"Codex CLI"},"dn":"GPT 5.4 (High)","e":"high","m":"gpt-5-4-2026-03-05","oid":"posttrainbench-d55107d7f11a93fe","rd":"2026-03-05","sid":"recA3c9MYtvW1Fxb8","sm":"gpt-5.4-2026-03-05_high","v":0.19},{"b":"proofbench","c":2.70347,"cfg":{"effort":"max"},"dn":"anthropic/claude-fable-5-1","e":"max","m":"claude-fable-5-1","metrics":{"latency_seconds":478.303},"oid":"proofbench-f7d23281db824ae3","rd":"2026-09-01","sm":"claude-fable-5-1_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":1},{"b":"proofbench","c":0.961648,"cfg":{"effort":"max"},"dn":"anthropic/claude-opus-5-5","e":"max","m":"claude-opus-5-5","metrics":{"latency_seconds":351.035},"oid":"proofbench-c4133cc97186d1e7","rd":"2026-09-22","sm":"claude-opus-5-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":1},{"b":"proofbench","c":0.558327,"cfg":{"effort":"max"},"dn":"anthropic/claude-sonnet-5-5","e":"max","m":"claude-sonnet-5-5","metrics":{"latency_seconds":347.583},"oid":"proofbench-64de02eb0f1b807c","rd":"2026-09-28","sm":"claude-sonnet-5-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":1},{"b":"proofbench","c":1.786125,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-opus-5","e":"max","m":"claude-opus-5","metrics":{"latency_seconds":590.886},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-ba72d8bcfa4170bc","rd":"2026-07-24","se":0.01,"sm":"claude-opus-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.99},{"b":"proofbench","c":1.71238,"dn":"openai/gpt-6-astra","m":"gpt-6-astra","metrics":{"latency_seconds":307.867},"oid":"proofbench-1b7c69c95afd1c03","rd":"2026-09-03","sm":"gpt-6-astra_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.99},{"b":"proofbench","c":0.233057,"dn":"openai/gpt-6.1-sol","m":"gpt-6-1-sol","metrics":{"latency_seconds":284.697},"oid":"proofbench-dbfc69dad7760b0c","rd":"2026-09-29","sm":"gpt-6.1-sol_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.99},{"b":"proofbench","c":0.782978,"dn":"google/gemini-4-argon","m":"gemini-4-argon","metrics":{"latency_seconds":380.246},"oid":"proofbench-5b65e40d0b3106d8","rd":"2026-09-30","sm":"gemini-4-argon_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.99},{"b":"proofbench","c":5.229282,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-fable-5","e":"max","m":"claude-fable-5","metrics":{"latency_seconds":822.739},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-df37d67946fc5516","rd":"2026-06-09","se":0.0219,"sm":"claude-fable-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.9500000000000001},{"b":"proofbench","c":0.474341,"dn":"kimi/kimi-k3.1-pro","m":"kimi-k3-1-pro","metrics":{"latency_seconds":496.158},"oid":"proofbench-c696c9026a162db7","rd":"2026-10-23","se":0.02876,"sm":"kimi-k3.1-pro_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.91},{"b":"proofbench","c":0.173874,"cfg":{"effort":"max"},"dn":"anthropic/claude-haiku-5-5","e":"max","m":"claude-haiku-5-5","metrics":{"latency_seconds":418.047},"oid":"proofbench-25a0ead4f4f86831","rd":"2026-10-07","sm":"claude-haiku-5-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.89},{"b":"proofbench","c":1.646802,"cfg":{"provider":"Moonshot AI"},"dn":"kimi/kimi-k3","m":"kimi-k3","metrics":{"latency_seconds":2070.511},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-39eb93ef85e14d5f","rd":"2026-07-16","se":0.0338,"sm":"kimi-k3_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.87},{"b":"proofbench","c":1.313345,"dn":"meta/muse-wonder","m":"muse-wonder-1-0","metrics":{"latency_seconds":929.053},"oid":"proofbench-9e8a50bc11fde8e2","rd":"2026-10-12","sm":"muse-wonder-1.0_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.84},{"b":"proofbench","c":1.438313,"cfg":{"effort":"max","provider":"OpenAI"},"dn":"openai/gpt-5.6-sol","e":"max","m":"gpt-5-6-sol","metrics":{"latency_seconds":520.405},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-d1af79bcaa7bffc6","rd":"2026-07-09","se":0.03775,"sm":"gpt-5.6-sol_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.8300000000000001},{"b":"proofbench","c":0.476321,"dn":"openai/gpt-6-sol","m":"gpt-6-sol","metrics":{"latency_seconds":304.509},"oid":"proofbench-45448130e9af7f78","rd":"2026-09-22","sm":"gpt-6-sol_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.8300000000000001},{"b":"proofbench","c":1.373684,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-sonnet-5","e":"max","m":"claude-sonnet-5","metrics":{"latency_seconds":898.079},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-f08d694141c9361e","rd":"2026-06-30","se":0.042300000000000004,"sm":"claude-sonnet-5_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.77},{"b":"proofbench","c":0.335717,"dn":"tencent/hy4-preview","m":"hy4-preview","metrics":{"latency_seconds":1373.837},"oid":"proofbench-5022bf04bfcfd693","rd":"2026-08-28","sm":"hy4-preview_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.75},{"b":"proofbench","c":1.274209,"cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"openai/gpt-5.6-terra","e":"xhigh","m":"gpt-5-6-terra","metrics":{"latency_seconds":777.341},"oid":"proofbench-4856aaac6c526c38","rd":"2026-07-09","se":0.04408,"sm":"gpt-5.6-terra_xhigh","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.74},{"b":"proofbench","c":0.24535,"dn":"xiaomi/mimo-v2.6-pro","m":"mimo-v2-6-pro","metrics":{"latency_seconds":3693.469},"oid":"proofbench-3f7b989ff0ecd748","rd":"2026-09-21","sm":"mimo-v2.6-pro","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.7000000000000001},{"b":"proofbench","c":2.293129,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-opus-4-8","e":"max","m":"claude-opus-4-8","metrics":{"latency_seconds":1166.322},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-8c10f4effaaec92f","rd":"2026-05-28","se":0.04648,"sm":"claude-opus-4-8_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.6900000000000001},{"b":"proofbench","c":0.041637,"dn":"openai/gpt-6-luna","m":"gpt-6-luna","metrics":{"latency_seconds":451.481},"oid":"proofbench-feae93773d9e6ff7","rd":"2026-09-22","sm":"gpt-6-luna_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.64},{"b":"proofbench","c":0.123277,"dn":"xiaomi/mimo-v2.6-flash","m":"mimo-v2-6-flash","metrics":{"latency_seconds":3769.808},"oid":"proofbench-c55fba567cf12d51","rd":"2026-09-21","sm":"mimo-v2.6-flash","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.63},{"b":"proofbench","c":0.123288,"cfg":{"effort":"max","provider":"OpenAI"},"dn":"openai/gpt-5.6-luna","e":"max","m":"gpt-5-6-luna","metrics":{"latency_seconds":513.687},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-2f8aae3c4c764d14","rd":"2026-07-09","se":0.049240000000000006,"sm":"gpt-5.6-luna_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.6},{"b":"proofbench","c":0.558508,"dn":"google/gemini-3.7-flash","m":"gemini-3-7-flash","metrics":{"latency_seconds":423.698},"oid":"proofbench-52435caec2633298","rd":"2026-08-13","sm":"gemini-3.7-flash_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.58},{"b":"proofbench","c":1.503984,"dn":"alibaba/qwen3.8-max","m":"qwen3-8-max","metrics":{"latency_seconds":1867.622},"oid":"proofbench-d4578dbeedaf51bb","rd":"2026-08-02","sm":"qwen3.8-max_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.58},{"b":"proofbench","c":0.479019,"cfg":{"effort":"max"},"dn":"meta/muse_spark_1_3_max","e":"max","m":"muse-spark-1-3","metrics":{"latency_seconds":568.696},"oid":"proofbench-25b0d1f0af0c8cc2","rd":"2026-09-02","sm":"muse-spark-1.3_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.58},{"b":"proofbench","c":3.202015,"cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"openai/gpt-5.4-2026-03-05","e":"xhigh","m":"gpt-5-4-2026-03-05","metrics":{"latency_seconds":2751.526},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-543c390bbac49ee0","rd":"2026-03-05","se":0.049640000000000004,"sm":"gpt-5.4-2026-03-05_xhigh","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.56},{"b":"proofbench","c":0.156045,"dn":"deepseek/deepseek-v4-flash-0731","m":"deepseek-v4-flash-0731","metrics":{"latency_seconds":933.981},"oid":"proofbench-5997e9c016fed0ab","rd":"2026-07-31","se":0.04989,"sm":"deepseek-v4-flash-0731_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.56},{"b":"proofbench","c":0.433712,"dn":"meta/muse_spark_1_3","m":"muse-spark-1-3","metrics":{"latency_seconds":534.06},"oid":"proofbench-0185c6e85305e637","rd":"2026-09-02","sm":"muse-spark-1.3_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.55},{"b":"proofbench","c":1.807786,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-opus-4-7","e":"max","m":"claude-opus-4-7","metrics":{"latency_seconds":679.708},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-57c3ac30dafd29ec","rd":"2026-04-16","se":0.04984,"sm":"claude-opus-4-7_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.54},{"b":"proofbench","c":0.126437,"dn":"deepseek/deepseek-v4.1-flash","m":"deepseek-v4-1-flash","metrics":{"latency_seconds":546.595},"oid":"proofbench-7305f600e9caafa0","rd":"2026-09-09","sm":"deepseek-v4.1-flash_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.54},{"b":"proofbench","c":0.824475,"dn":"grok/grok-4.6","m":"grok-4-6","metrics":{"latency_seconds":861.898},"oid":"proofbench-3c1560f6d035df8a","rd":"2026-08-12","se":0.05024,"sm":"grok-4.6_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.51},{"b":"proofbench","c":1.68179,"cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"openai/gpt-5.5","e":"xhigh","m":"gpt-5-5","metrics":{"latency_seconds":2953.443},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-ff50ed4128d45223","rd":"2026-04-23","se":0.05,"sm":"gpt-5.5_xhigh","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.5},{"b":"proofbench","c":2.703583,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-opus-4-6-thinking","e":"max","m":"claude-opus-4-6","metrics":{"latency_seconds":3973.892},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-5d17765cb54894a2","rd":"2026-02-05","se":0.05,"sm":"claude-opus-4-6_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.5},{"b":"proofbench","c":0.262008,"dn":"deepseek/deepseek-v4-pro-0813","m":"deepseek-v4-pro-0813","metrics":{"latency_seconds":732.365},"oid":"proofbench-63e1eac381fbc382","rd":"2026-08-13","se":0.05025,"sm":"deepseek-v4-pro-0813_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.5},{"b":"proofbench","c":2.081538,"cfg":{"effort":"max"},"dn":"zai/glm-5.3","e":"max","m":"glm-5-3","metrics":{"latency_seconds":2542.672},"oid":"proofbench-799e3dae1dcf8182","rd":"2026-08-14","sm":"glm-5.3_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.49},{"b":"proofbench","c":0.602872,"dn":"google/gemini-3.8-flash","m":"gemini-3-8-flash","metrics":{"latency_seconds":368.648},"oid":"proofbench-5f2655cf4ab721e4","rd":"2026-09-02","sm":"gemini-3.8-flash_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.48},{"b":"proofbench","c":2.162108,"cfg":{"effort":"max","provider":"Anthropic"},"dn":"anthropic/claude-sonnet-4-6","e":"max","m":"claude-sonnet-4-6","metrics":{"latency_seconds":5463.593},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-7f5275bc8176fb59","rd":"2026-02-17","se":0.049749999999999996,"sm":"claude-sonnet-4-6_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.45},{"b":"proofbench","c":0.442007,"dn":"meta/muse_spark_1_2","m":"muse-spark-1-2","metrics":{"latency_seconds":469.146},"oid":"proofbench-54beffbba0aa27e1","rd":"2026-08-05","sm":"muse-spark-1.2","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.43},{"b":"proofbench","c":0.714564,"dn":"stepfun/step-5-preview","m":"step-5-preview","metrics":{"latency_seconds":2313.592},"oid":"proofbench-24839db1e286f11b","rd":"2026-09-18","sm":"step-5-preview_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.42},{"b":"proofbench","c":0.519718,"cfg":{"effort":"xhigh","provider":"Meta"},"dn":"meta/muse_spark_1_1","e":"xhigh","m":"muse-spark-1-1","metrics":{"latency_seconds":1056.902},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-d1d3efb129f57bf7","rd":"2026-07-09","se":0.04902,"sm":"muse-spark-1.1","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.39},{"b":"proofbench","c":1.267043,"cfg":{"effort":"high","provider":"Anthropic"},"dn":"anthropic/claude-opus-4-5-20251101-thinking","m":"claude-opus-4-5-2025-11-01","metrics":{"latency_seconds":3757.671},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-162b5c1d7ad15eeb","rd":"2025-11-24","se":0.048,"sm":"claude-opus-4-5-20251101_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.36},{"b":"proofbench","c":0.522059,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3.6-flash","m":"gemini-3-6-flash","metrics":{"latency_seconds":346.937},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-6698ea1858abb516","rd":"2026-07-21","se":0.04824,"sm":"gemini-3.6-flash_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.36},{"b":"proofbench","c":0,"cfg":{"effort":"max","provider":"Zhipu AI"},"dn":"zai/glm-5.2","e":"max","m":"glm-5-2","metrics":{"latency_seconds":0},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-2fedabaf0bae6c5b","rd":"2026-06-16","se":0.047939999999999997,"sm":"glm-5.2_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.35000000000000003},{"b":"proofbench","c":0.573991,"cfg":{"effort":"high","provider":"SpaceXAI"},"dn":"grok/grok-4.5","e":"high","m":"grok-4-5","metrics":{"latency_seconds":668.86},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-d6f4d1e0ab6b017f","rd":"2026-07-08","se":0.04648,"sm":"grok-4.5_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.31},{"b":"proofbench","c":0.569863,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3.5-flash","e":"high","m":"gemini-3-5-flash","metrics":{"latency_seconds":394.476},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-1a9fea1374c122a9","rd":"2026-05-19","se":0.04648,"sm":"gemini-3.5-flash_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.31},{"b":"proofbench","c":2.462069,"cfg":{"provider":"Alibaba"},"dn":"alibaba/qwen3.7-max","m":"qwen3-7-max","metrics":{"latency_seconds":771.408},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-30094dd9723151dd","rd":"2026-05-19","se":0.04408,"sm":"qwen3.7-max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.26},{"b":"proofbench","c":0.784253,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3.1-pro-preview","e":"high","m":"gemini-3-1-pro-preview","metrics":{"latency_seconds":889.059},"oid":"proofbench-0af01f494f4af2f0","rd":"2026-02-19","se":0.04408,"sm":"gemini-3.1-pro-preview","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.26},{"b":"proofbench","c":0.787951,"dn":"grok/grok-4.7","m":"grok-4-7","metrics":{"latency_seconds":767.177},"oid":"proofbench-58f48e41660fdd49","rd":"2026-09-21","sm":"grok-4.7_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.26},{"b":"proofbench","c":0.584577,"cfg":{"provider":"Zhipu AI"},"dn":"zai/glm-5.1","m":"glm-5-1","metrics":{"latency_seconds":4755.746},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-10d90abbdce660f5","rd":"2026-04-07","se":0.04178,"sm":"glm-5.1","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.22222000000000003},{"b":"proofbench","c":0.089853,"cfg":{"provider":"Xiaomi"},"dn":"xiaomi/mimo-v2.5-pro","m":"mimo-v2-5-pro","metrics":{"latency_seconds":1177.352},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-a0d83489d23f1d36","se":0.04163,"sm":"mimo-v2.5-pro","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.22},{"b":"proofbench","c":0.249441,"cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"openai/gpt-5.4-mini-2026-03-17","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","metrics":{"latency_seconds":1410.671},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-9b83e2f082c2710e","rd":"2026-03-17","se":0.04073,"sm":"gpt-5.4-mini-2026-03-17_xhigh","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.21},{"b":"proofbench","c":0.224234,"cfg":{"effort":"max"},"dn":"zai/glm-5.3-flash","e":"max","m":"glm-5-3-flash","metrics":{"latency_seconds":4274.803},"oid":"proofbench-9bc741170f8a204a","rd":"2026-08-20","sm":"glm-5.3-flash_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.21},{"b":"proofbench","c":1.229076,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3-pro-preview","e":"high","m":"gemini-3-pro-preview","metrics":{"latency_seconds":10383.833},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-0dfc2db2aa6093f9","rd":"2025-11-18","se":0.04,"sm":"gemini-3-pro-preview","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.2},{"b":"proofbench","c":0.65341,"cfg":{"provider":"Anthropic"},"dn":"anthropic/claude-sonnet-4-5-20250929-thinking","m":"claude-sonnet-4-5-2025-09-29","metrics":{"latency_seconds":12300.813},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-ce2877ef7e21316b","rd":"2025-09-29","se":0.03923,"sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.19},{"b":"proofbench","c":0.419224,"cfg":{"provider":"MiniMax"},"dn":"minimax/MiniMax-M3","m":"minimax-m3","metrics":{"latency_seconds":718.66},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-b25f3ee9a7d1b6f9","rd":"2026-06-01","se":0.038610000000000005,"sm":"MiniMax-M3","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.18},{"b":"proofbench","c":0.852361,"cfg":{"effort":"high","provider":"OpenAI"},"dn":"openai/gpt-5-2025-08-07","e":"high","m":"gpt-5-2025-08-07","metrics":{"latency_seconds":4579.325},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-4618214002c3c2ce","rd":"2025-08-07","se":0.03842,"sm":"gpt-5-2025-08-07_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.18},{"b":"proofbench","dn":"reflection/Beam-1.0","m":"beam-1-0","metrics":{"latency_seconds":266.658},"oid":"proofbench-66f19c9594250461","rd":"2026-10-05","se":0.038610000000000005,"sm":"beam-1.0_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.18},{"b":"proofbench","c":0.067086,"cfg":{"provider":"Meta"},"dn":"meta/muse_spark","m":"muse-spark","metrics":{"latency_seconds":478.958},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-57ef15ef48aae753","rd":"2026-04-08","se":0.037559999999999996,"sm":"muse-spark","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.17},{"b":"proofbench","c":0.295811,"cfg":{"provider":"Moonshot AI"},"dn":"kimi/kimi-k2.6","m":"kimi-k2-6","metrics":{"latency_seconds":1386.754},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-a60ec8a78f4648c3","rd":"2026-04-20","se":0.03666,"sm":"kimi-k2.6","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.16},{"b":"proofbench","c":0.05707,"cfg":{"provider":"Xiaomi"},"dn":"xiaomi/mimo-v2.5","m":"mimo-v2-5","metrics":{"latency_seconds":1255.813},"oid":"proofbench-3d3a61afc18308ff","se":0.03685,"sm":"mimo-v2.5","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.16},{"b":"proofbench","c":0.142045,"cfg":{"effort":"max","provider":"DeepSeek"},"dn":"deepseek/deepseek-v4-pro","e":"max","m":"deepseek-v4-pro","metrics":{"latency_seconds":423.45},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-591112d6f376acb0","rd":"2026-04-24","se":0.03685,"sm":"deepseek-v4-pro_max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.16},{"b":"proofbench","c":2.298832,"dn":"alibaba/qwen3.8-27b","m":"qwen3-8-27b","metrics":{"latency_seconds":1453.872},"oid":"proofbench-fd95e4c00b05ead4","rd":"2026-08-14","sm":"qwen3.8-27b","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.16},{"b":"proofbench","c":0.327286,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3-flash-preview","e":"high","m":"gemini-3-flash-preview","metrics":{"latency_seconds":1141.548},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-d313244d56c8dab0","rd":"2025-12-17","se":0.035710000000000006,"sm":"gemini-3-flash-preview","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.15},{"b":"proofbench","c":1.059689,"cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"openai/gpt-5.2-2025-12-11","e":"xhigh","m":"gpt-5-2-2025-12-11","metrics":{"latency_seconds":1493.817},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-28cfa8655f64ce87","rd":"2025-12-11","se":0.035710000000000006,"sm":"gpt-5.2-2025-12-11_xhigh","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.15},{"b":"proofbench","c":0.31815,"cfg":{"provider":"SpaceXAI"},"dn":"grok/grok-4.20-0309-reasoning","m":"grok-4-20-0309-reasoning","metrics":{"latency_seconds":494.946},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-979d5c4bcd1d9125","rd":"2026-02-17","se":0.0347,"sm":"grok-4.20-0309-reasoning","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.14},{"b":"proofbench","c":0.132788,"cfg":{"effort":"high","provider":"Google"},"dn":"google/gemini-3.5-flash-lite","e":"high","m":"gemini-3-5-flash-lite","metrics":{"latency_seconds":290.614},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-27392bf56a8edfc5","rd":"2026-07-21","se":0.0338,"sm":"gemini-3.5-flash-lite","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.13},{"b":"proofbench","c":0.049218,"cfg":{"effort":"high","provider":"OpenAI"},"dn":"openai/gpt-5-nano-2025-08-07","e":"high","m":"gpt-5-nano-2025-08-07","metrics":{"latency_seconds":3729.837},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-9ea2c1ad48ab6a9f","rd":"2025-08-07","se":0.0325,"sm":"gpt-5-nano-2025-08-07_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.12},{"b":"proofbench","c":0.244122,"cfg":{"effort":"high","provider":"SpaceXAI"},"dn":"grok/grok-4.3","e":"high","m":"grok-4-3","metrics":{"latency_seconds":1643.019},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-98b7888a1d8d720c","rd":"2026-04-17","se":0.03129,"sm":"grok-4.3_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.11},{"b":"proofbench","c":0.642365,"dn":"mistralai/mistral-large-4","m":"mistral-large-4-0","metrics":{"latency_seconds":1258.806},"oid":"proofbench-7025ee229bc021fb","rd":"2026-10-06","sm":"mistral-large-4-0_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.1},{"b":"proofbench","c":0.22021,"cfg":{"effort":"high","provider":"OpenAI"},"dn":"openai/gpt-5-mini-2025-08-07","e":"high","m":"gpt-5-mini-2025-08-07","metrics":{"latency_seconds":3349.817},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-948a12be47953b9e","rd":"2025-08-07","se":0.028620000000000003,"sm":"gpt-5-mini-2025-08-07_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.09},{"b":"proofbench","c":0.527356,"cfg":{"effort":"high","provider":"OpenAI"},"dn":"openai/gpt-5.1-codex-max","e":"high","m":"gpt-5-1-codex-max","metrics":{"latency_seconds":3184.213},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-60dcfe4d495409cd","rd":"2025-11-19","se":0.028620000000000003,"sm":"gpt-5.1-codex-max","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.09},{"b":"proofbench","c":1.302032,"cfg":{"effort":"high","provider":"Mistral AI"},"dn":"mistralai/mistral-medium-3.5","e":"high","m":"mistral-medium-2604","metrics":{"latency_seconds":479.048},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-c80aac6b7a10bba3","rd":"2026-04-28","se":0.02876,"sm":"mistral-medium-2604","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.09},{"b":"proofbench","c":0.11773,"cfg":{"effort":"none","provider":"Fireworks AI"},"dn":"fireworks/deepseek-v3p2","e":"none","m":"fireworksdeepseek-v3p2","metrics":{"latency_seconds":3981.084},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-9e9d8c827398bee6","rd":"2025-12-01","se":0.02713,"sm":"fireworks/deepseek-v3p2","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.08},{"b":"proofbench","c":0.14404,"cfg":{"provider":"Zhipu AI"},"dn":"zai/glm-4.7","m":"glm-4-7","metrics":{"latency_seconds":3364.789},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-810ea0761d3b39db","rd":"2025-12-22","se":0.02375,"sm":"glm-4.7","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.06},{"b":"proofbench","c":0.082234,"cfg":{"effort":"0.99","provider":"Thinkingmachines"},"dn":"thinkingmachines/inkling-small","e":"0.99","m":"inkling-small","metrics":{"latency_seconds":507.301},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-b474012b269629fe","rd":"2026-07-15","se":0.023870000000000002,"sm":"inkling-small","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.06},{"b":"proofbench","c":0.015894,"cfg":{"effort":"high","provider":"OpenAI"},"dn":"openai/gpt-5.4-nano-2026-03-17","e":"high","m":"gpt-5-4-nano-2026-03-17","metrics":{"latency_seconds":525.432},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-c9a8381a21366623","rd":"2026-03-17","se":0.02179,"sm":"gpt-5.4-nano-2026-03-17_high","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.05},{"b":"proofbench","c":0.096578,"cfg":{"provider":"SpaceXAI"},"dn":"grok/grok-4-1-fast-reasoning","m":"grok-4-1-fast-reasoning","metrics":{"latency_seconds":1421.041},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-5286fbce511bb999","rd":"2025-11-19","se":0.0196,"sm":"grok-4-1-fast-reasoning","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.04},{"b":"proofbench","c":0.057497,"cfg":{"provider":"MiniMax"},"dn":"minimax/MiniMax-M2.5","m":"minimax-m2-5","metrics":{"latency_seconds":4514.143},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-b10529f1690c39ff","rd":"2026-02-12","se":0.0196,"sm":"MiniMax-M2.5","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.04},{"b":"proofbench","c":0.04293,"cfg":{"provider":"MiniMax"},"dn":"minimax/MiniMax-M2.7","m":"minimax-m2-7","metrics":{"latency_seconds":1634.068},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-18e666877b931e59","rd":"2026-03-18","se":0.01706,"sm":"MiniMax-M2.7","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.03},{"b":"proofbench","c":0.044105,"dn":"inception/mercury-2.5","m":"mercury-2-5","metrics":{"latency_seconds":201.725},"oid":"proofbench-2f5554d6ab6e6fa6","rd":"2026-09-08","sm":"mercury-2.5_unknown","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.03},{"b":"proofbench","cfg":{"provider":"Nvidia"},"dn":"nvidia/nemotron-3-ultra-550b-a55b","m":"nemotron-3-ultra","metrics":{"latency_seconds":2552.288},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-00a863192beb7160","rd":"2026-06-04","se":0.014070000000000001,"sm":"nemotron-3-ultra","src":"https://www.vals.ai/benchmarks/proof_bench","v":0.02},{"b":"proofbench","c":0.024998,"cfg":{"provider":"Poolside"},"dn":"poolside/laguna-xs.2","m":"laguna-xs-2","metrics":{"latency_seconds":315.721},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-946a199d5382bd11","rd":"2026-04-28","se":0,"sm":"laguna-xs.2","src":"https://www.vals.ai/benchmarks/proof_bench","v":0},{"b":"proofbench","c":0.067403,"cfg":{"provider":"Poolside"},"dn":"poolside/laguna-m.1","m":"laguna-m-1","metrics":{"latency_seconds":423.12},"notes":"ProofBench v1.0-era value; model absent from the v1.1 board (vals.ai re-grade 2026-08-14, methodology changed) and not re-run. Not comparable to v1.1 rows. Annotated 2026-08-18.","oid":"proofbench-ab341aa7398fd4c9","rd":"2026-04-28","se":0,"sm":"laguna-m.1","src":"https://www.vals.ai/benchmarks/proof_bench","v":0},{"b":"proofbench","c":0.198639,"cfg":{"provider":"Thinkingmachines"},"dn":"thinkingmachines/inkling","m":"inkling","metrics":{"latency_seconds":436.43},"notes":"ProofBench v1.1 re-grade (vals.ai, 2026-08-14): corrected Lean formalizations + hardened grader (rejects sorry/admit shortcuts, larger elaboration budgets). Not comparable to v1.0-era values. Updated 2026-08-18 after multi-agent verification.","oid":"proofbench-7466ba9af9cab9e4","rd":"2026-07-15","se":0,"sm":"Inkling","src":"https://www.vals.ai/benchmarks/proof_bench","v":0},{"b":"rli","dn":"Opus 5.5","m":"claude-opus-5-5","oid":"rli-998691805f600105","rd":"2026-09-22","sid":"rec3RdZkH2jPijNAA","sm":"claude-opus-5-5_unknown","v":0.2125},{"b":"rli","dn":"GPT 6 Astra","m":"gpt-6-astra","oid":"rli-c1c37146f30c4a4f","rd":"2026-09-03","sid":"recgQnocVAluxtaGu","sm":"gpt-6-astra_unknown","v":0.20829999999999999},{"b":"rli","dn":"Fable 5.1","m":"claude-fable-5-1","oid":"rli-0d1d655085b60a6a","rd":"2026-09-01","sid":"recjGFecys5ECoyQg","sm":"claude-fable-5-1_unknown","v":0.17920000000000003},{"b":"rli","dn":"Gemini Argon 4","m":"gemini-4-argon","oid":"rli-fce9a38fb80b3875","rd":"2026-09-30","sid":"recn7rYNMvwHrvYKu","sm":"gemini-4-argon_unknown","v":0.17079999999999998},{"b":"rli","dn":"Fable-5","m":"claude-fable-5","oid":"rli-2d696af89056c68d","rd":"2026-06-09","sid":"recuSPVAbyN1m9xpA","sm":"claude-fable-5","v":0.161},{"b":"rli","dn":"Fable 5","m":"claude-fable-5","oid":"rli-aecf3ef6fd574f47","rd":"2026-06-09","sid":"recViBWyhB5Kow9VY","sm":"claude-fable-5_unknown","v":0.1583},{"b":"rli","dn":"Opus 4.8","m":"claude-opus-4-8","oid":"rli-a514bf585af3d861","rd":"2026-05-28","sid":"reca1681E9WeV4Fcd","sm":"claude-opus-4-8_unknown","v":0.0833},{"b":"rli","dn":"Codex GPT 5.5","m":"gpt-5-5","oid":"rli-97e7723c19633452","rd":"2026-04-23","sid":"rec3Yv4ZA0c008npD","sm":"gpt-5.5_unknown","v":0.0625},{"b":"rli","dn":"Gemini 3.8 Flash","m":"gemini-3-8-flash","oid":"rli-9c06ecafcab08fab","rd":"2026-09-02","sid":"recZEtu0hO5eRfs1Y","sm":"gemini-3.8-flash_unknown","v":0.058300000000000005},{"b":"rli","dn":"Gemini 3.7 Flash","m":"gemini-3-7-flash","oid":"rli-b6f20c9ddc15ce77","rd":"2026-08-13","sid":"recIhfw1UfEy3uVSL","sm":"gemini-3.7-flash_unknown","v":0.05},{"b":"rli","dn":"claude-opus-4-6 (CoWork)","m":"claude-opus-4-6","oid":"rli-3b902f74413ab7ce","rd":"2026-02-05","sid":"reciJQ128sKYL8guC","sm":"claude-opus-4-6_unknown","v":0.0417},{"b":"rli","dn":"claude-opus-4-5-20251101-thinking","m":"claude-opus-4-5-2025-11-01","oid":"rli-8bab81519253baf0","rd":"2025-11-24","sid":"recOr0uhtoQjJ9idw","sm":"claude-opus-4-5-20251101_unknown","v":0.0375},{"b":"rli","cfg":{"effort":"medium"},"dn":"gpt-5.2-2025-12-11 (medium)","e":"medium","m":"gpt-5-2-2025-12-11","oid":"rli-1eae4fdc0dd48207","rd":"2025-12-11","sid":"recxOBMXzUoImRHvn","sm":"gpt-5.2-2025-12-11_medium","v":0.025019999999999997},{"b":"rli","dn":"gpt-5.2-2025-12-11 (default)","m":"gpt-5-2-2025-12-11","oid":"rli-0fe02b32705c9009","rd":"2025-12-11","sid":"rec5arqgc6HV2I2Vm","sm":"gpt-5.2-2025-12-11_unknown","v":0.020800000000000003},{"b":"rli","dn":"claude-4-5-Sonnet","m":"claude-sonnet-4-5-2025-09-29","oid":"rli-be4e7565d616b51f","rd":"2025-09-29","sid":"recTwto22Sfk92hl2","sm":"claude-sonnet-4-5-20250929_unknown","v":0.020800000000000003},{"b":"rli","dn":"gpt-5-2025-08-07","m":"gpt-5-2025-08-07","oid":"rli-75bb41a44741c069","rd":"2025-08-07","sid":"rec6bwbUoEHIUSUWL","sm":"gpt-5-2025-08-07_unknown","v":0.0167},{"b":"rli","dn":"gemini-3-pro-preview","m":"gemini-3-pro-preview","oid":"rli-e4ad458c559f4e55","rd":"2025-11-18","sid":"recJGPLMjez47bc8p","sm":"gemini-3-pro-preview","v":0.0125},{"b":"rli","dn":"gemini-2.5-pro-preview-06-05","m":"gemini-2-5-pro-preview-06-05","oid":"rli-5da0adcec74daed2","rd":"2025-06-05","sid":"rec9Y98phtupm7tbq","sm":"gemini-2.5-pro-preview-06-05","v":0.0083},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Fable 5 (Adaptive Reasoning, Max Effort, Opus 4.8 Fallback)","e":"max","m":"claude-fable-5","oid":"scicode-e5fc222441f01050","rd":"2026-06-09","sm":"claude-fable-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.609953703703704},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemini 3.1 Pro Preview","m":"gemini-3-1-pro-preview","oid":"scicode-a2f36d68bbc584cd","rd":"2026-02-19","sm":"gemini-3.1-pro-preview","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.586805555555556},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.4 (xhigh)","e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"scicode-e63e500b616fbc4e","rd":"2026-03-05","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.565972222222222},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.5 (Xhigh)","e":"xhigh","m":"gpt-5-5","oid":"scicode-0c3627256b5ed9af","rd":"2026-04-23","sm":"gpt-5.5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.55787037037037},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-5.5 (High)","e":"high","m":"gpt-5-5","oid":"scicode-0c4a02bd2993e7be","rd":"2026-04-23","sm":"gpt-5.5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.561342592592593},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 4.7 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-4-7","oid":"scicode-48e40876f4755d38","rd":"2026-04-16","sm":"claude-opus-4-7_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.545138888888889},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-5.5 (Medium)","e":"medium","m":"gpt-5-5","oid":"scicode-4722ff0a741e35b7","rd":"2026-04-23","sm":"gpt-5.5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.545138888888889},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 4.8 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-4-8","oid":"scicode-77c8306f4999bdc1","rd":"2026-05-28","sm":"claude-opus-4-8_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.534722222222222},{"b":"scicode","cfg":{"provider":"Kimi"},"dn":"Kimi K2.6","m":"kimi-k2-6","oid":"scicode-ab58971ea5924530","rd":"2026-04-20","sm":"kimi-k2.6","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.534722222222222},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.5 Flash (High)","e":"high","m":"gemini-3-5-flash","oid":"scicode-f9bb3ae51f6633c5","rd":"2026-05-19","sm":"gemini-3.5-flash_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.539351851851852},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-5.5 (low)","e":"low","m":"gpt-5-5","oid":"scicode-c6d87d0bb85a14d2","rd":"2026-04-23","sm":"gpt-5.5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","cfg":{"provider":"Meta"},"dn":"Muse Spark","m":"muse-spark","oid":"scicode-491a278c4e11b1ed","rd":"2026-04-08","sm":"muse-spark","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.515046296296296},{"b":"scicode","cfg":{"effort":"max","provider":"Z AI"},"dn":"GLM-5.2 (Max)","e":"max","m":"glm-5-2","oid":"scicode-ecbd5b2c865dd093","rd":"2026-06-16","sm":"glm-5.2_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.511574074074074},{"b":"scicode","cfg":{"provider":"Xiaomi"},"dn":"MiMo-V2.5-Pro","m":"mimo-v2-5-pro","oid":"scicode-df50510042beba06","sm":"mimo-v2.5-pro","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.502314814814815},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Pro (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-pro","oid":"scicode-00ba941a2f696c63","rd":"2026-04-24","sm":"deepseek-v4-pro_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.5},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.4 mini (Xhigh)","e":"xhigh","m":"gpt-5-4-mini-2026-03-17","oid":"scicode-4b1f8730eb3096a9","rd":"2026-03-17","sm":"gpt-5.4-mini-2026-03-17_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.520833333333333},{"b":"scicode","cfg":{"effort":"max","provider":"Alibaba"},"dn":"Qwen3.7 Max","e":"max","m":"qwen3-7-max","oid":"scicode-3e1dbd5eb13e9023","rd":"2026-05-19","sm":"qwen3.7-max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.49537037037037},{"b":"scicode","cfg":{"provider":"Kimi"},"dn":"Kimi K2.7 Code","m":"kimi-k2-7-code","oid":"scicode-9b0a5f9d62dd6764","rd":"2026-06-12","sm":"kimi-k2.7-code","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.478009259259259},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-5.5 (Non-reasoning)","e":"none","m":"gpt-5-5","oid":"scicode-3585d0849953fb21","rd":"2026-04-23","sm":"gpt-5.5_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.47337962962963},{"b":"scicode","cfg":{"effort":"high","provider":"xAI"},"dn":"Grok 4.3 (High)","e":"high","m":"grok-4-3","oid":"scicode-e53b04f9cec4c6e7","rd":"2026-04-17","sm":"grok-4.3_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.482638888888889},{"b":"scicode","cfg":{"provider":"MiniMax"},"dn":"MiniMax-M2.7","m":"minimax-m2-7","oid":"scicode-e1214057b33a328a","rd":"2026-03-18","sm":"MiniMax-M2.7","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.501157407407407},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.4 nano (Xhigh)","e":"xhigh","m":"gpt-5-4-nano-2026-03-17","oid":"scicode-45ca33470c5816c0","rd":"2026-03-17","sm":"gpt-5.4-nano-2026-03-17_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.472222222222222},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Sonnet 4.6 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-sonnet-4-6","oid":"scicode-f527c16233f86703","rd":"2026-02-17","sm":"claude-sonnet-4-6_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.467592592592593},{"b":"scicode","cfg":{"provider":"MiniMax"},"dn":"MiniMax-M3","m":"minimax-m3","oid":"scicode-7d943963fc39a441","rd":"2026-06-01","sm":"MiniMax-M3","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.47106481481481505},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Flash (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-flash","oid":"scicode-17393c4398b6d108","rd":"2026-04-24","sm":"deepseek-v4-flash_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.44907407407407407},{"b":"scicode","cfg":{"provider":"Z AI"},"dn":"GLM-5.1 (Reasoning)","m":"glm-5-1","oid":"scicode-52d603f4467ad035","rd":"2026-04-07","sm":"glm-5.1","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.447916666666667},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemma 4 31B (Reasoning)","m":"gemma-4-31b-it","oid":"scicode-53ee1755a8605112","rd":"2026-04-02","sm":"gemma-4-31b-it","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.454861111111111},{"b":"scicode","cfg":{"provider":"Anthropic"},"dn":"Claude 4.5 Haiku (Reasoning)","m":"claude-haiku-4-5-2025-10-01","oid":"scicode-98d5c54d02846111","rd":"2025-10-15","sm":"claude-haiku-4-5-20251001_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.42245370370370405},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemini 3.1 Flash-Lite","m":"gemini-3-1-flash-lite","oid":"scicode-a8c863bd80ccd241","rd":"2026-03-03","sm":"gemini-3.1-flash-lite","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.434027777777778},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.6 Plus","m":"qwen3-6-plus","oid":"scicode-1b1719c62fd1896d","rd":"2026-03-31","sm":"qwen3.6-plus","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.407407407407407},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemma 4 26B A4B (Reasoning)","m":"gemma-4-26b-a4b","oid":"scicode-00a8e8ac70e0c352","rd":"2026-04-02","sm":"gemma-4-26b-a4b","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4004629629629631},{"b":"scicode","cfg":{"provider":"NVIDIA"},"dn":"Nemotron 3 Ultra 550B A55B (Reasoning)","m":"nemotron-3-ultra","oid":"scicode-672d7e50ef2e5d7a","rd":"2026-06-04","sm":"nemotron-3-ultra","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.402777777777778},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"gpt-oss-120b (High)","e":"high","m":"openaigpt-oss-120b","oid":"scicode-fc8f0f7b770ff25d","rd":"2025-08-05","sm":"openai/gpt-oss-120b_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.340277777777778},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.6 35B A3B (Reasoning)","m":"qwen3-6-35b-a3b","oid":"scicode-3ca8206d7a426564","rd":"2026-04-14","sm":"qwen3.6-35b-a3b","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.365740740740741},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"gpt-oss-20b (High)","e":"high","m":"gpt-oss-20b","oid":"scicode-138f3ca1fd3e1d10","rd":"2025-08-05","sm":"gpt-oss-20b_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.388888888888889},{"b":"scicode","cfg":{"provider":"Meta"},"dn":"Llama 4 Maverick","m":"llama-4-maverick-17b-128e-instruct","oid":"scicode-9bb17fdc56639feb","rd":"2025-04-05","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.31712962962963},{"b":"scicode","cfg":{"provider":"Meta"},"dn":"Llama 4 Scout","m":"llama-4-scout-17b-16e-instruct","oid":"scicode-0e9f9f83a82b9799","rd":"2025-04-05","sm":"Llama-4-Scout-17B-16E-Instruct","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.212962962962963},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-sonnet-5","oid":"scicode-bedc262afa0bf5f6","rd":"2026-06-30","sm":"claude-sonnet-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.53587962962963},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-5.5 Instant (June 2026)","m":"gpt-5-5-instant","oid":"scicode-376c720328e34d59","rd":"2026-05-05","sm":"gpt-5.5-instant","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.525462962962963},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"scicode-6da80df91f718b4f","rd":"2025-06-17","sm":"gemini-2.5-pro","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4629629629629631},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-4.1 mini","m":"gpt-4-1-mini-2025-04-14","oid":"scicode-e74c20f7c178136d","rd":"2025-04-14","sm":"gpt-4.1-mini-2025-04-14","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.403935185185185},{"b":"scicode","cfg":{"provider":"Anthropic"},"dn":"Claude 4 Sonnet (Reasoning)","m":"claude-sonnet-4-2025-05-14","oid":"scicode-d03d0540e6caa12f","rd":"2025-05-22","sm":"claude-sonnet-4-20250514_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4004629629629631},{"b":"scicode","cfg":{"provider":"DeepSeek"},"dn":"DeepSeek V3 0324","m":"deepseek-v3-0324","oid":"scicode-37e617453578cabf","rd":"2025-03-24","sm":"DeepSeek-V3-0324","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.390046296296296},{"b":"scicode","cfg":{"provider":"DeepSeek"},"dn":"DeepSeek R1 (Jan '25)","m":"deepseek-r1","oid":"scicode-8c808a952ccf9c15","rd":"2025-01-20","sm":"DeepSeek-R1","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.3831018518518521},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 32B (Reasoning)","m":"qwen3-32b","oid":"scicode-eaea4937aa9e2dcf","rd":"2025-04-29","sm":"Qwen3-32B","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.354166666666667},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.5 9B (Reasoning)","m":"qwen3-5-9b","oid":"scicode-8136ff92c334883f","rd":"2026-02-24","sm":"qwen3.5-9B","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.295138888888889},{"b":"scicode","cfg":{"provider":"Anthropic"},"dn":"Claude 3.5 Haiku","m":"claude-3-5-haiku-2024-10-22","oid":"scicode-064cf962d9f71928","rd":"2024-10-22","sm":"claude-3-5-haiku-20241022","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.274305555555556},{"b":"scicode","cfg":{"provider":"Xiaomi"},"dn":"MiMo-V2-Flash (Non-reasoning)","m":"mimo-v2-flash","oid":"scicode-f3967d9d183de351","sm":"mimo-v2-flash","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.259259259259259},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-4.1 nano","m":"gpt-4-1-nano-2025-04-14","oid":"scicode-d79e0d4312b6fc57","rd":"2025-04-14","sm":"gpt-4.1-nano-2025-04-14","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.259259259259259},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemma 3 27B Instruct","m":"gemma-3-27b-it","oid":"scicode-7101178cd0ed9c84","rd":"2025-03-12","sm":"gemma-3-27b-it","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.232638888888889},{"b":"scicode","cfg":{"provider":"Meta"},"dn":"Llama 3.1 Instruct 8B","m":"llama-3-1-8b-instruct","oid":"scicode-209ccfd20489224e","rd":"2024-07-23","sm":"Llama-3.1-8B-Instruct","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.131944444444444},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-5.1 (high)","m":"gpt-5-1-2025-11-13","oid":"scicode-f830977c9a64cad6","rd":"2025-11-13","sm":"gpt-5.1-2025-11-13_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.43287037037037},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-5.6 Sol (High)","e":"high","m":"gpt-5-6-sol","oid":"scicode-f3185f4d7882c5b2","rd":"2026-07-09","sm":"gpt-5.6-sol_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.577546296296296},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-5.6 Sol (Medium)","e":"medium","m":"gpt-5-6-sol","oid":"scicode-0223f87cd480aaf0","rd":"2026-07-09","sm":"gpt-5.6-sol_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.574074074074074},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-5.6 Sol (Max)","e":"max","m":"gpt-5-6-sol","oid":"scicode-8859c00a5f2072f3","rd":"2026-07-09","sm":"gpt-5.6-sol_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.570601851851852},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.6 Sol (Xhigh)","e":"xhigh","m":"gpt-5-6-sol","oid":"scicode-306532f63cd102e2","rd":"2026-07-09","sm":"gpt-5.6-sol_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.570601851851852},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-5.6 Sol (Low)","e":"low","m":"gpt-5-6-sol","oid":"scicode-1be5c661b8684a33","rd":"2026-07-09","sm":"gpt-5.6-sol_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.563657407407407},{"b":"scicode","cfg":{"effort":"high","provider":"SpaceXAI"},"dn":"Grok 4.5 (High)","e":"high","m":"grok-4-5","oid":"scicode-a0e1aae57708f860","rd":"2026-07-08","sm":"grok-4.5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.549768518518518},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-5.6 Terra (Max)","e":"max","m":"gpt-5-6-terra","oid":"scicode-237f5f37ff4be159","rd":"2026-07-09","sm":"gpt-5.6-terra_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.549768518518518},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-5.6 Luna (Max)","e":"max","m":"gpt-5-6-luna","oid":"scicode-106d87d1e0a19086","rd":"2026-07-09","sm":"gpt-5.6-luna_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.53587962962963},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.6 Terra (Xhigh)","e":"xhigh","m":"gpt-5-6-terra","oid":"scicode-6ebf9a23462817c1","rd":"2026-07-09","sm":"gpt-5.6-terra_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.523148148148148},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-5.6 Luna (High)","e":"high","m":"gpt-5-6-luna","oid":"scicode-aa48dcfa70515a2a","rd":"2026-07-09","sm":"gpt-5.6-luna_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-5.6 Terra (High)","e":"high","m":"gpt-5-6-terra","oid":"scicode-4ab4e6ef5c008ce5","rd":"2026-07-09","sm":"gpt-5.6-terra_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.524305555555556},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-5.6 Luna (Xhigh)","e":"xhigh","m":"gpt-5-6-luna","oid":"scicode-c73da8d9f446649a","rd":"2026-07-09","sm":"gpt-5.6-luna_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.50462962962963},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-5.6 Terra (Medium)","e":"medium","m":"gpt-5-6-terra","oid":"scicode-0bfcbe6a7f9812c2","rd":"2026-07-09","sm":"gpt-5.6-terra_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.50462962962963},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-5.6 Terra (Low)","e":"low","m":"gpt-5-6-terra","oid":"scicode-8ba05ee10c15f962","rd":"2026-07-09","sm":"gpt-5.6-terra_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.49884259259259306},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-5.6 Luna (Medium)","e":"medium","m":"gpt-5-6-luna","oid":"scicode-4e002843562faf2d","rd":"2026-07-09","sm":"gpt-5.6-luna_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.467592592592593},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-5.6 Luna (Low)","e":"low","m":"gpt-5-6-luna","oid":"scicode-463773ab6d8b75e5","rd":"2026-07-09","sm":"gpt-5.6-luna_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.460648148148148},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-5 (high)","m":"gpt-5-2025-08-07","oid":"scicode-75220199a11b22bb","rd":"2025-08-07","sm":"gpt-5-2025-08-07_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.429398148148148},{"b":"scicode","cfg":{"provider":"OpenAI"},"dn":"GPT-5 mini (high)","m":"gpt-5-mini-2025-08-07","oid":"scicode-83a66669f1ea363c","rd":"2025-08-07","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.392361111111111},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive Reasoning, Max Effort)","e":"max","m":"claude-opus-5","oid":"scicode-3ee2e7b2f6924cd5","rd":"2026-07-24","sm":"claude-opus-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.563657407407407},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive Reasoning, Xhigh Effort)","e":"xhigh","m":"claude-opus-5","oid":"scicode-8963a2d2d445829d","rd":"2026-07-24","sm":"claude-opus-5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.549768518518518},{"b":"scicode","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive Reasoning, High Effort)","e":"high","m":"claude-opus-5","oid":"scicode-b0a0b35d3b752d76","rd":"2026-07-24","sm":"claude-opus-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.542824074074074},{"b":"scicode","cfg":{"effort":"medium","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive Reasoning, Medium Effort)","e":"medium","m":"claude-opus-5","oid":"scicode-ee5f0195e71f80b8","rd":"2026-07-24","sm":"claude-opus-5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.506944444444444},{"b":"scicode","cfg":{"effort":"low","provider":"Anthropic"},"dn":"Claude Opus 5 (Adaptive Reasoning, Low Effort)","e":"low","m":"claude-opus-5","oid":"scicode-feba99b0f868484f","rd":"2026-07-24","sm":"claude-opus-5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.48032407407407407},{"b":"scicode","cfg":{"provider":"Kimi"},"dn":"Kimi K3","m":"kimi-k3","oid":"scicode-8505ffb18c3bcb10","rd":"2026-07-16","sm":"kimi-k3_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.586805555555556},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark 1.1 (xhigh)","e":"xhigh","m":"muse-spark-1-1","oid":"scicode-134de844a8de28d0","rd":"2026-07-09","sm":"muse-spark-1.1","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.587962962962963},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.6 Flash (High)","e":"high","m":"gemini-3-6-flash","oid":"scicode-fb816c2b37e6771c","rd":"2026-07-21","sm":"gemini-3.6-flash_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.533564814814815},{"b":"scicode","cfg":{"provider":"SpaceXAI"},"dn":"Grok Build 0.1 0616","m":"grok-build-0-1","oid":"scicode-39cc019a9fd7b93e","rd":"2026-05-29","sm":"grok-build-0.1","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.502314814814815},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.7 Plus","m":"qwen3-7-plus","oid":"scicode-5cf894e381ceb268","rd":"2026-06-02","sm":"qwen3.7-plus","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.460648148148148},{"b":"scicode","cfg":{"provider":"Xiaomi"},"dn":"MiMo-V2.5","m":"mimo-v2-5","oid":"scicode-6f5591bd62ac547d","sm":"mimo-v2.5","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.430555555555556},{"b":"scicode","cfg":{"effort":"medium","provider":"Amazon"},"dn":"Nova 2.0 Pro Preview (medium)","e":"medium","m":"nova-2-0-pro-preview","oid":"scicode-fe4340c66b85582a","rd":"2025-12-02","sm":"nova-2.0-pro-preview_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.42708333333333304},{"b":"scicode","cfg":{"provider":"InclusionAI"},"dn":"Ring-2.6-1T","m":"ring-2-6-1t","oid":"scicode-88c91c2aab74a298","rd":"2026-05-14","sm":"Ring-2.6-1T","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.450231481481481},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 235B A22B 2507 (Reasoning)","m":"qwenqwen3-235b-a22b-thinking-2507","oid":"scicode-907968a05a5fafaf","rd":"2025-07-25","sm":"qwen/qwen3-235b-a22b-thinking-2507","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4143518518518521},{"b":"scicode","cfg":{"provider":"Deep Cogito"},"dn":"Cogito v2.1 (Reasoning)","m":"cogito-671b-v2-1","oid":"scicode-7f4772a27d42dadf","rd":"2025-11-19","sm":"cogito-671b-v2.1","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.409722222222222},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemini 3.5 Flash-Lite","m":"gemini-3-5-flash-lite","oid":"scicode-58e3ba2ddae45b45","rd":"2026-07-21","sm":"gemini-3.5-flash-lite","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.41319444444444403},{"b":"scicode","cfg":{"provider":"StepFun"},"dn":"Step 3.7 Flash","m":"step-3-7-flash","oid":"scicode-20554339d7f14f09","rd":"2026-05-29","sm":"step-3.7-flash","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4004629629629631},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Medium 3.5","m":"mistral-medium-2604","oid":"scicode-2ed027297534e894","rd":"2026-04-28","sm":"mistral-medium-2604","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.40162037037037},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Magistral Medium 1.2","m":"magistral-medium-2509","oid":"scicode-349b4d13ce3a5b17","rd":"2025-09-18","sm":"magistral-medium-2509","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.392361111111111},{"b":"scicode","cfg":{"provider":"Inception"},"dn":"Mercury 2","m":"mercury-2","oid":"scicode-56123aa72e3261a1","sm":"mercury-2","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.38657407407407407},{"b":"scicode","cfg":{"provider":"Cohere"},"dn":"Command A+","m":"command-a-plus-05-2026","oid":"scicode-c9d424b4208fc77a","rd":"2026-05-20","sm":"command-a-plus-05-2026","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.385416666666667},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Large 3","m":"mistral-large-2512","oid":"scicode-438b4f011a0013ee","rd":"2025-12-02","sm":"mistral-large-2512","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.365740740740741},{"b":"scicode","cfg":{"effort":"thinking","provider":"Arcee AI"},"dn":"Trinity Large Thinking","e":"thinking","m":"trinity-large-thinking","oid":"scicode-b7b9532053442c97","rd":"2026-04-01","sm":"trinity-large-thinking","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.40625},{"b":"scicode","cfg":{"provider":"NVIDIA"},"dn":"NVIDIA Nemotron 3 Super 120B A12B (Reasoning)","m":"nemotron-3-super","oid":"scicode-fea0641e572efdc3","rd":"2026-03-11","sm":"nemotron-3-super","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.35995370370370405},{"b":"scicode","cfg":{"provider":"DeepSeek"},"dn":"DeepSeek V3 (Dec '24)","m":"deepseek-v3","oid":"scicode-3f67c4a574adac3e","rd":"2024-12-26","sm":"DeepSeek-V3","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.357638888888889},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Magistral Small 1.2","m":"magistral-small-2509","oid":"scicode-d5af60fb0546fca9","rd":"2025-09-18","sm":"magistral-small-2509","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.351851851851852},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Medium 3.1","m":"mistral-medium-2508","oid":"scicode-92b1371d51839d80","rd":"2025-08-12","sm":"mistral-medium-2508","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.32407407407407407},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 Coder Next","m":"qwen3-coder-next","oid":"scicode-e68cbf4a57c6d631","rd":"2026-02-02","sm":"qwen3-coder-next","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.362268518518519},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Devstral Small 2","m":"devstral-small-2512","oid":"scicode-9adb0ce75af848da","rd":"2025-12-09","sm":"devstral-small-2512","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.288194444444444},{"b":"scicode","cfg":{"effort":"none","provider":"Amazon"},"dn":"Nova 2.0 Pro Preview (Non-reasoning)","e":"none","m":"nova-2-0-pro-preview","oid":"scicode-8fb3e02aa66e6636","rd":"2025-12-02","sm":"nova-2.0-pro-preview_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.28125},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Small 3.1","m":"mistral-small-2503","oid":"scicode-09e6346eac165d82","rd":"2025-03-17","sm":"mistral-small-2503","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.277777777777778},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Small 3.2","m":"mistral-small-2506","oid":"scicode-2951c4310e9da8f2","rd":"2025-06-20","sm":"mistral-small-2506","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.263888888888889},{"b":"scicode","cfg":{"provider":"Meta"},"dn":"Llama 3.3 Instruct 70B","m":"llama-3-3-70b-instruct","oid":"scicode-27cc17816950da58","rd":"2024-12-06","sm":"Llama-3.3-70B-Instruct","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.260416666666667},{"b":"scicode","cfg":{"provider":"IBM"},"dn":"Granite 4.1 30B","m":"granite-4-1-30b","oid":"scicode-08ea17f57be0ab28","rd":"2026-04-29","sm":"granite-4.1-30b","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.258101851851852},{"b":"scicode","cfg":{"provider":"Upstage"},"dn":"Solar Pro 3","m":"solar-pro-3","oid":"scicode-defcecc2952c1406","rd":"2026-01-27","sm":"solar-pro-3","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.25462962962963},{"b":"scicode","cfg":{"provider":"Google"},"dn":"Gemma 3 12B Instruct","m":"gemma-3-12b-it","oid":"scicode-4c01af294c66967e","rd":"2025-03-12","sm":"gemma-3-12b-it","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.16435185185185203},{"b":"scicode","cfg":{"provider":"Microsoft"},"dn":"Phi-4 Mini Instruct","m":"phi-4-mini-instruct","oid":"scicode-db43daf1ba69970d","rd":"2025-02-27","sm":"Phi-4-mini-instruct","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.107638888888889},{"b":"scicode","cfg":{"effort":"max","provider":"Kimi"},"dn":"Kimi K3 (Max)","e":"max","m":"kimi-k3","oid":"scicode-db78f6e3c950a025","rd":"2026-07-16","sm":"kimi-k3_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.594907407407407},{"b":"scicode","cfg":{"effort":"low","provider":"Kimi"},"dn":"Kimi K3 (Low)","e":"low","m":"kimi-k3","oid":"scicode-f81a88980d83f072","rd":"2026-07-16","sm":"kimi-k3_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.52662037037037},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Flash 0731 (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-flash-0731","oid":"scicode-6bca8b4d996677eb","rd":"2026-07-31","sm":"deepseek-v4-flash-0731_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.49884259259259306},{"b":"scicode","cfg":{"provider":"Kimi"},"dn":"Kimi K2.5 (Reasoning)","m":"kimi-k2-5","oid":"scicode-4caa7055dbe99e6f","rd":"2026-01-27","sm":"kimi-k2.5","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.48958333333333304},{"b":"scicode","cfg":{"provider":"Thinking Machines"},"dn":"Inkling Small","m":"inkling-small","oid":"scicode-b66b2fa1bde42730","rd":"2026-07-15","sm":"inkling-small","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.496527777777778},{"b":"scicode","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Sonnet 5 (Non-reasoning, High Effort)","e":"high","m":"claude-sonnet-5","oid":"scicode-40fd6e0629b7ebd4","rd":"2026-06-30","sm":"claude-sonnet-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.486111111111111},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-5.6 Sol (Non-reasoning)","e":"none","m":"gpt-5-6-sol","oid":"scicode-eef93a2cbea1d5e7","rd":"2026-07-09","sm":"gpt-5.6-sol_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.47106481481481505},{"b":"scicode","cfg":{"effort":"high","provider":"DeepSeek"},"dn":"DeepSeek V4 Pro (Reasoning, High Effort)","e":"high","m":"deepseek-v4-pro","oid":"scicode-8eaa76cb43659f31","rd":"2026-04-24","sm":"deepseek-v4-pro_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.46412037037037},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Thinking Machines"},"dn":"Inkling (Xhigh)","e":"xhigh","m":"inkling","oid":"scicode-8e20d27f6ec556e7","rd":"2026-07-15","sm":"Inkling_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.469907407407407},{"b":"scicode","cfg":{"provider":"Z AI"},"dn":"GLM-4.7 (Reasoning)","m":"glm-4-7","oid":"scicode-be843fee0653758a","rd":"2025-12-22","sm":"glm-4.7","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.451388888888889},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Flash (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-flash","oid":"scicode-3434a1979b6f1ba3","rd":"2026-04-24","sm":"deepseek-v4-flash_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.44907407407407407},{"b":"scicode","cfg":{"provider":"Anthropic"},"dn":"Claude 4.5 Sonnet (Reasoning)","m":"claude-sonnet-4-5-2025-09-29","oid":"scicode-5cd3eec50ef5f9a6","rd":"2025-09-29","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.457175925925926},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-5.6 Terra (Non-reasoning)","e":"none","m":"gpt-5-6-terra","oid":"scicode-aa7ea4ef8461b250","rd":"2026-07-09","sm":"gpt-5.6-terra_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.451388888888889},{"b":"scicode","cfg":{"effort":"high","provider":"DeepSeek"},"dn":"DeepSeek V4 Flash (Reasoning, High Effort)","e":"high","m":"deepseek-v4-flash","oid":"scicode-7417b27c41753f97","rd":"2026-04-24","sm":"deepseek-v4-flash_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.420138888888889},{"b":"scicode","cfg":{"provider":"DeepSeek"},"dn":"DeepSeek V3.1 Terminus (Reasoning)","m":"deepseek-v3-1-terminus","oid":"scicode-8f7ed0f3489a8a57","rd":"2025-09-22","sm":"DeepSeek-V3.1-Terminus","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.37962962962963},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-5.6 Luna (Non-reasoning)","e":"none","m":"gpt-5-6-luna","oid":"scicode-f6983376b31a2b8b","rd":"2026-07-09","sm":"gpt-5.6-luna_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.403935185185185},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"o3-mini (High)","e":"high","m":"o3-mini-2025-01-31","oid":"scicode-a3868a9ea45fcfd2","rd":"2025-01-31","sm":"o3-mini-2025-01-31_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.428240740740741},{"b":"scicode","cfg":{"effort":"thinking","provider":"DeepSeek"},"dn":"DeepSeek V3.2 (Reasoning)","e":"thinking","m":"deepseek-v3-2-exp","oid":"scicode-4a4b113af81bfdb1","rd":"2025-09-29","sm":"DeepSeek-V3.2-Exp_thinking","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.388888888888889},{"b":"scicode","cfg":{"effort":"low","provider":"Amazon"},"dn":"Nova 2.0 Pro Preview (low)","e":"low","m":"nova-2-0-pro-preview","oid":"scicode-3767baa438d9710a","rd":"2025-12-02","sm":"nova-2.0-pro-preview_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.38657407407407407},{"b":"scicode","cfg":{"provider":"Z AI"},"dn":"GLM-4.6 (Reasoning)","m":"glm-4-6","oid":"scicode-78b3941ec3110ac2","rd":"2025-09-30","sm":"glm-4.6","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.384259259259259},{"b":"scicode","cfg":{"effort":"none","provider":"Z AI"},"dn":"GLM-5.2 (Non-reasoning)","e":"none","m":"glm-5-2","oid":"scicode-b7511123a62d8992","rd":"2026-06-16","sm":"glm-5.2_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.361111111111111},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"gpt-oss-120b (low)","e":"low","m":"openaigpt-oss-120b","oid":"scicode-fcbe6731633f8746","rd":"2025-08-05","sm":"openai/gpt-oss-120b_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.35995370370370405},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Max, Default Fallback)","e":"max","m":"claude-fable-5-1","oid":"scicode-f91c948fcfde0d3c","rd":"2026-09-01","sm":"claude-fable-5-1_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.630787037037037},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-fable-5-1","oid":"scicode-9583047df2fd1fd5","rd":"2026-09-01","sm":"claude-fable-5-1_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.608796296296296},{"b":"scicode","cfg":{"effort":"medium","provider":"Google"},"dn":"Gemini 3.7 Flash (Medium)","e":"medium","m":"gemini-3-7-flash","oid":"scicode-5295e994f635cbe5","rd":"2026-08-13","sm":"gemini-3.7-flash_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.59837962962963},{"b":"scicode","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Fable 5.1 (High, Default Fallback)","e":"high","m":"claude-fable-5-1","oid":"scicode-8082b0d1e89a1638","rd":"2026-09-01","sm":"claude-fable-5-1_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.586805555555556},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.7 Flash (High)","e":"high","m":"gemini-3-7-flash","oid":"scicode-d744e55025ec25cb","rd":"2026-08-13","sm":"gemini-3.7-flash_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.571759259259259},{"b":"scicode","cfg":{"effort":"max","provider":"Z AI"},"dn":"GLM-5.3 (Max)","e":"max","m":"glm-5-3","oid":"scicode-b7cbfb47c245c9e7","rd":"2026-08-14","sm":"glm-5.3_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.590277777777778},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark 1.2 (Xhigh)","e":"xhigh","m":"muse-spark-1-2","oid":"scicode-47414a2244b679c4","rd":"2026-08-05","sm":"muse-spark-1.2_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.574074074074074},{"b":"scicode","cfg":{"effort":"low","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Low, Default Fallback)","e":"low","m":"claude-fable-5-1","oid":"scicode-ecc350a86a75f1c7","rd":"2026-09-01","sm":"claude-fable-5-1_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.56712962962963},{"b":"scicode","cfg":{"effort":"medium","provider":"Anthropic"},"dn":"Claude Fable 5.1 (Medium, Default Fallback)","e":"medium","m":"claude-fable-5-1","oid":"scicode-2eabf9e8b3ad8206","rd":"2026-09-01","sm":"claude-fable-5-1_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.563657407407407},{"b":"scicode","cfg":{"effort":"medium","provider":"SpaceXAI"},"dn":"Grok 4.6 (Medium)","e":"medium","m":"grok-4-6","oid":"scicode-5742f81361df1585","rd":"2026-08-12","sm":"grok-4.6_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.559027777777778},{"b":"scicode","cfg":{"effort":"medium","provider":"Google"},"dn":"Gemini 3.8 Flash (Medium)","e":"medium","m":"gemini-3-8-flash","oid":"scicode-86667792ba57ffb0","rd":"2026-09-02","sm":"gemini-3.8-flash_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.550925925925926},{"b":"scicode","cfg":{"effort":"low","provider":"Google"},"dn":"Gemini 3.8 Flash (Low)","e":"low","m":"gemini-3-8-flash","oid":"scicode-8818b51dd0eb52c2","rd":"2026-09-02","sm":"gemini-3.8-flash_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.549768518518518},{"b":"scicode","cfg":{"effort":"high","provider":"SpaceXAI"},"dn":"Grok 4.6 (High)","e":"high","m":"grok-4-6","oid":"scicode-0eed3e5147d297eb","rd":"2026-08-12","sm":"grok-4.6_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.564814814814815},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 3.8 Flash (High)","e":"high","m":"gemini-3-8-flash","oid":"scicode-64b59a29e3d729b2","rd":"2026-09-02","sm":"gemini-3.8-flash_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.565972222222222},{"b":"scicode","cfg":{"effort":"low","provider":"Google"},"dn":"Gemini 3.7 Flash (low)","e":"low","m":"gemini-3-7-flash","oid":"scicode-e1b97137dc31652a","rd":"2026-08-13","sm":"gemini-3.7-flash_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.53587962962963},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.8 Max","m":"qwen3-8-max","oid":"scicode-70438e06e88e4870","rd":"2026-08-02","sm":"qwen3.8-max_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.528935185185185},{"b":"scicode","cfg":{"effort":"xhigh","provider":"SpaceXAI"},"dn":"Grok 4.6 (Xhigh)","e":"xhigh","m":"grok-4-6","oid":"scicode-169ec8c76a641b9a","rd":"2026-08-12","sm":"grok-4.6_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.530092592592593},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Pro (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-pro","oid":"scicode-8535d3073a38c7a3","rd":"2026-04-24","sm":"deepseek-v4-pro_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.5},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4 Pro 0813 (Max)","e":"max","m":"deepseek-v4-pro-0813","oid":"scicode-84f23af3694c6a6a","rd":"2026-08-13","sm":"deepseek-v4-pro-0813_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.510416666666667},{"b":"scicode","cfg":{"effort":"low","provider":"SpaceXAI"},"dn":"Grok 4.6 (Low)","e":"low","m":"grok-4-6","oid":"scicode-7b40bd87a8fc9a69","rd":"2026-08-12","sm":"grok-4.6_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4942129629629631},{"b":"scicode","cfg":{"effort":"high","provider":"DeepSeek"},"dn":"DeepSeek V4 Pro (Reasoning, High Effort)","e":"high","m":"deepseek-v4-pro","oid":"scicode-72dd24f042bfb261","rd":"2026-04-24","sm":"deepseek-v4-pro_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.46412037037037},{"b":"scicode","cfg":{"provider":"Z AI"},"dn":"GLM-5.3-Flash","m":"glm-5-3-flash","oid":"scicode-f3cfbc935028e892","rd":"2026-08-20","sm":"glm-5.3-flash_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Alibaba"},"dn":"Qwen3.8 27B (Xhigh)","e":"xhigh","m":"qwen3-8-27b","oid":"scicode-bcaa497be1371a50","rd":"2026-08-14","sm":"qwen3.8-27b_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.466435185185185},{"b":"scicode","cfg":{"effort":"low","provider":"Alibaba"},"dn":"Qwen3.8 27B (Low)","e":"low","m":"qwen3-8-27b","oid":"scicode-6e837c41442e7086","rd":"2026-08-14","sm":"qwen3.8-27b_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.4004629629629631},{"b":"scicode","cfg":{"effort":"medium","provider":"Alibaba"},"dn":"Qwen3.8 27B (Medium)","e":"medium","m":"qwen3-8-27b","oid":"scicode-896c5e2157febc85","rd":"2026-08-14","sm":"qwen3.8-27b_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.390046296296296},{"b":"scicode","cfg":{"effort":"none","provider":"Alibaba"},"dn":"Qwen3.6 27B (Non-reasoning)","e":"none","m":"qwen3-6-27b","oid":"scicode-6e49aaabba83dd7b","rd":"2026-04-22","sm":"qwen3.6-27b_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.372685185185185},{"b":"scicode","cfg":{"effort":"none","provider":"Alibaba"},"dn":"Qwen3.5 122B A10B (Non-reasoning)","e":"none","m":"qwen3-5-122b-a10b","oid":"scicode-215c025fd37c6c43","rd":"2026-02-24","sm":"qwen3.5-122b-a10b_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.356481481481482},{"b":"scicode","cfg":{"effort":"none","provider":"Alibaba"},"dn":"Qwen3.8 27B (Non-reasoning)","e":"none","m":"qwen3-8-27b","oid":"scicode-2114ca2c4a99d55e","rd":"2026-08-14","sm":"qwen3.8-27b_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.362268518518519},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 30B A3B 2507 (Reasoning)","m":"qwen3-30b-a3b-thinking-2507","oid":"scicode-a9090b6404201788","rd":"2025-07-30","sm":"qwen3-30b-a3b-thinking-2507","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.329861111111111},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 14B (Reasoning)","m":"qwen3-14b","oid":"scicode-122bfba792ec7fbc","rd":"2025-04-29","sm":"qwen3-14b","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.315972222222222},{"b":"scicode","cfg":{"effort":"none","provider":"Alibaba"},"dn":"Qwen3.5 35B A3B (Non-reasoning)","e":"none","m":"qwen3-5-35b-a3b","oid":"scicode-bfb9ca3904df8c4e","rd":"2026-02-24","sm":"qwen3.5-35b-a3b_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.292824074074074},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3 8B (Reasoning)","m":"qwen3-8b","oid":"scicode-386fcdc2e9a72d46","rd":"2025-04-28","sm":"qwen3-8b","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.225694444444444},{"b":"scicode","cfg":{"effort":"none","provider":"Alibaba"},"dn":"Qwen3.6 35B A3B (Non-reasoning)","e":"none","m":"qwen3-6-35b-a3b","oid":"scicode-d72452602f0043bd","rd":"2026-04-14","sm":"qwen3.6-35b-a3b_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.0127314814814815},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-6 Astra (Max)","e":"max","m":"gpt-6-astra","oid":"scicode-dfcd11cec5e0a5d4","rd":"2026-09-03","sm":"gpt-6-astra_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.564814814814815},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-6 Astra (Xhigh)","e":"xhigh","m":"gpt-6-astra","oid":"scicode-3e8e18862aea7e2a","rd":"2026-09-03","sm":"gpt-6-astra_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.556712962962963},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-6 Astra (High)","e":"high","m":"gpt-6-astra","oid":"scicode-2964e8fbc4f5eb80","rd":"2026-09-03","sm":"gpt-6-astra_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.554398148148148},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-6 Astra (Medium)","e":"medium","m":"gpt-6-astra","oid":"scicode-3126e8b4efc386a3","rd":"2026-09-03","sm":"gpt-6-astra_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.541666666666667},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-6 Astra (Low)","e":"low","m":"gpt-6-astra","oid":"scicode-c2eda597bce890b4","rd":"2026-09-03","sm":"gpt-6-astra_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.540509259259259},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-6 Astra (Non-reasoning)","e":"none","m":"gpt-6-astra","oid":"scicode-ddccc8ea1d5bf6ee","rd":"2026-09-03","sm":"gpt-6-astra_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.534722222222222},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark 1.3 (Xhigh)","e":"xhigh","m":"muse-spark-1-3","oid":"scicode-34f6e5fc6c413793","rd":"2026-09-02","sm":"muse-spark-1.3_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.597222222222222},{"b":"scicode","cfg":{"provider":"StepFun"},"dn":"Step 5 Preview","m":"step-5-preview","oid":"scicode-fc4e182fb6c121d5","rd":"2026-09-18","sm":"step-5-preview_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.58912037037037},{"b":"scicode","cfg":{"effort":"max","provider":"Meta"},"dn":"Muse Spark 1.3 (Max)","e":"max","m":"muse-spark-1-3","oid":"scicode-f6ec94f258bba9d6","rd":"2026-09-02","sm":"muse-spark-1.3_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.587962962962963},{"b":"scicode","cfg":{"effort":"xhigh","provider":"SpaceXAI"},"dn":"Grok 4.7 (Xhigh)","e":"xhigh","m":"grok-4-7","oid":"scicode-e0fb5ea5eba45b20","rd":"2026-09-21","sm":"grok-4.7_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.574074074074074},{"b":"scicode","cfg":{"provider":"Alibaba"},"dn":"Qwen3.8 Max (0902)","m":"qwen3-8-max-0902","oid":"scicode-da30b50e232afc5f","rd":"2026-09-01","sm":"qwen3.8-max-0902_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.520833333333333},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4.1 Flash (Reasoning, Max Effort)","e":"max","m":"deepseek-v4-1-flash","oid":"scicode-c8ed17ee9e8785c6","rd":"2026-09-09","sm":"deepseek-v4.1-flash_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.518518518518518},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Opus 5.5 (Max, Default Fallback)","e":"max","m":"claude-opus-5-5","oid":"scicode-63faf8a1bc87828f","rd":"2026-09-22","sm":"claude-opus-5-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.668981481481482},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Anthropic"},"dn":"Claude Opus 5.5 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-opus-5-5","oid":"scicode-f99f8923d47e3ff8","rd":"2026-09-22","sm":"claude-opus-5-5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.650462962962963},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (Max, Default Fallback)","e":"max","m":"claude-sonnet-5-5","oid":"scicode-b79adfdbe7592981","rd":"2026-09-28","sm":"claude-sonnet-5-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.609953703703704},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (Xhigh, Default Fallback)","e":"xhigh","m":"claude-sonnet-5-5","oid":"scicode-e6fcc49bad16db41","rd":"2026-09-28","sm":"claude-sonnet-5-5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.572916666666667},{"b":"scicode","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (High, Default Fallback)","e":"high","m":"claude-sonnet-5-5","oid":"scicode-48343d1994d45da7","rd":"2026-09-28","sm":"claude-sonnet-5-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.537037037037037},{"b":"scicode","cfg":{"effort":"medium","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (Medium, Default Fallback)","e":"medium","m":"claude-sonnet-5-5","oid":"scicode-db80fc884853ede6","rd":"2026-09-28","sm":"claude-sonnet-5-5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.528935185185185},{"b":"scicode","cfg":{"effort":"low","provider":"Anthropic"},"dn":"Claude Sonnet 5.5 (Low, Default Fallback)","e":"low","m":"claude-sonnet-5-5","oid":"scicode-7d8fa1a36aa4ef08","rd":"2026-09-28","sm":"claude-sonnet-5-5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.490740740740741},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-6 Sol (Max)","e":"max","m":"gpt-6-sol","oid":"scicode-c265af107805d351","rd":"2026-09-22","sm":"gpt-6-sol_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.576388888888889},{"b":"scicode","cfg":{"effort":"high"},"dn":"GPT-6.1 Sol (High)","e":"high","m":"gpt-6-1-sol","oid":"scicode-d5faca7626bcfecb","rd":"2026-09-29","sm":"gpt-6.1-sol_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.55787037037037},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"GPT-6.1 Sol (Xhigh)","e":"xhigh","m":"gpt-6-1-sol","oid":"scicode-10e2fb3d23bc02b9","rd":"2026-09-29","sm":"gpt-6.1-sol_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.556712962962963},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"GPT-6 Sol (Xhigh)","e":"xhigh","m":"gpt-6-sol","oid":"scicode-93749d61ab324c04","rd":"2026-09-22","sm":"gpt-6-sol_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.550925925925926},{"b":"scicode","cfg":{"effort":"high"},"dn":"GPT-6 Sol (High)","e":"high","m":"gpt-6-sol","oid":"scicode-a214a077e8580168","rd":"2026-09-22","sm":"gpt-6-sol_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.548611111111111},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-6 Luna (Max)","e":"max","m":"gpt-6-luna","oid":"scicode-72040dfe66d8a371","rd":"2026-09-22","sm":"gpt-6-luna_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.546296296296296},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-6.1 Sol (Max)","e":"max","m":"gpt-6-1-sol","oid":"scicode-27d7d1faf497d26b","rd":"2026-09-29","sm":"gpt-6.1-sol_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.541666666666667},{"b":"scicode","cfg":{"effort":"medium"},"dn":"GPT-6 Sol (Medium)","e":"medium","m":"gpt-6-sol","oid":"scicode-9d4226d436d0b92d","rd":"2026-09-22","sm":"gpt-6-sol_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.538194444444444},{"b":"scicode","cfg":{"effort":"medium"},"dn":"GPT-6.1 Sol (Medium)","e":"medium","m":"gpt-6-1-sol","oid":"scicode-2dc8baed94c81f48","rd":"2026-09-29","sm":"gpt-6.1-sol_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.532407407407407},{"b":"scicode","cfg":{"effort":"low"},"dn":"GPT-6.1 Sol (Low)","e":"low","m":"gpt-6-1-sol","oid":"scicode-eebc67311067857f","rd":"2026-09-29","sm":"gpt-6.1-sol_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.532407407407407},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"GPT-6 Luna (Xhigh)","e":"xhigh","m":"gpt-6-luna","oid":"scicode-401429c710e22113","rd":"2026-09-22","sm":"gpt-6-luna_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.517361111111111},{"b":"scicode","cfg":{"effort":"medium"},"dn":"GPT-6 Luna (Medium)","e":"medium","m":"gpt-6-luna","oid":"scicode-52394c0dfede3e32","rd":"2026-09-22","sm":"gpt-6-luna_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.509259259259259},{"b":"scicode","cfg":{"effort":"high"},"dn":"GPT-6 Luna (High)","e":"high","m":"gpt-6-luna","oid":"scicode-2c153971c83f4de2","rd":"2026-09-22","sm":"gpt-6-luna_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.503472222222222},{"b":"scicode","cfg":{"effort":"low"},"dn":"GPT-6 Sol (Low)","e":"low","m":"gpt-6-sol","oid":"scicode-8282e42cf7593765","rd":"2026-09-22","sm":"gpt-6-sol_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.502314814814815},{"b":"scicode","cfg":{"effort":"none"},"dn":"GPT-6 Sol (Non-reasoning)","e":"none","m":"gpt-6-sol","oid":"scicode-b22c907e1e599ada","rd":"2026-09-22","sm":"gpt-6-sol_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.47337962962963},{"b":"scicode","cfg":{"effort":"low"},"dn":"GPT-6 Luna (Low)","e":"low","m":"gpt-6-luna","oid":"scicode-d5d70c7728c34e32","rd":"2026-09-22","sm":"gpt-6-luna_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.46875},{"b":"scicode","cfg":{"effort":"none"},"dn":"GPT-6 Luna (Non-reasoning)","e":"none","m":"gpt-6-luna","oid":"scicode-7e74d45e22b8693f","rd":"2026-09-22","sm":"gpt-6-luna_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.430555555555556},{"b":"scicode","cfg":{"effort":"high","provider":"Anthropic"},"dn":"Claude Opus 5.5 (High, Default Fallback)","e":"high","m":"claude-opus-5-5","oid":"scicode-54c236c5981a6eb4","rd":"2026-09-22","sm":"claude-opus-5-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.604166666666667},{"b":"scicode","cfg":{"effort":"medium"},"dn":"Claude Opus 5.5 (Medium, Default Fallback)","e":"medium","m":"claude-opus-5-5","oid":"scicode-01bfbdc0a566b930","rd":"2026-09-22","sm":"claude-opus-5-5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.592592592592593},{"b":"scicode","cfg":{"effort":"low"},"dn":"Claude Opus 5.5 (Low, Default Fallback)","e":"low","m":"claude-opus-5-5","oid":"scicode-9e7676e77ee3f134","rd":"2026-09-22","sm":"claude-opus-5-5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.585648148148148},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 4 Argon (High)","e":"high","m":"gemini-4-argon","oid":"scicode-5d2fbd94ad4fa605","rd":"2026-09-30","sm":"gemini-4-argon_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.618055555555556},{"b":"scicode","cfg":{"provider":"Xiaomi"},"dn":"MiMo-V2.6-Pro","m":"mimo-v2-6-pro","oid":"scicode-888d837703158c85","rd":"2026-09-21","sm":"mimo-v2.6-pro","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.608796296296296},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"Muse Spark 1.1 (Xhigh)","e":"xhigh","m":"muse-spark-1-1","oid":"scicode-d76e860dc32f1321","rd":"2026-07-09","sm":"muse-spark-1.1_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.587962962962963},{"b":"scicode","cfg":{"effort":"high"},"dn":"Grok 4.7 (High)","e":"high","m":"grok-4-7","oid":"scicode-a19dd3a427d1388f","rd":"2026-09-21","sm":"grok-4.7_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.577546296296296},{"b":"scicode","cfg":{"effort":"low"},"dn":"Grok 4.7 (Low)","e":"low","m":"grok-4-7","oid":"scicode-23fb822a484f0ece","rd":"2026-09-21","sm":"grok-4.7_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.548611111111111},{"b":"scicode","cfg":{"effort":"high"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, High Effort)","e":"high","m":"claude-sonnet-5","oid":"scicode-54aef7c2aeda8c58","rd":"2026-06-30","sm":"claude-sonnet-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.542824074074074},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Xhigh Effort)","e":"xhigh","m":"claude-sonnet-5","oid":"scicode-a8b25bf1ffdbfefc","rd":"2026-06-30","sm":"claude-sonnet-5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.540509259259259},{"b":"scicode","dn":"Qwen3.8 Max","m":"qwen3-8-max","oid":"scicode-577e1afb884392fd","rd":"2026-08-02","sm":"qwen3.8-max_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.532407407407407},{"b":"scicode","cfg":{"effort":"medium"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Medium Effort)","e":"medium","m":"claude-sonnet-5","oid":"scicode-1aebb5bd17569c1f","rd":"2026-06-30","sm":"claude-sonnet-5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","dn":"MiMo-V2.6-Flash","m":"mimo-v2-6-flash","oid":"scicode-1095fedca41064b5","rd":"2026-09-21","sm":"mimo-v2.6-flash","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.512731481481482},{"b":"scicode","cfg":{"effort":"low"},"dn":"Claude Sonnet 5 (Adaptive Reasoning, Low Effort)","e":"low","m":"claude-sonnet-5","oid":"scicode-f939f88c7f4acf5b","rd":"2026-06-30","sm":"claude-sonnet-5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.501157407407407},{"b":"scicode","cfg":{"effort":"high","provider":"Meta"},"dn":"Muse Glimmer (High)","e":"high","m":"muse-glimmer","oid":"scicode-f7624d64dbf9fe36","rd":"2026-08-10","sm":"muse-glimmer_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.44907407407407407},{"b":"scicode","cfg":{"effort":"low"},"dn":"GLM-5.3 (Low)","e":"low","m":"glm-5-3","oid":"scicode-2366045dbffbd52b","rd":"2026-08-14","sm":"glm-5.3_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.420138888888889},{"b":"scicode","cfg":{"effort":"none"},"dn":"DeepSeek V4 Pro 0813 (Non-reasoning)","e":"none","m":"deepseek-v4-pro-0813","oid":"scicode-b8fe573db10e70d9","rd":"2026-08-13","sm":"deepseek-v4-pro-0813_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.399305555555556},{"b":"scicode","cfg":{"effort":"high"},"dn":"GPT-5 mini (High)","e":"high","m":"gpt-5-mini-2025-08-07","oid":"scicode-e896d864a06a7584","rd":"2025-08-07","sm":"gpt-5-mini-2025-08-07_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.390046296296296},{"b":"scicode","dn":"Mercury 2.5","m":"mercury-2-5","oid":"scicode-ca53a9da431d8e49","rd":"2026-09-08","sm":"mercury-2.5_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.385416666666667},{"b":"scicode","cfg":{"effort":"none"},"dn":"DeepSeek V4.1 Flash (Non-reasoning)","e":"none","m":"deepseek-v4-1-flash","oid":"scicode-24eac8642beda1be","rd":"2026-09-09","sm":"deepseek-v4.1-flash_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.35532407407407407},{"b":"scicode","cfg":{"effort":"max","provider":"Meta"},"dn":"Muse Wonder (Max)","e":"max","m":"muse-wonder-1-0","oid":"scicode-4b1ec865bf45e11e","rd":"2026-10-12","sm":"muse-wonder-1.0_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.603009259259259},{"b":"scicode","cfg":{"effort":"max","provider":"Anthropic"},"dn":"Claude Haiku 5.5 (Max)","e":"max","m":"claude-haiku-5-5","oid":"scicode-45fd1afd56387e3a","rd":"2026-10-07","sm":"claude-haiku-5-5_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.549768518518518},{"b":"scicode","cfg":{"provider":"Mistral"},"dn":"Mistral Large 4 Preview","m":"mistral-large-4-0","oid":"scicode-845f74c86ea8efeb","rd":"2026-10-06","sm":"mistral-large-4-0_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.541666666666667},{"b":"scicode","cfg":{"effort":"xhigh"},"dn":"Claude Haiku 5.5 (Xhigh)","e":"xhigh","m":"claude-haiku-5-5","oid":"scicode-9f83da4bb701eae3","rd":"2026-10-07","sm":"claude-haiku-5-5_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.517361111111111},{"b":"scicode","cfg":{"effort":"low"},"dn":"Claude Haiku 5.5 (Low)","e":"low","m":"claude-haiku-5-5","oid":"scicode-de5e77bdf6ffa554","rd":"2026-10-07","sm":"claude-haiku-5-5_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.491898148148148},{"b":"scicode","cfg":{"effort":"medium"},"dn":"Claude Haiku 5.5 (Medium)","e":"medium","m":"claude-haiku-5-5","oid":"scicode-e1fd2b41acdf2ead","rd":"2026-10-07","sm":"claude-haiku-5-5_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.48958333333333304},{"b":"scicode","cfg":{"effort":"high"},"dn":"Claude Haiku 5.5 (High)","e":"high","m":"claude-haiku-5-5","oid":"scicode-a2b5728db3c11579","rd":"2026-10-07","sm":"claude-haiku-5-5_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.487268518518519},{"b":"scicode","cfg":{"effort":"xhigh","provider":"Meta"},"dn":"Muse Spark Open (Xhigh)","e":"xhigh","m":"muse-spark-open-816b-a42b","oid":"scicode-4fbd942fb69557b8","rd":"2026-10-15","sm":"Muse-Spark-Open-816B-A42B_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.584490740740741},{"b":"scicode","cfg":{"effort":"high","provider":"Google"},"dn":"Gemini 4 Carbon (High)","e":"high","m":"gemini-4-carbon","oid":"scicode-3df08358ba0dcc6d","rd":"2026-10-28","sm":"gemini-4-carbon_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.642361111111111},{"b":"scicode","cfg":{"provider":"Kimi"},"dn":"Kimi K3.1 Pro","m":"kimi-k3-1-pro","oid":"scicode-220a59a31ce2052e","rd":"2026-10-23","sm":"kimi-k3.1-pro_unknown","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.607638888888889},{"b":"scicode","cfg":{"effort":"max","provider":"DeepSeek"},"dn":"DeepSeek V4.1 Flash 1017 (Max)","e":"max","m":"deepseek-v4-1-flash-1017","oid":"scicode-f1c738e9c08e1afa","rd":"2026-10-17","sm":"deepseek-v4.1-flash-1017_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.559027777777778},{"b":"scicode","cfg":{"effort":"max","provider":"OpenAI"},"dn":"GPT-6.1 Terra (Max)","e":"max","m":"gpt-6-1-terra","oid":"scicode-03769570f56639b0","rd":"2026-10-26","sm":"gpt-6.1-terra_max","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.550925925925926},{"b":"scicode","cfg":{"effort":"xhigh","provider":"OpenAI"},"dn":"GPT-6.1 Terra (Xhigh)","e":"xhigh","m":"gpt-6-1-terra","oid":"scicode-1ef7be775126ba15","rd":"2026-10-26","sm":"gpt-6.1-terra_xhigh","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.533564814814815},{"b":"scicode","cfg":{"effort":"low","provider":"OpenAI"},"dn":"GPT-6.1 Terra (Low)","e":"low","m":"gpt-6-1-terra","oid":"scicode-305c06f6ff877c0f","rd":"2026-10-26","sm":"gpt-6.1-terra_low","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.524305555555556},{"b":"scicode","cfg":{"effort":"medium","provider":"OpenAI"},"dn":"GPT-6.1 Terra (Medium)","e":"medium","m":"gpt-6-1-terra","oid":"scicode-8d05c5a367fcc7b9","rd":"2026-10-26","sm":"gpt-6.1-terra_medium","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","cfg":{"effort":"high","provider":"OpenAI"},"dn":"GPT-6.1 Terra (High)","e":"high","m":"gpt-6-1-terra","oid":"scicode-e3ddf02678dc7e93","rd":"2026-10-26","sm":"gpt-6.1-terra_high","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.516203703703704},{"b":"scicode","cfg":{"effort":"none","provider":"OpenAI"},"dn":"GPT-6.1 Terra (Non-reasoning)","e":"none","m":"gpt-6-1-terra","oid":"scicode-e3e194ed86f705d4","rd":"2026-10-26","sm":"gpt-6.1-terra_none","src":"https://artificialanalysis.ai/evaluations/scicode","v":0.478009259259259},{"b":"science_qa","dn":"BLIP-2","m":"blip2-opt-2-7b","oid":"science_qa-c18ea0c748572a4f","rd":"2023-02-06","sid":"reckMv8O8r13skE3P","sm":"blip2-opt-2.7b","src":"https://scienceqa.github.io/leaderboard.html","v":0.7417},{"b":"science_qa","dn":"GPT-3 (0-shot)","m":"text-davinci-001","oid":"science_qa-deb43d9e752fc3b9","rd":"2022-01-27","sid":"recL8mZMJvVlv3aVD","sm":"text-davinci-001","src":"https://scienceqa.github.io/leaderboard.html","v":0.7404},{"b":"science_qa","dn":"GPT-3 (2-shot)","m":"text-davinci-001","oid":"science_qa-7f967ef9173ef263","rd":"2022-01-27","sid":"rec6clTwByWcSogei","sm":"text-davinci-001","src":"https://scienceqa.github.io/leaderboard.html","v":0.7397},{"b":"science_qa","dn":"LLaMA2-13B","m":"llama-2-13b","oid":"science_qa-815444edc989f8d2","rd":"2023-07-18","sid":"rec4f1Hv3tHKZwpM3","sm":"Llama-2-13b","src":"https://scienceqa.github.io/leaderboard.html","v":0.5578},{"b":"science_qa","dn":"LLaMA1-13B","m":"llama-13b","oid":"science_qa-e02e71686b316c9c","rd":"2023-02-24","sid":"reckv5UwxSlzuqqxU","sm":"LLaMA-13B","src":"https://scienceqa.github.io/leaderboard.html","v":0.4333},{"b":"science_qa","dn":"LLaMA2-7B","m":"llama-2-7b","oid":"science_qa-f765be7aa8e9ea89","rd":"2023-07-18","sid":"recst4vtvBflTNIt5","sm":"Llama-2-7b","src":"https://scienceqa.github.io/leaderboard.html","v":0.4308},{"b":"science_qa","dn":"LLaMA1-7B","m":"llama-7b","oid":"science_qa-3262cc91de13322f","rd":"2023-02-24","sid":"recOWTFKlhYYIQazU","sm":"LLaMA-7B","src":"https://scienceqa.github.io/leaderboard.html","v":0.3619},{"b":"science_qa","dn":"Phi-3.5-Vision 4.2b","m":"phi-3-5-vision-instruct","oid":"science_qa-ec3285ead92bcbaa","rd":"2024-08-16","sid":"rec0npY7rc711egAe","sm":"Phi-3.5-vision-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.913},{"b":"science_qa","dn":"MM1-3B-Chat 3.6b","m":"mm1-3b-chat","oid":"science_qa-6a88437c5e49d74f","rd":"2024-03-14","sid":"recFXgM2b1MxXJ0GW","sm":"MM1-3B-Chat","src":"http://arxiv.org/abs/2404.14219","v":0.694},{"b":"science_qa","dn":"MM1-7B-Chat 7.6b","m":"mm1-7b-chat","oid":"science_qa-77c5aed6141c0a0b","rd":"2024-03-14","sid":"recOtCxq6HmvnIucp","sm":"MM1-7B-Chat","src":"http://arxiv.org/abs/2404.14219","v":0.726},{"b":"science_qa","dn":"LLaVA-1.6 Vicuna-7b","m":"llava-v1-6-vicuna-7b","oid":"science_qa-82f8883387f29b23","rd":"2024-01-31","sid":"recUUhaZd0PNJBznr","sm":"llava-v1.6-vicuna-7b","src":"http://arxiv.org/abs/2404.14219","v":0.706},{"b":"science_qa","dn":"LLaVA-Next Llama3-8b","m":"llama3-llava-next-8b","oid":"science_qa-5a8933e65c89a439","rd":"2024-04-20","sid":"rec4WiZfQpgoA4nv2","sm":"llama3-llava-next-8b","src":"http://arxiv.org/abs/2404.14219","v":0.737},{"b":"science_qa","dn":"Qwen-VL-Chat 9.6b","m":"qwen-vl-chat","oid":"science_qa-2617cad34b4a1f84","rd":"2023-08-20","sid":"recxtuT8rYMP8wp87","sm":"Qwen-VL-Chat","src":"http://arxiv.org/abs/2404.14219","v":0.672},{"b":"science_qa","dn":"Claude 3 haiku","m":"claude-3-haiku-2024-03-07","oid":"science_qa-c2d883d17efbcd14","rd":"2024-03-07","sid":"reccDSd3p2ZJ5XIgq","sm":"claude-3-haiku-20240307","src":"http://arxiv.org/abs/2404.14219","v":0.72},{"b":"science_qa","dn":"Gemini 1.0 Pro V","m":"gemini-1-0-pro-vision","oid":"science_qa-2b89730c97bba15e","rd":"2024-01-04","sid":"recSwZQTEaQUGeWS6","sm":"gemini-1.0-pro-vision","src":"http://arxiv.org/abs/2404.14219","v":0.797},{"b":"science_qa","dn":"GPT-4O 2024-05-13","m":"gpt-4o-2024-05-13","oid":"science_qa-d8b62923b4b7f882","rd":"2024-05-13","sid":"rec0F1EWXSFmyhqsG","sm":"gpt-4o-2024-05-13","src":"http://arxiv.org/abs/2404.14219","v":0.885},{"b":"science_qa","dn":"Falcon2-11B VLM","m":"falcon-11b-vlm","oid":"science_qa-4ccf75bc7734a837","rd":"2024-05-21","sid":"rec6Px3f23pIK410C","sm":"falcon-11B-vlm","src":"http://arxiv.org/abs/2407.14885","v":0.749},{"b":"science_qa","dn":"LLaVA1.6-Vicuna-7B","m":"llava-v1-6-vicuna-7b","oid":"science_qa-cdc2d74fc696b50d","rd":"2024-01-31","sid":"recLQWTFD7gZEc4Pd","sm":"llava-v1.6-vicuna-7b","src":"http://arxiv.org/abs/2407.14885","v":0.701},{"b":"science_qa","dn":"LLaVA1.6-Vicuna-13B","m":"llava-v1-6-vicuna-13b","oid":"science_qa-1dc8873cff69b134","rd":"2024-01-31","sid":"recMzgZXL1CtfmhvN","sm":"llava-v1.6-vicuna-13b","src":"http://arxiv.org/abs/2407.14885","v":0.736},{"b":"science_qa","dn":"LLaVA1.6-Mistral-7B","m":"llava-v1-6-mistral-7b","oid":"science_qa-af1855bdab4310fc","rd":"2024-01-31","sid":"recHkwW8HkY5EvWuw","sm":"llava-v1.6-mistral-7b","src":"http://arxiv.org/abs/2407.14885","v":0.728},{"b":"science_qa","dn":"InstructBLIP-7B (Vicuna)","m":"instructblip-vicuna-7b","oid":"science_qa-984f6279d39dbca3","rd":"2023-05-22","sid":"recrqpiBC2wq4jXQS","sm":"instructblip-vicuna-7b","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.605},{"b":"science_qa","dn":"InstructBLIP-13B (Vicuna)","m":"instructblip-vicuna-13b","oid":"science_qa-fd9a74b3256c5a89","rd":"2023-12-25","sid":"recfjqnJ0LLGYRCzA","sm":"instructblip-vicuna-13b","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.631},{"b":"science_qa","dn":"Qwen-VL-Chat","m":"qwen-vl-chat","oid":"science_qa-06eba70e73af0eaf","rd":"2023-08-20","sid":"recvidEkZ2TJ7PLNe","sm":"Qwen-VL-Chat","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.682},{"b":"science_qa","dn":"InternVL-7B","m":"internvl-chat-vit-6b-vicuna-7b","oid":"science_qa-89715f57b15ca184","rd":"2023-12-25","sid":"recwNwS3zW5ZnI8my","sm":"InternVL-Chat-ViT-6B-Vicuna-7B","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.662},{"b":"science_qa","dn":"InternVL-13B","m":"internvl-chat-vit-6b-vicuna-13b","oid":"science_qa-d0492f755a2777f2","rd":"2024-08-16","sid":"recLjjaIU4o6tkfTL","sm":"InternVL-Chat-ViT-6B-Vicuna-13B","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.701},{"b":"science_qa","dn":"LLaVA-1.5-7B","m":"llava-v1-5-7b","oid":"science_qa-51595f544dd5dba1","rd":"2023-10-05","sid":"rec0ww4tyLYeEjUZ3","sm":"llava-v1.5-7b","src":"https://proceedings.neurips.cc/paper_files/paper/2024/file/81a60d18e010b27b36cd465c6604b915-Paper-Conference.pdf","v":0.668},{"b":"simplebench","cfg":{"effort":"max"},"e":"max","m":"claude-fable-5","oid":"simplebench-174cd8cfdf350ffe","rd":"2026-06-09","sid":"recLu3km4ejdL7kz3","sm":"claude-fable-5_max","v":0.8190000000000001},{"b":"simplebench","m":"claude-opus-5","oid":"simplebench-3994e0245ab7fd2c","rd":"2026-07-24","sid":"recrn9TfQ9EAU0tm6","sm":"claude-opus-5_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.8059999999999999},{"b":"simplebench","m":"gemini-3-1-pro-preview","oid":"simplebench-aa0f0ed20e86c34b","rd":"2026-02-19","sid":"recgewVkDueXvkVnM","sm":"gemini-3.1-pro-preview","src":"https://simple-bench.com/","v":0.7959999999999999},{"b":"simplebench","m":"gpt-5-5-pro","oid":"simplebench-8615bf6d117a17d0","rd":"2026-04-23","sid":"recCDyGH0xqsGV5P3","sm":"gpt-5.5-pro_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.7690000000000001},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-5-pro","oid":"simplebench-7aea877cf64c0bd0","rd":"2026-04-23","sid":"recjW01mLP3BOi97I","sm":"gpt-5.5-pro_high","v":0.7690000000000001},{"b":"simplebench","m":"gemini-3-5-flash","oid":"simplebench-a64ee08b7f4ccd82","rd":"2026-05-19","sid":"recT00msMRRERjlhG","sm":"gemini-3.5-flash_unknown","v":0.767},{"b":"simplebench","m":"gemini-3-pro-preview","oid":"simplebench-00deb515ba278d8d","rd":"2025-11-18","sid":"recyUT7yjrhcHvcWI","sm":"gemini-3-pro-preview","src":"https://simple-bench.com/","v":0.7640000000000001},{"b":"simplebench","m":"grok-4-6","oid":"simplebench-7785e7b1aaf1804f","rd":"2026-08-12","sid":"reciU38L8LpB1VoOo","sm":"grok-4.6_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.7590000000000001},{"b":"simplebench","m":"muse-spark-1-2","oid":"simplebench-afe2823cbebe8687","rd":"2026-08-05","sid":"recS0SvkmpuX3aKKC","sm":"muse-spark-1.2","src":"https://lmcouncil.ai/benchmarks","v":0.745},{"b":"simplebench","m":"gpt-5-4-pro-2026-03-05","oid":"simplebench-c93815188e787a5c","rd":"2026-03-05","sid":"reck4FhiJDV2Du9GI","sm":"gpt-5.4-pro-2026-03-05_unknown","src":"https://simple-bench.com/","v":0.741},{"b":"simplebench","m":"gpt-5-6-sol-proxhigh","oid":"simplebench-74bd1434ac2275e2","rd":"2026-07-09","sid":"recLvVjubvjSAUEPo","sm":"gpt-5.6-sol_proxhigh","src":"https://lmcouncil.ai/benchmarks","v":0.7170000000000001},{"b":"simplebench","m":"gpt-5-6-sol-prounknown","oid":"simplebench-6f97df1d30650994","rd":"2026-07-09","sid":"recF9cVAXsbbbYZfc","sm":"gpt-5.6-sol_prounknown","v":0.717},{"b":"simplebench","m":"qwen3-7-max","oid":"simplebench-6494fb4019e15de0","rd":"2026-05-19","sid":"recoFFvBl4kIhe27W","sm":"qwen3.7-max","v":0.7040000000000001},{"b":"simplebench","m":"grok-4-5","oid":"simplebench-a13f6faff92d8b52","rd":"2026-07-08","sid":"recvoyWD46brAhdsp","sm":"grok-4.5_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.7000000000000001},{"b":"simplebench","m":"gpt-5-5","oid":"simplebench-7e5e27d999fb41e5","rd":"2026-04-23","sid":"recjiKXnBPSb5IusO","sm":"gpt-5.5_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.6900000000000001},{"b":"simplebench","m":"claude-opus-4-6","oid":"simplebench-14dd6c9dbd24e5d9","rd":"2026-02-05","sid":"rec730nvcPbPg0QuS","sm":"claude-opus-4-6","src":"https://simple-bench.com/","v":0.6759999999999999},{"b":"simplebench","m":"claude-opus-4-8","oid":"simplebench-5600cbfa4dcb9e24","rd":"2026-05-28","sid":"rec6BxLe6yXuWZnUJ","sm":"claude-opus-4-8_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.648},{"b":"simplebench","m":"gpt-5-6-sol","oid":"simplebench-9f1066d635c0c622","rd":"2026-07-09","sid":"recYPLPHkE9223KxD","sm":"gpt-5.6-sol_unknown","v":0.648},{"b":"simplebench","cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-6-sol","oid":"simplebench-bc089e934926e2b4","rd":"2026-07-09","sid":"reczyLGkripRzVWw8","sm":"gpt-5.6-sol_xhigh","src":"https://lmcouncil.ai/benchmarks","v":0.648},{"b":"simplebench","m":"qwen3-6-max-preview","oid":"simplebench-e91280a59d88eb9e","rd":"2026-04-20","sid":"recdMCAKq8wbA4FAN","sm":"qwen3.6-max-preview","v":0.63},{"b":"simplebench","m":"gemini-2-5-pro-preview-06-05","oid":"simplebench-9dde538d12c7ff01","rd":"2025-06-05","sid":"recViPEOMOrSB6VKK","sm":"gemini-2.5-pro-preview-06-05","src":"https://simple-bench.com/","v":0.624},{"b":"simplebench","m":"claude-opus-4-5-2025-11-01","oid":"simplebench-4c5c1ccfbd68b9ad","rd":"2025-11-24","sid":"reclKBQ393HPkOMed","sm":"claude-opus-4-5-20251101","src":"https://simple-bench.com/","v":0.62},{"b":"simplebench","m":"claude-opus-4-5-2025-11-01","oid":"simplebench-9bf7f7d5d2a753fa","rd":"2025-11-24","sid":"recC875h5FNEGSV9i","sm":"claude-opus-4-5-20251101_unknown","v":0.62},{"b":"simplebench","m":"claude-opus-4-7","oid":"simplebench-656c13378c2421a9","rd":"2026-04-16","sid":"recpuyXgr9SSKBpUy","sm":"claude-opus-4-7_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.617},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-pro-2025-10-06","oid":"simplebench-cdd10e40fad064d8","rd":"2025-10-07","sid":"recbQ9dn731JxU1au","sm":"gpt-5-pro-2025-10-06_high","src":"https://simple-bench.com/","v":0.616},{"b":"simplebench","m":"gpt-5-pro-2025-10-06","oid":"simplebench-2516074ae96cb430","rd":"2025-10-07","sid":"recXHUgdw3J3S0dYf","sm":"gpt-5-pro-2025-10-06_unknown","v":0.616},{"b":"simplebench","m":"gemini-3-flash-preview","oid":"simplebench-e70825b2b2ae7325","rd":"2025-12-17","sid":"rectQAsXBWhS8ne4o","sm":"gemini-3-flash-preview","src":"https://simple-bench.com/","v":0.611},{"b":"simplebench","m":"deepseek-v4-flash-0731","oid":"simplebench-b97b01c18f28b13a","rd":"2026-07-31","sid":"reciZGK1zgaNuGKdO","sm":"deepseek-v4-flash-0731_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.611},{"b":"simplebench","cfg":{"effort":"max"},"e":"max","m":"kimi-k3","oid":"simplebench-f9f5c65e057cb48f","rd":"2026-07-16","sid":"recl8BAY5cq6WQkPy","sm":"kimi-k3_max","src":"https://lmcouncil.ai/benchmarks","v":0.6070000000000001},{"b":"simplebench","m":"claude-sonnet-5","oid":"simplebench-7befb72ef9d632bb","rd":"2026-06-30","sid":"recV9xJYjOKNzHcea","sm":"claude-sonnet-5_unknown","v":0.606},{"b":"simplebench","m":"grok-4-0709","oid":"simplebench-1c08c838b8c5dcdf","rd":"2025-07-09","sid":"reccvzfzhvNO8LxS5","sm":"grok-4-0709","src":"https://simple-bench.com/","v":0.605},{"b":"simplebench","m":"claude-opus-4-1-2025-08-05","oid":"simplebench-7a6a473292ba42f4","rd":"2025-08-05","sid":"recNq0JntySeETz1j","sm":"claude-opus-4-1-20250805","src":"https://simple-bench.com/","v":0.6},{"b":"simplebench","m":"claude-opus-4-1-2025-08-05","oid":"simplebench-ccd4e930f2597f8f","rd":"2025-08-05","sid":"reclvi58OvdQ1U4f2","sm":"claude-opus-4-1-20250805_unknown","v":0.6},{"b":"simplebench","m":"claude-opus-4-20250514-12k","oid":"simplebench-954de5c0b1679599","rd":"2025-05-22","sid":"recCa8wyF9haBALeE","sm":"claude-opus-4-20250514_12K","src":"https://simple-bench.com/","v":0.588},{"b":"simplebench","m":"claude-opus-4-2025-05-14","oid":"simplebench-9de00cecc0be2707","rd":"2025-05-22","sid":"recpufOkYTK86bXJp","sm":"claude-opus-4-20250514_unknown","v":0.588},{"b":"simplebench","m":"glm-5-2","oid":"simplebench-cd04af566aa0b71f","rd":"2026-06-16","sid":"recJoicmmdxhXOEIM","sm":"glm-5.2_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.588},{"b":"simplebench","m":"kimi-k2-7-code","oid":"simplebench-5aed7bdcf508657e","rd":"2026-06-12","sid":"recMlNcNAtdEKVODv","sm":"kimi-k2.7-code","src":"https://lmcouncil.ai/benchmarks","v":0.579},{"b":"simplebench","m":"gpt-5-2-pro-2025-12-11","notes":"5.2 Pro xHigh","oid":"simplebench-5ceef094cca7a9d7","rd":"2025-12-11","sid":"rec8C0VsHASRk2heI","sm":"gpt-5.2-pro-2025-12-11","src":"https://simple-bench.com/","v":0.574},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-2025-08-07","oid":"simplebench-50126a6a930af33b","rd":"2025-08-07","sid":"recYzgCOm2JdsaO4v","sm":"gpt-5-2025-08-07_high","src":"https://simple-bench.com/","v":0.5670000000000001},{"b":"simplebench","m":"grok-4-1-fast-non-reasoning","oid":"simplebench-3baa64d17c07ea3e","rd":"2025-11-19","sid":"recEjVoUb45jOJuzf","sm":"grok-4-1-fast-non-reasoning","src":"https://simple-bench.com/","v":0.56},{"b":"simplebench","m":"grok-4-1-fast-reasoning","oid":"simplebench-f88e7ba71711af60","rd":"2025-11-19","sid":"recCw9orwjBjIgRjF","sm":"grok-4-1-fast-reasoning","v":0.56},{"b":"simplebench","m":"glm-5-1","oid":"simplebench-38d0516c2bdae799","rd":"2026-04-07","sid":"recDcxKsGJyZp9pTu","sm":"glm-5.1","v":0.551},{"b":"simplebench","m":"claude-sonnet-4-5-20250929-12k","oid":"simplebench-80692dfb1e5219fc","rd":"2025-09-29","sid":"recWdgpLTxKnAX8LL","sm":"claude-sonnet-4-5-20250929_12K","src":"https://simple-bench.com/","v":0.543},{"b":"simplebench","m":"claude-sonnet-4-5-2025-09-29","oid":"simplebench-64b4113c4b6da0a8","rd":"2025-09-29","sid":"rec3Wopwym5wv8br0","sm":"claude-sonnet-4-5-20250929_unknown","v":0.543},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-1-2025-11-13","oid":"simplebench-635168a86a3312f5","rd":"2025-11-13","sid":"recoGupfEEaPNxcDe","sm":"gpt-5.1-2025-11-13_high","src":"https://simple-bench.com/","v":0.532},{"b":"simplebench","m":"glm-5","oid":"simplebench-adaca1f4bcd0c545","rd":"2026-02-11","sid":"recBj1oZpLXvopQw1","sm":"glm-5","v":0.532},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"o3-2025-04-16","oid":"simplebench-76f15b2f4b9bbe70","rd":"2025-04-16","sid":"recI5aJlmr2NrvE0V","sm":"o3-2025-04-16_high","src":"https://simple-bench.com/","v":0.531},{"b":"simplebench","m":"deepseek-v3-2-speciale","oid":"simplebench-4a1ce95d16e41895","rd":"2025-12-01","sid":"recADTeXCQKXTUdkk","sm":"DeepSeek-V3.2-Speciale","v":0.526},{"b":"simplebench","m":"gemini-2-5-pro-exp-03-25","oid":"simplebench-cd255bb4998737e1","rd":"2025-03-25","sid":"reccAb0YstbYurrte","sm":"gemini-2.5-pro-exp-03-25","src":"https://simple-bench.com/","v":0.516},{"b":"simplebench","m":"gemini-2-5-pro-preview-03-25","oid":"simplebench-90222b9bccf1dff5","rd":"2025-03-31","sid":"recpb2PdfImmfmr5y","sm":"gemini-2.5-pro-preview-03-25","v":0.516},{"b":"simplebench","m":"inkling","oid":"simplebench-d9697762a6f5acb1","rd":"2026-07-15","sid":"rect1J5rw9Pr5Q4Ww","sm":"Inkling","src":"https://lmcouncil.ai/benchmarks","v":0.5},{"b":"simplebench","m":"gpt-5-6-terra","oid":"simplebench-c0f2433d7dae24f6","rd":"2026-07-09","sid":"recOLqv7N05mPCScD","sm":"gpt-5.6-terra_unknown","v":0.489},{"b":"simplebench","cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-6-terra","oid":"simplebench-ea108de7260188aa","rd":"2026-07-09","sid":"recFGmom3i9zQS6Wk","sm":"gpt-5.6-terra_xhigh","src":"https://lmcouncil.ai/benchmarks","v":0.489},{"b":"simplebench","m":"zai-orgglm-4-7","oid":"simplebench-90a3e8104b959e5d","rd":"2025-12-22","sid":"recmjT6x061caVJFv","sm":"zai-org/glm-4.7","v":0.47700000000000004},{"b":"simplebench","m":"glm-4-7","oid":"simplebench-baccc4c04ce58b06","rd":"2025-12-22","sid":"rec2gsPE5AL8Jzaf2","sm":"glm-4.7","src":"https://simple-bench.com/","v":0.477},{"b":"simplebench","m":"gpt-5-6-luna","oid":"simplebench-200596978e8a8a19","rd":"2026-07-09","sid":"recNyzZlNJm3bgwYP","sm":"gpt-5.6-luna_unknown","v":0.468},{"b":"simplebench","m":"kimi-k2-5","oid":"simplebench-9f1515337ba0666c","rd":"2026-01-27","sid":"reckCYspTZj7974Qy","sm":"kimi-k2.5","v":0.46799999999999997},{"b":"simplebench","cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-6-luna","oid":"simplebench-4c14c237c21081ee","rd":"2026-07-09","sid":"rec9GiL2Ji035I3dZ","sm":"gpt-5.6-luna_xhigh","src":"https://lmcouncil.ai/benchmarks","v":0.46799999999999997},{"b":"simplebench","m":"claude-3-7-sonnet-20250219-12k","oid":"simplebench-4de6fdefb23760ab","rd":"2025-02-24","sid":"reckxfqYaBADs0cCf","sm":"claude-3-7-sonnet-20250219_12K","src":"https://simple-bench.com/","v":0.464},{"b":"simplebench","m":"claude-3-7-sonnet-2025-02-19","oid":"simplebench-5eaf02d0d1bb9869","rd":"2025-02-24","sid":"recopfv8laJSj5Q6H","sm":"claude-3-7-sonnet-20250219_unknown","v":0.46399999999999997},{"b":"simplebench","m":"deepseek-v4-flash","oid":"simplebench-c6f6910198dfccab","rd":"2026-04-24","sid":"rec90NXP3QjQ994fW","sm":"deepseek-v4-flash_unknown","src":"https://lmcouncil.ai/benchmarks","v":0.46299999999999997},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"gpt-5-2-2025-12-11","oid":"simplebench-fcd664c2018027a9","rd":"2025-12-11","sid":"recqx2zZ9hxht3Y6n","sm":"gpt-5.2-2025-12-11_high","src":"https://simple-bench.com/","v":0.458},{"b":"simplebench","m":"gpt-5-2-2025-12-11","oid":"simplebench-b01a88bbce5f9337","rd":"2025-12-11","sid":"recrV1K3KLv9dtzZB","sm":"gpt-5.2-2025-12-11_unknown","v":0.45799999999999996},{"b":"simplebench","m":"minimax-m3","oid":"simplebench-db60e2345de00053","rd":"2026-06-01","sid":"recfi5aOi68UorAf5","sm":"MiniMax-M3","src":"https://lmcouncil.ai/benchmarks","v":0.45799999999999996},{"b":"simplebench","m":"claude-sonnet-4-20250514-12k","oid":"simplebench-caac324d6c584f1f","rd":"2025-05-22","sid":"recOF5ZoFqZHYxpwD","sm":"claude-sonnet-4-20250514_12K","src":"https://simple-bench.com/","v":0.455},{"b":"simplebench","m":"claude-sonnet-4-2025-05-14","oid":"simplebench-73a29263d7284b6b","rd":"2025-05-22","sid":"recAkNOio9nZhF4CL","sm":"claude-sonnet-4-20250514_unknown","v":0.455},{"b":"simplebench","m":"claude-3-7-sonnet-2025-02-19","oid":"simplebench-6480c3012fff5688","rd":"2025-02-24","sid":"rectGr4imJANQzE34","sm":"claude-3-7-sonnet-20250219","src":"https://simple-bench.com/","v":0.449},{"b":"simplebench","m":"o1-preview-2024-09-12","oid":"simplebench-c89d00ed0276f7f9","rd":"2024-09-12","sid":"rec5At0uBqsEuLVVH","sm":"o1-preview-2024-09-12","src":"https://simple-bench.com/","v":0.417},{"b":"simplebench","m":"claude-3-5-sonnet-2024-10-22","oid":"simplebench-17584f47e7414654","rd":"2024-10-22","sid":"rec55sHDKD0gxxg5R","sm":"claude-3-5-sonnet-20241022","src":"https://simple-bench.com/","v":0.414},{"b":"simplebench","m":"gemini-2-5-flash","oid":"simplebench-d8ea1e269e00abb5","rd":"2025-06-17","sid":"reciIN7KQ8EiMs6s9","sm":"gemini-2.5-flash","src":"https://simple-bench.com/","v":0.412},{"b":"simplebench","m":"deepseek-r1-0528","oid":"simplebench-25c444c4de2affd0","rd":"2025-05-28","sid":"rec345Y86bjDNWZmV","sm":"DeepSeek-R1-0528","src":"https://simple-bench.com/","v":0.408},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"o1-2024-12-17","oid":"simplebench-e0e05a49984805a0","rd":"2024-12-17","sid":"rec6j46lH0wCisYVL","sm":"o1-2024-12-17_high","src":"https://simple-bench.com/","v":0.401},{"b":"simplebench","m":"deepseek-v3-1","oid":"simplebench-e445d96d16b36603","rd":"2025-08-21","sid":"recAiu3FFJglQ0FdI","sm":"DeepSeek-V3.1","src":"https://simple-bench.com/","v":0.4},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"o4-mini-2025-04-16","oid":"simplebench-2d2796726fd95ee5","rd":"2025-04-16","sid":"recodGIOZHwIsosFv","sm":"o4-mini-2025-04-16_high","src":"https://simple-bench.com/","v":0.387},{"b":"simplebench","cfg":{"effort":"medium"},"e":"medium","m":"o1-2024-12-17","oid":"simplebench-e3a2ece2adcb8fd6","rd":"2024-12-17","sid":"recCGePx18ITu5c6O","sm":"o1-2024-12-17_medium","src":"https://simple-bench.com/","v":0.367},{"b":"simplebench","m":"grok-3","oid":"simplebench-74924620c47db038","rd":"2025-04-09","sid":"recoLH9cFM9CmL37j","sm":"grok-3","src":"https://simple-bench.com/","v":0.361},{"b":"simplebench","m":"qwen3-6-flash","oid":"simplebench-9b721f9e02804b2b","rd":"2026-04-27","sid":"recPZCSfWzjzM8LFv","sm":"qwen3.6-flash","v":0.352},{"b":"simplebench","m":"gpt-4-5-preview-2025-02-27","oid":"simplebench-9bb8dcbb3e997d76","rd":"2025-02-27","sid":"recNFMYJcGpWOttKB","sm":"gpt-4.5-preview-2025-02-27","src":"https://simple-bench.com/","v":0.345},{"b":"simplebench","m":"gemini-exp-1206","oid":"simplebench-ecf65b02e110e208","rd":"2024-12-06","sid":"rechDCACKgdAKATMz","sm":"gemini-exp-1206","src":"https://simple-bench.com/","v":0.311},{"b":"simplebench","m":"qwen3-235b-a22b","oid":"simplebench-8be592afe9192b0e","rd":"2025-04-29","sid":"recZifiu78SQWQAFX","sm":"qwen3-235b-a22b","src":"https://simple-bench.com/","v":0.31},{"b":"simplebench","m":"deepseek-r1","oid":"simplebench-b99730fe9f058a7f","rd":"2025-01-20","sid":"recVHI1CZTBTCgTmM","sm":"DeepSeek-R1","src":"https://simple-bench.com/","v":0.309},{"b":"simplebench","m":"gemini-2-0-flash-thinking-exp-01-21","notes":"Assuming later of two experimental versions","oid":"simplebench-6588d3a6762d6d6c","rd":"2025-01-21","sid":"recuXcLAsfUz4Bx9u","sm":"gemini-2.0-flash-thinking-exp-01-21","src":"https://simple-bench.com/","v":0.307},{"b":"simplebench","m":"llama-4-maverick-17b-128e-instruct","oid":"simplebench-7cc3e01f7c61fbdd","rd":"2025-04-05","sid":"recXdYB88sX3J1uDn","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://simple-bench.com/","v":0.277},{"b":"simplebench","m":"claude-3-5-sonnet-2024-06-20","oid":"simplebench-8c63fa1377342bc9","rd":"2024-06-20","sid":"recmO9zJFuB6oSrz8","sm":"claude-3-5-sonnet-20240620","src":"https://simple-bench.com/","v":0.275},{"b":"simplebench","m":"deepseek-v3-0324","oid":"simplebench-941def793b783251","rd":"2025-03-24","sid":"rec0QIxD9Dbi1PU3M","sm":"DeepSeek-V3-0324","src":"https://simple-bench.com/","v":0.272},{"b":"simplebench","m":"gemini-1-5-pro-002","oid":"simplebench-003c45ef79e9b2bf","rd":"2024-09-24","sid":"recJPOyD1j4nEFjGF","sm":"gemini-1.5-pro-002","src":"https://simple-bench.com/","v":0.271},{"b":"simplebench","m":"gpt-4-1-2025-04-14","oid":"simplebench-c2d341ca9303eeaf","rd":"2025-04-14","sid":"recXeUQqhdaSXasO2","sm":"gpt-4.1-2025-04-14","src":"https://simple-bench.com/","v":0.27},{"b":"simplebench","m":"kimi-k2-instruct","oid":"simplebench-51db8bdf0b8629f5","rd":"2025-07-12","sid":"recX4og5KAbGbOhwv","sm":"Kimi-K2-Instruct","src":"https://simple-bench.com/","v":0.263},{"b":"simplebench","m":"gpt-4-turbo-2024-04-09","oid":"simplebench-2868188ccb1d93b5","rd":"2024-04-09","sid":"reczQMtVDyVdqr3Fw","sm":"gpt-4-turbo-2024-04-09","src":"https://simple-bench.com/","v":0.251},{"b":"simplebench","m":"claude-3-opus-2024-02-29","oid":"simplebench-5f68f956ce6e33a2","rd":"2024-02-29","sid":"reckWZi9nZmUCMK0d","sm":"claude-3-opus-20240229","src":"https://simple-bench.com/","v":0.235},{"b":"simplebench","m":"llama-3-1-405b-instruct","oid":"simplebench-2fd0dd8a266f705c","rd":"2024-07-23","sid":"recHm5YlK5f56Qa1p","sm":"Llama-3.1-405B-Instruct","src":"https://simple-bench.com/","v":0.23},{"b":"simplebench","cfg":{"effort":"high"},"e":"high","m":"o3-mini-2025-01-31","oid":"simplebench-f2b7363c737f65ca","rd":"2025-01-31","sid":"recsKDEJe3WUTzlXB","sm":"o3-mini-2025-01-31_high","src":"https://simple-bench.com/","v":0.228},{"b":"simplebench","m":"grok-2-1212","oid":"simplebench-97d1c9c34dd6f442","rd":"2024-12-12","sid":"recU76IB1e4qQPMy0","sm":"grok-2-1212","src":"https://simple-bench.com/","v":0.227},{"b":"simplebench","m":"mistral-large-2407","oid":"simplebench-720ccf4df363ea97","rd":"2024-07-24","sid":"recqB8MS2LaGRGShV","sm":"mistral-large-2407","src":"https://simple-bench.com/","v":0.225},{"b":"simplebench","m":"gpt-oss-120b","oid":"simplebench-05c973b3a8357e23","rd":"2025-08-05","sid":"recFVmMzFAlgDScJC","sm":"gpt-oss-120b","src":"https://simple-bench.com/","v":0.221},{"b":"simplebench","m":"llama-3-3-70b-instruct","oid":"simplebench-a7c72689f6306789","rd":"2024-12-06","sid":"recSRP5IhOYxwzPxz","sm":"Llama-3.3-70B-Instruct","src":"https://simple-bench.com/","v":0.199},{"b":"simplebench","m":"deepseek-v3","oid":"simplebench-5f57e4bdfa774dc8","rd":"2024-12-26","sid":"recc5o0Xjz6mlrIKU","sm":"DeepSeek-V3","src":"https://simple-bench.com/","v":0.189},{"b":"simplebench","m":"gemini-2-0-flash-exp","oid":"simplebench-2d4e7b3a9d4547e6","rd":"2024-12-11","sid":"recpuJCN7xqjCU4hn","sm":"gemini-2.0-flash-exp","src":"https://simple-bench.com/","v":0.189},{"b":"simplebench","cfg":{"effort":"medium"},"e":"medium","m":"o1-mini-2024-09-12","notes":"Assuming medium reasoning effort","oid":"simplebench-a5ab457b4a1c8e9a","rd":"2024-09-12","sid":"recY4JPURbdldMOQR","sm":"o1-mini-2024-09-12_medium","src":"https://simple-bench.com/","v":0.181},{"b":"simplebench","m":"gpt-4o-2024-08-06","oid":"simplebench-b43f12312a352987","rd":"2024-08-06","sid":"receDGtlCRZB7DW1z","sm":"gpt-4o-2024-08-06","src":"https://simple-bench.com/","v":0.178},{"b":"simplebench","m":"c4ai-command-r-plus-08-2024","oid":"simplebench-e2606059a956675a","rd":"2024-08-30","sid":"recMrDg2K2bDD6w9J","sm":"c4ai-command-r-plus-08-2024","src":"https://simple-bench.com/","v":0.174},{"b":"simplebench","m":"gpt-4o-mini-2024-07-18","oid":"simplebench-b9b6a9cb866c14b9","rd":"2024-07-18","sid":"recRSFVLtD45NI9Tm","sm":"gpt-4o-mini-2024-07-18","src":"https://simple-bench.com/","v":0.107},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-11-06","e":"max","evaluated_at":"2026-11-06T00:59:23.000Z","m":"kimi-k3-1-pro-together","oid":"simpleqa_verified-9e3db460bec92266","rd":"2026-10-23","se":0.015341165254026547,"sid":"HumfsXVh5BZWuTo6Z8Gqnf","sm":"kimi-k3.1-pro-together_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHumfsXVh5BZWuTo6Z8Gqnf.eval","v":0.622},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T20:57:35.000Z","m":"doubao-seed-evolving","oid":"simpleqa_verified-4eeb7085d27ba2ac","rd":"2026-11-02","se":0.015610338967577953,"sid":"PWieWgMJqvCcm7zkN9iYqH","sm":"doubao-seed-evolving_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPWieWgMJqvCcm7zkN9iYqH.eval","v":0.419},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-11-02","e":"high","evaluated_at":"2026-11-02T13:47:17.000Z","m":"mistral-large-4-0-oss","oid":"simpleqa_verified-30a4527848e1658d","rd":"2026-10-29","se":0.013190830072364589,"sid":"JYSfRXRddi6hYzpnSD3G7V","sm":"mistral-large-4-0-oss_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FJYSfRXRddi6hYzpnSD3G7V.eval","v":0.224},{"b":"simpleqa_verified","d":"2026-10-30","evaluated_at":"2026-10-30T18:28:17.000Z","m":"seed-1-6","oid":"simpleqa_verified-09d7b8b9076ea3c9","rd":"2025-06-11","se":0.01199449323097345,"sid":"jn6ts6NmQFtBGauL9n5i82","sm":"seed-1.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fjn6ts6NmQFtBGauL9n5i82.eval","v":0.174},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-30","e":"high","evaluated_at":"2026-10-30T12:30:09.000Z","m":"doubao-seed-2-1-turbo-260628","oid":"simpleqa_verified-31b96f457d176d41","rd":"2026-06-23","se":0.01445683229480096,"sid":"9wS2NJacFaJp9tkq8FSRyF","sm":"doubao-seed-2.1-turbo-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9wS2NJacFaJp9tkq8FSRyF.eval","v":0.297},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:56.000Z","m":"doubao-seed-2-0-lite-260428","oid":"simpleqa_verified-4b305ed72a764aee","rd":"2026-04-28","se":0.014696631960792619,"sid":"3rTiRa2THK8MWXtVJZQHxS","sm":"doubao-seed-2.0-lite-260428_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3rTiRa2THK8MWXtVJZQHxS.eval","v":0.315},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:39.000Z","m":"doubao-seed-2-0-pro-260215","oid":"simpleqa_verified-911b4c74a7cb59e7","rd":"2026-02-14","se":0.015459721957493382,"sid":"GvNEYw523w8aUy63WpgovD","sm":"doubao-seed-2.0-pro-260215_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGvNEYw523w8aUy63WpgovD.eval","v":0.394},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-28","e":"high","evaluated_at":"2026-10-28T23:55:29.000Z","m":"doubao-seed-2-1-pro-260628","oid":"simpleqa_verified-52306bd2bd36678f","rd":"2026-06-23","se":0.0157301760460091,"sid":"7TdGnviq5zZhNSXK5nJSAL","sm":"doubao-seed-2.1-pro-260628_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7TdGnviq5zZhNSXK5nJSAL.eval","v":0.447},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-10-26","e":"max","evaluated_at":"2026-10-26T20:00:37.000Z","m":"gpt-6-1-terra","oid":"simpleqa_verified-13b357c56d611f17","rd":"2026-10-26","se":0.01564978964446219,"sid":"Bixeio8A6SejZtUzFXFSXK","sm":"gpt-6.1-terra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FBixeio8A6SejZtUzFXFSXK.eval","v":0.573},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-10-19","e":"max","evaluated_at":"2026-10-19T22:38:54.000Z","m":"deepseek-v4-1-flash-1017","oid":"simpleqa_verified-04d0f6896b752409","rd":"2026-10-17","se":0.01564032031704017,"sid":"Ei4XWaCH5drYUnDRo2MYCW","sm":"deepseek-v4.1-flash-1017_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FEi4XWaCH5drYUnDRo2MYCW.eval","v":0.425},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-10-15","e":"xhigh","evaluated_at":"2026-10-15T18:52:04.000Z","m":"muse-spark-open-816b-a42b","oid":"simpleqa_verified-9b30d55e0bf49f14","rd":"2026-10-15","se":0.01568087656637504,"sid":"dNuxLdV4DWqfSvtTq6NnK8","sm":"Muse-Spark-Open-816B-A42B_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdNuxLdV4DWqfSvtTq6NnK8.eval","v":0.566},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-13","e":"high","evaluated_at":"2026-10-13T17:15:14.000Z","m":"gemini-4-argon","oid":"simpleqa_verified-82719abbf9953ae6","rd":"2026-09-30","se":0.013512312258920845,"sid":"iq4Y9tXpiyLP25yotySxJW","sm":"gemini-4-argon_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fiq4Y9tXpiyLP25yotySxJW.eval","v":0.76},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-10-13","e":"max","evaluated_at":"2026-10-13T12:59:33.000Z","m":"deepseek-v4-1-flash","oid":"simpleqa_verified-fc5f8b5b6a3c4cec","rd":"2026-09-09","se":0.015658997547870177,"sid":"3JJ4ChNxMo8CGAh3HrKNrw","sm":"deepseek-v4.1-flash_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3JJ4ChNxMo8CGAh3HrKNrw.eval","v":0.429},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-10-09","e":"max","evaluated_at":"2026-10-09T14:36:06.000Z","m":"claude-haiku-5-5","oid":"simpleqa_verified-53df48f1ffca8db7","rd":"2026-10-07","se":0.013473586661967093,"sid":"NwnrUMtveqHDYB2Um4ycmA","sm":"claude-haiku-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNwnrUMtveqHDYB2Um4ycmA.eval","v":0.238},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-10-09","e":"high","evaluated_at":"2026-10-09T14:12:09.000Z","m":"mistral-large-4-0","oid":"simpleqa_verified-53431bb413098a20","rd":"2026-10-06","se":0.012655439943366657,"sid":"hvNNKdnzLvr9X8zVxWKcfk","sm":"mistral-large-4-0_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhvNNKdnzLvr9X8zVxWKcfk.eval","v":0.2},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T22:00:34.000Z","m":"gpt-6-1-sol","oid":"simpleqa_verified-3fcbb95dbe3b028b","rd":"2026-09-29","se":0.013895037677965185,"sid":"dfT5VD7cQug8XW6GQHyJUF","sm":"gpt-6.1-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdfT5VD7cQug8XW6GQHyJUF.eval","v":0.739},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-29","e":"max","evaluated_at":"2026-09-29T15:15:14.000Z","m":"claude-sonnet-5-5","oid":"simpleqa_verified-c63c335c5920a769","rd":"2026-09-28","se":0.015780495050030086,"sid":"GtJ5yd3ZPQ7HVcm6Q9ssQz","sm":"claude-sonnet-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGtJ5yd3ZPQ7HVcm6Q9ssQz.eval","v":0.465},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T22:14:05.000Z","m":"gpt-6-luna","oid":"simpleqa_verified-fe7693c780e0e33e","rd":"2026-09-22","se":0.015583544104177553,"sid":"HX9sRVWJtZiNvneALnVzs8","sm":"gpt-6-luna_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FHX9sRVWJtZiNvneALnVzs8.eval","v":0.414},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-09-22","e":"xhigh","evaluated_at":"2026-09-22T21:49:22.000Z","m":"grok-4-7","oid":"simpleqa_verified-c252e0c771f13d0c","rd":"2026-09-21","se":0.015704987954361718,"sid":"6izDCw5ygCLZJEKPS8jV5T","sm":"grok-4.7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6izDCw5ygCLZJEKPS8jV5T.eval","v":0.56},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T19:59:24.000Z","m":"gpt-6-sol","oid":"simpleqa_verified-179778fff0155a4f","rd":"2026-09-22","se":0.015452824654081525,"sid":"donMeEKBZZdqpQsojKWJEm","sm":"gpt-6-sol_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdonMeEKBZZdqpQsojKWJEm.eval","v":0.607},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-22","e":"max","evaluated_at":"2026-09-22T18:27:39.000Z","m":"claude-opus-5-5","oid":"simpleqa_verified-286cda131c2e8325","rd":"2026-09-22","se":0.014174516461485273,"sid":"CgLfiGcNwMPVzxr8GdNUAK","sm":"claude-opus-5-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCgLfiGcNwMPVzxr8GdNUAK.eval","v":0.722},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-09-02","e":"xhigh","evaluated_at":"2026-09-02T18:04:58.000Z","m":"qwen3-8-max-0902","oid":"simpleqa_verified-fc7913b4ba9cceca","rd":"2026-09-01","se":0.01579621855130273,"sid":"nBcLJJomVoeg2zh5F8Mnsu","sm":"qwen3.8-max-0902_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnBcLJJomVoeg2zh5F8Mnsu.eval","v":0.473},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-09-02","e":"high","evaluated_at":"2026-09-02T17:15:36.000Z","m":"gemini-3-8-flash","oid":"simpleqa_verified-d83fec4bc4861235","rd":"2026-09-02","se":0.014539683710535193,"sid":"gZ4bwozTKtpmZTqnqX3a33","sm":"gemini-3.8-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgZ4bwozTKtpmZTqnqX3a33.eval","v":0.697},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-09-01","e":"max","evaluated_at":"2026-09-01T18:31:59.000Z","m":"claude-fable-5-1","oid":"simpleqa_verified-18dfb3398762dee9","rd":"2026-09-01","se":0.014385511563477433,"sid":"ExyL3GN8s34tizrXV3VmS3","sm":"claude-fable-5-1_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FExyL3GN8s34tizrXV3VmS3.eval","v":0.708},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T22:02:07.000Z","m":"gpt-4o-mini-2024-07-18","oid":"simpleqa_verified-f8806b044fab65e0","rd":"2024-07-18","se":0.008728527206074758,"sid":"DLr586bZUHpghAf8m3GRSm","sm":"gpt-4o-mini-2024-07-18","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDLr586bZUHpghAf8m3GRSm.eval","v":0.083},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T22:02:02.000Z","m":"gpt-4o-2024-08-06","oid":"simpleqa_verified-3047b9b6b0ca52fb","rd":"2024-08-06","se":0.01387777332977422,"sid":"bDFqfWxMG7gLDq4Mz8PRTP","sm":"gpt-4o-2024-08-06","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbDFqfWxMG7gLDq4Mz8PRTP.eval","v":0.26},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T22:01:58.000Z","m":"gpt-4-1-nano-2025-04-14","oid":"simpleqa_verified-0f3ff3c7154f69ed","rd":"2025-04-14","se":0.007513751157474971,"sid":"A4JEeJR5LQfLR3Hb8z7kL2","sm":"gpt-4.1-nano-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FA4JEeJR5LQfLR3Hb8z7kL2.eval","v":0.06},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T22:01:51.000Z","m":"gpt-4-1-mini-2025-04-14","oid":"simpleqa_verified-820b0cb4c8a81d44","rd":"2025-04-14","se":0.010534798620855644,"sid":"gYV4CJTjSd7VonfzMmnwcw","sm":"gpt-4.1-mini-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgYV4CJTjSd7VonfzMmnwcw.eval","v":0.127},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T22:01:48.000Z","m":"gpt-4-1-2025-04-14","oid":"simpleqa_verified-50b737b0b46648e0","rd":"2025-04-14","se":0.014645596385722692,"sid":"avdYKfKUuY6EvfVdxftBE8","sm":"gpt-4.1-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FavdYKfKUuY6EvfVdxftBE8.eval","v":0.311},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-31","e":"high","evaluated_at":"2026-08-31T22:01:45.000Z","m":"o3-mini-2025-01-31","oid":"simpleqa_verified-ed0fffffba96b47e","rd":"2025-01-31","se":0.011389500459665504,"sid":"UCba4H3NNtUiWXMisWnh9g","sm":"o3-mini-2025-01-31_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FUCba4H3NNtUiWXMisWnh9g.eval","v":0.153},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-31","e":"high","evaluated_at":"2026-08-31T22:01:34.000Z","m":"o1-2024-12-17","oid":"simpleqa_verified-167e87cb93afbb04","rd":"2024-12-17","se":0.01556667341859915,"sid":"LQfu4ngArcgMEyQZxuR7r4","sm":"o1-2024-12-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLQfu4ngArcgMEyQZxuR7r4.eval","v":0.411},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T19:21:58.000Z","m":"muse-spark-1-1","oid":"simpleqa_verified-eb2dffd1977c354b","rd":"2026-07-09","se":0.015665429072238852,"sid":"bCdwpRXgZtvQLYC4wA9JXr","sm":"muse-spark-1.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbCdwpRXgZtvQLYC4wA9JXr.eval","v":0.5778894472361809},{"b":"simpleqa_verified","d":"2026-08-31","evaluated_at":"2026-08-31T18:18:52.000Z","m":"claude-3-opus-2024-02-29","oid":"simpleqa_verified-c3d31b91ac9844ed","rd":"2024-02-29","se":0.010499249222407964,"sid":"NxVp4VF8uQbL5S6hx6HFRB","sm":"claude-3-opus-20240229","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FNxVp4VF8uQbL5S6hx6HFRB.eval","v":0.126},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-30","e":"max","evaluated_at":"2026-08-30T14:57:43.000Z","m":"gpt-6-astra","oid":"simpleqa_verified-2bdfcfdd7657298e","rd":"2026-09-03","se":0.013588548437881386,"sid":"PppwnX7EjXx2SeTcFN8Ujn","sm":"gpt-6-astra_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPppwnX7EjXx2SeTcFN8Ujn.eval","v":0.756},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-28","e":"max","evaluated_at":"2026-08-28T17:00:06.000Z","m":"glm-5-3","oid":"simpleqa_verified-82bd627ae1be497c","rd":"2026-08-14","se":0.015560917136921659,"sid":"TY8R8FV7zCwByEipkqyg2P","sm":"glm-5.3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTY8R8FV7zCwByEipkqyg2P.eval","v":0.41},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T21:29:42.000Z","m":"qwen3-5-plus","oid":"simpleqa_verified-f5c02d6e233a0e14","rd":"2026-02-16","se":0.013777178222916077,"sid":"jrawnM2wHFbVfJFJsap4kS","sm":"qwen3.5-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjrawnM2wHFbVfJFJsap4kS.eval","v":0.2535070140280561},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:36:49.000Z","m":"muse-spark-1-2","oid":"simpleqa_verified-e05762c211e1c5b6","rd":"2026-08-05","se":0.015518791563606454,"sid":"mLLmVVvo6t23MM5yxq7ryW","sm":"muse-spark-1.2_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmLLmVVvo6t23MM5yxq7ryW.eval","v":0.6030150753768844},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:36:45.000Z","m":"inkling","oid":"simpleqa_verified-735f32986ad0b20d","rd":"2026-07-15","se":0.015518757419066513,"sid":"bpnzC6CCi3hWuKCfqX4KEY","sm":"Inkling_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbpnzC6CCi3hWuKCfqX4KEY.eval","v":0.403},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:36:29.000Z","m":"qwen3-235b-a22b-thinking-2507","oid":"simpleqa_verified-3078c5b441fe741f","rd":"2025-07-25","se":0.015535255725079345,"sid":"4ACEC4x26HWRGiAepHGw43","sm":"Qwen3-235B-A22B-Thinking-2507","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4ACEC4x26HWRGiAepHGw43.eval","v":0.4044044044044044},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:36:21.000Z","m":"glm-5-1","oid":"simpleqa_verified-5582c71ec69a8661","rd":"2026-04-07","se":0.014987482264363972,"sid":"R2fMFAtRXah864hMifdQGV","sm":"glm-5.1","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FR2fMFAtRXah864hMifdQGV.eval","v":0.34},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:36:17.000Z","m":"inkling-small","oid":"simpleqa_verified-1c61798235ca6c6d","rd":"2026-07-15","se":0.012436787112179482,"sid":"eJfbDKqeg9kNXFoxpeV7HG","sm":"inkling-small_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FeJfbDKqeg9kNXFoxpeV7HG.eval","v":0.191},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:36:12.000Z","m":"kimi-k3","oid":"simpleqa_verified-fed599295763e7a4","rd":"2026-07-16","se":0.015818160898606836,"sid":"mZLWt2JSYvJyGhMvji7CT8","sm":"kimi-k3_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmZLWt2JSYvJyGhMvji7CT8.eval","v":0.506},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:36:05.000Z","m":"glm-5-2","oid":"simpleqa_verified-30ce8c1f13274dee","rd":"2026-06-16","se":0.015008706182121802,"sid":"WgjFZzojpiv54XqUhvfuUG","sm":"glm-5.2_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWgjFZzojpiv54XqUhvfuUG.eval","v":0.342},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:36:03.000Z","m":"kimi-k2-7-code","oid":"simpleqa_verified-194d5602053a4dd9","rd":"2026-06-12","se":0.015231776226264846,"sid":"5AijYw5bFJtSuahnbzNo6Z","sm":"kimi-k2.7-code","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5AijYw5bFJtSuahnbzNo6Z.eval","v":0.365},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:36:02.000Z","m":"glm-4-7","oid":"simpleqa_verified-497a1dc8aadc5cc1","rd":"2025-12-22","se":0.014782913600996744,"sid":"aYau6EG7CCaWVSyVq3vvQc","sm":"glm-4.7","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaYau6EG7CCaWVSyVq3vvQc.eval","v":0.322},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:35:45.000Z","m":"kimi-k2-5","oid":"simpleqa_verified-2be1157e671ea423","rd":"2026-01-27","se":0.015019206922357007,"sid":"RbXXg4JEbTSBnzw3CMSHnD","sm":"kimi-k2.5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRbXXg4JEbTSBnzw3CMSHnD.eval","v":0.343},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:34:09.000Z","m":"qwen3-7-max","oid":"simpleqa_verified-017a5eb477f449b7","rd":"2026-05-19","se":0.015737359397180177,"sid":"DMjUrG6eZEqwcQ4SqUoViu","sm":"qwen3.7-max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDMjUrG6eZEqwcQ4SqUoViu.eval","v":0.5576730190571715},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:34:06.000Z","m":"qwen3-6-plus","oid":"simpleqa_verified-42de4709473f1d8d","rd":"2026-03-31","se":0.015718302238794238,"sid":"4LWNorP8Fy8V8doxirNFcH","sm":"qwen3.6-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4LWNorP8Fy8V8doxirNFcH.eval","v":0.44144144144144143},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:33:58.000Z","m":"qwen3-6-max-preview","oid":"simpleqa_verified-af2caac12d646f6d","rd":"2026-04-20","se":0.015822434816352416,"sid":"iyPiR8GUuyNNA6Fssziq32","sm":"qwen3.6-max-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FiyPiR8GUuyNNA6Fssziq32.eval","v":0.5200400801603207},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:33:42.000Z","m":"qwen3-6-flash","oid":"simpleqa_verified-3599e8d4e6a1db7d","rd":"2026-04-27","se":0.011579970321213651,"sid":"FoMSS4FmH3VxpsvZiUJPCv","sm":"qwen3.6-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFoMSS4FmH3VxpsvZiUJPCv.eval","v":0.15915915915915915},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:32:07.000Z","m":"qwen3-5-flash","oid":"simpleqa_verified-6c1459e776f1566b","rd":"2026-02-25","se":0.012737194890497593,"sid":"K8zTckSXLKLesoLBv24TWc","sm":"qwen3.5-flash","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FK8zTckSXLKLesoLBv24TWc.eval","v":0.2032032032032032},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:32:02.000Z","m":"qwen3-max-2025-09-23","oid":"simpleqa_verified-a899cf69aa337cb1","rd":"2025-09-24","se":0.01582226675546796,"sid":"fXG2GPt5F25tHRTPhAoZ4K","sm":"qwen3-max-2025-09-23","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FfXG2GPt5F25tHRTPhAoZ4K.eval","v":0.4874874874874875},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:30:55.000Z","m":"qwen3-8-max","oid":"simpleqa_verified-29c6a72c6d3b8f4f","rd":"2026-08-02","se":0.015763390640483554,"sid":"bfftd5EzUK9yAekafEofvK","sm":"qwen3.8-max_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fbfftd5EzUK9yAekafEofvK.eval","v":0.458},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:30:35.000Z","m":"gemini-3-5-flash","oid":"simpleqa_verified-c50b8db917527d66","rd":"2026-05-19","se":0.014965960710224383,"sid":"GJ6YsSuzyRD9YXe6SBwvsu","sm":"gemini-3.5-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGJ6YsSuzyRD9YXe6SBwvsu.eval","v":0.662},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:30:29.000Z","m":"deepseek-v4-pro","oid":"simpleqa_verified-3f3544549b611796","rd":"2026-04-24","se":0.015806515332355317,"sid":"Sq9xqkVTmFvELpQy9KH5TM","sm":"deepseek-v4-pro_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSq9xqkVTmFvELpQy9KH5TM.eval","v":0.46993987975951906},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:30:25.000Z","m":"gemini-3-7-flash","oid":"simpleqa_verified-c0e8e10480338c9f","rd":"2026-08-13","se":0.01460648312734278,"sid":"84ARuGyhQsjgh8umRmBvas","sm":"gemini-3.7-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F84ARuGyhQsjgh8umRmBvas.eval","v":0.692},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:30:20.000Z","m":"deepseek-v4-pro-0813","oid":"simpleqa_verified-e19597ec6cd0d04c","rd":"2026-08-13","se":0.015808394665706597,"sid":"2xfFf6KZAp2gUWYhHGkFgs","sm":"deepseek-v4-pro-0813_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2xfFf6KZAp2gUWYhHGkFgs.eval","v":0.5290581162324649},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:30:16.000Z","m":"deepseek-v4-flash-0731","oid":"simpleqa_verified-766e1feada75b818","rd":"2026-07-31","se":0.01495531772489948,"sid":"TqFmiGvqozt7TSZYmhgF8m","sm":"deepseek-v4-flash-0731_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTqFmiGvqozt7TSZYmhgF8m.eval","v":0.33633633633633636},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:30:09.000Z","m":"gemini-3-6-flash","oid":"simpleqa_verified-7293d825bc194c01","rd":"2026-07-21","se":0.014965960710224383,"sid":"F83pa7Gfsb4UNVttTnS5Hk","sm":"gemini-3.6-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FF83pa7Gfsb4UNVttTnS5Hk.eval","v":0.662},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:30:07.000Z","m":"gemma-4-31b-it","oid":"simpleqa_verified-85d0f75e3a8ccaa7","rd":"2026-04-02","se":0.009658016218524242,"sid":"D5cbXacPnJ8akq2jAtifEQ","sm":"gemma-4-31b-it","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FD5cbXacPnJ8akq2jAtifEQ.eval","v":0.104},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:30:06.000Z","m":"gemini-3-flash-preview","oid":"simpleqa_verified-35fb1e7b67d2ae81","rd":"2025-12-17","se":0.014899597242811567,"sid":"7cc6cDCXjsPjtKt36bbeMR","sm":"gemini-3-flash-preview_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7cc6cDCXjsPjtKt36bbeMR.eval","v":0.668},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:29:53.000Z","m":"grok-4-6","oid":"simpleqa_verified-ec6ce4d735f1b248","rd":"2026-08-12","se":0.01581774956184353,"sid":"TpHYtuPG6kYZFvMK7Z6Hqj","sm":"grok-4.6_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTpHYtuPG6kYZFvMK7Z6Hqj.eval","v":0.493},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:29:44.000Z","m":"grok-4-20-0309-reasoning","oid":"simpleqa_verified-48886da4311a3bc6","rd":"2026-02-17","se":0.014526080235459494,"sid":"GVm7khGvtdnQKGfkgAHD8q","sm":"grok-4.20-0309-reasoning","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FGVm7khGvtdnQKGfkgAHD8q.eval","v":0.302},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:29:33.000Z","m":"grok-4-3","oid":"simpleqa_verified-e83dd270e3425bce","rd":"2026-04-17","se":0.014899597242811567,"sid":"Y39ZRp553ULpUDBV2XrBbc","sm":"grok-4.3_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FY39ZRp553ULpUDBV2XrBbc.eval","v":0.332},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:29:30.000Z","m":"grok-4-5","oid":"simpleqa_verified-020d1dffd51a7f42","rd":"2026-07-08","se":0.015810153729833274,"sid":"7HWW3sjRstKzb9JpcnvLaV","sm":"grok-4.5_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7HWW3sjRstKzb9JpcnvLaV.eval","v":0.483},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:29:19.000Z","m":"grok-4-6","oid":"simpleqa_verified-d04bf918537cc37b","rd":"2026-08-12","se":0.015815471195292575,"sid":"dKCWFaoGCWJTnD5CPWQGru","sm":"grok-4.6_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdKCWFaoGCWJTnD5CPWQGru.eval","v":0.489},{"b":"simpleqa_verified","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:29:17.000Z","m":"o4-mini-2025-04-16","oid":"simpleqa_verified-2077401efcf28857","rd":"2025-04-16","se":0.012559527926707347,"sid":"Dzky9brdTFbSoVd2zTKiUp","sm":"o4-mini-2025-04-16_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FDzky9brdTFbSoVd2zTKiUp.eval","v":0.196},{"b":"simpleqa_verified","cfg":{"effort":"low"},"d":"2026-08-27","e":"low","evaluated_at":"2026-08-27T19:29:12.000Z","m":"gpt-5-2-2025-12-11","oid":"simpleqa_verified-fdc92fbf6f9cc4f9","rd":"2025-12-11","se":0.014853842487270312,"sid":"Cyw3GA9a64AmT4pqLAhtTF","sm":"gpt-5.2-2025-12-11_low","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCyw3GA9a64AmT4pqLAhtTF.eval","v":0.328},{"b":"simpleqa_verified","cfg":{"effort":"medium"},"d":"2026-08-27","e":"medium","evaluated_at":"2026-08-27T19:29:07.000Z","m":"gpt-5-2-2025-12-11","oid":"simpleqa_verified-2eccf2683ce74b13","rd":"2025-12-11","se":0.014842213153411162,"sid":"esr5dcSuHiVoteo6vZfHjP","sm":"gpt-5.2-2025-12-11_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fesr5dcSuHiVoteo6vZfHjP.eval","v":0.327},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:51.000Z","m":"o3-2025-04-16","oid":"simpleqa_verified-0dbf2f460b8b812a","rd":"2025-04-16","se":0.015818160898606833,"sid":"CxHSa2gY7ySiPPGTZEZaRt","sm":"o3-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCxHSa2gY7ySiPPGTZEZaRt.eval","v":0.494},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:50.000Z","m":"gpt-5-4-nano-2026-03-17","oid":"simpleqa_verified-0546cd4d8afbb8a7","rd":"2026-03-17","se":0.010169287802713345,"sid":"3QBsxKkbF2R2LEXTd6HDrJ","sm":"gpt-5.4-nano-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F3QBsxKkbF2R2LEXTd6HDrJ.eval","v":0.117},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:50.000Z","m":"o4-mini-2025-04-16","oid":"simpleqa_verified-952f78aed215bd2b","rd":"2025-04-16","se":0.012361586015103758,"sid":"gaGdPvvfvAw7MjP4HaVQCT","sm":"o4-mini-2025-04-16_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgaGdPvvfvAw7MjP4HaVQCT.eval","v":0.188},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:49.000Z","m":"gpt-5-2025-08-07","oid":"simpleqa_verified-fd559bb447aaa6c9","rd":"2025-08-07","se":0.015819268290576817,"sid":"gTLurbyKU63i2opsGkJGwh","sm":"gpt-5-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FgTLurbyKU63i2opsGkJGwh.eval","v":0.501},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:49.000Z","m":"gpt-5-4-mini-2026-03-17","oid":"simpleqa_verified-41e9a7638a604420","rd":"2026-03-17","se":0.014414290540008265,"sid":"Z3pjKgAULY9j9wmzQpdqZG","sm":"gpt-5.4-mini-2026-03-17_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZ3pjKgAULY9j9wmzQpdqZG.eval","v":0.294},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:37.000Z","m":"gpt-5-2-2025-12-11","oid":"simpleqa_verified-2d3ec8f4d3cfe9ef","rd":"2025-12-11","se":0.015019206922357007,"sid":"Xp4qkZ32zjDeSYii2eQpnm","sm":"gpt-5.2-2025-12-11_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXp4qkZ32zjDeSYii2eQpnm.eval","v":0.343},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:28:25.000Z","m":"gpt-5-5","oid":"simpleqa_verified-5419076d59ff7070","rd":"2026-04-23","se":0.015275252316519465,"sid":"TijCafTTJUFrJ6jPGvBbgq","sm":"gpt-5.5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FTijCafTTJUFrJ6jPGvBbgq.eval","v":0.63},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:28:24.000Z","m":"gpt-5-4-pro-2026-03-05","oid":"simpleqa_verified-aad97723985ff783","rd":"2026-03-05","se":0.015775927227262426,"sid":"RfSJbuuuMkeVHQj7xrth4u","sm":"gpt-5.4-pro-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRfSJbuuuMkeVHQj7xrth4u.eval","v":0.463},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-27","e":"high","evaluated_at":"2026-08-27T19:28:24.000Z","m":"gpt-5-1-2025-11-13","oid":"simpleqa_verified-f34ecc36d00cefa4","rd":"2025-11-13","se":0.015806639423035177,"sid":"QCLnRy69NBH9ip3VYtQneh","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQCLnRy69NBH9ip3VYtQneh.eval","v":0.48},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:28:19.000Z","m":"claude-sonnet-4-5-20250929-59k","oid":"simpleqa_verified-e51d64c191bc2971","rd":"2025-09-29","se":0.014593284892852741,"sid":"XQEGUQjpXupjdyKoGuSmum","sm":"claude-sonnet-4-5-20250929_59K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FXQEGUQjpXupjdyKoGuSmum.eval","v":0.307},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:28:16.000Z","m":"gpt-5-4-2026-03-05","oid":"simpleqa_verified-f294501fd15e77fa","rd":"2026-03-05","se":0.015743152379585443,"sid":"AzCkm282PLhDdLJ8DnAQQc","sm":"gpt-5.4-2026-03-05_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAzCkm282PLhDdLJ8DnAQQc.eval","v":0.451},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:28:11.000Z","m":"claude-sonnet-5","oid":"simpleqa_verified-1b4fae484aee31a2","rd":"2026-06-30","se":0.014865395385928355,"sid":"Qw4tGw38pk4sdsEehhrLSR","sm":"claude-sonnet-5_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FQw4tGw38pk4sdsEehhrLSR.eval","v":0.329},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:28:02.000Z","m":"claude-opus-4-5-20251101-32k","oid":"simpleqa_verified-292ff6f77faefb59","rd":"2025-11-24","se":0.0157606915901365,"sid":"CA5LW4phRnHSsZUGXEm8H6","sm":"claude-opus-4-5-20251101_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FCA5LW4phRnHSsZUGXEm8H6.eval","v":0.457},{"b":"simpleqa_verified","d":"2026-08-27","evaluated_at":"2026-08-27T19:27:58.000Z","m":"claude-haiku-4-5-20251001-32k","oid":"simpleqa_verified-49d8d436086c0b79","rd":"2025-10-15","se":0.010499249222407964,"sid":"VhBM9HgJ2RuSVwXriRXMVJ","sm":"claude-haiku-4-5-20251001_32K","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FVhBM9HgJ2RuSVwXriRXMVJ.eval","v":0.126},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:27:55.000Z","m":"gpt-5-2-2025-12-11","oid":"simpleqa_verified-7bff52a4c86bd469","rd":"2025-12-11","se":0.015283736211823095,"sid":"E3325ytkSNYP8ez53VA4fE","sm":"gpt-5.2-2025-12-11_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FE3325ytkSNYP8ez53VA4fE.eval","v":0.371},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:27:45.000Z","m":"claude-opus-4-6","oid":"simpleqa_verified-2a202dfab7ca7ccf","rd":"2026-02-05","se":0.015790799515836722,"sid":"85fhMxLaebWtLoi6oLpTbi","sm":"claude-opus-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F85fhMxLaebWtLoi6oLpTbi.eval","v":0.47},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:27:42.000Z","m":"claude-sonnet-5","oid":"simpleqa_verified-0875175704de3b5c","rd":"2026-06-30","se":0.014955087918653461,"sid":"LPo74T9L3CawnKX5LLUrSZ","sm":"claude-sonnet-5_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLPo74T9L3CawnKX5LLUrSZ.eval","v":0.337},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:27:37.000Z","m":"claude-sonnet-4-6","oid":"simpleqa_verified-77c8a48cadb623fa","rd":"2026-02-17","se":0.014853842487270312,"sid":"Zs5fCvNzaz8LhqgwDSAztU","sm":"claude-sonnet-4-6_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FZs5fCvNzaz8LhqgwDSAztU.eval","v":0.328},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-27","e":"xhigh","evaluated_at":"2026-08-27T19:27:26.000Z","m":"claude-opus-4-7","oid":"simpleqa_verified-0e0bf2be2ec34f4f","rd":"2026-04-16","se":0.015810153729833274,"sid":"2c4rvmf5N7QXTugrFLMZ83","sm":"claude-opus-4-7_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2c4rvmf5N7QXTugrFLMZ83.eval","v":0.517},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-27","e":"max","evaluated_at":"2026-08-27T19:27:24.000Z","m":"claude-opus-4-8","oid":"simpleqa_verified-713450b82d96790c","rd":"2026-05-28","se":0.015790799515836722,"sid":"ViC6pxQZUVS3E9M9iEubF7","sm":"claude-opus-4-8_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FViC6pxQZUVS3E9M9iEubF7.eval","v":0.53},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-10","e":"max","evaluated_at":"2026-08-10T23:44:30.000Z","m":"gpt-5-6-sol","oid":"simpleqa_verified-a973ff90f9f8e7a7","rd":"2026-07-09","se":0.014539683710535193,"sid":"UnRUSuZzm2X27TUbpeMPva","sm":"gpt-5.6-sol_max","v":0.697},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-10","e":"max","evaluated_at":"2026-08-10T23:44:21.000Z","m":"gpt-5-6-terra","oid":"simpleqa_verified-82562222794a5291","rd":"2026-07-09","se":0.015672320237336324,"sid":"DienatPbgfgn3EzM7Gvzxr","sm":"gpt-5.6-terra_max","v":0.432},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-10","e":"max","evaluated_at":"2026-08-10T23:44:20.000Z","m":"gpt-5-6-luna","oid":"simpleqa_verified-3daa13e9b4b5ecf6","rd":"2026-07-09","se":0.015560917136921657,"sid":"CeDghdBk8CqtiFSbUzxNH5","sm":"gpt-5.6-luna_max","v":0.41},{"b":"simpleqa_verified","cfg":{"effort":"max"},"d":"2026-08-10","e":"max","evaluated_at":"2026-08-10T23:44:04.000Z","m":"claude-opus-5","oid":"simpleqa_verified-88c3a2ff248c8b5a","rd":"2026-07-24","se":0.01550610974549837,"sid":"Zftz2yk4X5NF28JQzjN3hZ","sm":"claude-opus-5_max","v":0.599},{"b":"simpleqa_verified","cfg":{"effort":"xhigh"},"d":"2026-08-10","e":"xhigh","evaluated_at":"2026-08-10T23:44:03.000Z","m":"claude-fable-5","oid":"simpleqa_verified-b12cac20bfbb5782","rd":"2026-06-09","se":0.014399942998441292,"sid":"SEnvgtunRa5GwAtU2sHa87","sm":"claude-fable-5_xhigh","v":0.707},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-10","e":"high","evaluated_at":"2026-08-10T22:42:23.000Z","m":"claude-sonnet-4-6","oid":"simpleqa_verified-0eea8a27dfcd23ba","rd":"2026-02-17","se":0.015139491543780595,"sid":"LJyKf8Xy4dymxNHbGLwNdA","sm":"claude-sonnet-4-6_high","v":0.355},{"b":"simpleqa_verified","d":"2026-08-10","evaluated_at":"2026-08-10T22:42:10.000Z","m":"claude-sonnet-4-5-2025-09-29","oid":"simpleqa_verified-9234d343e5d35089","rd":"2025-09-29","se":0.013454070462577935,"sid":"MhqeuRVx4JcRfxPQDYPq7Z","sm":"claude-sonnet-4-5-20250929","v":0.237},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-10","e":"high","evaluated_at":"2026-08-10T21:35:39.000Z","m":"gemini-3-1-pro-preview","oid":"simpleqa_verified-71bc4ee9d5d52ad2","rd":"2026-02-19","se":0.013963164754810065,"sid":"VGeF8Xac4WcSAGgSXVdEBi","sm":"gemini-3.1-pro-preview_high","v":0.735},{"b":"simpleqa_verified","d":"2026-08-10","evaluated_at":"2026-08-10T21:35:30.000Z","m":"kimi-k2-6","oid":"simpleqa_verified-1b47fa6ba274c851","rd":"2026-04-20","se":0.015080663991563052,"sid":"hHnQVatVTbQjM4sfrjw7GH","sm":"kimi-k2.6","v":0.349},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-10","e":"high","evaluated_at":"2026-08-10T21:35:29.000Z","m":"gpt-5-mini-2025-08-07","oid":"simpleqa_verified-1014a816b1f06442","rd":"2025-08-07","se":0.013019735539307761,"sid":"8d5RuFTTJKuJ5iJYmPZVJe","sm":"gpt-5-mini-2025-08-07_high","v":0.216},{"b":"simpleqa_verified","cfg":{"effort":"high"},"d":"2026-08-10","e":"high","evaluated_at":"2026-08-10T21:35:25.000Z","m":"gpt-5-nano-2025-08-07","oid":"simpleqa_verified-24a498a3d570ba6f","rd":"2025-08-07","se":0.010169287802713343,"sid":"KgUCEfVCyRnY8B5kVChyiD","sm":"gpt-5-nano-2025-08-07_high","v":0.117},{"b":"simpleqa_verified","d":"2026-08-10","evaluated_at":"2026-08-10T21:26:22.000Z","m":"claude-haiku-4-5-2025-10-01","oid":"simpleqa_verified-13ec8b8130e305be","rd":"2025-10-15","se":0.010709373963527967,"sid":"J2kUEjR9r6Spd594qDNTTf","sm":"claude-haiku-4-5-20251001","v":0.132},{"b":"spatialviz_bench","dn":"Qwen2.5-VL-72B-Instruct","m":"qwen2-5-vl-72b-instruct","oid":"spatialviz_bench-2ad60a064aada03e","rd":"2024-09-19","sm":"Qwen2.5-VL-72B-Instruct","src":"https://arxiv.org/abs/2507.07610","v":0.3331},{"b":"spatialviz_bench","dn":"Llama-4-Maverick-17B-128E-Instruct","m":"llama-4-maverick-17b-128e-instruct","oid":"spatialviz_bench-0f20027f67e14217","rd":"2025-04-05","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://arxiv.org/abs/2507.07610","v":0.3178},{"b":"spatialviz_bench","dn":"Llama-4-Scout-17B-16E-Instruct","m":"llama-4-scout-17b-16e-instruct","oid":"spatialviz_bench-6d2af6968b3d5fab","rd":"2025-04-05","sm":"Llama-4-Scout-17B-16E-Instruct","src":"https://arxiv.org/abs/2507.07610","v":0.34240000000000004},{"b":"spatialviz_bench","dn":"o1","m":"o1-2024-12-17","oid":"spatialviz_bench-67dae65f2095588e","rd":"2024-12-17","sm":"o1-2024-12-17_unknown","src":"https://arxiv.org/abs/2507.07610","v":0.4136},{"b":"spatialviz_bench","dn":"Claude-3.7-sonnet","m":"claude-3-7-sonnet-2025-02-19","oid":"spatialviz_bench-a9722edcacd0890e","rd":"2025-02-24","sm":"claude-3-7-sonnet-20250219_unknown","src":"https://arxiv.org/abs/2507.07610","v":0.33899999999999997},{"b":"spatialviz_bench","dn":"Gemini-2.5-flash","m":"gemini-2-5-flash","oid":"spatialviz_bench-a87eb0b8be7e6fe3","rd":"2025-06-17","sm":"gemini-2.5-flash","src":"https://arxiv.org/abs/2507.07610","v":0.3686},{"b":"spatialviz_bench","dn":"Gemini-2.5-pro","m":"gemini-2-5-pro","oid":"spatialviz_bench-816092b48cec2eba","rd":"2025-06-17","sm":"gemini-2.5-pro","src":"https://arxiv.org/abs/2507.07610","v":0.4466},{"b":"spatialviz_bench","dn":"Qwen-VL-max","m":"qwen-vl-max","oid":"spatialviz_bench-140c45640d1a3900","rd":"2024-01-18","sm":"Qwen-VL-Max","src":"https://arxiv.org/abs/2507.07610","v":0.32030000000000003},{"b":"superglue","dn":"GPT-3 Few-Shot","m":"text-davinci-001","oid":"superglue-4fa612880f057e24","rd":"2022-01-27","sid":"recjdKkQvD43gCZIG","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.718},{"b":"superglue","dn":"T5-Base","m":"t5-base","oid":"superglue-9a6abcd246d7bb8e","sid":"recZEQUJTXyiuueTC","sm":"T5-Base","src":"http://arxiv.org/abs/2101.03961","v":0.751},{"b":"superglue","dn":"Switch-Base","m":"switch-base","oid":"superglue-4958e4bd12a22449","sid":"rec295HTE28FPSMIh","sm":"Switch-Base","src":"http://arxiv.org/abs/2101.03961","v":0.733},{"b":"superglue","dn":"T5-Large","m":"t5-large","oid":"superglue-e8562f82bc85dd3e","sid":"recyMbNGI1OZRdoDb","sm":"T5-Large","src":"http://arxiv.org/abs/2101.03961","v":0.827},{"b":"superglue","dn":"Switch-Large","m":"switch-large","oid":"superglue-c2e67ea66a6ab176","sid":"recOy50NpQz1hM303","sm":"Switch-Large","src":"http://arxiv.org/abs/2101.03961","v":0.847},{"b":"superglue","dn":"T5-Small","m":"t5-small","oid":"superglue-10f998910d3ba326","sid":"rectWEngA0ZxOhjXS","sm":"T5-Small","src":"https://arxiv.org/pdf/1910.10683","v":0.633},{"b":"superglue","dn":"T5-Base","m":"t5-base","oid":"superglue-dd0735b773b3f56a","sid":"recKOjji7dPv8CMeY","sm":"T5-Base","src":"https://arxiv.org/pdf/1910.10683","v":0.762},{"b":"superglue","dn":"T5-Large","m":"t5-large","oid":"superglue-68ef22ddf97fe3b2","sid":"recn2AEOdhT6ox1s3","sm":"T5-Large","src":"https://arxiv.org/pdf/1910.10683","v":0.823},{"b":"superglue","dn":"T5-3B","m":"t5-3b","oid":"superglue-f9a6fbf0d0832f68","sid":"recsyCrajtvMBWsU7","sm":"T5-3B","src":"https://arxiv.org/pdf/1910.10683","v":0.864},{"b":"superglue","dn":"T5-11B","m":"t5-11b","oid":"superglue-3f5c4396f834f40b","sid":"recyISVSc8fWnwPWJ","sm":"T5-11B","src":"https://arxiv.org/pdf/1910.10683","v":0.889},{"b":"surface_evolver_bench","c":19.43522,"cfg":{"effort":"high","provider":"claude"},"dn":"claude-fable-5 (high)","e":"high","m":"claude-fable-5","metrics":{"total_tokens":933342},"oid":"surface_evolver_bench-a30abc9fa1a5eb79","rd":"2026-06-09","sm":"claude-fable-5_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.9500000000000001},{"b":"surface_evolver_bench","c":9.40775925,"cfg":{"effort":"xhigh","provider":"openai"},"dn":"gpt-5.6-sol (xhigh)","e":"xhigh","m":"gpt-5-6-sol","metrics":{"total_tokens":1415604},"oid":"surface_evolver_bench-6abaf519df02173b","rd":"2026-07-09","sm":"gpt-5.6-sol_xhigh","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.93125},{"b":"surface_evolver_bench","c":13.825813,"cfg":{"effort":"high","provider":"openai"},"dn":"gpt-5.5 (high)","e":"high","m":"gpt-5-5","metrics":{"total_tokens":2033603},"oid":"surface_evolver_bench-61e60129d2c2b487","rd":"2026-04-23","sm":"gpt-5.5_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.88125},{"b":"surface_evolver_bench","c":12.14,"cfg":{"effort":"max","provider":"kimi"},"dn":"kimi-k3","e":"max","m":"kimi-k3","metrics":{"total_tokens":3700000},"oid":"surface_evolver_bench-66bf984b8f9884f3","rd":"2026-07-16","sm":"kimi-k3_max","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.93},{"b":"surface_evolver_bench","c":21.961645,"cfg":{"effort":"high","provider":"claude"},"dn":"claude-opus-4.8 (high)","e":"high","m":"claude-opus-4-8","metrics":{"total_tokens":2699281},"oid":"surface_evolver_bench-9240312f9d828c67","rd":"2026-05-28","sm":"claude-opus-4-8_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.875},{"b":"surface_evolver_bench","c":7.58990275,"cfg":{"effort":"xhigh","provider":"openai"},"dn":"gpt-5.6-terra (xhigh)","e":"xhigh","m":"gpt-5-6-terra","metrics":{"total_tokens":2127318},"oid":"surface_evolver_bench-bffeecec4b83bbd6","rd":"2026-07-09","sm":"gpt-5.6-terra_xhigh","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.8375},{"b":"surface_evolver_bench","c":9.52131,"cfg":{"effort":"medium","provider":"openai"},"dn":"gpt-5.5 (medium)","e":"medium","m":"gpt-5-5","metrics":{"total_tokens":1637911},"oid":"surface_evolver_bench-a3bfc16fba7fc949","rd":"2026-04-23","sm":"gpt-5.5_medium","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.8125},{"b":"surface_evolver_bench","c":4.610306,"cfg":{"effort":"high","provider":"grok"},"dn":"grok-4.5 (high)","e":"high","m":"grok-4-5","metrics":{"total_tokens":3055609},"oid":"surface_evolver_bench-ef76c50fc2814a20","rd":"2026-07-08","sm":"grok-4.5_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.74375},{"b":"surface_evolver_bench","c":4.0321230474,"cfg":{"provider":"kimi"},"dn":"kimi-k2.7-code","m":"kimi-k2-7-code","metrics":{"total_tokens":5324288},"oid":"surface_evolver_bench-b345ef7235e5fab8","rd":"2026-06-12","sm":"kimi-k2.7-code","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.4875},{"b":"surface_evolver_bench","c":4.646197132,"cfg":{"effort":"high","provider":"z-ai"},"dn":"glm-5.2 (high)","e":"high","m":"glm-5-2","metrics":{"total_tokens":2543170},"oid":"surface_evolver_bench-97d531265fcc5303","rd":"2026-06-16","sm":"glm-5.2_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.55625},{"b":"surface_evolver_bench","c":9.727075000000001,"cfg":{"effort":"none","provider":"claude"},"dn":"claude-opus-4.8 (none)","e":"none","m":"claude-opus-4-8","metrics":{"total_tokens":1455083},"oid":"surface_evolver_bench-e11deb4893df334d","rd":"2026-05-28","sm":"claude-opus-4-8_none","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.68125},{"b":"surface_evolver_bench","c":3.857008705,"cfg":{"effort":"high","provider":"deepseek"},"dn":"deepseek-v4-pro (high)","e":"high","m":"deepseek-v4-pro","metrics":{"total_tokens":4495515},"oid":"surface_evolver_bench-d6afdf121b9baa65","rd":"2026-04-24","sm":"deepseek-v4-pro_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.4},{"b":"surface_evolver_bench","c":1.00794085,"cfg":{"effort":"medium","provider":"openai"},"dn":"gpt-5.6-luna (medium)","e":"medium","m":"gpt-5-6-luna","metrics":{"total_tokens":1243407},"oid":"surface_evolver_bench-ec915dc5c86923bd","rd":"2026-07-09","sm":"gpt-5.6-luna_medium","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.61875},{"b":"surface_evolver_bench","c":2.07806376,"cfg":{"provider":"minimax"},"dn":"minimax-m3","m":"minimax-m3","metrics":{"total_tokens":5547086},"oid":"surface_evolver_bench-75325dd38f36c8cd","rd":"2026-06-01","sm":"MiniMax-M3","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.5499999999999999},{"b":"surface_evolver_bench","c":2.1925831197,"cfg":{"provider":"qwen"},"dn":"qwen3.6-35b-a3b","m":"qwen3-6-35b-a3b","metrics":{"total_tokens":4037755},"oid":"surface_evolver_bench-8d7c44c204a41c7a","rd":"2026-04-14","sm":"qwen3.6-35b-a3b","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.44375000000000003},{"b":"surface_evolver_bench","c":0.21557616000000002,"cfg":{"provider":"gemma"},"dn":"gemma-4-31b-it (none)","m":"gemma-4-31b-it","metrics":{"total_tokens":1419535},"oid":"surface_evolver_bench-1e5734581de96f19","rd":"2026-04-02","sm":"gemma-4-31b-it","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.30624999999999997},{"b":"surface_evolver_bench","c":0.08923236999999999,"cfg":{"provider":"openai"},"dn":"gpt-oss-120b","m":"gpt-oss-120b","metrics":{"total_tokens":1475754},"oid":"surface_evolver_bench-3807c31eac433772","rd":"2025-08-05","sm":"gpt-oss-120b","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.25},{"b":"surface_evolver_bench","c":12.1421778,"cfg":{"provider":"kimi"},"dn":"kimi-k3","m":"kimi-k3","metrics":{"total_tokens":3710719},"oid":"surface_evolver_bench-0c31a6f9e110a16c","rd":"2026-07-16","sm":"kimi-k3_unknown","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.9500000000000001},{"b":"surface_evolver_bench","c":6.6382038,"cfg":{"effort":"medium","provider":"gemini"},"dn":"gemini-3.5-flash (medium)","e":"medium","m":"gemini-3-5-flash","metrics":{"total_tokens":3108560},"oid":"surface_evolver_bench-48bad78d74fd1149","rd":"2026-05-19","sm":"gemini-3.5-flash_medium","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.58125},{"b":"surface_evolver_bench","c":11.938944000000001,"cfg":{"effort":"medium","provider":"claude"},"dn":"claude-sonnet-5 (medium)","e":"medium","m":"claude-sonnet-5","metrics":{"total_tokens":3783644},"oid":"surface_evolver_bench-8e76675fc7c962c7","rd":"2026-06-30","sm":"claude-sonnet-5_medium","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.6},{"b":"surface_evolver_bench","c":6.348204,"cfg":{"effort":"high","provider":"mistral"},"dn":"mistral-medium-3-5 (high)","e":"high","m":"mistral-medium-2604","metrics":{"total_tokens":2760172},"oid":"surface_evolver_bench-c47fe03af150f274","rd":"2026-04-28","sm":"mistral-medium-2604","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.26875},{"b":"surface_evolver_bench","c":0.4947199,"cfg":{"provider":"poolside"},"dn":"laguna-m.1","m":"laguna-m-1","metrics":{"total_tokens":3092659},"oid":"surface_evolver_bench-999a74d7e7d4064a","rd":"2026-04-28","sm":"laguna-m.1","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.15625},{"b":"surface_evolver_bench","c":1.24719325,"cfg":{"provider":"arcee"},"dn":"trinity-large-thinking","m":"trinity-large-thinking","metrics":{"total_tokens":2601726},"oid":"surface_evolver_bench-bf9628cd830df80c","rd":"2026-04-01","sm":"trinity-large-thinking","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.15625},{"b":"surface_evolver_bench","c":2.49833875,"cfg":{"effort":"high","provider":"meta"},"dn":"muse-spark-1.1 (high)","e":"high","m":"muse-spark-1-1","metrics":{"total_tokens":3800825},"oid":"surface_evolver_bench-ff33303e5d4f208f","rd":"2026-07-09","sm":"muse-spark-1.1_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.525},{"b":"surface_evolver_bench","c":9.0487613,"cfg":{"effort":"xhigh","provider":"qwen"},"dn":"qwen3.8-27b (xhigh)","e":"xhigh","m":"qwen3-8-27b","metrics":{"total_tokens":4703787},"oid":"surface_evolver_bench-435e8e93e9a835d2","rd":"2026-08-14","sm":"qwen3.8-27b_xhigh","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.45},{"b":"surface_evolver_bench","c":0.30853474999999997,"cfg":{"effort":"max","provider":"z-ai"},"dn":"glm-5.3-flash (max)","e":"max","m":"glm-5-3-flash","metrics":{"total_tokens":2619902},"oid":"surface_evolver_bench-18b182a9d3af9d6d","rd":"2026-08-20","sm":"glm-5.3-flash_max","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.525},{"b":"surface_evolver_bench","c":2.5981149,"cfg":{"effort":"high","provider":"gemini"},"dn":"gemini-3.8-flash (high)","e":"high","m":"gemini-3-8-flash","metrics":{"total_tokens":3212425},"oid":"surface_evolver_bench-9173e3998d3b74c5","rd":"2026-09-02","sm":"gemini-3.8-flash_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.76875},{"b":"surface_evolver_bench","c":2.1277077120000003,"cfg":{"effort":"high","provider":"deepseek"},"dn":"deepseek-v4.1-flash (high)","e":"high","m":"deepseek-v4-1-flash","metrics":{"total_tokens":3347576},"oid":"surface_evolver_bench-28a3bf5a33e1d51d","rd":"2026-09-09","sm":"deepseek-v4.1-flash_high","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.4625},{"b":"surface_evolver_bench","c":6.0171848,"cfg":{"provider":"mistral"},"dn":"mistral-large-4-0","m":"mistral-large-4-0","metrics":{"total_tokens":5653545},"oid":"surface_evolver_bench-55895d0b5de6cdfe","rd":"2026-10-06","sm":"mistral-large-4-0_unknown","src":"https://yhenon.github.io/surface-evolver-llm-eval/","v":0.28125},{"b":"swe_bench_verified","cfg":{"effort":"max"},"d":"2026-06-25","e":"max","evaluated_at":"2026-06-25T13:13:06.902Z","m":"glm-5-2","oid":"swe_bench_verified-38f394c625eb49b0","rd":"2026-06-16","se":0.018656911901656907,"sid":"4AfhcmYVNrw6gM5u8CQsyy","sm":"glm-5.2_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F4AfhcmYVNrw6gM5u8CQsyy.eval","v":0.787},{"b":"swe_bench_verified","d":"2026-06-18","evaluated_at":"2026-06-18T17:48:14.587Z","m":"qwen3-7-max","oid":"swe_bench_verified-5064b837966f8c07","rd":"2026-05-19","se":0.01906835447169652,"sid":"83hhDhcT9UUrZ2NiUbWSUU","sm":"qwen3.7-max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F83hhDhcT9UUrZ2NiUbWSUU.eval","v":0.7727272727272727},{"b":"swe_bench_verified","cfg":{"effort":"max"},"d":"2026-06-18","e":"max","evaluated_at":"2026-06-18T17:43:54.039Z","m":"deepseek-v4-pro","oid":"swe_bench_verified-587676020123c5a0","rd":"2026-04-24","se":0.01897828052158487,"sid":"WUk3f44JhxtAJaSxi6gFZR","sm":"deepseek-v4-pro_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FWUk3f44JhxtAJaSxi6gFZR.eval","v":0.7763975155279503},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-06-01","e":"high","evaluated_at":"2026-06-01T10:22:55.131Z","m":"gemini-3-5-flash","oid":"swe_bench_verified-a64a0055206abaaa","rd":"2026-05-19","se":0.018422420250424888,"sid":"dwRzFGTghYEy4HNZajJXoc","sm":"gemini-3.5-flash_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FdwRzFGTghYEy4HNZajJXoc.eval","v":0.7933884297520661},{"b":"swe_bench_verified","d":"2026-05-28","evaluated_at":"2026-05-28T10:24:53.000Z","m":"qwen3-6-max-preview","oid":"swe_bench_verified-3cd208d46e9529d6","rd":"2026-04-20","se":0.01924895909142592,"sid":"FkDMVLdEnJDR5PyUCT6BNQ","sm":"qwen3.6-max-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFkDMVLdEnJDR5PyUCT6BNQ.eval","v":0.7665289256198347},{"b":"swe_bench_verified","d":"2026-05-15","evaluated_at":"2026-05-15T10:07:59.494Z","m":"glm-5-1","oid":"swe_bench_verified-a942215f8dfa31a7","rd":"2026-04-07","se":0.0199,"sid":"glm-5.1-swe-bench-frankenstein","sm":"glm-5.1","v":0.7417},{"b":"swe_bench_verified","d":"2026-05-14","evaluated_at":"2026-05-14T17:28:26.954Z","m":"qwen3-6-plus","oid":"swe_bench_verified-191b0ffa357d6492","rd":"2026-03-31","se":0.02246855652298021,"sid":"8HV4HP9EM5KuLSwzifuLye","sm":"qwen3.6-plus","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8HV4HP9EM5KuLSwzifuLye.eval","v":0.5785123966942148},{"b":"swe_bench_verified","d":"2026-05-08","evaluated_at":"2026-05-08T10:21:24.110Z","m":"kimi-k2-6","oid":"swe_bench_verified-d0e74a2ba7f18848","rd":"2026-04-20","se":0.01924895909142592,"sid":"bUPtAK7spEwvmiC8pi6afc","sm":"kimi-k2.6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FbUPtAK7spEwvmiC8pi6afc.eval","v":0.7665289256198347},{"b":"swe_bench_verified","cfg":{"effort":"xhigh"},"d":"2026-04-24","e":"xhigh","evaluated_at":"2026-04-24T13:52:36.000Z","m":"gpt-5-5-pre-release","oid":"swe_bench_verified-96d99ab1db7481e1","rd":"2026-04-23","se":0.018000198869416815,"sid":"LbhPRqUkXrs5KJ7vyT4CR7","sm":"gpt-5.5-pre-release_xhigh","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLbhPRqUkXrs5KJ7vyT4CR7.eval","v":0.8057851239669421},{"b":"swe_bench_verified","cfg":{"effort":"max"},"d":"2026-04-20","e":"max","evaluated_at":"2026-04-20T13:16:09.316Z","m":"claude-opus-4-7","oid":"swe_bench_verified-b95100d2297e4cef","rd":"2026-04-16","se":0.016901169179529604,"sid":"nCmGWWMBip2s9AVEZNYXDX","sm":"claude-opus-4-7_max","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FnCmGWWMBip2s9AVEZNYXDX.eval","v":0.8347107438016529},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-03-06","e":"high","evaluated_at":"2026-03-06T11:07:43.396Z","m":"gpt-5-4-2026-03-05","oid":"swe_bench_verified-2b60cbc4dfe6e936","rd":"2026-03-05","se":0.019189407003691976,"sid":"i9HTMbAxcwsrTf2w2FuJ6N","sm":"gpt-5.4-2026-03-05_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2Fi9HTMbAxcwsrTf2w2FuJ6N.eval","v":0.768595041322314},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-02-25","e":"high","evaluated_at":"2026-02-25T10:37:06.792Z","m":"gpt-5-3-codex","oid":"swe_bench_verified-05e137be8b363fcc","rd":"2026-02-05","se":0.019756739501473306,"sid":"6w5buMbgeEqtRoE55cLZVv","sm":"gpt-5.3-codex_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F6w5buMbgeEqtRoE55cLZVv.eval","v":0.7479338842975206},{"b":"swe_bench_verified","d":"2026-02-24","evaluated_at":"2026-02-24T13:34:48.126Z","m":"gemini-3-1-pro-preview-customtools","oid":"swe_bench_verified-de387d959e975af9","rd":"2026-02-19","se":0.019537213360116365,"sid":"8QQQWDgmmEsmQVUJWcxx4P","sm":"gemini-3.1-pro-preview-customtools","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F8QQQWDgmmEsmQVUJWcxx4P.eval","v":0.756198347107438},{"b":"swe_bench_verified","d":"2026-02-21","evaluated_at":"2026-02-21T10:32:51.288Z","m":"claude-sonnet-4-6","oid":"swe_bench_verified-c3fcbc44b69a2944","rd":"2026-02-17","se":0.01964818270139032,"sid":"KEdgiMmtJKHxgGiS82agYj","sm":"claude-sonnet-4-6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKEdgiMmtJKHxgGiS82agYj.eval","v":0.7520661157024794},{"b":"swe_bench_verified","d":"2026-02-18","evaluated_at":"2026-02-18T20:12:36.725Z","m":"gemini-3-flash-preview","oid":"swe_bench_verified-eb86928cb3068295","rd":"2025-12-17","se":0.019593002139508684,"sid":"jxoyMNmTnYirjLJN6wRNmG","sm":"gemini-3-flash-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FjxoyMNmTnYirjLJN6wRNmG.eval","v":0.7541322314049587},{"b":"swe_bench_verified","d":"2026-02-18","evaluated_at":"2026-02-18T20:07:57.705Z","m":"claude-opus-4-6","oid":"swe_bench_verified-9790644ee6fd0a76","rd":"2026-02-05","se":0.01862353730305288,"sid":"2hH2GAPUFaJR7pfVKhhicg","sm":"claude-opus-4-6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F2hH2GAPUFaJR7pfVKhhicg.eval","v":0.7871900826446281},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-02-18","e":"high","evaluated_at":"2026-02-18T12:36:35.180Z","m":"gpt-5-1-2025-11-13","oid":"swe_bench_verified-f0de83dde2faf424","rd":"2025-11-13","se":0.02122974956319103,"sid":"YoHtZpMNdsRE6qdj2K4Ucj","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FYoHtZpMNdsRE6qdj2K4Ucj.eval","v":0.6797520661157025},{"b":"swe_bench_verified","d":"2026-02-17","evaluated_at":"2026-02-17T14:43:08.990Z","m":"kimi-k2-5","oid":"swe_bench_verified-ce3ec54ad7a5e474","rd":"2026-01-27","se":0.020017832064928274,"sid":"5wVjUb3yn6bdjPSfPEMuiZ","sm":"kimi-k2.5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5wVjUb3yn6bdjPSfPEMuiZ.eval","v":0.737603305785124},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-02-17","e":"high","evaluated_at":"2026-02-17T01:48:11.460Z","m":"gpt-5-1-2025-11-13","oid":"swe_bench_verified-8946d37ba5a7876e","rd":"2025-11-13","se":0.021568422716496703,"sid":"G5hTYKeE5buuWzwRQK7EgS","sm":"gpt-5.1-2025-11-13_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FG5hTYKeE5buuWzwRQK7EgS.eval","v":0.6590909090909091},{"b":"swe_bench_verified","d":"2026-02-15","evaluated_at":"2026-02-15T15:40:10.848Z","m":"glm-5","oid":"swe_bench_verified-46af229fb28c3c13","rd":"2026-02-11","se":0.020894141602415234,"sid":"hKkJEu4ueLiNWsTtrJ78PS","sm":"glm-5","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FhKkJEu4ueLiNWsTtrJ78PS.eval","v":0.7207792207792207},{"b":"swe_bench_verified","d":"2026-02-13","evaluated_at":"2026-02-13T18:40:15.413Z","m":"gemini-2-5-pro","oid":"swe_bench_verified-3f21c191d59cfedf","rd":"2025-06-17","se":0.02251275673889269,"sid":"KL724uHFGbiXj4o5PHyLn2","sm":"gemini-2.5-pro","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FKL724uHFGbiXj4o5PHyLn2.eval","v":0.5755693581780539},{"b":"swe_bench_verified","d":"2026-02-13","evaluated_at":"2026-02-13T10:43:27.233Z","m":"gemini-3-pro-preview","oid":"swe_bench_verified-98c705a6ab7053a5","rd":"2025-11-18","se":0.02021641171153417,"sid":"5XdYZBG5Wy8DUMRqcYv4fn","sm":"gemini-3-pro-preview","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F5XdYZBG5Wy8DUMRqcYv4fn.eval","v":0.7293388429752066},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-02-12","e":"high","evaluated_at":"2026-02-12T13:34:00.865Z","m":"gpt-5-2-2025-12-11","oid":"swe_bench_verified-de97f21f70a0be15","rd":"2025-12-11","se":0.020017832064928277,"sid":"9owdbpnWJ5pkQQC3GWKP5L","sm":"gpt-5.2-2025-12-11_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F9owdbpnWJ5pkQQC3GWKP5L.eval","v":0.737603305785124},{"b":"swe_bench_verified","cfg":{"effort":"medium"},"d":"2026-02-12","e":"medium","evaluated_at":"2026-02-12T03:46:13.664Z","m":"o3-2025-04-16","oid":"swe_bench_verified-b4a981ec0cae49c3","rd":"2025-04-16","se":0.022072335658124498,"sid":"AKt7PcE8LQtoMWVuxwMidk","sm":"o3-2025-04-16_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FAKt7PcE8LQtoMWVuxwMidk.eval","v":0.6231884057971014},{"b":"swe_bench_verified","d":"2026-02-11","evaluated_at":"2026-02-11T21:33:53.502Z","m":"claude-opus-4-1-2025-08-05","oid":"swe_bench_verified-f785708a358ef94f","rd":"2025-08-05","se":0.020118245554677085,"sid":"cA6Mq67yaynLhY8F5kes3c","sm":"claude-opus-4-1-20250805","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcA6Mq67yaynLhY8F5kes3c.eval","v":0.7334710743801653},{"b":"swe_bench_verified","d":"2026-02-11","evaluated_at":"2026-02-11T15:05:15.089Z","m":"gpt-4o-2024-11-20","oid":"swe_bench_verified-5c39a35c759b9400","rd":"2024-11-20","se":0.021042625979133336,"sid":"PncXWJm7ZCY3qv982pKoRy","sm":"gpt-4o-2024-11-20","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FPncXWJm7ZCY3qv982pKoRy.eval","v":0.30991735537190085},{"b":"swe_bench_verified","d":"2026-02-10","evaluated_at":"2026-02-10T22:23:09.876Z","m":"claude-opus-4-6","oid":"swe_bench_verified-ec74a669e1f465aa","rd":"2026-02-05","se":0.019537213360116365,"sid":"aQiAg6mt8HPHXjKcJvax2d","sm":"claude-opus-4-6","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FaQiAg6mt8HPHXjKcJvax2d.eval","v":0.756198347107438},{"b":"swe_bench_verified","d":"2026-02-08","evaluated_at":"2026-02-08T17:41:23.879Z","m":"gpt-4-1-2025-04-14","oid":"swe_bench_verified-fe563b273f95da11","rd":"2025-04-14","se":0.022835863746433965,"sid":"SheUt9cNpkSevy4r6Gsr3u","sm":"gpt-4.1-2025-04-14","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FSheUt9cNpkSevy4r6Gsr3u.eval","v":0.48541666666666666},{"b":"swe_bench_verified","cfg":{"effort":"high"},"d":"2026-02-06","e":"high","evaluated_at":"2026-02-06T14:23:51.990Z","m":"gpt-5-2025-08-07","oid":"swe_bench_verified-b2cc081130d8500a","rd":"2025-08-07","se":0.02006832181710289,"sid":"C85Bnci9xVzq5jmt8zNnbw","sm":"gpt-5-2025-08-07_high","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FC85Bnci9xVzq5jmt8zNnbw.eval","v":0.7355371900826446},{"b":"swe_bench_verified","d":"2026-02-06","evaluated_at":"2026-02-06T07:21:39.976Z","m":"claude-opus-4-2025-05-14","oid":"swe_bench_verified-2ba1cd96be1a048e","rd":"2025-05-22","se":0.02071754535128056,"sid":"RbcvuVChvxK6SMQbPK8iRn","sm":"claude-opus-4-20250514","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FRbcvuVChvxK6SMQbPK8iRn.eval","v":0.7066115702479339},{"b":"swe_bench_verified","d":"2026-02-05","evaluated_at":"2026-02-05T22:06:33.964Z","m":"claude-opus-4-5-2025-11-01","oid":"swe_bench_verified-32696a8b77686627","rd":"2025-11-24","se":0.01924895909142592,"sid":"mKQKxkpGDrFHBPBYR3mBrB","sm":"claude-opus-4-5-20251101","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FmKQKxkpGDrFHBPBYR3mBrB.eval","v":0.7665289256198347},{"b":"swe_bench_verified","d":"2026-02-05","evaluated_at":"2026-02-05T16:52:49.622Z","m":"claude-sonnet-4-5-2025-09-29","oid":"swe_bench_verified-1f6d5e1b6fe608ce","rd":"2025-09-29","se":0.020587234937983246,"sid":"Lti64yMEsBWSzgEMFCSVhp","sm":"claude-sonnet-4-5-20250929","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FLti64yMEsBWSzgEMFCSVhp.eval","v":0.7128099173553719},{"b":"swe_bench_verified","cfg":{"effort":"medium"},"d":"2026-02-05","e":"medium","evaluated_at":"2026-02-05T13:39:28.137Z","m":"gpt-5-2025-08-07","oid":"swe_bench_verified-c8f33fba5c6dd2c1","rd":"2025-08-07","se":0.020542754003902382,"sid":"FCCkMNVHcjLLUDeNFqTGRL","sm":"gpt-5-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FFCCkMNVHcjLLUDeNFqTGRL.eval","v":0.7148760330578512},{"b":"swe_bench_verified","d":"2026-02-04","evaluated_at":"2026-02-04T18:05:21.852Z","m":"claude-3-7-sonnet-2025-02-19","oid":"swe_bench_verified-6eaea3c6084f07a4","rd":"2025-02-24","se":0.022198466971122266,"sid":"cnM8U6moeqrp74wnXRj2Jj","sm":"claude-3-7-sonnet-20250219","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2FcnM8U6moeqrp74wnXRj2Jj.eval","v":0.609504132231405},{"b":"swe_bench_verified","cfg":{"effort":"medium"},"d":"2026-02-01","e":"medium","evaluated_at":"2026-02-01T11:28:37.125Z","m":"gpt-5-mini-2025-08-07","oid":"swe_bench_verified-1981ae7b4026cc07","rd":"2025-08-07","se":0.021749601139671306,"sid":"7nvdQy3AwKtoay2FBvfST5","sm":"gpt-5-mini-2025-08-07_medium","trace":"https://logs.epoch.ai/inspect-viewer/36231d6d/viewer.html?log_file=https%3A%2F%2Flogs.epoch.ai%2Finspect_ai_logs%2F7nvdQy3AwKtoay2FBvfST5.eval","v":0.6466942148760331},{"b":"terminalbench","cfg":{"agent":"NexAU-AHE"},"d":"2026-04-23","dn":"GPT-5.5","m":"gpt-5-5","oid":"terminalbench-b632e168a7e19fce","rd":"2026-04-23","se":0.020892351283,"sid":"recdZ2Aq4dSe9MUj6","sm":"gpt-5.5_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.847191011236},{"b":"terminalbench","cfg":{"agent":"Capy"},"d":"2026-04-23","dn":"GPT-5.5","m":"gpt-5-5","oid":"terminalbench-b0e5dbb1969afa58","rd":"2026-04-23","se":0.021123213002000003,"sid":"recGwko1gzRhNX0ub","sm":"gpt-5.5_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.8314606741570001},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2026-04-23","dn":"GPT-5.5","m":"gpt-5-5","oid":"terminalbench-74cdb5a2a4ca0c73","rd":"2026-04-23","se":0.022241606381,"sid":"recp64pC4XBYynZdL","sm":"gpt-5.5_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.822471910112},{"b":"terminalbench","cfg":{"agent":"Codex"},"d":"2026-04-23","dn":"GPT-5.5","m":"gpt-5-5","oid":"terminalbench-2b87cb6fdf88891b","rd":"2026-04-23","se":0.022000000000000002,"sid":"recJA7K9RNsgzOaAU","sm":"gpt-5.5_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.8200000000000001},{"b":"terminalbench","cfg":{"agent":"ForgeCode"},"d":"2026-03-12","dn":"GPT-5.4","m":"gpt-5-4-2026-03-05","oid":"terminalbench-1eaed8db5f444b0d","rd":"2026-03-05","se":0.02,"sid":"rec4q8Z4CrKFnEAuK","sm":"gpt-5.4-2026-03-05_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.818},{"b":"terminalbench","cfg":{"agent":"TongAgents"},"d":"2026-02-19","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"terminalbench-264ca13b97fe0ed1","rd":"2026-02-19","se":0.026242804005000002,"sid":"recWIDb8VVaXoNXGP","sm":"gemini-3.1-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.8022471910110001},{"b":"terminalbench","cfg":{"agent":"WOZCODE"},"d":"2026-04-16","dn":"Claude Opus 4.7","m":"claude-opus-4-7","oid":"terminalbench-bc3b937b32847192","rd":"2026-04-16","se":0.021123213002000003,"sid":"rec1OgwoZk8uGh0cf","sm":"claude-opus-4-7_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.8022471910110001},{"b":"terminalbench","cfg":{"agent":"ForgeCode"},"d":"2026-03-12","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-4fe72d8cbb95107e","rd":"2026-02-05","se":0.016,"sid":"reciHkuyWIdoQzFVZ","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.798},{"b":"terminalbench","cfg":{"agent":"SageAgent"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-f039e76f0ec619c4","rd":"2026-02-05","se":0.02202247191,"sid":"recaq5BH3KfrPDhfy","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7842696629210001},{"b":"terminalbench","cfg":{"agent":"Forge Code"},"d":"2026-03-02","m":"gemini-3-1-pro-preview","oid":"terminalbench-2a7c060261fd408e","rd":"2026-02-19","se":0.018,"sid":"recIsprMd6SFvZAf1","sm":"gemini-3.1-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.784},{"b":"terminalbench","cfg":{"agent":"ForgeCode"},"d":"2026-03-02","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"terminalbench-8168932c6e87a371","rd":"2026-02-19","se":0.018000000000000002,"sid":"rec64ox7LJjoKhMmd","sm":"gemini-3.1-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.784},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-263ae7d9d3d3d529","rd":"2026-02-05","se":0.021577527622,"sid":"recMJkgMHRSOjLKpR","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.773033707865},{"b":"terminalbench","cfg":{"agent":"Meta-Harness"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-c3e460a06ac83161","rd":"2026-02-05","se":0.024324615274,"sid":"rec0JCWY63Blk9hvu","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.76404494382},{"b":"terminalbench","cfg":{"agent":"CodeBrain-1.5"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-c94a31bfe8dfb04e","rd":"2026-02-05","se":0.020303715883,"sid":"recu2IkOMIWvkSCuc","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7584269662920001},{"b":"terminalbench","cfg":{"agent":"Codelia"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-ece265422eb2b99d","rd":"2026-02-05","se":0.021577527622,"sid":"recDp7GQrLzo9PXAN","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.757303370787},{"b":"terminalbench","cfg":{"agent":"Capy"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-c3514d29028dbc7a","rd":"2026-02-05","se":0.023922527823,"sid":"recL5sLwUUzYQRQHC","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.752808988764},{"b":"terminalbench","cfg":{"agent":"Simple Codex"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-a53e6194306fea32","rd":"2026-02-05","se":0.023922527823,"sid":"recQGdQu18B2G90mW","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7505617977529999},{"b":"terminalbench","cfg":{"agent":"Terminus-KIRA"},"d":"2026-02-19","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"terminalbench-82f87a88701985a4","rd":"2026-02-19","se":0.025682394863000002,"sid":"recwQlLxqWmKqLyki","sm":"gemini-3.1-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.748314606742},{"b":"terminalbench","cfg":{"agent":"Terminus-KIRA"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-ce259d8892e03a5d","rd":"2026-02-05","se":0.025964111461,"sid":"reccEEVJ98pyKkKic","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.747191011236},{"b":"terminalbench","cfg":{"agent":"Mux"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-5f02fe0d75aef794","rd":"2026-02-05","se":0.025109481272,"sid":"recG4Not08o5icKST","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7460674157299999},{"b":"terminalbench","cfg":{"agent":"MAYA-V2"},"d":"2026-02-05","dn":"Claude 4.6 Opus","m":"claude-opus-4-6","oid":"terminalbench-eafda30f2eec18d0","rd":"2026-02-05","se":0.02208661072,"sid":"recbCEp3NVGK1vNTl","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7207865168540001},{"b":"terminalbench","cfg":{"agent":"TongAgents"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-3ae728c8530844be","rd":"2026-02-05","se":0.026791493391999998,"sid":"recsbGWmj9mRLEypD","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7191011235960001},{"b":"terminalbench","cfg":{"agent":"spoox-o-m"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-622e68759a36bb7d","rd":"2026-02-05","se":0.024915582762000003,"sid":"recE1erZH43qiwt01","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.7146067415730001},{"b":"terminalbench","cfg":{"agent":"CodeBrain-1"},"d":"2026-02-10","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-64077d314d4a736c","rd":"2026-02-05","se":0.026000000000000002,"sid":"reckRlNCDoPJqQ51J","sm":"gpt-5.3-codex","v":0.703},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2026-02-05","m":"claude-opus-4-6","oid":"terminalbench-1f9dc9e1a040fa8a","rd":"2026-02-05","se":0.025,"sid":"recZd8as5p49jd05r","sm":"claude-opus-4-6","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.699},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-b1a850aca1e876ab","rd":"2026-02-05","se":0.024720163409000004,"sid":"recm47gxTJYtnhFi9","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.698876404494},{"b":"terminalbench","cfg":{"agent":"Ante"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-8ec8bbe7533840a3","rd":"2025-11-18","se":0.020886618713,"sid":"recel7iZnjcoDWjFt","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.6942571785270001},{"b":"terminalbench","cfg":{"agent":"IndusAGI Coding Agent"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-cc2617c244d12ec0","rd":"2026-02-05","se":0.022792046745,"sid":"rec0sW71nWWpW9Szk","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.69138576779},{"b":"terminalbench","cfg":{"agent":"Crux"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-2bf0505d79702879","rd":"2026-02-05","sid":"recAnM8GuIbCLfEhc","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.66872659176},{"b":"terminalbench","cfg":{"agent":"Mux"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-392adbb3bd36e8f8","rd":"2026-02-05","se":0.025301893904,"sid":"rec3SssCd9VRZn61b","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.6651685393260001},{"b":"terminalbench","cfg":{"agent":"Deep Agents"},"d":"2026-01-14","dn":"GPT-5.2-Codex","m":"gpt-5-2-codex","oid":"terminalbench-d56de37c12cbc25f","rd":"2025-12-18","se":0.030621004558,"sid":"recwxCGE7farYLuja","sm":"gpt-5.2-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.6651685393260001},{"b":"terminalbench","cfg":{"agent":"clnkr"},"d":"2026-04-23","dn":"GPT-5.5","m":"gpt-5-5","oid":"terminalbench-311092ad9da62115","rd":"2026-04-23","se":0.024915582762000003,"sid":"recmgPXDch3lXoWZy","sm":"gpt-5.5_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.660674157303},{"b":"terminalbench","cfg":{"agent":"SageAgent"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-318b7a9dffc92dcf","rd":"2025-11-18","se":0.020658909863,"sid":"recfAdtO655s2ZUhC","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.651685393258},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2025-12-11","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"terminalbench-e16a624a966d0b93","rd":"2025-12-11","se":0.027681817951,"sid":"recgRH031sZYfPDqe","sm":"gpt-5.2-2025-12-11_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.6494382022470001},{"b":"terminalbench","cfg":{"agent":"Droid","effort":"medium"},"d":"2025-12-24","e":"medium","m":"gpt-5-2-2025-12-11","oid":"terminalbench-31ad04e8c672369d","rd":"2025-12-11","se":0.028,"sid":"rechjHEUQLlqCTIXc","sm":"gpt-5.2-2025-12-11_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.649},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2026-02-05","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"terminalbench-c2b850b8d4266d49","rd":"2026-02-05","se":0.027151126845,"sid":"recAMbJKjZNVkGjpL","sm":"gpt-5.3-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.647191011236},{"b":"terminalbench","cfg":{"agent":"Junie CLI"},"d":"2025-12-17","dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"terminalbench-72e331a0945789a3","rd":"2025-12-17","se":0.027856468377,"sid":"recUBHBNuIOfkcKRX","sm":"gemini-3-flash-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.642696629213},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-86d1feba46facb7f","rd":"2025-11-24","se":0.026971909527,"sid":"rec6dIIsNVTXVuRDv","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.631460674157},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2025-12-11","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-efdfa97e9e74db3d","rd":"2025-11-24","se":0.027,"sid":"recuiEXbA7NpkFohL","sm":"claude-opus-4-5-20251101","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.631},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-12-11","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"terminalbench-955b232ae33dc4ac","rd":"2025-12-11","se":0.030034647333,"sid":"recKka6IlJnGPWGok","sm":"gpt-5.2-2025-12-11_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.629213483146},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-851f559c770dfaec","rd":"2026-02-05","se":0.026609854054,"sid":"recX6PzRl9e6dU4jS","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.629213483146},{"b":"terminalbench","cfg":{"agent":"Codex CLI","effort":"medium"},"d":"2025-12-18","e":"medium","m":"gpt-5-2-2025-12-11","oid":"terminalbench-35635587672a5c0a","rd":"2025-12-11","se":0.03,"sid":"recumDtosjSEJ2D3d","sm":"gpt-5.2-2025-12-11_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.629},{"b":"terminalbench","cfg":{"agent":"CodeBrain-1.5"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-ed3f131f9d31365c","rd":"2025-11-18","se":0.026057340168,"sid":"recc4ldyhTauum9Fy","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.622471910112},{"b":"terminalbench","cfg":{"agent":"CodeBrain-1"},"d":"2026-02-05","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-4eb39cec651f36e5","rd":"2025-11-18","se":0.026000000000000002,"sid":"reczLTVTgCc3wuoOJ","sm":"gemini-3-pro-preview","v":0.622},{"b":"terminalbench","cfg":{"agent":"II-Agent"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-ee4b6058c0829200","rd":"2025-11-18","se":0.028202524731,"sid":"rectX4lsyYs6OmUsy","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.6179775280900001},{"b":"terminalbench","cfg":{"agent":"hookele"},"d":"2025-11-12","dn":"GPT-5.1-Codex-Mini","m":"gpt-5-1-codex-mini","oid":"terminalbench-0a031c9a4d2839de","rd":"2025-11-12","se":0.018944446656,"sid":"recPGQsdEihz8Ckv3","sm":"gpt-5.1-codex-mini","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.615730337079},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2026-02-19","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"terminalbench-38bfa2012281304e","rd":"2026-02-19","se":0.041200272349000006,"sid":"rechRUph4VPi4BkPa","sm":"gemini-3.1-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.614232209738},{"b":"terminalbench","cfg":{"agent":"Droid"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-b5573ddc1cc225e3","rd":"2025-11-18","se":0.027856468377,"sid":"recNP2GjuvuiMcEyv","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.611235955056},{"b":"terminalbench","cfg":{"agent":"Mux"},"d":"2025-12-11","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"terminalbench-7525d1baee28745b","rd":"2025-12-11","sid":"recpYruU4E3J5eQwz","sm":"gpt-5.2-2025-12-11_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.606741573034},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-11-19","dn":"GPT-5.1-Codex-Max","m":"gpt-5-1-codex-max","oid":"terminalbench-a4bbe00b04ee25ca","rd":"2025-11-19","se":0.026609854054,"sid":"recaaVycAGesmFJ32","sm":"gpt-5.1-codex-max","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.604494382022},{"b":"terminalbench","cfg":{"agent":"Letta Code"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-a2c7a5f9410526ec","rd":"2025-11-24","se":0.024124409274000003,"sid":"recpSQWuI4flKPHHO","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.591011235955},{"b":"terminalbench","cfg":{"agent":"Letta Code"},"d":"2025-12-17","m":"claude-opus-4-5-20251101-128k","oid":"terminalbench-bf5fcdd9c2ab221c","rd":"2025-11-24","se":0.024,"sid":"reciL106EieXnavcX","sm":"claude-opus-4-5-20251101_128K","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.591},{"b":"terminalbench","cfg":{"agent":"Mux"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-a2d15450a1fae23e","rd":"2025-11-24","sid":"recJewQ720QCUWUI0","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.584269662921},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2026-02-05","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"terminalbench-3d687994619de164","rd":"2026-02-05","se":0.029049635593,"sid":"recsCMkghG4mKBFQJ","sm":"claude-opus-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.579775280899},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-11-16","m":"gpt-5-1-codex","oid":"terminalbench-03c2d25e833fb341","rd":"2025-11-12","se":0.029,"sid":"recninlGInIspn658","sm":"gpt-5.1-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.578},{"b":"terminalbench","cfg":{"agent":"Crux"},"d":"2025-11-12","dn":"GPT-5.1-Codex","m":"gpt-5-1-codex","oid":"terminalbench-41fd989e21726024","rd":"2025-11-12","se":0.029381642209,"sid":"recMan57K3dKSqHWB","sm":"gpt-5.1-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.577528089888},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-d8ee772ecdba9b5f","rd":"2025-11-24","se":0.025301893904,"sid":"recculHGd5t2GvQt9","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.577528089888},{"b":"terminalbench","cfg":{"agent":"Grok CLI"},"d":"2026-03-10","dn":"Grok 4.20 Reasoning","m":"grok-4-20","oid":"terminalbench-75cf8120a1938517","rd":"2026-02-17","sid":"recWSbNb07JV8aOj5","sm":"grok-4-20","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.573033707865},{"b":"terminalbench","cfg":{"agent":"grok-cli"},"d":"2026-04-02","dn":"Grok 4.20 Reasoning","m":"grok-4-20","oid":"terminalbench-b734b9648c63902b","rd":"2026-02-17","sid":"recauvZ8Z2shdiV8Y","sm":"grok-4-20","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.573},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-e9b8626869b8f157","rd":"2025-11-18","se":0.024720163409000004,"sid":"rec4G1Mf13x9ynwlw","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.5685393258429999},{"b":"terminalbench","cfg":{"agent":"Letta Code"},"d":"2025-11-18","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"terminalbench-b42b902014ac6c9f","rd":"2025-11-18","se":0.029709938903,"sid":"rec3LvdlngMWgVNX7","sm":"gemini-3-pro-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.559550561798},{"b":"terminalbench","cfg":{"agent":"Goose"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-0a380645e05c91ef","rd":"2025-11-24","se":0.026265894906,"sid":"recpIcBUBke0D7mNd","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.543258426966},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-12-11","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"terminalbench-a647426460bd7464","rd":"2025-12-11","se":0.028910174223,"sid":"recu0QWuZH20pvT2b","sm":"gpt-5.2-2025-12-11_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.540449438202},{"b":"terminalbench","cfg":{"agent":"Letta Code"},"d":"2025-11-12","dn":"GPT-5.1-Codex","m":"gpt-5-1-codex","oid":"terminalbench-f3e2e3e1ea1e0ce5","rd":"2025-11-12","se":0.02837397023,"sid":"recdETGOmZWS2iKAe","sm":"gpt-5.1-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.534831460674},{"b":"terminalbench","cfg":{"agent":"Simplai Agent"},"d":"2026-02-17","dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","oid":"terminalbench-5826770cac5bda42","rd":"2026-02-17","se":0.028300280743,"sid":"recDxT0PT7JaamTSf","sm":"claude-sonnet-4-6_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.534269662921},{"b":"terminalbench","cfg":{"agent":"Simplai Agent"},"d":"2026-05-14","dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","oid":"terminalbench-27d65e77d4ca05eb","rd":"2026-02-17","se":0.027999999999999997,"sid":"recLi0k6lS2WZTy3O","sm":"claude-sonnet-4-6","v":0.534},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2026-02-11","dn":"GLM 5","m":"glm-5","oid":"terminalbench-301efc79adde66fa","rd":"2026-02-11","se":0.026265894906,"sid":"recuBaChFTWpnR5jj","sm":"glm-5","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.524157303371},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-bfa85822219434e4","rd":"2025-11-24","se":0.025301893904,"sid":"recub09g3exhXqHL0","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.521348314607},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-18eda6184f57f40a","rd":"2025-11-24","se":0.028882201141000003,"sid":"recA5YQhqDEEEZMYJ","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.519101123596},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-12-17","dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"terminalbench-2a57249a0488dd2d","rd":"2025-12-17","se":0.031299813473,"sid":"reclYJoF7jvlAegKw","sm":"gemini-3-flash-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.516853932584},{"b":"terminalbench","cfg":{"agent":"OpenCode"},"d":"2025-11-24","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"terminalbench-880e4a7651da3779","rd":"2025-11-24","sid":"rec7Aip1YsA0Xpjtk","sm":"claude-opus-4-5-20251101_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.516853932584},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-12-23","m":"gemini-3-flash-preview","oid":"terminalbench-b39eae69c95d4d18","rd":"2025-12-17","se":0.03,"sid":"recXnaoWnwYiDCTLY","sm":"gemini-3-flash-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.51},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-08-07","dn":"GPT-5","m":"gpt-5-2025-08-07","oid":"terminalbench-2673b768796883be","rd":"2025-08-07","se":0.028973014913,"sid":"rec9i8Ic0KTT2yBKw","sm":"gpt-5-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.49606741573},{"b":"terminalbench","cfg":{"agent":"Codex CLI","effort":"medium"},"d":"2025-11-04","e":"medium","m":"gpt-5-2025-08-07","oid":"terminalbench-fbd9b975905aa9fb","rd":"2025-08-07","se":0.029,"sid":"rec6jIJxArc92pivy","sm":"gpt-5-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.496},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"medium"},"d":"2025-11-16","e":"medium","m":"gpt-5-1-2025-11-13","notes":"Assuming medium based on GPT-5 being run at medium.","oid":"terminalbench-1279691e4e4f217e","rd":"2025-11-13","se":0.028,"sid":"rec8JKbUbg1BqQUuc","sm":"gpt-5.1-2025-11-13_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.476},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"medium"},"d":"2025-11-16","e":"medium","m":"gpt-5-1-2025-11-13","notes":"Assuming medium based on GPT-5 being run at medium.","oid":"terminalbench-f73178b98fb59793","rd":"2025-11-13","se":0.028,"sid":"recTEUNVY8Ta5gHeC","sm":"gpt-5.1-2025-11-13_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.476},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-12","dn":"GPT-5.1","m":"gpt-5-1-2025-11-13","oid":"terminalbench-38cc236c69ef7074","rd":"2025-11-13","se":0.027616039281,"sid":"recNG2YeioSnRQkBL","sm":"gpt-5.1-2025-11-13_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.475842696629},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-12-17","dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"terminalbench-02334d2e90b0f72f","rd":"2025-12-17","se":0.030355882645,"sid":"recEx94YIfACLb1n0","sm":"gemini-3-flash-preview","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.47415730337100004},{"b":"terminalbench","cfg":{"agent":"CAMEL-AI"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-5962a859d21119e8","rd":"2025-09-29","se":0.023513565589,"sid":"recUr1bsa1LcRX043","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.465168539326},{"b":"terminalbench","cfg":{"agent":"IndusAGI Coding Agent"},"d":"2026-03-18","dn":"MiniMax M2.7","m":"minimax-m2-7","oid":"terminalbench-89ca202b87cf227d","rd":"2026-03-18","sid":"recVgdB7STmvB8TFK","sm":"MiniMax-M2.7","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.45056179775300004},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-09-15","dn":"GPT-5-Codex","m":"gpt-5-codex","oid":"terminalbench-438166eddce09789","rd":"2025-09-15","se":0.027054198168,"sid":"recAFFDfAzTiAM5HO","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.443258426966},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-11-04","m":"gpt-5-codex","oid":"terminalbench-c1319c320dac5fa1","rd":"2025-09-15","se":0.027,"sid":"rec1qBXfk7ZcFVViF","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.443},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-08-07","dn":"GPT-5","m":"gpt-5-2025-08-07","oid":"terminalbench-ff263ae962a1bfd6","rd":"2025-08-07","se":0.029872734308000002,"sid":"recJMDLwtXhVgVmhO","sm":"gpt-5-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.438202247191},{"b":"terminalbench","cfg":{"agent":"OpenHands","effort":"medium"},"d":"2025-11-02","e":"medium","m":"gpt-5-2025-08-07","notes":"Medium thinking used in other GPT-5 evaluations, assuming the same here.","oid":"terminalbench-f651e173883ecb21","rd":"2025-08-07","se":0.03,"sid":"recimrZ9t1kWdq9df","sm":"gpt-5-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.438},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-09-15","dn":"GPT-5-Codex","m":"gpt-5-codex","oid":"terminalbench-3e8dc42ba70260c2","rd":"2025-09-15","se":0.028734895694000003,"sid":"reciHggGNoFXEb71Z","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.434269662921},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"gpt-5-codex","oid":"terminalbench-754cad549ec21f2c","rd":"2025-09-15","se":0.029,"sid":"recXSpZpHeeVjiI3u","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.434},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2026-01-27","dn":"Kimi K2.5","m":"kimi-k2-5","oid":"terminalbench-4f8176ea84d9d553","rd":"2026-01-27","se":0.028777059907,"sid":"recwAteBLwH7AXt3F","sm":"kimi-k2.5","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.43202247191},{"b":"terminalbench","cfg":{"agent":"Crux"},"d":"2025-11-12","dn":"GPT-5.1-Codex-Mini","m":"gpt-5-1-codex-mini","oid":"terminalbench-6c1bfc7574a4fd3b","rd":"2025-11-12","se":0.030034647333,"sid":"rec9XiiLe8VVBrkm0","sm":"gpt-5.1-codex-mini","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.431460674157},{"b":"terminalbench","cfg":{"agent":"Goose"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-94f7b6fad1469ca5","rd":"2025-09-29","se":0.025870546791,"sid":"recUDNI3Kg8XVDBC8","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.431460674157},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-11-17","m":"gpt-5-1-codex-mini","oid":"terminalbench-e92892a8f786916f","rd":"2025-11-12","se":0.03,"sid":"recJBKBBdOIEodbRT","sm":"gpt-5.1-codex-mini","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.431},{"b":"terminalbench","cfg":{"agent":"Harness Agent"},"d":"2026-03-18","dn":"MiniMax M2.7 Highspeed","m":"minimax-m2-7","oid":"terminalbench-e195e274846089de","rd":"2026-03-18","se":0.029381642209,"sid":"recP18tiQzw7yeuDF","sm":"MiniMax-M2.7","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.42921348314600005},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-61d50ede528f250f","rd":"2025-09-29","se":0.027885470433999998,"sid":"recKeFvLcPnLgoUmH","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.42808988764},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-19d778270c5b01a7","rd":"2025-09-29","se":0.028,"sid":"recZH7p8iYiyYB0Ch","sm":"claude-sonnet-4-5-20250929","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.428},{"b":"terminalbench","cfg":{"agent":"MAYA"},"d":"2026-01-04","dn":"Claude 4.5 Sonnet","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-158b31283b4b4960","rd":"2025-09-29","sid":"recdkO8Qjk07Jy6x3","sm":"claude-sonnet-4-5-20250929_unknown","v":0.42700000000000005},{"b":"terminalbench","cfg":{"agent":"cchuter"},"d":"2026-02-12","dn":"minimax-m2.5","m":"minimax-m2-5","oid":"terminalbench-5bcc2f6a77e48f48","rd":"2026-02-12","se":0.028245483667000002,"sid":"recVaqQHEH5MbcNtU","sm":"MiniMax-M2.5","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.42696629213500004},{"b":"terminalbench","cfg":{"agent":"MAYA-V2"},"d":"2025-09-29","dn":"Claude 4.5 Sonnet","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-736cf30b054e8c76","rd":"2025-09-29","sid":"rece9vUo1ww7GL6EO","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.42696629213500004},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-df6f57889312fd6c","rd":"2025-09-29","se":0.028,"sid":"reclDcpO9SUv4Tpau","sm":"claude-sonnet-4-5-20250929","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.426},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-c886143461030400","rd":"2025-09-29","se":0.027711002796,"sid":"recJeZzLs4VeGn2nV","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.425842696629},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-be0a25daddbe5bb6","rd":"2025-09-29","se":0.028051650379000002,"sid":"recmWHKFsA4OK8Ti1","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.42528089887599996},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-78def1586a8a5860","rd":"2025-09-29","se":0.028,"sid":"recfCRv3Mp3k41IHO","sm":"claude-sonnet-4-5-20250929","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.425},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2026-02-12","dn":"Minimax m2.5","m":"minimax-m2-5","oid":"terminalbench-4e08f1ad70f257a2","rd":"2026-02-12","se":0.026242804005000002,"sid":"rec2efDw2CQRGldo9","sm":"MiniMax-M2.5","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.422471910112},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-09-15","dn":"GPT-5-Codex","m":"gpt-5-codex","oid":"terminalbench-549530cb3bd8ad7e","rd":"2025-09-15","se":0.027681817951,"sid":"recRmzMIVDOO2eEBY","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.41348314606700004},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"gpt-5-codex","oid":"terminalbench-22a26eedb4157c38","rd":"2025-09-15","se":0.028,"sid":"rec9sfNDeDk0cbJQL","sm":"gpt-5-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.413},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-11-04","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-7513f4a5968a0185","rd":"2025-09-29","se":0.029,"sid":"recDripAbQH8CuooL","sm":"claude-sonnet-4-5-20250929","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.401},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-09-29","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"terminalbench-42f402ab689b1f3a","rd":"2025-09-29","se":0.029153793818,"sid":"recBr0bdFWEQJlw24","sm":"claude-sonnet-4-5-20250929_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.40056179775300005},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-12-01","dn":"DeepSeek-V3.2","m":"deepseekdeepseek-v3-2","oid":"terminalbench-de0c813db3bdcbae","rd":"2025-12-01","se":0.027506058599999997,"sid":"recxR7bQuL8biEr1p","sm":"deepseek/deepseek-v3.2","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.395505617978},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-d125a115780ecfdc","rd":"2025-08-05","se":0.026,"sid":"recZqrNbY83D2rMna","sm":"claude-opus-4-1-20250805","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.38},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-05","dn":"Claude Opus 4.1","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-a87416c5b141745b","rd":"2025-08-05","se":0.026242804005000002,"sid":"receznXnwcy2WCuyz","sm":"claude-opus-4-1-20250805_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.379775280899},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-17","m":"gpt-5-1-codex","oid":"terminalbench-adc4f3cc94002e75","rd":"2025-11-12","se":0.032,"sid":"rec65Jj1nz0W6LZFC","sm":"gpt-5.1-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.369},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-24fd373aa9946a6a","rd":"2025-08-05","se":0.027,"sid":"recO3hSKFRHSXO0v0","sm":"claude-opus-4-1-20250805","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.369},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-08-05","dn":"Claude Opus 4.1","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-58caa0d8e4ea0b39","rd":"2025-08-05","se":0.026971909527,"sid":"rechRJ6Uae8I0mHXa","sm":"claude-opus-4-1-20250805_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.368539325843},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-12","dn":"GPT-5.1-Codex","m":"gpt-5-1-codex","oid":"terminalbench-e7ad659e11ddec12","rd":"2025-11-12","se":0.032366291433,"sid":"reclqVW78N7ZdBWbk","sm":"gpt-5.1-codex","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.368539325843},{"b":"terminalbench","cfg":{"agent":"Crux"},"d":"2025-12-23","dn":"MiniMax M2.1","m":"minimax-m2-1","oid":"terminalbench-6a17304c992cc859","rd":"2025-12-23","se":0.028882201141000003,"sid":"recoAqoyZqRsoB6Kc","sm":"MiniMax-M2.1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.36629213483100004},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"thinking"},"d":"2025-11-06","dn":"Kimi K2 Thinking","e":"thinking","m":"kimi-k2-thinking","oid":"terminalbench-6c66f11080299f33","rd":"2025-11-06","se":0.028202524731,"sid":"recM5WAExwpby8B8E","sm":"kimi-k2-thinking","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.35730337078700003},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-11","m":"kimi-k2-thinking","oid":"terminalbench-5c20ef32d4ebb4d9","rd":"2025-11-06","se":0.028,"sid":"rec2gS7bw3o4JruIn","sm":"kimi-k2-thinking","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.357},{"b":"terminalbench","cfg":{"agent":"Goose"},"d":"2025-10-15","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-222b4e8dd1bdafda","rd":"2025-10-15","se":0.028882201141000003,"sid":"recYFtwkZx7UpcnlV","sm":"claude-haiku-4-5-20251001_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.35505617977500004},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"medium"},"d":"2025-10-31","e":"medium","m":"gpt-5-2025-08-07","oid":"terminalbench-305a06fd389a8d80","rd":"2025-08-07","se":0.031,"sid":"recbqMxsZI87hk2Zv","sm":"gpt-5-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.352},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-07","dn":"GPT-5","m":"gpt-5-2025-08-07","oid":"terminalbench-2def8ecbb6bc0f8f","rd":"2025-08-07","se":0.030554939746,"sid":"rec1Juuim3oNht6yT","sm":"gpt-5-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.35168539325799997},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-6b1b940cfdd2a633","rd":"2025-08-05","se":0.025,"sid":"rec1kbUd5yf48B3vK","sm":"claude-opus-4-1-20250805","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.351},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-05","dn":"Claude Opus 4.1","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-e7256c44be1420cd","rd":"2025-08-05","se":0.025109481272,"sid":"recCL4Qu6Dd73ew1S","sm":"claude-opus-4-1-20250805_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.350561797753},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-08-05","dn":"Claude Opus 4.1","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-bc9f40f66ebc2052","rd":"2025-08-05","se":0.028993931543000003,"sid":"recNswzdzYnWC135z","sm":"claude-opus-4-1-20250805_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.34831460674200004},{"b":"terminalbench","cfg":{"agent":"spoox-o-m"},"d":"2025-08-07","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-a024d2cf6324dc3f","rd":"2025-08-07","se":0.027151126845,"sid":"recM8hndoEYq6Jlbf","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.34831460674200004},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-11-04","m":"claude-opus-4-1-2025-08-05","oid":"terminalbench-89c05a8df50d0857","rd":"2025-08-05","se":0.029,"sid":"recIh4lviUhDM9aux","sm":"claude-opus-4-1-20250805","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.348},{"b":"terminalbench","cfg":{"agent":"spoox-m"},"d":"2025-12-24","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-7d08fb4266cdc4e5","rd":"2025-08-07","se":0.027000000000000003,"sid":"rec3mqkLQJWWwGiiW","sm":"gpt-5-mini-2025-08-07_unknown","v":0.348},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-07","dn":"GPT-5","m":"gpt-5-2025-08-07","oid":"terminalbench-fd0ddc0777bc7c97","rd":"2025-08-07","se":0.029049635593,"sid":"recGz2hxdobFUq7NN","sm":"gpt-5-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.339325842697},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent","effort":"medium"},"d":"2025-11-03","e":"medium","m":"gpt-5-2025-08-07","oid":"terminalbench-83b52001ac993ccb","rd":"2025-08-07","se":0.029,"sid":"recN22yvosEoDYoao","sm":"gpt-5-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.339},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-12-22","dn":"GLM 4.7","m":"glm-4-7","oid":"terminalbench-ee24c1bfcbacb411","rd":"2025-12-22","se":0.028414299458,"sid":"rec1VQj4GryD6eaGI","sm":"glm-4.7","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.333895131086},{"b":"terminalbench","cfg":{"agent":"Crux"},"d":"2025-12-22","dn":"GLM 4.7","m":"glm-4-7","oid":"terminalbench-89970bcc8ccc51d1","rd":"2025-12-22","se":0.024523186857,"sid":"recG5DDGxMEXJrTWs","sm":"glm-4.7","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.33258426966300003},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-06-17","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"terminalbench-e5c2b228918d9c3a","rd":"2025-06-17","se":0.030215762425,"sid":"recy9TahZMrNgdLAU","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.32640449438200003},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"gemini-2-5-pro","oid":"terminalbench-97b5521ebedae978","rd":"2025-06-17","se":0.03,"sid":"rect0YAzqrcZe6qxi","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.326},{"b":"terminalbench","cfg":{"agent":"Codex CLI","effort":"medium"},"d":"2025-11-04","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-eef4f06366255aac","rd":"2025-08-07","se":0.03,"sid":"recNTuh2AD5nZEqvj","sm":"gpt-5-mini-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.319},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-08-07","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-6801b6217a7297ed","rd":"2025-08-07","se":0.029987513007,"sid":"recTuTGBX7vknCWdE","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.318539325843},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-27","dn":"MiniMax M2","m":"minimax-m2","oid":"terminalbench-0d793cced8f0a9e5","rd":"2025-10-27","se":0.027407927502999998,"sid":"recjRYjr04mfk7kN1","sm":"MiniMax-M2","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.300374531835},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-10-15","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-c80afac083e81e29","rd":"2025-10-15","se":0.025133613407,"sid":"rec7okc2EI7syaupc","sm":"claude-haiku-4-5-20251001_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.298314606742},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-d62c4fe1626d1c66","rd":"2025-10-15","se":0.025,"sid":"recjYJxgaZXKRW8nE","sm":"claude-haiku-4-5-20251001","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.298},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-12-23","dn":"MiniMax M2.1","m":"minimax-m2-1","oid":"terminalbench-ce8a4bc386deb2ab","rd":"2025-12-23","se":0.029049635593,"sid":"rec4F7W15iJy3hjQh","sm":"MiniMax-M2.1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.292134831461},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-08-07","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-6eaefb347b491d8f","rd":"2025-08-07","se":0.027506058599999997,"sid":"recZeMXA8WRW0RPTl","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.292134831461},{"b":"terminalbench","cfg":{"agent":"OpenHands","effort":"medium"},"d":"2025-11-02","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-aa7270e3ee3a9aa1","rd":"2025-08-07","se":0.028,"sid":"recqpHHQIxrdzWQ2e","sm":"gpt-5-mini-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.292},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-15","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-e7ed481bb45ab275","rd":"2025-10-15","se":0.028755985528000003,"sid":"recTafF90BHib6N1S","sm":"claude-haiku-4-5-20251001_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.283146067416},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-0e42789efd552d2b","rd":"2025-10-15","se":0.029,"sid":"recBFoPPbvHLqQkIF","sm":"claude-haiku-4-5-20251001","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.283},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-01","m":"kimi-k2-instruct","oid":"terminalbench-1aaf7e471ef2543b","rd":"2025-07-12","se":0.025,"sid":"recxOKDiLuVBjbuwV","sm":"Kimi-K2-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.278},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-07-11","dn":"Kimi K2 Instruct","m":"kimi-k2-instruct","oid":"terminalbench-be18f82f9c4501f1","rd":"2025-07-12","se":0.025216558437000002,"sid":"recE77NwwL0EUyZ8C","sm":"Kimi-K2-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.277902621723},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-10-15","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-a36f7ff964ac904f","rd":"2025-10-15","se":0.027754722523000003,"sid":"recPD1bwT2Dp0439M","sm":"claude-haiku-4-5-20251001_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.275280898876},{"b":"terminalbench","cfg":{"agent":"Claude Code"},"d":"2025-11-04","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-07f92bade823c7da","rd":"2025-10-15","se":0.028,"sid":"reca6RNNkwrVNSBgQ","sm":"claude-haiku-4-5-20251001","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.275},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"grok-4-0709","oid":"terminalbench-58023a2368e16bb9","rd":"2025-07-09","se":0.031,"sid":"recSY6n1UG1WaG3BJ","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.272},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-07-09","dn":"Grok 4","m":"grok-4-0709","oid":"terminalbench-9739d2d2fba5532b","rd":"2025-07-09","se":0.030515232205,"sid":"recItJ3vEdOIZoUy3","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.27191011236},{"b":"terminalbench","cfg":{"agent":"Dakou Agent"},"d":"2025-07-22","dn":"Qwen 3 Coder 480B","m":"qwen3-coder-480b-a35b-instruct","oid":"terminalbench-efcad5f75529b02e","rd":"2025-07-31","se":0.025682394863000002,"sid":"recOAMwEh5spGhfvw","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.27191011236},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-07-11","dn":"Kimi K2 Instruct","m":"kimi-k2-instruct","oid":"terminalbench-a6e8cb65bb22b3d0","rd":"2025-07-12","se":0.026971909527,"sid":"recrMHVuQYDBd8BrS","sm":"Kimi-K2-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.267415730337},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"kimi-k2-instruct","oid":"terminalbench-58efc1dd6e9a4bdf","rd":"2025-07-12","se":0.027,"sid":"recZ35cLEBZqtObMV","sm":"Kimi-K2-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.267},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"gemini-2-5-pro","oid":"terminalbench-9517b80c30960dcb","rd":"2025-06-17","se":0.025,"sid":"reczBJ8FZoe7Ihdp1","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.261},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-06-17","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"terminalbench-79aa0b9bc74b35a2","rd":"2025-06-17","se":0.025492854303,"sid":"recCkXuWxF2axfuj5","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.260674157303},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-28","dn":"Grok Code Fast 1","m":"grok-code-fast-1","oid":"terminalbench-8c916e2bf2f60628","rd":"2025-08-28","se":0.026242804005000002,"sid":"recNNY0F5gd3Q1Agm","sm":"grok-code-fast-1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.258426966292},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"grok-code-fast-1","oid":"terminalbench-0e3480bc59674c23","rd":"2025-08-28","se":0.026,"sid":"recJFHV7PmdXOIqt5","sm":"grok-code-fast-1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.258},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"grok-4-0709","oid":"terminalbench-063135425af49572","rd":"2025-07-09","se":0.029,"sid":"rec2SbVuabwABv4Ag","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.254},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"qwen3-coder-480b-a35b-instruct","oid":"terminalbench-b56b630d7d08a4aa","rd":"2025-07-31","se":0.026,"sid":"rec6ayRIToMb9XXDx","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.254},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-07-09","dn":"Grok 4","m":"grok-4-0709","oid":"terminalbench-6da95f8b67b36859","rd":"2025-07-09","se":0.029216110513000004,"sid":"recBtnN0lWlUnjstK","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.25393258427},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-07-22","dn":"Qwen 3 Coder 480B","m":"qwen3-coder-480b-a35b-instruct","oid":"terminalbench-535b76b1a5987bbe","rd":"2025-07-31","se":0.025870546791,"sid":"recGuRUrIjf13P4BW","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.25393258427},{"b":"terminalbench","cfg":{"agent":"little-coder"},"d":"2026-04-16","dn":"Qwen3.6-35B-A3B","m":"qwen3-6-35b-a3b","oid":"terminalbench-fb1784bc86222e57","rd":"2026-04-14","se":0.03205049478,"sid":"recl1nPnCE0RX3UaB","sm":"qwen3.6-35b-a3b","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.24625468164800002},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-09-30","dn":"GLM 4.6","m":"glm-4-6","oid":"terminalbench-119c011336dae0dd","rd":"2025-09-30","se":0.024446156239,"sid":"rec4pCN6lbxLbFW1G","sm":"glm-4.6","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.24531835206},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-01","m":"glm-4-6","oid":"terminalbench-6df6e0a1dbaca340","rd":"2025-09-30","se":0.024,"sid":"recHvj6HO3PYp6H5Z","sm":"glm-4.6","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.245},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-07","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-486d5bd03361c682","rd":"2025-08-07","se":0.025301893904,"sid":"rec1GeHQE2AB6dCLS","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.24044943820200002},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"medium"},"d":"2025-10-31","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-b64bdeb86daefa4d","rd":"2025-08-07","se":0.025,"sid":"recmHdcWhKi0CzznQ","sm":"gpt-5-mini-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.24},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-01","m":"qwen3-coder-480b-a35b-instruct","oid":"terminalbench-025e487c0b3163f9","rd":"2025-07-31","se":0.028,"sid":"recjc6bHW1H4vwXQE","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.239},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-07-22","dn":"Qwen 3 Coder 480B","m":"qwen3-coder-480b-a35b-instruct","oid":"terminalbench-e52ad81f5b9b284e","rd":"2025-07-31","se":0.028010799023,"sid":"recjHgjD6DEjBnG1l","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.23895131086100002},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-07-09","dn":"Grok 4","m":"grok-4-0709","oid":"terminalbench-d17fbf3a628617f8","rd":"2025-07-09","se":0.029049635593,"sid":"recEfZ44svjk4Sx9h","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.231460674157},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"grok-4-0709","oid":"terminalbench-be2a880c80f8c422","rd":"2025-07-09","se":0.029,"sid":"rec8wkritwB2vOG3E","sm":"grok-4-0709","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.231},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-07","dn":"GPT-5-Mini","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-f15fb2ba06b5cf37","rd":"2025-08-07","se":0.026242804005000002,"sid":"rec27cVB8sQ24sFVx","sm":"gpt-5-mini-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.22247191011200002},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent","effort":"medium"},"d":"2025-11-03","e":"medium","m":"gpt-5-mini-2025-08-07","oid":"terminalbench-65e6900a630efe49","rd":"2025-08-07","se":0.026,"sid":"recnD7LCzSjD2bMOB","sm":"gpt-5-mini-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.222},{"b":"terminalbench","cfg":{"agent":"spoox-o-m"},"d":"2025-08-07","dn":"GPT-5-Nano","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-ce5e97b01e6b5e04","rd":"2025-08-07","se":0.028030030607,"sid":"recrRurMbK55MKkEl","sm":"gpt-5-nano-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.21797752808999998},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-11-04","m":"gemini-2-5-pro","oid":"terminalbench-4554b3a47fe7788b","rd":"2025-06-17","se":0.029,"sid":"rec263O7vZj3zvozA","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.196},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-06-17","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"terminalbench-f7768c6f29d81966","rd":"2025-06-17","se":0.028586831205,"sid":"recWtJcU6RNCsTkXu","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.195505617978},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-01","m":"gpt-oss-120b","oid":"terminalbench-e2b6af01f623ad2d","rd":"2025-08-05","se":0.027,"sid":"recQOII1RUaWlRx1a","sm":"gpt-oss-120b","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.187},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-05","dn":"GPT-OSS-120B","m":"gpt-oss-120b","oid":"terminalbench-604b2a80c61ae4f5","rd":"2025-08-05","se":0.026599726612,"sid":"recfG0w8YzGI925GU","sm":"gpt-oss-120b_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.18689138576800002},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"gemini-2-5-flash-preview-09-2025","oid":"terminalbench-64459c0261081efd","rd":"2025-09-25","se":0.025,"sid":"rec5CS4eSqb3DAWFd","sm":"gemini-2.5-flash-preview-09-2025","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.171},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-06-17","dn":"Gemini 2.5 Flash","m":"gemini-2-5-flash","oid":"terminalbench-0b601c274fbe3978","rd":"2025-06-17","se":0.024915582762000003,"sid":"rec0Cj5jRitxcYSEk","sm":"gemini-2.5-flash","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.170786516854},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"gemini-2-5-flash-preview-09-2025","oid":"terminalbench-0f9715d533b7e212","rd":"2025-09-25","se":0.024,"sid":"recPFqIRxpwoxFWNM","sm":"gemini-2.5-flash-preview-09-2025","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.169},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-06-17","dn":"Gemini 2.5 Flash","m":"gemini-2-5-flash","oid":"terminalbench-c338851b6008521a","rd":"2025-06-17","se":0.023718928139000003,"sid":"recDRkAqIJ73QPIlZ","sm":"gemini-2.5-flash","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.16853932584300002},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-06-17","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"terminalbench-2f57b8d1f35fd8ff","rd":"2025-06-17","se":0.027506058599999997,"sid":"recObzKzzxSqsVaZU","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.16404494381999998},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-06-17","dn":"Gemini 2.5 Flash","m":"gemini-2-5-flash","oid":"terminalbench-f831136faf888b8f","rd":"2025-06-17","se":0.023922527823,"sid":"recU2DOtVn4N0Dais","sm":"gemini-2.5-flash","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.16404494381999998},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"gemini-2-5-pro","oid":"terminalbench-e3b52bf969545793","rd":"2025-06-17","se":0.028,"sid":"recHlyAnoUE2kdSdW","sm":"gemini-2.5-pro","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.164},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"gemini-2-5-flash-preview-09-2025","oid":"terminalbench-19675a68730d65a9","rd":"2025-09-25","se":0.024,"sid":"recqa4Q4kJzmyqp3p","sm":"gemini-2.5-flash-preview-09-2025","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.164},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-06-17","dn":"Gemini 2.5 Flash","m":"gemini-2-5-flash","oid":"terminalbench-8dff78780fc31fe0","rd":"2025-06-17","se":0.023158525428000003,"sid":"reco0rRGUbz2Jmh98","sm":"gemini-2.5-flash","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.154494382022},{"b":"terminalbench","cfg":{"agent":"Gemini CLI"},"d":"2025-11-04","m":"gemini-2-5-flash-preview-09-2025","oid":"terminalbench-c9dffd2d5d4ed7b1","rd":"2025-09-25","se":0.023,"sid":"recr9kiUX6cDggDYx","sm":"gemini-2.5-flash-preview-09-2025","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.154},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-10-31","m":"grok-code-fast-1","oid":"terminalbench-52455da75bbb1841","rd":"2025-08-28","se":0.025,"sid":"rech8rVcJ0mZR84p6","sm":"grok-code-fast-1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.142},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"gpt-oss-120b","oid":"terminalbench-1603a5d1d75e23f7","rd":"2025-08-05","se":0.023,"sid":"reczhxfX0o9hQLBlA","sm":"gpt-oss-120b","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.142},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-05","dn":"GPT-OSS-120B","m":"gpt-oss-120b","oid":"terminalbench-1a3e7e7744340bb0","rd":"2025-08-05","se":0.022673522558000002,"sid":"rec7kV7Cz3bwsrDiB","sm":"gpt-oss-120b_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.141573033708},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-28","dn":"Grok Code Fast 1","m":"grok-code-fast-1","oid":"terminalbench-9221212ba789d8a8","rd":"2025-08-28","se":0.025492854303,"sid":"recBxL7VWpbeKKA30","sm":"grok-code-fast-1","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.141573033708},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-10-15","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-cce571b1a370ea52","rd":"2025-10-15","se":0.027329168928,"sid":"recjGQE5ZmhLlLmjS","sm":"claude-haiku-4-5-20251001_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.139325842697},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-11-02","m":"claude-haiku-4-5-2025-10-01","oid":"terminalbench-e25754efc2afbf3c","rd":"2025-10-15","se":0.027,"sid":"rec2PAXFlaUXgbf7W","sm":"claude-haiku-4-5-20251001","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.139},{"b":"terminalbench","cfg":{"agent":"Codex CLI","effort":"medium"},"d":"2025-11-04","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-9746461db0e9aacc","rd":"2025-08-07","se":0.023,"sid":"recPmu1Q5fQdfQ0Qg","sm":"gpt-5-nano-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.115},{"b":"terminalbench","cfg":{"agent":"Codex CLI"},"d":"2025-08-07","dn":"GPT-5-Nano","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-f3471de7d1947541","rd":"2025-08-07","se":0.022673522558000002,"sid":"rec1pKVvhlreqGD54","sm":"gpt-5-nano-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.11460674157299999},{"b":"terminalbench","cfg":{"agent":"OpenHands","effort":"medium"},"d":"2025-11-02","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-f95e86443532dcd4","rd":"2025-08-07","se":0.021,"sid":"reckUrafyTwxe61sP","sm":"gpt-5-nano-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.099},{"b":"terminalbench","cfg":{"agent":"OpenHands"},"d":"2025-08-07","dn":"GPT-5-Nano","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-7c77fb64a2dd8fde","rd":"2025-08-07","se":0.021123213002000003,"sid":"recSn7LNDEYNombkC","sm":"gpt-5-nano-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.098876404494},{"b":"terminalbench","cfg":{"agent":"little-coder"},"d":"2026-03-02","dn":"Qwen3.5-9B","m":"qwen3-5-9b","oid":"terminalbench-6f7a48abe1cfb6f3","rd":"2026-02-24","se":0.023513565589,"sid":"recn5iAj0XJsNdjeG","sm":"qwen3.5-9B","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.092134831461},{"b":"terminalbench","cfg":{"agent":"Terminus 2","effort":"medium"},"d":"2025-10-31","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-15b877a1504e48d1","rd":"2025-08-07","se":0.019,"sid":"rec1PtzzU1cAULZYv","sm":"gpt-5-nano-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.079},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-07","dn":"GPT-5-Nano","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-a375e3e0c29b7062","rd":"2025-08-07","se":0.018686687071,"sid":"recmaThICuBZ4f3Um","sm":"gpt-5-nano-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.078651685393},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent","effort":"medium"},"d":"2025-11-03","e":"medium","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-f2783149e486dd87","rd":"2025-08-07","se":0.019,"sid":"rec9SLPT2R9rByofO","sm":"gpt-5-nano-2025-08-07_medium","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.07},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-07","dn":"GPT-5-Nano","m":"gpt-5-nano-2025-08-07","oid":"terminalbench-ce15ec0414343093","rd":"2025-08-07","se":0.019198745909,"sid":"recuhoyHUTRuz9tSy","sm":"gpt-5-nano-2025-08-07_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.06966292134799999},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-11-03","m":"gpt-oss-20b","oid":"terminalbench-8fa6c14f6650b82e","rd":"2025-08-05","se":0.014,"sid":"recXJetcV0C5gLRl3","sm":"gpt-oss-20b","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.034},{"b":"terminalbench","cfg":{"agent":"Mini-SWE-Agent"},"d":"2025-08-05","dn":"GPT-OSS-20B","m":"gpt-oss-20b","oid":"terminalbench-1fe84dd43bb0cd29","rd":"2025-08-05","se":0.013575563422000001,"sid":"recpGfJBPRXbdXmbV","sm":"gpt-oss-20b_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.033707865169},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-11-01","m":"gpt-oss-20b","oid":"terminalbench-2fce92a6f7721e01","rd":"2025-08-05","se":0.015,"sid":"recaaiHI5eYtINDFb","sm":"gpt-oss-20b","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.031},{"b":"terminalbench","cfg":{"agent":"Terminus 2"},"d":"2025-08-05","dn":"GPT-OSS-20B","m":"gpt-oss-20b","oid":"terminalbench-5b40a181a7a1c092","rd":"2025-08-05","se":0.015186814718,"sid":"recOw2KPlGf4Z8s97","sm":"gpt-oss-20b_unknown","src":"https://www.tbench.ai/leaderboard/terminal-bench/2.0","v":0.030711610487},{"b":"the_agent_company","c":6.34,"dn":"OpenHands + Claude 3.5 Sonnet","m":"claude-3-5-sonnet-2024-10-22","metrics":{"steps":29.17},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team. Assuming Claude version based on dates.","oid":"the_agent_company-f5fc1d75a8813cb6","rd":"2024-10-22","sid":"recsHmIt77Rr4LD4D","sm":"claude-3-5-sonnet-20241022","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-sonnet-20241022","v":0.344},{"b":"the_agent_company","c":0.79,"dn":"Gemini 2.0 Flash","m":"gemini-2-0-flash-001","metrics":{"steps":39.85},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-4471593c4f024230","rd":"2025-02-05","sid":"recg0UBT3qOUyHeaR","sm":"gemini-2.0-flash-001","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-gemini-2.0-flash","v":0.19},{"b":"the_agent_company","c":1.29,"dn":"OpenHands + GPT 4o","m":"gpt-4o-2024-11-20","metrics":{"steps":14.55},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team. Assuming latest 4o version based on date.","oid":"the_agent_company-e1e561758e4499cb","rd":"2024-11-20","sid":"recyTMvFANtWFk7jZ","sm":"gpt-4o-2024-11-20","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-gpt-4o-2024-08-06","v":0.167},{"b":"the_agent_company","c":3.21,"dn":"OpenHands + Llama-3.1-405b","m":"llama-3-1-405b-instruct","metrics":{"steps":22.895},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-06d804a53f111b03","rd":"2024-07-23","sid":"recpu7npNorRgBdGT","sm":"Llama-3.1-405B-Instruct","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-llama-3.1-405b","v":0.141},{"b":"the_agent_company","c":0.93,"dn":"OpenHands + LLama-3.3-70b","m":"llama-3-1-70b-instruct","metrics":{"steps":20.93},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-1b6db3f8c98482f5","rd":"2024-07-23","sid":"recjM5TXNeULyzAM4","sm":"Llama-3.1-70B-Instruct","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-llama-3.3-70b","v":0.128},{"b":"the_agent_company","c":1.53,"dn":"OpenHands + Qwen-2.5-72b","m":"qwen2-5-72b-instruct","metrics":{"steps":23.99},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-073c8399aa0898ce","rd":"2024-09-19","sid":"recJN85mxDtJTm4xa","sm":"qwen2.5-72b-instruct","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-qwen2.5-72b","v":0.118},{"b":"the_agent_company","c":6.78,"dn":"OpenHands + Gemini 1.5 Pro","m":"gemini-1-5-pro-002","metrics":{"steps":22.1},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team. Assuming pro-002 version based on dates.","oid":"the_agent_company-4883b4662858eef4","rd":"2024-09-24","sid":"recSEDEJf5qf4uo36","sm":"gemini-1.5-pro-002","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-gemini-1.5-pro","v":0.08},{"b":"the_agent_company","c":0.83,"dn":"OpenHands + Llama-3.1-70b","m":"llama-3-1-70b-instruct","metrics":{"steps":19.18},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-b3de1a310b991b14","rd":"2024-07-23","sid":"rechUIH1qLtkmlKsq","sm":"Llama-3.1-70B-Instruct","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-llama-3.1-70b","v":0.065},{"b":"the_agent_company","c":1.55,"dn":"OpenHands + Amazon Nova Pro V1:0","m":"amazon-nova-pro-v10","metrics":{"steps":19.59},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-fcb7e54fdc7e7bd3","rd":"2024-12-03","sid":"recsCakwmRnBzeLtI","sm":"amazon.nova-pro-v1:0","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-nova-pro-v1%3A0","v":0.057},{"b":"the_agent_company","c":0.28,"dn":"OpenHands + Qwen-2-72b","m":"qwen2-72b-instruct","metrics":{"steps":23.7},"notes":"Submission has open-source code, submission patch generations reproduced by Agent Company team","oid":"the_agent_company-268c22c4ea23a1f6","rd":"2024-06-07","sid":"rec0ig6A3JLlKRUc5","sm":"qwen2-72b-instruct","src":"https://github.com/TheAgentCompany/experiments/tree/main/evaluation/1.0.0/20241217_OpenHands-0.14.2-qwen2-72b","v":0.042},{"b":"the_agent_company","dn":"MUSE + Gemini 2.5 Flash","m":"gemini-2-5-flash-preview-09-2025","notes":"Unclear exactly what version of Gemini 2.5 Flash used (defaulted to most recent before evaluation date)","oid":"the_agent_company-0bbea75b8669421c","rd":"2025-09-25","sid":"rec66F710Pnp62F6d","sm":"gemini-2.5-flash-preview-09-2025","src":"https://the-agent-company.com/#/leaderboard","v":0.518},{"b":"the_agent_company","c":0.4,"dn":"TTE-MatrixAgent + Deepseek-V3.2","m":"deepseek-v3-2-exp","metrics":{"steps":29.91},"notes":"Not confident on organization name.","oid":"the_agent_company-b6078a0fca13e0b2","rd":"2025-09-29","sid":"recAxkUUD4IoswYaG","sm":"DeepSeek-V3.2-Exp","src":"https://the-agent-company.com/#/leaderboard","v":0.524},{"b":"the_agent_company","c":1.63,"dn":"OpenHands-Versa + Claude Sonnet 4","m":"claude-sonnet-4-2025-05-14","metrics":{"steps":46.45},"oid":"the_agent_company-373964decb3d5853","rd":"2025-05-22","sid":"recs2VYZXpRvV4pYW","sm":"claude-sonnet-4-20250514","src":"https://the-agent-company.com/#/leaderboard","v":0.432},{"b":"the_agent_company","c":3.7,"dn":"OpenHands-Versa + Claude 3.7 Sonnet","m":"claude-3-7-sonnet-2025-02-19","metrics":{"steps":52.73},"oid":"the_agent_company-aa324bc43decc308","rd":"2025-02-24","sid":"recaCvCBsz9PBv8Rr","sm":"claude-3-7-sonnet-20250219","src":"https://the-agent-company.com/#/leaderboard","v":0.402},{"b":"the_agent_company","c":4.23,"dn":"OpenHands + Gemini 2.5 Pro","m":"gemini-2-5-pro-preview-05-06","metrics":{"steps":27.23},"notes":"Unclear exactly what version of Gemini 2.5 Pro used (defaulted to most recent before evaluation date)","oid":"the_agent_company-40135bd8a0424e8f","rd":"2025-05-06","sid":"recU1309NhQSjUsBh","sm":"gemini-2.5-pro-preview-05-06","src":"https://the-agent-company.com/#/leaderboard","v":0.393},{"b":"the_agent_company","c":4.05,"dn":"OpenHands + Claude 3.7 Sonnet","m":"claude-3-7-sonnet-2025-02-19","metrics":{"steps":27.78},"oid":"the_agent_company-15b68f3ad37181cb","rd":"2025-02-24","sid":"recI14oHMROiMrTxp","sm":"claude-3-7-sonnet-20250219","src":"https://the-agent-company.com/#/leaderboard","v":0.364},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Claude 2","m":"claude-2-0","oid":"trivia_qa-04560bdf904ce851","rd":"2023-07-11","sid":"recTmyjydTD3zEmrN","sm":"claude-2.0","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.875},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Claude instant","m":"claude-instant-1-1","oid":"trivia_qa-694b853f36c1e031","sid":"rec2o8utHtyekSr2n","sm":"claude-instant-1.1","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.789},{"b":"trivia_qa","dn":"GPT-4-0613","m":"gpt-4-0613","oid":"trivia_qa-ecea4116ff60e566","rd":"2023-06-13","sid":"recLawKW7ayTST6hX","sm":"gpt-4-0613","src":"https://paperswithcode.com/sota/question-answering-on-triviaqa","v":0.848},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Claude 1.3","m":"claude-1-3","oid":"trivia_qa-03c4305c86906c2c","rd":"2023-04-18","sid":"recqOhSu3mm8qtu8u","sm":"claude-1.3","src":"https://www-cdn.anthropic.com/5c49cc247484cecf107c699baf29250302e5da70/claude-2-model-card.pdf","v":0.867},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"PaLM 2-L","m":"palm-2-l","oid":"trivia_qa-69c223d009fa1414","rd":"2023-05-17","sid":"rec2eB4hK2TZtrSFg","sm":"PaLM 2-L","src":"https://arxiv.org/pdf/2305.10403","v":0.861},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"trivia_qa-d230d36645afb82b","rd":"2023-07-18","sid":"recSXmx6Jt3xz96O1","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2307.09288v2","v":0.85},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"trivia_qa-07261b11734f368b","rd":"2023-05-17","sid":"reczS1Jlz0DzFtnq1","sm":"PaLM 2-M","src":"https://arxiv.org/pdf/2305.10403","v":0.817},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"PaLM-540B","m":"palm-540b","oid":"trivia_qa-21df1f14246cefd9","rd":"2022-04-04","sid":"recjb3O3PZmDHTwPt","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.769},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"PaLM-540B","m":"palm-540b","oid":"trivia_qa-452e6b2c29b60bc0","rd":"2022-04-04","sid":"recrfwOWCS1C7TstM","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.814},{"b":"trivia_qa","cfg":{"shots":"few"},"dn":"PaLM-540B","m":"palm-540b","oid":"trivia_qa-11c9587f819ffe0c","rd":"2022-04-04","sid":"recQGR1k0JWfvm4YO","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.814},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"trivia_qa-0fbfa2bccdeff69f","rd":"2023-05-17","sid":"rec1H51pEEne0IdGs","sm":"PaLM 2-S","src":"https://arxiv.org/pdf/2305.10403","v":0.752},{"b":"trivia_qa","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-c3a0ffb75a491ad8","rd":"2022-01-27","sid":"rech95qTb390zRaA1","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.712},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-d2209b4c94345886","rd":"2022-01-27","sid":"rec80FIbMEIFtngNq","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.68},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-06d81cad6633b868","rd":"2022-01-27","sid":"recjbZ5iX41fMNKTS","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.643},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Gemma 2B","m":"gemma-2b","oid":"trivia_qa-21c23116ef00f602","rd":"2024-02-21","sid":"recLZAYwaYmir4fkG","sm":"gemma-2b","src":"https://arxiv.org/pdf/2403.08295","v":0.532},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Gemma 7B","m":"gemma-7b","oid":"trivia_qa-b645433f66a95279","rd":"2024-02-21","sid":"recUuVZqNjnF8nZ5p","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.634},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"trivia_qa-ac085eea4d72c411","rd":"2023-09-27","sid":"recWGZr0UHYNvgMbt","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2403.08295","v":0.625},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"trivia_qa-1ab4269194e210d0","rd":"2023-07-18","sid":"rec3x5YjE2OF68lfJ","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.721},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"trivia_qa-e78989049877e5a6","rd":"2023-07-18","sid":"recFncBMpsvN2mFPe","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2403.08295","v":0.796},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"trivia_qa-67796ddf65e91b09","rd":"2024-05-07","sid":"recvyWDwCUBlqF0P5","sm":"DeepSeek-V2","src":"https://arxiv.org/pdf/2412.19437","v":0.8},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"trivia_qa-97cc3a7156ec50f2","rd":"2024-09-19","sid":"rec53JZuaKLqzEwju","sm":"Qwen2.5-72B","src":"https://arxiv.org/pdf/2412.19437","v":0.719},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"trivia_qa-5702947160bb1c30","rd":"2024-12-26","sid":"rec9b6hIEDSUXwyOk","sm":"DeepSeek-V3","src":"https://arxiv.org/pdf/2412.19437","v":0.829},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"trivia_qa-ba7563f41bb8744a","rd":"2024-07-23","sid":"recNrDeTdobewAoHU","sm":"Llama-3.1-405B","src":"https://arxiv.org/pdf/2412.19437","v":0.827},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Phi-3-mini","m":"phi-3-mini-4k-instruct","oid":"trivia_qa-1d3c4cf6aac46a08","rd":"2024-04-23","sid":"recXplA8GrPMhfKW1","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.64},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Phi-3-small","m":"phi-3-small-8k-instruct","oid":"trivia_qa-1ca46c67356d82e7","rd":"2024-04-23","sid":"recXHoTSIBIjrb2hr","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.581},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Phi-3-medium","m":"phi-3-medium-128k-instruct","oid":"trivia_qa-a53b6f4fd284c3a3","rd":"2024-04-23","sid":"recLjsOhngDpVoxi4","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.739},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Phi-2","m":"phi-2","oid":"trivia_qa-252fdf856beba186","rd":"2023-12-12","sid":"recJZ6lcpMRWGlYfC","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.452},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"trivia_qa-cdab09f9c577e7ea","rd":"2023-09-27","sid":"reccS3oFHW9PBkukw","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.752},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Gemma 7b","m":"gemma-7b","oid":"trivia_qa-01dca6166fc15a1f","rd":"2024-02-21","sid":"recxcFhfalF9shRhu","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.723},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"trivia_qa-cda8c60cbbf194c1","rd":"2024-04-18","sid":"recdU5vKsuTRE1e60","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.677},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"trivia_qa-315edda358df3652","rd":"2023-12-11","sid":"recX29PfnrbG9x67s","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.822},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"trivia_qa-1b290cda93793b5f","rd":"2023-11-06","sid":"rec9zhI3QCvrHtHw2","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.858},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"trivia_qa-ce2363708e845bfa","rd":"2022-01-27","sid":"rec9QQGCXZgygnfBz","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.643},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"GLaM","m":"glam-moe","oid":"trivia_qa-42d44b1320640c47","rd":"2021-12-13","sid":"recpfngtEfxU1D6zc","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.713},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"trivia_qa-20ebbab1c6f6ff4a","rd":"2022-01-27","sid":"recEreiw1NzbvJYEK","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.68},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"GLaM","m":"glam-moe","oid":"trivia_qa-76b8c372418274eb","rd":"2021-12-13","sid":"rec5nxJnKiiqNIm4u","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.758},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-b49ad1a3e3b370ff","rd":"2022-01-27","sid":"recPrYWrsvsjKbhpk","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.712},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Gopher","m":"gopher-280b","oid":"trivia_qa-0bfa3f6cc006cffd","rd":"2021-12-08","sid":"recTPeBzo4moFIytC","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.06905","v":0.571},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"GLaM","m":"glam-moe","oid":"trivia_qa-4df638883782f759","rd":"2021-12-13","sid":"recMagk4MHW8hCgKS","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.758},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-059961eddba44256","rd":"2022-01-27","sid":"recTEzt7DQe7MTvSz","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.643},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-16aa30e81ffda722","rd":"2022-01-27","sid":"recbxIZpvmvJV4tOs","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.68},{"b":"trivia_qa","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"trivia_qa-ad9dfcbc8b824f11","rd":"2022-01-27","sid":"recNaxMPrhoe9tttx","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.712},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"MPT 7B","m":"mpt-7b","oid":"trivia_qa-7a8c6085052c3ad1","rd":"2023-05-05","sid":"recxHPOudfnEYsSKo","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.557},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"MPT 30B","m":"mpt-30b","oid":"trivia_qa-512879caf6b85021","rd":"2023-06-22","sid":"recZdKly7cWCpB6z4","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.68},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"Falcon 7B","m":"falcon-7b","oid":"trivia_qa-fb8d94c143c43c93","rd":"2023-04-24","sid":"recSAMMSyXiWsKmQM","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.526},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"Falcon 40B","m":"falcon-40b","oid":"trivia_qa-5d64f0a828ca1d55","rd":"2023-03-15","sid":"recSSBi5NOmlpF1Wf","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.746},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"trivia_qa-13532189ed5d688f","rd":"2023-02-24","sid":"recCLEbg8Fr5L4iUB","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.633},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"trivia_qa-732085e7e7ba37c9","rd":"2023-02-24","sid":"rec2WVnSxDCxf1TaT","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.701},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"trivia_qa-d9b4b7eb074a5c0f","rd":"2023-02-24","sid":"rectI7ZO1EbE1psOu","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.787},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"trivia_qa-2e9510f2e643e17a","rd":"2023-02-24","sid":"recvGyvO0ekg8Xnyf","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.817},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"trivia_qa-3daf14750a562415","rd":"2023-07-18","sid":"rec3QdcfMo7naAw5B","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.658},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"trivia_qa-6d33e37832d7a078","rd":"2023-07-18","sid":"recCxf9PmXq2B1Tgv","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.731},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"trivia_qa-21b549f758d9fccb","rd":"2023-07-18","sid":"recQWr69MKaXBNEOe","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.81},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"trivia_qa-2540f548279cd869","rd":"2023-07-18","sid":"recBs16PqLYX6M8Ct","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.824},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"MPT 7B","m":"mpt-7b","oid":"trivia_qa-f24a0b5066258fd4","rd":"2023-05-05","sid":"recF1NDi4Po3tFG0O","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.596},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"MPT 30B","m":"mpt-30b","oid":"trivia_qa-0e52b85ac23e30e0","rd":"2023-06-22","sid":"recJwCErRtLEN58c3","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.713},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"Falcon 7B","m":"falcon-7b","oid":"trivia_qa-5ef1f47d83cd779e","rd":"2023-04-24","sid":"recq3CUWtharnjJU3","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.568},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"Falcon 40B","m":"falcon-40b","oid":"trivia_qa-dcd44de4f072f7c4","rd":"2023-03-15","sid":"recMXULQHaMWEPQi9","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.786},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"trivia_qa-d65204a0924ab1c6","rd":"2023-02-24","sid":"rec8GQLFVuPKD6gyb","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.674},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"trivia_qa-14856eed30fcf7f3","rd":"2023-02-24","sid":"recRwFCEXvySQabBx","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.744},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"trivia_qa-58ecb2fbd8f70429","rd":"2023-02-24","sid":"rec8onltex0qQybgU","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.807},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"trivia_qa-5140a28140ddfc5f","rd":"2023-02-24","sid":"rec9BGXGx4cOBdEgJ","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.845},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"trivia_qa-7680d978c91aba16","rd":"2023-07-18","sid":"recCatGFAGRahbmdQ","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.689},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"trivia_qa-a0c72b68c2e25bb3","rd":"2023-07-18","sid":"recbu9TKWP3KR1Zlu","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.772},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"trivia_qa-ebaf0a540be0613f","rd":"2023-07-18","sid":"recDUk2ykCxMIEdvS","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.833},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"trivia_qa-95422f09356d0fb2","rd":"2023-07-18","sid":"recu7hSGTkBIUK6uz","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.85},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"MPT 7B","m":"mpt-7b","oid":"trivia_qa-d951347df44f1455","rd":"2023-05-05","sid":"recAg3ij56SMsYKCs","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.612},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"MPT 30B","m":"mpt-30b","oid":"trivia_qa-f2bf2243a5f7b960","rd":"2023-06-22","sid":"rec1R50MzTdbjFMas","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.733},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"trivia_qa-2ec0641c8ae4788b","rd":"2023-04-24","sid":"rec432VRSzNpUr2vd","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.646},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"trivia_qa-de4a2f9bf4383eaa","rd":"2023-03-15","sid":"recaRROdzmQnlf6Ri","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.799},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"trivia_qa-5a62d08c4ec6375a","rd":"2023-02-24","sid":"reclnkTiXtySsU5iU","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.704},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"trivia_qa-acfbf02d03d273e9","rd":"2023-02-24","sid":"recxTxb1Z8P7JOZv9","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.771},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"trivia_qa-3f9cc6067c3d1709","rd":"2023-02-24","sid":"rec4UT13sFHK39weH","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.838},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"trivia_qa-f3d2531972a56deb","rd":"2023-02-24","sid":"recpk7XDG6nKdNpIg","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.859},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"trivia_qa-29e8cb17086f5512","rd":"2023-07-18","sid":"recB9XYISuhZ0mvnt","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.721},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"trivia_qa-b3a61c057407cf15","rd":"2023-07-18","sid":"recMqqlRlGjkDv2qu","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.796},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"trivia_qa-6e72900c96329736","rd":"2023-07-18","sid":"recJdaeqGegzSAa4P","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.845},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"trivia_qa-7132374282f83f2c","rd":"2023-07-18","sid":"recaPm1UihmalxzSJ","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.876},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"MPT 7B","m":"mpt-7b","oid":"trivia_qa-39e4b4cd5d2a0383","rd":"2023-05-05","sid":"rechg030AkzmiWBU4","sm":"mpt-7b","src":"http://arxiv.org/abs/2307.09288","v":0.616},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"MPT 30B","m":"mpt-30b","oid":"trivia_qa-3453cba692d2b20b","rd":"2023-06-22","sid":"reccDYpXS53UyENQM","sm":"mpt-30b","src":"http://arxiv.org/abs/2307.09288","v":0.736},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Falcon 7B","m":"falcon-7b","oid":"trivia_qa-48e20e7029b28bfa","rd":"2023-04-24","sid":"recOKNmFGqwz432S7","sm":"falcon-7b","src":"http://arxiv.org/abs/2307.09288","v":0.611},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Falcon 40B","m":"falcon-40b","oid":"trivia_qa-1fbe211283c118cb","rd":"2023-03-15","sid":"recr4IlXlA3vTmsO5","sm":"falcon-40b","src":"http://arxiv.org/abs/2307.09288","v":0.796},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 1 7B","m":"llama-7b","oid":"trivia_qa-89d8efc8fa10dba0","rd":"2023-02-24","sid":"recm141PCpwLC6YtJ","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2307.09288","v":0.71},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 1 13B","m":"llama-13b","oid":"trivia_qa-f351aa0fe1678b87","rd":"2023-02-24","sid":"recb5fYdoFf6gvCeq","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2307.09288","v":0.779},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 1 33B","m":"llama-33b","oid":"trivia_qa-705d4f4dc5a63f76","rd":"2023-02-24","sid":"recbxBZXIp802OvtJ","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2307.09288","v":0.836},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 1 65B","m":"llama-65b","oid":"trivia_qa-88c718695de16096","rd":"2023-02-24","sid":"recWNmgYCvQokgf0f","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2307.09288","v":0.86},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 2 7B","m":"llama-2-7b","oid":"trivia_qa-6f4a178355d046d2","rd":"2023-07-18","sid":"recx2RC7OcrgdI4q1","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2307.09288","v":0.737},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 2 13B","m":"llama-2-13b","oid":"trivia_qa-fc28f8a61c866a9c","rd":"2023-07-18","sid":"recp8ob2tSNVHQ3xu","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2307.09288","v":0.794},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 2 34B","m":"llama-2-34b","oid":"trivia_qa-66aa166b9314914c","rd":"2023-07-18","sid":"rec7vgP607mIOWpG9","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2307.09288","v":0.846},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLAMA 2 70B","m":"llama-2-70b-hf","oid":"trivia_qa-941513c8cda8197b","rd":"2023-07-18","sid":"recrVtUWFAXNf8Px7","sm":"Llama-2-70b-hf","src":"http://arxiv.org/abs/2307.09288","v":0.875},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"Gopher 280B","m":"gopher-280b","oid":"trivia_qa-00b541cd9f71ef88","rd":"2021-12-08","sid":"rec7K8WKjBBVw4Nsp","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.435},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"trivia_qa-81e2ff25c2d9ca9d","rd":"2022-03-29","sid":"recFzzrC4aa3N4f2U","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.554},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"trivia_qa-fb1c436903e1f0c4","rd":"2023-02-24","sid":"rec8VC85NYcvUmEEb","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.5},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"trivia_qa-7fbaf7807e5341d0","rd":"2023-02-24","sid":"recaX2Uhmvyw9wvhN","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.566},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLaMA 33B","m":"llama-33b","oid":"trivia_qa-c3a500016c501fdb","rd":"2023-02-24","sid":"recpBHyQn5yoic5UD","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.651},{"b":"trivia_qa","cfg":{"shots":"0"},"dn":"LLaMA 65B","m":"llama-65b","oid":"trivia_qa-48d2f9d21fd4af36","rd":"2023-02-24","sid":"recH0bUglM0szsv3e","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.682},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLaMA 7B","m":"llama-7b","oid":"trivia_qa-3d50a77a5c70d301","rd":"2023-02-24","sid":"rec2LNih9QFLaecPj","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.534},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLaMA 13B","m":"llama-13b","oid":"trivia_qa-f66ca472c1dffa18","rd":"2023-02-24","sid":"rec5ASfpNuRqmCRrk","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.605},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLaMA 33B","m":"llama-33b","oid":"trivia_qa-bcd17b7d14bb300b","rd":"2023-02-24","sid":"recGlW4KYaVMMt6p6","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.679},{"b":"trivia_qa","cfg":{"shots":"1"},"dn":"LLaMA 65B","m":"llama-65b","oid":"trivia_qa-1487c725d16ca44d","rd":"2023-02-24","sid":"recys1yf6UaLOaiii","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.716},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Gopher 280B","m":"gopher-280b","oid":"trivia_qa-c8bbfdfc49a02e10","rd":"2021-12-08","sid":"recYzKbqStsGq5bEP","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.57},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"trivia_qa-d9e234d6aab1db5c","rd":"2022-03-29","sid":"recwZkD2OfndyATZ9","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.641},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA 7B","m":"llama-7b","oid":"trivia_qa-ba2e624ff33cc3e4","rd":"2023-02-24","sid":"rec5trLGrVUUU8h9s","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.563},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA 13B","m":"llama-13b","oid":"trivia_qa-8f00a95feb585859","rd":"2023-02-24","sid":"recJV7XYKa7kWSPU1","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.631},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA 33B","m":"llama-33b","oid":"trivia_qa-99a07421aff0bae5","rd":"2023-02-24","sid":"recwHiE8DvowuL4XI","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.699},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"LLaMA 65B","m":"llama-65b","oid":"trivia_qa-7dbd405a3dc5009e","rd":"2023-02-24","sid":"recglHj9DVjwTu3Zp","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.726},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Gopher 280B","m":"gopher-280b","oid":"trivia_qa-3b54195866f17ad8","rd":"2021-12-08","sid":"recHEkMoe2cRJB06H","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2302.13971","v":0.572},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"trivia_qa-da014128bb8fbc48","rd":"2022-03-29","sid":"recOh6y9PPVkfB63f","sm":"Chinchilla (70B)","src":"http://arxiv.org/abs/2302.13971","v":0.646},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLaMA 7B","m":"llama-7b","oid":"trivia_qa-b14b5b108533e8f0","rd":"2023-02-24","sid":"recGEjtYDWobf8nrj","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2302.13971","v":0.576},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLaMA 13B","m":"llama-13b","oid":"trivia_qa-47447059bd9f5665","rd":"2023-02-24","sid":"recJ8krqb6kapQNSk","sm":"LLaMA-13B","src":"http://arxiv.org/abs/2302.13971","v":0.64},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLaMA 33B","m":"llama-33b","oid":"trivia_qa-3b619a7f0d7d4cd7","rd":"2023-02-24","sid":"recnsnupizKu5Yg1D","sm":"LLaMA-33B","src":"http://arxiv.org/abs/2302.13971","v":0.704},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"LLaMA 65B","m":"llama-65b","oid":"trivia_qa-a8d53ac54592881f","rd":"2023-02-24","sid":"rec5qM6IHhVkLP5Mw","sm":"LLaMA-65B","src":"http://arxiv.org/abs/2302.13971","v":0.73},{"b":"trivia_qa","cfg":{"shots":"64"},"dn":"Gopher 280B","m":"gopher-280b","oid":"trivia_qa-9897fa67f161da13","rd":"2021-12-08","sid":"recEiFOxxUkVpTMNz","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.11446","v":0.571},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Claude Instant 1.1","m":"claude-instant-1-1","oid":"trivia_qa-5607b1b8afcb9dd9","sid":"recit7IX6ZlepTNtS","sm":"claude-instant-1.1","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.789},{"b":"trivia_qa","cfg":{"shots":"5"},"dn":"Claude Instant 1.2","m":"claude-instant-1-2","oid":"trivia_qa-75f83c22b71cce4a","rd":"2023-08-09","sid":"recEOw6eMJ9XwFZTY","sm":"claude-instant-1.2","src":"https://www.anthropic.com/news/releasing-claude-instant-1-2","v":0.787},{"b":"vending_bench_2","dn":"Gemini 4 Carbon","m":"gemini-4-carbon","oid":"vending_bench_2-065141ae931fb775","rd":"2026-10-28","sid":"recgYseBc2iH5xCmj","sm":"gemini-4-carbon_unknown","v":16484.20666666667},{"b":"vending_bench_2","dn":"GPT-6 Astra","m":"gpt-6-astra","oid":"vending_bench_2-14337fdb538412d8","rd":"2026-09-03","sid":"recIHrj7IZwngZzmG","sm":"gpt-6-astra_unknown","v":15514.701666666668},{"b":"vending_bench_2","dn":"GPT-6 Sol","m":"gpt-6-sol","oid":"vending_bench_2-24c739e01962887d","rd":"2026-09-22","sid":"recOsfANgIuve4p8J","sm":"gpt-6-sol_unknown","v":14427.848333333333},{"b":"vending_bench_2","dn":"Gemini 4 Argon","m":"gemini-4-argon","oid":"vending_bench_2-8fabb32986a605d5","rd":"2026-09-30","sid":"recnHKJey4BfNPh1f","sm":"gemini-4-argon_unknown","v":13718.158333333333},{"b":"vending_bench_2","dn":"Claude Opus 5","m":"claude-opus-5","oid":"vending_bench_2-30230b4f05adecea","rd":"2026-07-24","sid":"rec7OZVF0EN5vImL0","sm":"claude-opus-5_unknown","v":11181.87333333333},{"b":"vending_bench_2","dn":"Claude Sonnet 5.5","m":"claude-sonnet-5-5","oid":"vending_bench_2-8ffe3f738eb96ba1","rd":"2026-09-28","sid":"recuISzHM54q0s0VU","sm":"claude-sonnet-5-5_unknown","v":11165.27833333333},{"b":"vending_bench_2","dn":"Claude Opus 4.7","m":"claude-opus-4-7","oid":"vending_bench_2-3cedc9608b0af8dc","rd":"2026-04-16","sid":"rec63xBZ1e3RaKLVW","sm":"claude-opus-4-7_unknown","v":10936.763333333334},{"b":"vending_bench_2","dn":"Grok 4.7","m":"grok-4-7","oid":"vending_bench_2-c2d23acae05ca62b","rd":"2026-09-21","sid":"recqlIgpcwFxxpDZ8","sm":"grok-4.7_unknown","v":10536.831666666667},{"b":"vending_bench_2","dn":"GPT-5.6 Sol","m":"gpt-5-6-sol","oid":"vending_bench_2-889a90dacb2a9e83","rd":"2026-07-09","sid":"rec7Hn7Mhf7zcb5Ks","sm":"gpt-5.6-sol_unknown","v":9619.367999999999},{"b":"vending_bench_2","dn":"Muse Wonder","m":"muse-wonder-1-0","oid":"vending_bench_2-b6330c6a3a79533d","rd":"2026-10-12","sid":"recmNq9KpI3nIDTWU","sm":"muse-wonder-1.0_unknown","v":9432.18},{"b":"vending_bench_2","dn":"Claude Opus 5.5","m":"claude-opus-5-5","oid":"vending_bench_2-b60cc659dd82707d","rd":"2026-09-22","sid":"recBWEIN3JLagof2u","sm":"claude-opus-5-5_unknown","v":9235.248333333331},{"b":"vending_bench_2","dn":"GPT-6.1 Terra","m":"gpt-6-1-terra","oid":"vending_bench_2-429894abb9961c5a","rd":"2026-10-26","sid":"recjygL7zK33Q0cOC","sm":"gpt-6.1-terra_unknown","v":9107.458333333334},{"b":"vending_bench_2","dn":"Grok 4.6","m":"grok-4-6","oid":"vending_bench_2-7da4a04aab97ae6d","rd":"2026-08-12","sid":"recykaG8clpLwMQNZ","sm":"grok-4.6_unknown","v":9047.032500000001},{"b":"vending_bench_2","dn":"GLM-5.2","m":"glm-5-2","oid":"vending_bench_2-314525887fb3ebc4","rd":"2026-06-16","sid":"recFnUWa6JKxlpxJK","sm":"glm-5.2_unknown","v":8313.778333333335},{"b":"vending_bench_2","dn":"GLM-5.3","m":"glm-5-3","oid":"vending_bench_2-bf33a25e317abdbf","rd":"2026-08-14","sid":"recneOkHK8PAgzy1Y","sm":"glm-5.3_unknown","v":8163.609999999998},{"b":"vending_bench_2","dn":"Claude Opus 4.6","m":"claude-opus-4-6","oid":"vending_bench_2-e4d7e03fea618383","rd":"2026-02-05","sid":"recQ2Z14OWWnwOYgc","sm":"claude-opus-4-6_unknown","v":8017.594},{"b":"vending_bench_2","dn":"GPT-5.5","m":"gpt-5-5","oid":"vending_bench_2-f034bfcc929ebf81","rd":"2026-04-23","sid":"recg2odq5qkknQWX6","sm":"gpt-5.5_unknown","v":7523.843999999999},{"b":"vending_bench_2","dn":"GPT-5.6 Terra","m":"gpt-5-6-terra","oid":"vending_bench_2-13f5911ac79c33c6","rd":"2026-07-09","sid":"recLAjZ4HYB97nV26","sm":"gpt-5.6-terra_unknown","v":7343.206},{"b":"vending_bench_2","dn":"Claude Sonnet 4.6","m":"claude-sonnet-4-6","oid":"vending_bench_2-1eda9a1f971f6f1b","rd":"2026-02-17","sid":"rec1wYB2vM7FcJJFR","sm":"claude-sonnet-4-6_unknown","v":7204.139999999999},{"b":"vending_bench_2","dn":"Muse Spark 1.1","m":"muse-spark-1-1","oid":"vending_bench_2-7d2366cc30f97050","rd":"2026-07-09","sid":"recWZ6XgxFaZmibEx","sm":"muse-spark-1.1","v":6520.474999999999},{"b":"vending_bench_2","dn":"Claude Sonnet 5","m":"claude-sonnet-5","oid":"vending_bench_2-93e7916b96cb5102","rd":"2026-06-30","sid":"rec57E8P1Gu85Bppk","sm":"claude-sonnet-5_unknown","v":6377.701666666665},{"b":"vending_bench_2","dn":"Kimi K2.6","m":"kimi-k2-6","oid":"vending_bench_2-e6a3441ba472df0a","rd":"2026-04-20","sid":"recHNRpMfTLV74TOv","sm":"kimi-k2.6","v":6204.570000000001},{"b":"vending_bench_2","dn":"GPT-5.4","m":"gpt-5-4-2026-03-05","oid":"vending_bench_2-fb6ea1314fa695d6","rd":"2026-03-05","sid":"recJ1F0E7NEV8ifLR","sm":"gpt-5.4-2026-03-05_unknown","v":6144.176},{"b":"vending_bench_2","dn":"GPT-5.3-Codex","m":"gpt-5-3-codex","oid":"vending_bench_2-052a315c7cfe8de0","rd":"2026-02-05","sid":"recEwBQDbfSWm1MVs","sm":"gpt-5.3-codex","v":5940.124000000002},{"b":"vending_bench_2","dn":"Claude Opus 4.8 - High","m":"claude-opus-4-8","oid":"vending_bench_2-061c8d7a88d4a48c","rd":"2026-05-28","sid":"recCnok9JvFk9Yosw","sm":"claude-opus-4-8_unknown","v":5787.428},{"b":"vending_bench_2","cfg":{"effort":"high"},"dn":"Claude Fable 5 - High","e":"high","m":"claude-fable-5","oid":"vending_bench_2-e80f583bdeb9c9cc","rd":"2026-06-09","sid":"recpV3xliDr5Mm3RV","sm":"claude-fable-5_high","v":5680.258},{"b":"vending_bench_2","dn":"GLM-5.1","m":"glm-5-1","oid":"vending_bench_2-a8b78cdc48d52cd1","rd":"2026-04-07","sid":"reccQA2Me2YFH2uoo","sm":"glm-5.1","v":5634.406},{"b":"vending_bench_2","dn":"Gemini 3 Pro","m":"gemini-3-pro-preview","oid":"vending_bench_2-42a1a8e44a282ab6","rd":"2025-11-18","sid":"reck4G5LvBjIvc9sI","sm":"gemini-3-pro-preview","v":5478.158000000001},{"b":"vending_bench_2","dn":"Claude Fable 5.1","m":"claude-fable-5-1","oid":"vending_bench_2-a808408ad75f442d","rd":"2026-09-01","sid":"rec8LApQjhWHbouha","sm":"claude-fable-5-1_unknown","v":5421.556666666666},{"b":"vending_bench_2","dn":"Gemini 3.5 Flash","m":"gemini-3-5-flash","oid":"vending_bench_2-cc8d91ca4d837078","rd":"2026-05-19","sid":"rec5vYUb0zF2R4mUz","sm":"gemini-3.5-flash_unknown","v":5396.418333333334},{"b":"vending_bench_2","dn":"Kimi K3 (Moonshot)","m":"kimi-k3","oid":"vending_bench_2-42c9f9cda83b401f","rd":"2026-07-16","sid":"recYBYYTXhJssG2iO","sm":"kimi-k3_unknown","v":5165.035000000001},{"b":"vending_bench_2","dn":"Qwen 3.6 Plus","m":"qwen3-6-plus","oid":"vending_bench_2-0b84ad4a16d5e510","rd":"2026-03-31","sid":"recPQSBvNZDIIMmiC","sm":"qwen3.6-plus","v":5114.872000000001},{"b":"vending_bench_2","dn":"Gemini 3.8 Flash","m":"gemini-3-8-flash","oid":"vending_bench_2-45c86087cb6d5df8","rd":"2026-09-02","sid":"recMM8D3w2MMvKJ7p","sm":"gemini-3.8-flash_unknown","v":5093.793333333332},{"b":"vending_bench_2","dn":"Kimi K2.7 Code","m":"kimi-k2-7-code","oid":"vending_bench_2-53ebc5c31d07ada9","rd":"2026-06-12","sid":"rec3kjMcwIY0FFwTU","sm":"kimi-k2.7-code","v":5082.936},{"b":"vending_bench_2","cfg":{"effort":"low"},"dn":"Claude Fable 5 - Low","e":"low","m":"claude-fable-5","oid":"vending_bench_2-86053df2fa0a6319","rd":"2026-06-09","sid":"recafcEMcOdb67ldo","sm":"claude-fable-5_low","v":5018.519999999999},{"b":"vending_bench_2","dn":"Claude Opus 4.5","m":"claude-opus-4-5-2025-11-01","oid":"vending_bench_2-38a1d8c9607e07a2","rd":"2025-11-24","sid":"recxXzuUEqT3XSx5J","sm":"claude-opus-4-5-20251101_unknown","v":4967.056666666668},{"b":"vending_bench_2","cfg":{"effort":"max"},"dn":"Claude Fable 5 - Max","e":"max","m":"claude-fable-5","oid":"vending_bench_2-cc9aa65c8613aeb2","rd":"2026-06-09","sid":"recueSmV7urc8MR6w","sm":"claude-fable-5_max","v":4966.639999999998},{"b":"vending_bench_2","dn":"Grok 4.20","m":"grok-4-20","oid":"vending_bench_2-66fa4d581c5f627a","rd":"2026-02-17","sid":"recoe0ewL0dGWOxew","sm":"grok-4-20","v":4662.846666666667},{"b":"vending_bench_2","dn":"Claude Fable 5 - None","m":"claude-fable-5","oid":"vending_bench_2-b44f396993cdbaec","rd":"2026-06-09","sid":"recDIKtviJ5CZFZss","sm":"claude-fable-5","v":4529.944000000001},{"b":"vending_bench_2","dn":"GLM-5","m":"glm-5","oid":"vending_bench_2-ac5d0e97e9d11fed","rd":"2026-02-11","sid":"recN461q2BojFEYnc","sm":"glm-5","v":4432.122000000001},{"b":"vending_bench_2","cfg":{"effort":"medium"},"dn":"Claude Fable 5 - Medium","e":"medium","m":"claude-fable-5","oid":"vending_bench_2-1f5d3dfdf4c29be8","rd":"2026-06-09","sid":"reclKMbMHOf4EuCRo","sm":"claude-fable-5_medium","v":4339.812500000001},{"b":"vending_bench_2","cfg":{"effort":"max"},"dn":"Qwen 3.6 Max","e":"max","m":"qwen3-6-max-preview","oid":"vending_bench_2-bbe9b0eb16ae2b70","rd":"2026-04-20","sid":"recbsgfANkzVvQKHR","sm":"qwen3.6-max-preview","v":4254.1866666666665},{"b":"vending_bench_2","dn":"GPT-5.6 Luna","m":"gpt-5-6-luna","oid":"vending_bench_2-44efa0afed4df4b9","rd":"2026-07-09","sid":"recpCYTd0B6Cw2RF9","sm":"gpt-5.6-luna_unknown","v":4094.7120000000004},{"b":"vending_bench_2","dn":"Grok 4.5","m":"grok-4-5","oid":"vending_bench_2-5ce76180bb3f605d","rd":"2026-07-08","sid":"recQQQU8uQqzi2UwP","sm":"grok-4.5_unknown","v":3887.4259999999986},{"b":"vending_bench_2","dn":"Claude Sonnet 4.5","m":"claude-sonnet-4-5-2025-09-29","oid":"vending_bench_2-579b04e2f7cb09dc","rd":"2025-09-29","sid":"rechxcfDDQebfQxJh","sm":"claude-sonnet-4-5-20250929_unknown","v":3838.7400000000002},{"b":"vending_bench_2","dn":"Gemini 3.1 Pro Custom Tools","m":"gemini-3-1-pro-preview-customtools","oid":"vending_bench_2-98c227d06bf5fd4e","rd":"2026-02-19","sid":"recqpFFE7xXXe4Fs4","sm":"gemini-3.1-pro-preview-customtools","v":3774.254},{"b":"vending_bench_2","dn":"Gemini 3 Flash","m":"gemini-3-flash-preview","oid":"vending_bench_2-d05d81289daeb264","rd":"2025-12-17","sid":"recmbjpHUnRtEAE3N","sm":"gemini-3-flash-preview","v":3634.715999999999},{"b":"vending_bench_2","dn":"Claude Haiku 5.5","m":"claude-haiku-5-5","oid":"vending_bench_2-55db651a1487a982","rd":"2026-10-07","sid":"recsGf5Dytd9fKaBg","sm":"claude-haiku-5-5_unknown","v":3602.4633333333345},{"b":"vending_bench_2","dn":"GPT-5.2","m":"gpt-5-2-2025-12-11","oid":"vending_bench_2-27b93a753180dd43","rd":"2025-12-11","sid":"recmFLuBVs82r3Qti","sm":"gpt-5.2-2025-12-11_unknown","v":3591.3259999999996},{"b":"vending_bench_2","dn":"Deepseek V4 Pro","m":"deepseek-v4-pro","oid":"vending_bench_2-d0715845aa9ad77f","rd":"2026-04-24","sid":"recDV2KJb9lEWnMRy","sm":"deepseek-v4-pro_unknown","v":3284.519999999999},{"b":"vending_bench_2","cfg":{"effort":"max"},"dn":"Claude Opus 4.8 - Max","e":"max","m":"claude-opus-4-8","oid":"vending_bench_2-6c255741355bf483","rd":"2026-05-28","sid":"recfkcn8Cobr5tTyM","sm":"claude-opus-4-8_max","v":2992.344},{"b":"vending_bench_2","dn":"GLM-4.7","m":"glm-4-7","oid":"vending_bench_2-60f571f225575dae","rd":"2025-12-22","sid":"recI3WrdCbyuxFzet","sm":"glm-4.7","v":2376.816},{"b":"vending_bench_2","dn":"MiniMax-M3","m":"minimax-m3","oid":"vending_bench_2-7a83e1256a9004ba","rd":"2026-06-01","sid":"recLnH79xdVpRgeFP","sm":"MiniMax-M3","v":2157.7716666666665},{"b":"vending_bench_2","dn":"GPT-5.1","m":"gpt-5-1-2025-11-13","oid":"vending_bench_2-33a58e8c1719e193","rd":"2025-11-13","sid":"recagoHf0txa1WAaK","sm":"gpt-5.1-2025-11-13_unknown","v":1473.432},{"b":"vending_bench_2","dn":"Kimi K2.5","m":"kimi-k2-5","oid":"vending_bench_2-920e5e6737c1b8d2","rd":"2026-01-27","sid":"recgtFidMQMV7uDk8","sm":"kimi-k2.5","v":1198.4579999999999},{"b":"vending_bench_2","dn":"Grok 4.1 Fast","m":"grok-4-1-fast-reasoning","oid":"vending_bench_2-68e68dfae1622d46","rd":"2025-11-19","sid":"rec2lIHWKPMPdPuC8","sm":"grok-4-1-fast-reasoning","v":1106.6280000000002},{"b":"vending_bench_2","dn":"DeepSeek-V3.2","m":"deepseek-v3-2-exp","oid":"vending_bench_2-88466a6df16643d3","rd":"2025-09-29","sid":"recPR6i6sW30btXAG","sm":"DeepSeek-V3.2-Exp_unknown","v":1034.0040000000001},{"b":"vending_bench_2","dn":"Gemini 3.1 Pro","m":"gemini-3-1-pro-preview","oid":"vending_bench_2-89b4ca213eb93528","rd":"2026-02-19","sid":"rechXSwF5OJwdRGSH","sm":"gemini-3.1-pro-preview","v":911.2139999999997},{"b":"vending_bench_2","dn":"Gemini 2.5 Pro","m":"gemini-2-5-pro","oid":"vending_bench_2-585f23c89421dff2","rd":"2025-06-17","sid":"reckWpebEbdZnRciG","sm":"gemini-2.5-pro","v":573.6420000000003},{"b":"vending_bench_2","dn":"Gemini 2.5 Flash","m":"gemini-2-5-flash","oid":"vending_bench_2-f5a4a76af495f757","rd":"2025-06-17","sid":"recDbBoGij5RJMMUs","sm":"gemini-2.5-flash","v":548.836},{"b":"vending_bench_2","dn":"Qwen 3.5 35B A3B","m":"qwen3-5-flash","oid":"vending_bench_2-dada89f14e8871fb","rd":"2026-02-25","sid":"recS5F2c6cB3C72Fg","sm":"qwen3.5-flash","v":462.68500000000023},{"b":"vending_bench_2","dn":"Claude Haiku 4.5","m":"claude-haiku-4-5-2025-10-01","oid":"vending_bench_2-b1f5a3d2fd918e9f","rd":"2025-10-15","sid":"recJMmB8L2R2uCKOe","sm":"claude-haiku-4-5-20251001_unknown","v":458.88599999999985},{"b":"vending_bench_2","dn":"Qwen 3.5 27B","m":"qwen3-5-27b","oid":"vending_bench_2-0d31863b5c18317a","rd":"2026-02-24","sid":"recGMz3TCXWvSvJQ3","sm":"qwen3.5-27B","v":201.97599999999994},{"b":"vending_bench_2","dn":"MiniMax-M2","m":"minimax-m2","oid":"vending_bench_2-d13811eeb847eb71","rd":"2025-10-27","sid":"recpPBFKwqSc8Pg0L","sm":"MiniMax-M2","v":160.5959999999999},{"b":"vending_bench_2","cfg":{"effort":"max"},"dn":"Qwen3 Max","e":"max","m":"qwen3-max-2025-09-23","oid":"vending_bench_2-b623bc50e5094af1","rd":"2025-09-24","sid":"reccxfT5gQPzlmTgk","sm":"qwen3-max-2025-09-23","v":71.56500000000018},{"b":"vending_bench_2","dn":"Grok 4.3","m":"grok-4-3","oid":"vending_bench_2-3efdbc4b782774ec","rd":"2026-04-17","sid":"recWYJGQWpejO0gZW","sm":"grok-4.3_unknown","v":35.26166666666665},{"b":"vending_bench_2","dn":"Qwen 3.5 Plus","m":"qwen3-5-plus","oid":"vending_bench_2-b0a82aac66f61018","rd":"2026-02-16","sid":"recqI1CnWzPdpjA0C","sm":"qwen3.5-plus","v":0.5440000000000417},{"b":"vending_bench_2","cfg":{"effort":"thinking"},"dn":"Qwen3 235B A22B Thinking","e":"thinking","m":"qwen3-235b-a22b-thinking-2507","oid":"vending_bench_2-72d9ffee91b92796","rd":"2025-07-25","sid":"recnw8f4F4o6CYFuK","sm":"Qwen3-235B-A22B-Thinking-2507","v":-11.340000000000078},{"b":"vending_bench_2","dn":"GPT-OSS-120b","m":"gpt-oss-120b","oid":"vending_bench_2-a6d7f6e331f4c775","rd":"2025-08-05","sid":"recnj9dBUFHBxMRTC","sm":"gpt-oss-120b","v":-21.532000000000053},{"b":"vending_bench_2","dn":"MiniMax-M2.5","m":"minimax-m2-5","oid":"vending_bench_2-4a5ec14c017b8745","rd":"2026-02-12","sid":"recRGKGULrMrYYJr0","sm":"MiniMax-M2.5","v":-23.15999999999999},{"b":"vending_bench_2","dn":"GPT-5 mini","m":"gpt-5-mini-2025-08-07","oid":"vending_bench_2-0a3e60dda9b74cc6","rd":"2025-08-07","sid":"recrSY56pcQ9ZdRxY","sm":"gpt-5-mini-2025-08-07_unknown","v":-31.18399999999995},{"b":"video_mme","dn":"video-SALMONN 2+","m":"video-salmonn-2plus","oid":"video_mme-11100a83b78e0e36","rd":"2025-06-18","sid":"recMr6dqjQaA5cSL1","sm":"video-SALMONN-2plus","v":0.797},{"b":"video_mme","dn":"Gemini 1.5 Pro","m":"gemini-1-5-pro-001-feb24","oid":"video_mme-2137f4c483f26578","rd":"2024-02-15","sid":"rec5BPMcD8LfZ4vl5","sm":"gemini-1.5-pro-001-feb24","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.75},{"b":"video_mme","dn":"Gemini 1.5 Pro","m":"gemini-1-5-pro-001","oid":"video_mme-641e7962d7fe4866","rd":"2024-05-14","sid":"reck8Bdyl7VsIEVIf","sm":"gemini-1.5-pro-001","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.75},{"b":"video_mme","dn":"AdaReTaKe","m":"qwen2-5-vl-72b-instruct","notes":"This result is for the Qwen2.5-VL-72B-Instruct model augmented by the AdaReTaKe system.","oid":"video_mme-1d1031f1f3a9a748","rd":"2024-09-19","sid":"recyqau4pcAgM8IdV","sm":"Qwen2.5-VL-72B-Instruct","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.735},{"b":"video_mme","dn":"InternVL2.5","m":"internvl2-5-78b","oid":"video_mme-0b3f21d029956935","rd":"2024-12-06","sid":"recttiVwZcyyA09a1","sm":"InternVL2_5-78B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.721},{"b":"video_mme","dn":"GPT-4o","m":"gpt-4o-2024-11-20","oid":"video_mme-65667f94243ecf08","rd":"2024-11-20","sid":"recOD4UAd0CyEyws2","sm":"gpt-4o-2024-11-20","v":0.7190000000000001},{"b":"video_mme","dn":"GPT-4o","m":"gpt-4o-2024-08-06","oid":"video_mme-2f6c2c256c89bb01","rd":"2024-08-06","sid":"reciIWRDoigviHQq5","sm":"gpt-4o-2024-08-06","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.719},{"b":"video_mme","dn":"Qwen2-VL","m":"qwen2-vl-72b-instruct","oid":"video_mme-42d7768da5e8be1e","rd":"2024-08-29","sid":"recNTV6TuKBZxUPMJ","sm":"Qwen2-VL-72B-Instruct","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.712},{"b":"video_mme","dn":"LLaVA-Video","m":"llava-video-72b-qwen2","oid":"video_mme-67197652d724a0df","rd":"2024-09-02","sid":"rec33J119RJErottD","sm":"LLaVA-Video-72B-Qwen2","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.706},{"b":"video_mme","dn":"LinVT","m":"linvt","oid":"video_mme-4112df96b30992f8","rd":"2024-12-06","sid":"recN2BfY04EUYvH4S","sm":"LinVT","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.703},{"b":"video_mme","dn":"Gemini 1.5 Flash","m":"gemini-1-5-flash-001","oid":"video_mme-35e713203adae1b0","rd":"2024-05-23","sid":"receUlT4WLLzwVco3","sm":"gemini-1.5-flash-001","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.703},{"b":"video_mme","dn":"Aria","m":"aria","oid":"video_mme-58d530f71d0b422b","rd":"2024-09-30","sid":"rec2drA44hjKaGmOS","sm":"Aria","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.676},{"b":"video_mme","dn":"ViLAMP","m":"vilamp-llava-qwen","oid":"video_mme-e02f61b5761b6899","rd":"2025-05-01","sid":"recLjfSysntta3ETO","sm":"ViLAMP-llava-qwen","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.675},{"b":"video_mme","dn":"Oryx-1.5","m":"oryx-1-5-32b","oid":"video_mme-8a4dfa2a9c21b8ba","rd":"2024-10-22","sid":"recTT20ObJAdT0UBT","sm":"Oryx-1.5-32B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.673},{"b":"video_mme","dn":"LLaVA-OneVision","m":"llava-onevision-72b","oid":"video_mme-e4b3c454c2f10a37","rd":"2024-08-06","sid":"recuqUAsbZZ7qK9OD","sm":"LLaVA-OneVision 72B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.663},{"b":"video_mme","dn":"VideoLLaMA 3","m":"videollama3-7b","oid":"video_mme-5561309cdc8db4da","rd":"2024-01-22","sid":"recp8umU5rkUZgRrN","sm":"VideoLLAMA3-7B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.662},{"b":"video_mme","dn":"QuoTA","m":"llava-video-7b-qwen2","notes":"This result is for the LLaVA-Video-7B model augmented by the QuoTA pipeline.","oid":"video_mme-d05c46d0945347af","rd":"2024-09-02","sid":"recqH96EsjBT6A8oI","sm":"LLaVA-Video-7B-Qwen2","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.659},{"b":"video_mme","dn":"TPO","m":"llava-video-7b-qwen2-tpo","oid":"video_mme-0a9f0a177aae8aad","rd":"2025-01-19","sid":"recvTrkDVgOaC2rW0","sm":"LLaVA-Video-7B-Qwen2-TPO","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.656},{"b":"video_mme","dn":"VideoChat-Flash","m":"videochat-flash-qwen2-7b-res448","oid":"video_mme-4c5e812c67121f1e","rd":"2025-01-11","sid":"rec1d4OPDkM5E2gva","sm":"VideoChat-Flash-Qwen2-7B_res448","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.653},{"b":"video_mme","dn":"GPT-4o mini","m":"gpt-4o-mini-2024-07-18","oid":"video_mme-4d01656ff5b1688c","rd":"2024-07-18","sid":"recXpV4vq3BApmF47","sm":"gpt-4o-mini-2024-07-18","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.648},{"b":"video_mme","dn":"ByteVideoLLM","m":"bytevideollm-14b","oid":"video_mme-2c006af2795f26ec","rd":"2024-10-13","sid":"recMPYOeXatYatbmF","sm":"ByteVideoLLM-14B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.646},{"b":"video_mme","dn":"NVILA","m":"nvila-8b","oid":"video_mme-155efde2e7080c71","rd":"2024-12-10","sid":"recvDeYZzDKLYzJYX","sm":"NVILA-8B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.642},{"b":"video_mme","dn":"LiveCC","m":"livecc-7b-instruct","oid":"video_mme-43c0bc7ed38504ac","rd":"2024-04-12","sid":"recnjdgH8IWzUvB9e","sm":"LiveCC-7B-Instruct","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.641},{"b":"video_mme","dn":"ReTaKe","m":"qwen2-vl-7b-instruct","notes":"This result is for the qwen2_vl_instruct augmented with the ReTaKe method.","oid":"video_mme-f44e11f5648172fe","rd":"2024-08-29","sid":"recWRrzvp9jl8TpKx","sm":"Qwen2-VL-7B-Instruct","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.639},{"b":"video_mme","dn":"MiniCPM-o 2.6","m":"minicpm-o-2-6","oid":"video_mme-3bcc7947a33dce2f","rd":"2025-01-12","sid":"recxfLZgXLg3UfPxr","sm":"MiniCPM-o-2_6","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.639},{"b":"video_mme","dn":"VideoLLaMA 2","m":"videollama2-7b","oid":"video_mme-1e14291b382b2510","rd":"2024-06-12","sid":"reco3UQ0Kh0ZHWbUs","sm":"VideoLLAMA2-7B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.624},{"b":"video_mme","dn":"InternVL2","m":"internvl2-40b","oid":"video_mme-3a90d0c03860fe15","rd":"2024-07-08","sid":"recXqZ2P5EMCNApk5","sm":"InternVL2-40B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.612},{"b":"video_mme","dn":"MiniCPM-V 2.6","m":"minicpm-v-2-6","oid":"video_mme-a4f20abee6b03ae4","rd":"2024-08-03","sid":"recEF1QQG9rG72v8p","sm":"MiniCPM-V-2_6","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.609},{"b":"video_mme","dn":"Claude 3.5 Sonnet","m":"claude-3-5-sonnet-2024-06-20","oid":"video_mme-369efac9037f2680","rd":"2024-06-20","sid":"recjfbtBgqksInEwl","sm":"claude-3-5-sonnet-20240620","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.6},{"b":"video_mme","dn":"Claude 3.5 Sonnet","m":"claude-3-5-sonnet-2024-10-22","oid":"video_mme-9399d379bcdbb08c","rd":"2024-10-22","sid":"recgtjJXzBCTKdKog","sm":"claude-3-5-sonnet-20241022","v":0.6},{"b":"video_mme","dn":"GPT-4V","m":"gpt-4-1106-vision-preview","oid":"video_mme-162f334dea1ddcfe","rd":"2023-11-06","sid":"rec0rSn6Z7fJHJWM6","sm":"gpt-4-1106-vision-preview","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.599},{"b":"video_mme","dn":"mPLUG-Owl3","m":"mplug-owl3-7b-241101","oid":"video_mme-1c756f0c4a6b813c","rd":"2024-11-26","sid":"recLqfawDPsiPdaVY","sm":"mPLUG-Owl3-7B-241101","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.593},{"b":"video_mme","dn":"TimeMarker","m":"timemarker","oid":"video_mme-e5e2af7a70dda31c","rd":"2024-11-27","sid":"recd1Q3oRJQEjjQds","sm":"TimeMarker","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.573},{"b":"video_mme","dn":"VITA 1.5","m":"vita-1-5","oid":"video_mme-36d1a66702b2ae82","rd":"2024-12-20","sid":"recLvOBE3RjvQPOLA","sm":"VITA-1.5","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.561},{"b":"video_mme","dn":"Kangaroo","m":"kangaroo","oid":"video_mme-e5506844fb406e98","rd":"2024-11-13","sid":"recoBNEr7nHVeo2XN","sm":"kangaroo","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.56},{"b":"video_mme","dn":"VITA","m":"vita","oid":"video_mme-bec481c21b1e9207","rd":"2024-08-12","sid":"recMBoQfiQKjSw8Ha","sm":"VITA","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.558},{"b":"video_mme","dn":"Video-XL","m":"video-xl-7b","oid":"video_mme-79417b457e871d9e","rd":"2024-10-17","sid":"reczNMfNWxBOyIgHe","sm":"Video-XL-7B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.555},{"b":"video_mme","dn":"Video-CCAM","m":"video-ccam-7b-v1-2","oid":"video_mme-ed4c8387e57317ca","rd":"2024-09-29","sid":"recAiAIsYwfNhL0uc","sm":"Video-CCAM-7B-v1.2","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.532},{"b":"video_mme","dn":"Long-LLaVA","m":"long-llava-qwen2-7b","oid":"video_mme-ccca8eacf60a266a","rd":"2024-08-30","sid":"rec8UNeR00oNQulzZ","sm":"long-llava-qwen2-7b","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.529},{"b":"video_mme","dn":"LongVA","m":"longva-7b","oid":"video_mme-669dd2ddfedf03e1","rd":"2024-06-13","sid":"recXnvdHbEEIlEstD","sm":"LongVA-7B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.526},{"b":"video_mme","dn":"Qwen-VL-Max","m":"qwen-vl-max","oid":"video_mme-c9dc83a96c309f07","rd":"2024-01-18","sid":"recick6WPjNRBf1pn","sm":"Qwen-VL-Max","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.513},{"b":"video_mme","dn":"InternVL-Chat-V1.5","m":"internvl-chat-v1-5","oid":"video_mme-cd4ca9e625f21d09","rd":"2024-04-18","sid":"recZYnngnbStc73AO","sm":"InternVL-Chat-V1-5","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.507},{"b":"video_mme","dn":"SliME","m":"slime-llama3-8b","oid":"video_mme-eb8c4019a5650878","rd":"2024-06-02","sid":"recWVEWgg3xqBoD9K","sm":"SliME-Llama3-8B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.453},{"b":"video_mme","dn":"ShareGeminiXMU","m":"chat-uni-vi-7b-v1-5-100k-sg-wv","oid":"video_mme-4b7b4d37d4c91d42","rd":"2024-06-20","sid":"rec1HuNrKMcoSfrAv","sm":"Chat-Uni-Vi-7B-v1.5 + 100k SG-WV","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.432},{"b":"video_mme","dn":"Qwen-VL-Chat","m":"qwen-vl-chat","oid":"video_mme-f2c79e83120d91fa","rd":"2023-08-20","sid":"recVxdjuUpSEf1kyV","sm":"Qwen-VL-Chat","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.411},{"b":"video_mme","dn":"Chat-UniVi-v1.5","m":"chat-univi-7b-v1-5","oid":"video_mme-becdbb9ab8032d36","rd":"2024-04-23","sid":"reculoAu9RneDOhhM","sm":"Chat-UniVi-7B-v1.5","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.406},{"b":"video_mme","dn":"ShareGPT4Video","m":"sharegpt4video-8b","oid":"video_mme-cf5e76f49a3bfe9f","rd":"2024-05-27","sid":"recXDO9BjzFQSKlr7","sm":"sharegpt4video-8b","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.399},{"b":"video_mme","dn":"Video-LLaVA","m":"video-llava-7b","oid":"video_mme-d5f6b81be5da8007","rd":"2023-11-17","sid":"recmRFidEf4Es3i1U","sm":"Video-LLaVA-7B","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.399},{"b":"video_mme","dn":"VideoChat2-Mistral","m":"video-chat2-mistral","oid":"video_mme-4318e6828264d4c5","rd":"2023-11-29","sid":"recACe67sCHSfYlk1","sm":"video_chat2_mistral","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.395},{"b":"video_mme","dn":"ST-LLM","m":"st-llm","oid":"video_mme-3a6dec7efef3d907","rd":"2024-03-28","sid":"recXDGFkKjvk7Ky7X","sm":"ST-LLM","src":"https://video-mme.github.io/home_page.html#leaderboard","v":0.379},{"b":"vpct","m":"gemini-3-pro-preview","oid":"vpct-cafe95a7d211f59e","rd":"2025-11-18","sid":"recG05ZRFsIS6Np5A","sm":"gemini-3-pro-preview","src":"https://cbrower.dev/vpct","v":0.91},{"b":"vpct","cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"vpct-9789301064fbac4c","rd":"2025-12-11","sid":"rec44ii4M100mBK2S","sm":"gpt-5.2-2025-12-11_xhigh","src":"https://cbrower.dev/vpct","v":0.84},{"b":"vpct","m":"gemini-3-flash-preview","oid":"vpct-f7669f27962afd20","rd":"2025-12-17","sid":"recvXJM6wzywb9IOE","sm":"gemini-3-flash-preview","src":"https://cbrower.dev/vpct","v":0.726},{"b":"vpct","cfg":{"effort":"high"},"e":"high","m":"gpt-5-2-2025-12-11","oid":"vpct-edc9c926933a83b6","rd":"2025-12-11","sid":"reckX2AKbtEWNzbZK","sm":"gpt-5.2-2025-12-11_high","src":"https://cbrower.dev/vpct","v":0.67},{"b":"vpct","cfg":{"effort":"high"},"e":"high","m":"gpt-5-2025-08-07","oid":"vpct-a299ed0f087b99b8","rd":"2025-08-07","sid":"recr3Z2yR195jbjtt","sm":"gpt-5-2025-08-07_high","src":"https://cbrower.dev/vpct","v":0.66},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-2025-08-07","oid":"vpct-4e12c3eabc363453","rd":"2025-08-07","sid":"recsN4vpvg4hEFivY","sm":"gpt-5-2025-08-07_medium","src":"https://cbrower.dev/vpct","v":0.632},{"b":"vpct","cfg":{"effort":"high"},"e":"high","m":"gpt-5-1-2025-11-13","oid":"vpct-519b5670905549c6","rd":"2025-11-13","sid":"recozf8fQmQQsCQnw","sm":"gpt-5.1-2025-11-13_high","src":"https://cbrower.dev/vpct","v":0.587},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"o4-mini-2025-04-16","notes":"Assuming medium reasoning effort","oid":"vpct-8c81c346c7097b00","rd":"2025-04-16","sid":"recMIWcdIKL4MZxYy","sm":"o4-mini-2025-04-16_medium","src":"https://cbrower.dev/vpct","v":0.575},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-1-2025-11-13","oid":"vpct-ee7f63c04e24c59e","rd":"2025-11-13","sid":"reczaxzhyt2q07c7C","sm":"gpt-5.1-2025-11-13_medium","src":"https://cbrower.dev/vpct","v":0.533},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"o3-2025-04-16","notes":"Assuming medium reasoning effort","oid":"vpct-f2e226e6953fdd15","rd":"2025-04-16","sid":"recFneRecBUyPfN2q","sm":"o3-2025-04-16_medium","src":"https://cbrower.dev/vpct","v":0.52},{"b":"vpct","m":"gemini-2-5-pro-preview-03-25","oid":"vpct-f5395733d6eeab1f","rd":"2025-03-31","sid":"recbwV5ELYHiGvFL7","sm":"gemini-2.5-pro-preview-03-25","src":"https://cbrower.dev/vpct","v":0.48},{"b":"vpct","m":"gemini-2-5-pro-preview-06-05","oid":"vpct-5b8d608904d834b1","rd":"2025-06-05","sid":"rec52Pw8uNshiCO40","sm":"gemini-2.5-pro-preview-06-05","src":"https://cbrower.dev/vpct","v":0.464},{"b":"vpct","m":"gemini-2-5-flash-preview-09-2025","oid":"vpct-6c18032c2202fcef","rd":"2025-09-25","sid":"recUuPqxD6I72fBY3","sm":"gemini-2.5-flash-preview-09-2025","src":"https://cbrower.dev/vpct","v":0.462},{"b":"vpct","m":"gpt-4-5-preview-2025-02-27","oid":"vpct-8af82398a67ee14e","rd":"2025-02-27","sid":"recmQTMUVSEqkDA69","sm":"gpt-4.5-preview-2025-02-27","src":"https://cbrower.dev/vpct","v":0.45},{"b":"vpct","m":"gemini-robotics-er-1-5-preview","oid":"vpct-3c1b048ab7c0f325","rd":"2025-09-26","sid":"recjtzyakPMsEmkjI","sm":"gemini-robotics-er-1.5-preview","src":"https://cbrower.dev/vpct","v":0.408},{"b":"vpct","m":"gemini-2-5-pro-preview-05-06","oid":"vpct-f81003fa7cec8022","rd":"2025-05-06","sid":"recJ6k9ReFhyFGQtA","sm":"gemini-2.5-pro-preview-05-06","src":"https://cbrower.dev/vpct","v":0.405},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-mini-2025-08-07","oid":"vpct-5f0298b0bb608525","rd":"2025-08-07","sid":"rec6APYO4FYKyHDtb","sm":"gpt-5-mini-2025-08-07_medium","src":"https://cbrower.dev/vpct","v":0.402},{"b":"vpct","m":"gpt-4o-2024-11-20","oid":"vpct-ab492c02e2519438","rd":"2024-11-20","sid":"recM0nqnoo0kn90F5","sm":"gpt-4o-2024-11-20","src":"https://cbrower.dev/vpct","v":0.4},{"b":"vpct","m":"claude-opus-4-5-20251101-32k","notes":"Hasn't been added to leaderboard, but twitter thread confirms it.","oid":"vpct-a1f45959b9295602","rd":"2025-11-24","sid":"recx5AzuT2zNCCxlX","sm":"claude-opus-4-5-20251101_32K","src":"https://x.com/ChaseBrowe32432/status/1993038782975951149","v":0.4},{"b":"vpct","m":"claude-sonnet-4-5-20250929-32k","oid":"vpct-bd459394d6202c63","rd":"2025-09-29","sid":"recrRiFpB4Yu07ICW","sm":"claude-sonnet-4-5-20250929_32K","src":"https://cbrower.dev/vpct","v":0.398},{"b":"vpct","m":"claude-3-7-sonnet-2025-02-19","oid":"vpct-dc78c2486b062167","rd":"2025-02-24","sid":"recM5woYQkKuJ3OW3","sm":"claude-3-7-sonnet-20250219","src":"https://cbrower.dev/vpct","v":0.39},{"b":"vpct","cfg":{"effort":"high"},"e":"high","m":"gpt-5-mini-2025-08-07","oid":"vpct-c82c03f2992bf07b","rd":"2025-08-07","sid":"recV0qj04cOD1aAYJ","sm":"gpt-5-mini-2025-08-07_high","src":"https://cbrower.dev/vpct","v":0.39},{"b":"vpct","m":"gemini-2-5-flash-preview-04-17","oid":"vpct-56f8cd3044dd7cb0","rd":"2025-04-17","sid":"recDPItICe3yrLPoM","sm":"gemini-2.5-flash-preview-04-17","src":"https://cbrower.dev/vpct","v":0.38},{"b":"vpct","m":"claude-opus-4-20250514-16k","oid":"vpct-0c34351ad57acc23","rd":"2025-05-22","sid":"recR0EgymqTMRoVGM","sm":"claude-opus-4-20250514_16K","src":"https://cbrower.dev/vpct","v":0.38},{"b":"vpct","m":"claude-sonnet-4-5-2025-09-29","oid":"vpct-6f2f41e00dd654df","rd":"2025-09-29","sid":"recCCodoLB4uRw9uE","sm":"claude-sonnet-4-5-20250929","src":"https://cbrower.dev/vpct","v":0.38},{"b":"vpct","cfg":{"effort":"high"},"e":"high","m":"gpt-5-nano-2025-08-07","oid":"vpct-89cf1adc38fd7b93","rd":"2025-08-07","sid":"rec4oPG9VFZ9Hj1Nu","sm":"gpt-5-nano-2025-08-07_high","src":"https://cbrower.dev/vpct","v":0.372},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"o1-2024-12-17","notes":"Assuming medium reasoning effort for o1","oid":"vpct-43ef1fbaecdd8ded","rd":"2024-12-17","sid":"recKCsz6Np1hFa9Io","sm":"o1-2024-12-17_medium","src":"https://cbrower.dev/vpct","v":0.37},{"b":"vpct","cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-nano-2025-08-07","oid":"vpct-28f0191569fc14ac","rd":"2025-08-07","sid":"rec1Ak2IO7DaZW9Sm","sm":"gpt-5-nano-2025-08-07_medium","src":"https://cbrower.dev/vpct","v":0.354},{"b":"vpct","m":"claude-3-7-sonnet-20250219-64k","oid":"vpct-09cc9b803f58d7f6","rd":"2025-02-24","sid":"recoN8YNFbyWyRXtj","sm":"claude-3-7-sonnet-20250219_64K","src":"https://cbrower.dev/vpct","v":0.35},{"b":"vpct","m":"claude-opus-4-1-2025-08-05","oid":"vpct-16e01bd0b57b7129","rd":"2025-08-05","sid":"recW1ZN2O62BUbQkk","sm":"claude-opus-4-1-20250805","src":"https://cbrower.dev/vpct","v":0.35},{"b":"vpct","m":"gpt-4o-mini-2024-07-18","oid":"vpct-bef0dc52e4e643fb","rd":"2024-07-18","sid":"rec9Z8ze3bQlJ2HJ8","sm":"gpt-4o-mini-2024-07-18","src":"https://cbrower.dev/vpct","v":0.34},{"b":"vpct","m":"claude-sonnet-4-20250514-32k","oid":"vpct-f63dcca480fe4b12","rd":"2025-05-22","sid":"recw9Cr5pR8QbSQVR","sm":"claude-sonnet-4-20250514_32K","src":"https://cbrower.dev/vpct","v":0.34},{"b":"vpct","m":"claude-3-5-sonnet-2024-10-22","oid":"vpct-7ab814d078afb880","rd":"2024-10-22","sid":"recLSdu00YI6KGPj1","sm":"claude-3-5-sonnet-20241022","src":"https://cbrower.dev/vpct","v":0.33},{"b":"vpct","m":"claude-opus-4-2025-05-14","oid":"vpct-61931772b66658da","rd":"2025-05-22","sid":"recJ544ON0UcQQyf3","sm":"claude-opus-4-20250514","src":"https://cbrower.dev/vpct","v":0.33},{"b":"vpct","m":"claude-opus-4-1-20250805-16k","oid":"vpct-5995b83269c36a2d","rd":"2025-08-05","sid":"recK5mNmRO2BPjc77","sm":"claude-opus-4-1-20250805_16K","src":"https://cbrower.dev/vpct","v":0.33},{"b":"vpct","m":"claude-3-5-sonnet-2024-06-20","oid":"vpct-f32181d8c55e3290","rd":"2024-06-20","sid":"recYLAYmihUQYov44","sm":"claude-3-5-sonnet-20240620","src":"https://cbrower.dev/vpct","v":0.33},{"b":"vpct","m":"claude-sonnet-4-2025-05-14","oid":"vpct-44ea6b5f5df772f7","rd":"2025-05-22","sid":"recGWkeppRnxmGMKI","sm":"claude-sonnet-4-20250514","src":"https://cbrower.dev/vpct","v":0.3},{"b":"vpct","m":"gemini-2-5-flash-lite-preview-09-2025","oid":"vpct-33534e65a709d8a9","rd":"2025-09-25","sid":"recOdPVRW9wrLgR8D","sm":"gemini-2.5-flash-lite-preview-09-2025","src":"https://cbrower.dev/vpct","v":0.3},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1837.24,"lo":1803.38,"m":"claude-opus-5-5","metrics":{"votes":1887},"oid":"webdev_arena-26dc079dc3ecf169","rd":"2026-09-22","sid":"recmgLSDio5a8hfWC","sm":"claude-opus-5-5_max","src":"https://web.lmarena.ai/leaderboard","v":1820.31},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1816.76,"lo":1783.81,"m":"gpt-6-astra","metrics":{"votes":2281},"oid":"webdev_arena-2f83444497156b57","rd":"2026-09-03","sid":"rec5pxBoPZ09DjlAl","sm":"gpt-6-astra_max","src":"https://web.lmarena.ai/leaderboard","v":1800.28},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1804.45,"lo":1768.08,"m":"claude-sonnet-5-5","metrics":{"votes":1531},"oid":"webdev_arena-d4408a2261ef4237","rd":"2026-09-28","sid":"recbYf0qhP7q08OwS","sm":"claude-sonnet-5-5_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1786.26},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1777.82,"lo":1739.63,"m":"gpt-6-1-sol","metrics":{"votes":1264},"oid":"webdev_arena-6d1952cec27c5c8a","rd":"2026-09-29","sid":"recxV1YByh3vPMhLz","sm":"gpt-6.1-sol_max","src":"https://web.lmarena.ai/leaderboard","v":1758.72},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1771.65,"lo":1744.44,"m":"claude-fable-5-1","metrics":{"votes":3036},"oid":"webdev_arena-9c99d37f49b5785f","rd":"2026-09-01","sid":"recW7c6YS46DN6aYk","sm":"claude-fable-5-1_max","src":"https://web.lmarena.ai/leaderboard","v":1758.05},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1717.81,"lo":1679.82,"m":"claude-sonnet-5-5","metrics":{"votes":1205},"oid":"webdev_arena-905bbc9b8ac98c4a","rd":"2026-09-28","sid":"recToBiKpCTvPAk25","sm":"claude-sonnet-5-5_high","src":"https://web.lmarena.ai/leaderboard","v":1698.82},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1701.14,"lo":1677.53,"m":"gpt-6-sol","metrics":{"votes":3197},"oid":"webdev_arena-174de5f7b5e93b7b","rd":"2026-09-22","sid":"rec7T9l5XF7aKZFWM","sm":"gpt-6-sol_max","src":"https://web.lmarena.ai/leaderboard","v":1689.33},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1694.48,"lo":1679.51,"m":"claude-opus-5","metrics":{"votes":12087},"oid":"webdev_arena-014a126d0e9ea826","rd":"2026-07-24","sid":"recNUvyfQc6Bik8Uv","sm":"claude-opus-5_max","src":"https://web.lmarena.ai/leaderboard","v":1686.99},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1692.77,"lo":1666.24,"m":"gemini-4-argon","metrics":{"votes":2422},"oid":"webdev_arena-1bb9ac2375ca2f7b","rd":"2026-09-30","sid":"recwdOhldUOnpwUJK","sm":"gemini-4-argon_high","src":"https://web.lmarena.ai/leaderboard","v":1679.5},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1685.47,"lo":1663.05,"m":"kimi-k3","metrics":{"votes":4547},"oid":"webdev_arena-dbab4f98262659f0","rd":"2026-07-16","sid":"recDg44iIUtAe2RhI","sm":"kimi-k3_max","src":"https://web.lmarena.ai/leaderboard","v":1674.26},{"b":"webdev_arena","hi":1683.04,"lo":1658.11,"m":"qwen3-8-max","metrics":{"votes":3221},"oid":"webdev_arena-3530177cdda9357c","rd":"2026-08-02","sid":"recTdVm2qqlGWHUKV","sm":"qwen3.8-max_unknown","src":"https://web.lmarena.ai/leaderboard","v":1670.58},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1667.4,"lo":1653.5,"m":"claude-opus-5","metrics":{"votes":12566},"oid":"webdev_arena-c0ff7b07d33a7dc6","rd":"2026-07-24","sid":"recbSCzHPr3UAWM3V","sm":"claude-opus-5_high","src":"https://web.lmarena.ai/leaderboard","v":1660.45},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1664.07,"lo":1639.8,"m":"muse-spark-1-3","metrics":{"votes":2972},"oid":"webdev_arena-5562832b53d99b5d","rd":"2026-09-02","sid":"recLl8orHc32h5fTt","sm":"muse-spark-1.3_max","src":"https://web.lmarena.ai/leaderboard","v":1651.94},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1649.29,"lo":1625.76,"m":"grok-4-7","metrics":{"votes":3216},"oid":"webdev_arena-5f94f2ad05b5fd22","rd":"2026-09-21","sid":"recxFONeSIPSZlVez","sm":"grok-4.7_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1637.52},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1635.44,"lo":1620.53,"m":"claude-fable-5","metrics":{"votes":10081},"oid":"webdev_arena-52fc36956514c240","rd":"2026-06-09","sid":"recldtI0Pj1ckCP0y","sm":"claude-fable-5_high","src":"https://web.lmarena.ai/leaderboard","v":1627.98},{"b":"webdev_arena","hi":1634.46,"lo":1618.06,"m":"claude-fable-5","metrics":{"votes":8382},"oid":"webdev_arena-5873af7fd4dc933b","rd":"2026-06-09","sid":"recSi4QzdnavF3cTZ","sm":"claude-fable-5_unknown","src":"https://web.lmarena.ai/leaderboard","v":1626.26},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1637.13,"lo":1609.11,"m":"muse-spark-1-3","metrics":{"votes":2123},"oid":"webdev_arena-e687bac1cd00c11d","rd":"2026-09-02","sid":"recJf8XU8Ck0yrLX9","sm":"muse-spark-1.3_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1623.12},{"b":"webdev_arena","hi":1630.66,"lo":1606.17,"m":"mimo-v2-6-pro","metrics":{"votes":2734},"oid":"webdev_arena-77095dfa0023dcd3","rd":"2026-09-21","sid":"recTM8TiwWWDP1p7Z","sm":"mimo-v2.6-pro","src":"https://web.lmarena.ai/leaderboard","v":1618.41},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1627.79,"lo":1607.49,"m":"grok-4-6","metrics":{"votes":4282},"oid":"webdev_arena-229fdd377e4b5801","rd":"2026-08-12","sid":"recD2YH5MiuyOD1HA","sm":"grok-4.6_high","src":"https://web.lmarena.ai/leaderboard","v":1617.64},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1624.28,"lo":1610.36,"m":"gpt-5-6-sol","metrics":{"votes":11916},"oid":"webdev_arena-683940916e985a51","rd":"2026-07-09","sid":"recG5CUO9e2GOfiiz","sm":"gpt-5.6-sol_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1617.32},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1625.29,"lo":1603.61,"m":"glm-5-3","metrics":{"votes":3725},"oid":"webdev_arena-21de3c0c102ab0f8","rd":"2026-08-14","sid":"recGUmCEUF1DKCZhV","sm":"glm-5.3_max","src":"https://web.lmarena.ai/leaderboard","v":1614.45},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1631.08,"lo":1596.82,"m":"deepseek-v4-1-flash","metrics":{"votes":1361},"oid":"webdev_arena-27c5fbb3b83b23c3","rd":"2026-09-09","sid":"recUYd9mGJeTahnjd","sm":"deepseek-v4.1-flash_max","src":"https://web.lmarena.ai/leaderboard","v":1613.95},{"b":"webdev_arena","hi":1619.06,"lo":1595.13,"m":"glm-5-3-flash","metrics":{"votes":2925},"oid":"webdev_arena-4bc71d4b4b5498ab","rd":"2026-08-20","sid":"recnySn6GWU64r5vn","sm":"glm-5.3-flash_unknown","src":"https://web.lmarena.ai/leaderboard","v":1607.09},{"b":"webdev_arena","hi":1602.77,"lo":1583.81,"m":"qwen3-8-27b","metrics":{"votes":4913},"oid":"webdev_arena-f031a169c289a4f6","rd":"2026-08-14","sid":"rec67p0NIVtVh8Fw2","sm":"qwen3.8-27b","src":"https://web.lmarena.ai/leaderboard","v":1593.29},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1607.98,"lo":1578.51,"m":"glm-5-2","metrics":{"votes":1994},"oid":"webdev_arena-26f6e32bf3538c66","rd":"2026-06-16","sid":"recluUH4AYBO78ZCx","sm":"glm-5.2_max","src":"https://web.lmarena.ai/leaderboard","v":1593.25},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1599.36,"lo":1575.17,"m":"gemini-3-7-flash","metrics":{"votes":3007},"oid":"webdev_arena-3483b2295683262b","rd":"2026-08-13","sid":"recu8zmVvQ8mmZVig","sm":"gemini-3.7-flash_high","src":"https://web.lmarena.ai/leaderboard","v":1587.26},{"b":"webdev_arena","cfg":{"effort":"max"},"e":"max","hi":1590.96,"lo":1572.94,"m":"gpt-6-luna","metrics":{"votes":5577},"oid":"webdev_arena-74bdf85d8c529eee","rd":"2026-09-22","sid":"recRcRpkzCQ1tA6zz","sm":"gpt-6-luna_max","src":"https://web.lmarena.ai/leaderboard","v":1581.95},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1590.48,"lo":1570.59,"m":"deepseek-v4-pro-0813","metrics":{"votes":4258},"oid":"webdev_arena-691805cf6dfccb67","rd":"2026-08-13","sid":"rec65iBl9WzLAM7JB","sm":"deepseek-v4-pro-0813_high","src":"https://web.lmarena.ai/leaderboard","v":1580.53},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1589.75,"lo":1570.65,"m":"deepseek-v4-flash-0731","metrics":{"votes":4723},"oid":"webdev_arena-2bf9c8ee8d4f93ac","rd":"2026-07-31","sid":"recsLYLBxCXjXMVIU","sm":"deepseek-v4-flash-0731_high","src":"https://web.lmarena.ai/leaderboard","v":1580.2},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1582.93,"lo":1557.37,"m":"step-5-preview","metrics":{"votes":2506},"oid":"webdev_arena-a0b21cf334564723","rd":"2026-09-18","sid":"rec83p1OAiaoNleRJ","sm":"step-5-preview_high","src":"https://web.lmarena.ai/leaderboard","v":1570.15},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1580.25,"lo":1555.5,"m":"gemini-3-8-flash","metrics":{"votes":2732},"oid":"webdev_arena-b0b253168f131b09","rd":"2026-09-02","sid":"recHMg6xBTMJBE8o8","sm":"gemini-3.8-flash_high","src":"https://web.lmarena.ai/leaderboard","v":1567.88},{"b":"webdev_arena","hi":1576.37,"lo":1557.33,"m":"claude-opus-4-7","metrics":{"votes":5270},"oid":"webdev_arena-7a3f4750737e5bd4","rd":"2026-04-16","sid":"recokJu1lFXavDpwj","sm":"claude-opus-4-7_unknown","src":"https://web.lmarena.ai/leaderboard","v":1566.85},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1565.35,"lo":1552.31,"m":"claude-opus-4-8","metrics":{"votes":13787},"oid":"webdev_arena-9504c4b005898794","rd":"2026-05-28","sid":"recixLxQOXu4FgGzS","sm":"claude-opus-4-8_high","src":"https://web.lmarena.ai/leaderboard","v":1558.83},{"b":"webdev_arena","hi":1563.34,"lo":1551.4,"m":"claude-opus-4-7","metrics":{"votes":15914},"oid":"webdev_arena-6c7e014f5655e10f","rd":"2026-04-16","sid":"rec1WC4J4E7jHLpxy","sm":"claude-opus-4-7","src":"https://web.lmarena.ai/leaderboard","v":1557.37},{"b":"webdev_arena","hi":1570.13,"lo":1542.44,"m":"claude-opus-4-6","metrics":{"votes":2553},"oid":"webdev_arena-e4098bc6af21a959","rd":"2026-02-05","sid":"rec42aGstj0e6Rset","sm":"claude-opus-4-6_unknown","src":"https://web.lmarena.ai/leaderboard","v":1556.29},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1561.2,"lo":1549.23,"m":"claude-opus-4-7","metrics":{"votes":16387},"oid":"webdev_arena-f3b443fa60e8811a","rd":"2026-04-16","sid":"recMCbMMbkma3QX9K","sm":"claude-opus-4-7_high","src":"https://web.lmarena.ai/leaderboard","v":1555.22},{"b":"webdev_arena","hi":1562.67,"lo":1547.54,"m":"grok-4-5","metrics":{"votes":8133},"oid":"webdev_arena-a2278b147a5e5a0d","rd":"2026-07-08","sid":"rec0CzO93QZUWjPJ1","sm":"grok-4.5_unknown","src":"https://web.lmarena.ai/leaderboard","v":1555.1},{"b":"webdev_arena","hi":1568.57,"lo":1535.88,"m":"claude-opus-4-8","metrics":{"votes":1578},"oid":"webdev_arena-f248407679c323f3","rd":"2026-05-28","sid":"recOyBSiqPlt4m1jX","sm":"claude-opus-4-8_unknown","src":"https://web.lmarena.ai/leaderboard","v":1552.22},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1552.21,"lo":1541.03,"m":"claude-opus-4-6","metrics":{"votes":18332},"oid":"webdev_arena-749493d8b9622ad3","rd":"2026-02-05","sid":"recmfqfRYiNWDU56a","sm":"claude-opus-4-6_high","src":"https://web.lmarena.ai/leaderboard","v":1546.62},{"b":"webdev_arena","hi":1549.34,"lo":1533.75,"m":"muse-spark-1-1","metrics":{"votes":7549},"oid":"webdev_arena-60df5344038aee11","rd":"2026-07-09","sid":"rec7oAaI9pcEcm0Bz","sm":"muse-spark-1.1","src":"https://web.lmarena.ai/leaderboard","v":1541.55},{"b":"webdev_arena","cfg":{"effort":"none"},"e":"none","hi":1545.26,"lo":1532.36,"m":"claude-opus-4-8","metrics":{"votes":12654},"oid":"webdev_arena-28cf72d628f2819a","rd":"2026-05-28","sid":"rec8wkAhd66Iij2V5","sm":"claude-opus-4-8_none","src":"https://web.lmarena.ai/leaderboard","v":1538.81},{"b":"webdev_arena","hi":1542.59,"lo":1531.67,"m":"claude-opus-4-6","metrics":{"votes":19536},"oid":"webdev_arena-9236e43a3a1cdd9c","rd":"2026-02-05","sid":"recMRozRLKk7DABCI","sm":"claude-opus-4-6","src":"https://web.lmarena.ai/leaderboard","v":1537.13},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1544.62,"lo":1529.28,"m":"gemini-3-6-flash","metrics":{"votes":8259},"oid":"webdev_arena-51a8cad39671607c","rd":"2026-07-21","sid":"recsNwzTq8uZ9KnQl","sm":"gemini-3.6-flash_high","src":"https://web.lmarena.ai/leaderboard","v":1536.95},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1543.87,"lo":1529.57,"m":"claude-sonnet-5","metrics":{"votes":9296},"oid":"webdev_arena-9c64a418f66cdfd8","rd":"2026-06-30","sid":"recP9GzaDk8uq0RYS","sm":"claude-sonnet-5_high","src":"https://web.lmarena.ai/leaderboard","v":1536.72},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1548.32,"lo":1520.43,"m":"muse-spark-1-2","metrics":{"votes":2065},"oid":"webdev_arena-9dc01f143a1d314f","rd":"2026-08-05","sid":"reczZhOn2A3wQ2kJo","sm":"muse-spark-1.2_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1534.38},{"b":"webdev_arena","hi":1540.33,"lo":1515.33,"m":"gemini-3-6-flash","metrics":{"votes":2768},"oid":"webdev_arena-a7b562fb73d1440e","rd":"2026-07-21","sid":"recHyZi6cCqCKYq3D","sm":"gemini-3.6-flash_unknown","src":"https://web.lmarena.ai/leaderboard","v":1527.83},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1529,"lo":1513.49,"m":"gpt-5-6-terra","metrics":{"votes":7694},"oid":"webdev_arena-181f22e7732aac1d","rd":"2026-07-09","sid":"recOxZA3Y5CtVv4tH","sm":"gpt-5.6-terra_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1521.25},{"b":"webdev_arena","hi":1526.24,"lo":1515.73,"m":"claude-sonnet-4-6","metrics":{"votes":21642},"oid":"webdev_arena-253e6a5d04b6ebe3","rd":"2026-02-17","sid":"recEydvPq3RwtFg6j","sm":"claude-sonnet-4-6","src":"https://web.lmarena.ai/leaderboard","v":1520.99},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1526.48,"lo":1511.27,"m":"gpt-5-6-luna","metrics":{"votes":7828},"oid":"webdev_arena-e89fbfe9a2d1d547","rd":"2026-07-09","sid":"recOBEMJmw7esCPWt","sm":"gpt-5.6-luna_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1518.87},{"b":"webdev_arena","hi":1524.7,"lo":1509.54,"m":"qwen3-7-max","metrics":{"votes":8363},"oid":"webdev_arena-cc2db6f17980ceed","rd":"2026-05-19","sid":"recGyDjr4OoaFgvGL","sm":"qwen3.7-max","src":"https://web.lmarena.ai/leaderboard","v":1517.12},{"b":"webdev_arena","hi":1528.6,"lo":1496.47,"m":"muse-spark","metrics":{"votes":1608},"oid":"webdev_arena-79a03727af385c8b","rd":"2026-04-08","sid":"recRiDSRbOFjYObWz","sm":"muse-spark","src":"https://web.lmarena.ai/leaderboard","v":1512.53},{"b":"webdev_arena","cfg":{"effort":"xhigh"},"e":"xhigh","hi":1516.23,"lo":1504.06,"m":"gpt-5-5","metrics":{"votes":14946},"oid":"webdev_arena-1dec31e1661c179b","rd":"2026-04-23","sid":"recumoLWu0fLBcJe2","sm":"gpt-5.5_xhigh","src":"https://web.lmarena.ai/leaderboard","v":1510.15},{"b":"webdev_arena","hi":1516.18,"lo":1501.37,"m":"kimi-k2-6","metrics":{"votes":9585},"oid":"webdev_arena-92fffccb819ca1f7","rd":"2026-04-20","sid":"recLmGhmz9Qq5pJ29","sm":"kimi-k2.6","src":"https://web.lmarena.ai/leaderboard","v":1508.77},{"b":"webdev_arena","hi":1515.2,"lo":1501.38,"m":"glm-5-1","metrics":{"votes":10220},"oid":"webdev_arena-f97b3dfd5c138a1e","rd":"2026-04-07","sid":"reczQ82qDNkFeSOqt","sm":"glm-5.1","src":"https://web.lmarena.ai/leaderboard","v":1508.29},{"b":"webdev_arena","hi":1519.85,"lo":1492.91,"m":"gemini-3-5-flash","metrics":{"votes":2213},"oid":"webdev_arena-974779751bfa69db","rd":"2026-05-19","sid":"rec8SXNMSwKDPVpJj","sm":"gemini-3.5-flash_unknown","src":"https://web.lmarena.ai/leaderboard","v":1506.38},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1507.47,"lo":1492.6,"m":"gemini-3-5-flash","metrics":{"votes":8931},"oid":"webdev_arena-ac9e278aec93015b","rd":"2026-05-19","sid":"recBmp1LntSN6qKM0","sm":"gemini-3.5-flash_high","src":"https://web.lmarena.ai/leaderboard","v":1500.04},{"b":"webdev_arena","hi":1503.27,"lo":1486.52,"m":"claude-opus-4-5-20251101-32k","metrics":{"votes":10592},"oid":"webdev_arena-a0145e5e5fc8ea87","rd":"2025-11-24","sid":"recQLSl4DM4fz7VAt","sm":"claude-opus-4-5-20251101_32K","src":"https://web.lmarena.ai/leaderboard","v":1494.89},{"b":"webdev_arena","cfg":{"effort":"medium"},"e":"medium","hi":1498.41,"lo":1484.7,"m":"gemini-3-5-flash","metrics":{"votes":11412},"oid":"webdev_arena-1350c5effbdb1351","rd":"2026-05-19","sid":"recLqW9q7NaFnzS7x","sm":"gemini-3.5-flash_medium","src":"https://web.lmarena.ai/leaderboard","v":1491.55},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1493.01,"lo":1480.95,"m":"gpt-5-5","metrics":{"votes":17020},"oid":"webdev_arena-77476e5f61b26315","rd":"2026-04-23","sid":"recBkymoiC9MVMivU","sm":"gpt-5.5_high","src":"https://web.lmarena.ai/leaderboard","v":1486.98},{"b":"webdev_arena","hi":1493.15,"lo":1480.25,"m":"minimax-m3","metrics":{"votes":12686},"oid":"webdev_arena-2beb44dbca4c7d7d","rd":"2026-06-01","sid":"recHxyTnN62z1YBDE","sm":"MiniMax-M3","src":"https://web.lmarena.ai/leaderboard","v":1486.7},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1497,"lo":1463,"m":"gpt-5-2-2025-12-11","metrics":{"votes":1647},"oid":"webdev_arena-2b1d276985ec72d8","rd":"2025-12-11","sid":"recDCbjVONLP8GhN2","sm":"gpt-5.2-2025-12-11_high","src":"https://web.lmarena.ai/leaderboard","v":1480},{"b":"webdev_arena","hi":1491.82,"lo":1466.3,"m":"qwen3-6-max-preview","metrics":{"votes":2601},"oid":"webdev_arena-95dd53f256954928","rd":"2026-04-20","sid":"recYpwnD5SNup5J30","sm":"qwen3.6-max-preview","src":"https://web.lmarena.ai/leaderboard","v":1479.06},{"b":"webdev_arena","hi":1480.85,"lo":1469,"m":"mimo-v2-5-pro","metrics":{"votes":15843},"oid":"webdev_arena-1fdd8abe10ca6306","sid":"recW041lycMijl2lH","sm":"mimo-v2.5-pro","src":"https://web.lmarena.ai/leaderboard","v":1474.92},{"b":"webdev_arena","hi":1482.38,"lo":1463.06,"m":"kimi-k2-7-code","metrics":{"votes":4683},"oid":"webdev_arena-319b24fe003ce312","rd":"2026-06-12","sid":"recTViJ1YYcJ4mXVa","sm":"kimi-k2.7-code","src":"https://web.lmarena.ai/leaderboard","v":1472.72},{"b":"webdev_arena","hi":1475.46,"lo":1460.9,"m":"claude-opus-4-5-2025-11-01","metrics":{"votes":12899},"oid":"webdev_arena-1988b040db53c18b","rd":"2025-11-24","sid":"recxJC6KaFZO5Yo47","sm":"claude-opus-4-5-20251101","src":"https://web.lmarena.ai/leaderboard","v":1468.18},{"b":"webdev_arena","hi":1473.78,"lo":1453.48,"m":"deepseek-v4-pro","metrics":{"votes":3956},"oid":"webdev_arena-b2d40a3540b8f41d","rd":"2026-04-24","sid":"recyFGtGaq14s9gSg","sm":"deepseek-v4-pro_unknown","src":"https://web.lmarena.ai/leaderboard","v":1463.63},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1470.1,"lo":1457.12,"m":"deepseek-v4-pro","metrics":{"votes":12405},"oid":"webdev_arena-a3e60a01b944e12e","rd":"2026-04-24","sid":"reccESwrA2ewuVNOg","sm":"deepseek-v4-pro_high","src":"https://web.lmarena.ai/leaderboard","v":1463.61},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1482.43,"lo":1443.55,"m":"gpt-5-4-2026-03-05","metrics":{"votes":1277},"oid":"webdev_arena-914b25489a234c5a","rd":"2026-03-05","sid":"recVFgIXz9lscMYLN","sm":"gpt-5.4-2026-03-05_high","src":"https://web.lmarena.ai/leaderboard","v":1462.99},{"b":"webdev_arena","hi":1466.25,"lo":1454.81,"m":"qwen3-6-plus","metrics":{"votes":17562},"oid":"webdev_arena-48fdfa306f55dcf7","rd":"2026-03-31","sid":"recH4S2A84nuiTNtP","sm":"qwen3.6-plus","src":"https://web.lmarena.ai/leaderboard","v":1460.53},{"b":"webdev_arena","hi":1463.81,"lo":1451.72,"m":"gpt-5-5","metrics":{"votes":15012},"oid":"webdev_arena-3d7ae51e14eb7825","rd":"2026-04-23","sid":"recA3VwnfymOn47qK","sm":"gpt-5.5_unknown","src":"https://web.lmarena.ai/leaderboard","v":1457.76},{"b":"webdev_arena","hi":1451.8,"lo":1441.48,"m":"gemini-3-1-pro-preview","metrics":{"votes":22891},"oid":"webdev_arena-1b3960c2536fd95c","rd":"2026-02-19","sid":"reczlQxaml8buBx3t","sm":"gemini-3.1-pro-preview","src":"https://web.lmarena.ai/leaderboard","v":1446.64},{"b":"webdev_arena","hi":1489.11,"lo":1404.13,"m":"gemini-3-5-flash-lite","metrics":{"votes":228},"oid":"webdev_arena-514d9891f813837b","rd":"2026-07-21","sid":"recCFLA2191NzuA8o","sm":"gemini-3.5-flash-lite","src":"https://web.lmarena.ai/leaderboard","v":1446.62},{"b":"webdev_arena","cfg":{"effort":"none"},"e":"none","hi":1452.22,"lo":1439.22,"m":"deepseek-v4-pro","metrics":{"votes":13183},"oid":"webdev_arena-e08a2ae16ec6b2ad","rd":"2026-04-24","sid":"recXt285XtIYNHwks","sm":"deepseek-v4-pro_none","src":"https://web.lmarena.ai/leaderboard","v":1445.72},{"b":"webdev_arena","cfg":{"effort":"medium"},"e":"medium","hi":1461.33,"lo":1423.97,"m":"gpt-5-4-2026-03-05","metrics":{"votes":1316},"oid":"webdev_arena-6cb0461564f122c9","rd":"2026-03-05","sid":"recmdmHBSo9VpJQUs","sm":"gpt-5.4-2026-03-05_medium","src":"https://web.lmarena.ai/leaderboard","v":1442.65},{"b":"webdev_arena","hi":1447.32,"lo":1430.22,"m":"gemini-3-pro-preview","metrics":{"votes":13573},"oid":"webdev_arena-5a5a0b8499ff8a11","rd":"2025-11-18","sid":"rec7tzKLSS6JTykAP","sm":"gemini-3-pro-preview","src":"https://web.lmarena.ai/leaderboard","v":1438.77},{"b":"webdev_arena","hi":1443.32,"lo":1431,"m":"mimo-v2-5","metrics":{"votes":14117},"oid":"webdev_arena-e4737e8f27300c61","sid":"recXGMs0h0m6ZmhkY","sm":"mimo-v2.5","src":"https://web.lmarena.ai/leaderboard","v":1437.16},{"b":"webdev_arena","hi":1441.67,"lo":1431.07,"m":"kimi-k2-5","metrics":{"votes":19042},"oid":"webdev_arena-682f42f12bc3a69a","rd":"2026-01-27","sid":"reckf0B91qmnqnW8u","sm":"kimi-k2.5","src":"https://web.lmarena.ai/leaderboard","v":1436.37},{"b":"webdev_arena","hi":1444.25,"lo":1427.56,"m":"glm-5","metrics":{"votes":7056},"oid":"webdev_arena-7ce06053904832cd","rd":"2026-02-11","sid":"recjSg0H9Q5BhD7ez","sm":"glm-5","src":"https://web.lmarena.ai/leaderboard","v":1435.91},{"b":"webdev_arena","hi":1447.06,"lo":1422.34,"m":"glm-4-7","metrics":{"votes":3811},"oid":"webdev_arena-c8abac928d7d4590","rd":"2025-12-22","sid":"reczgCME2AOVMdLPt","sm":"glm-4.7","src":"https://lmarena.ai/leaderboard/webdev","v":1434.7},{"b":"webdev_arena","hi":1441.77,"lo":1425.21,"m":"mimo-v2-pro","metrics":{"votes":6963},"oid":"webdev_arena-8b2f6fe53d0c1609","sid":"rec76Jd5R8VLqCjHj","sm":"mimo-v2-pro","src":"https://web.lmarena.ai/leaderboard","v":1433.49},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1438.07,"lo":1423.79,"m":"deepseek-v4-flash","metrics":{"votes":9493},"oid":"webdev_arena-086bc70a09cf6ad3","rd":"2026-04-24","sid":"rec0leyLoZG7Y0EXR","sm":"deepseek-v4-flash_high","src":"https://web.lmarena.ai/leaderboard","v":1430.93},{"b":"webdev_arena","cfg":{"effort":"medium"},"e":"medium","hi":1435.27,"lo":1403.78,"m":"gpt-5-2025-08-07","metrics":{"votes":3032},"oid":"webdev_arena-cc42c3af31bbe845","rd":"2025-08-07","sid":"rec0jRmpNdEDdrg8X","sm":"gpt-5-2025-08-07_medium","src":"https://web.lmarena.ai/leaderboard","v":1419.52},{"b":"webdev_arena","hi":1439.52,"lo":1393.77,"m":"gpt-5-2-2025-12-11","metrics":{"votes":1055},"oid":"webdev_arena-c17171f593d38eb1","rd":"2025-12-11","sid":"rechsikgiOhrgerUx","sm":"gpt-5.2-2025-12-11_unknown","src":"https://web.lmarena.ai/leaderboard","v":1416.64},{"b":"webdev_arena","hi":1417.32,"lo":1402.65,"m":"inkling","metrics":{"votes":8970},"oid":"webdev_arena-7ae97d94ce68e65f","rd":"2026-07-15","sid":"recyrSEvcUeFlfbw4","sm":"Inkling","src":"https://web.lmarena.ai/leaderboard","v":1409.99},{"b":"webdev_arena","hi":1415.95,"lo":1397.21,"m":"inkling-small","metrics":{"votes":5058},"oid":"webdev_arena-45f098e5bb1a1b17","rd":"2026-07-15","sid":"rectYAkBI8QJesGbq","sm":"inkling-small","src":"https://web.lmarena.ai/leaderboard","v":1406.58},{"b":"webdev_arena","cfg":{"effort":"none"},"e":"none","hi":1417.93,"lo":1393.38,"m":"kimi-k2-5","metrics":{"votes":3095},"oid":"webdev_arena-65554f24fb136726","rd":"2026-01-27","sid":"recCxTl1iUkSuSlhf","sm":"kimi-k2.5_none","src":"https://web.lmarena.ai/leaderboard","v":1405.66},{"b":"webdev_arena","hi":1404.18,"lo":1393.66,"m":"qwen3-5-plus","metrics":{"votes":19922},"oid":"webdev_arena-710191d475c7abc6","rd":"2026-02-16","sid":"recHlzBI0TSSPPNFr","sm":"qwen3.5-plus","src":"https://web.lmarena.ai/leaderboard","v":1398.92},{"b":"webdev_arena","hi":1403.42,"lo":1391.62,"m":"minimax-m2-7","metrics":{"votes":15194},"oid":"webdev_arena-5b02649374804c4c","rd":"2026-03-18","sid":"recgWNjrRWlmCExNr","sm":"MiniMax-M2.7","src":"https://web.lmarena.ai/leaderboard","v":1397.52},{"b":"webdev_arena","cfg":{"effort":"high"},"e":"high","hi":1403.87,"lo":1390.5,"m":"gpt-5-4-mini-2026-03-17","metrics":{"votes":11279},"oid":"webdev_arena-19b31e55450ab300","rd":"2026-03-17","sid":"recILmM7G8gL0RC8d","sm":"gpt-5.4-mini-2026-03-17_high","src":"https://web.lmarena.ai/leaderboard","v":1397.18},{"b":"webdev_arena","cfg":{"effort":"medium"},"e":"medium","hi":1409,"lo":1379,"m":"gpt-5-2-2025-12-11","metrics":{"votes":1641},"oid":"webdev_arena-a2bac2044d91d904","rd":"2025-12-11","sid":"recSZOpvwfvrDdO4h","sm":"gpt-5.2-2025-12-11_medium","src":"https://web.lmarena.ai/leaderboard","v":1394},{"b":"webdev_arena","cfg":{"effort":"medium"},"e":"medium","hi":1403.69,"lo":1380.37,"m":"gpt-5-1-2025-11-13","metrics":{"votes":4751},"oid":"webdev_arena-1e2fa477ec551ff4","rd":"2025-11-13","sid":"recoxpFCpzEImK5i4","sm":"gpt-5.1-2025-11-13_medium","src":"https://web.lmarena.ai/leaderboard","v":1392.03},{"b":"webdev_arena","hi":1400,"lo":1382,"m":"claude-sonnet-4-5-20250929-32k","metrics":{"votes":8616},"oid":"webdev_arena-ffa34d2a011d02e4","rd":"2025-09-29","sid":"rectf5pzQGRAR1sZY","sm":"claude-sonnet-4-5-20250929_32K","src":"https://lmarena.ai/leaderboard/webdev","v":1391},{"b":"webdev_arena","hi":1399.83,"lo":1378.11,"m":"claude-opus-4-1-2025-08-05","metrics":{"votes":6769},"oid":"webdev_arena-5c6247c987305cd9","rd":"2025-08-05","sid":"recmHyx49gXYQP1ER","sm":"claude-opus-4-1-20250805_unknown","src":"https://web.lmarena.ai/leaderboard","v":1388.97},{"b":"webdev_arena","hi":1397.18,"lo":1377.78,"m":"minimax-m2-1","metrics":{"votes":7281},"oid":"webdev_arena-032e9eac7a8bd0b0","rd":"2025-12-23","sid":"recijJiojPfhRLBaH","sm":"MiniMax-M2.1","src":"https://web.lmarena.ai/leaderboard","v":1387.48},{"b":"webdev_arena","hi":1399.53,"lo":1373.94,"m":"gpt-5-4-2026-03-05","metrics":{"votes":2493},"oid":"webdev_arena-0e1eed60d17cf76f","rd":"2026-03-05","sid":"reclcQbSEOI2eiySg","sm":"gpt-5.4-2026-03-05_unknown","src":"https://web.lmarena.ai/leaderboard","v":1386.74},{"b":"webdev_arena","hi":1395,"lo":1377,"m":"claude-opus-4-1-2025-08-05","metrics":{"votes":8210},"oid":"webdev_arena-676582443ea87829","rd":"2025-08-05","sid":"recDMjkTx9oZNAOzz","sm":"claude-opus-4-1-20250805","src":"https://web.lmarena.ai/leaderboard","v":1386},{"b":"webdev_arena","hi":1392.86,"lo":1378.13,"m":"claude-sonnet-4-5-2025-09-29","metrics":{"votes":15150},"oid":"webdev_arena-77fbdcac9425054d","rd":"2025-09-29","sid":"recUrHg93mpJgakI0","sm":"claude-sonnet-4-5-20250929","src":"https://web.lmarena.ai/leaderboard","v":1385.49},{"b":"webdev_arena","hi":1392.61,"lo":1375.65,"m":"minimax-m2-5","metrics":{"votes":7186},"oid":"webdev_arena-395ba903f1f03b8a","rd":"2026-02-12","sid":"reckw2GD2bAw3IoI1","sm":"MiniMax-M2.5","src":"https://web.lmarena.ai/leaderboard","v":1384.13},{"b":"webdev_arena","hi":1388.45,"lo":1377.79,"m":"gemini-3-flash-preview","metrics":{"votes":20911},"oid":"webdev_arena-7370ca5bba7f3b0a","rd":"2025-12-17","sid":"rec8wVMb44iQySeae","sm":"gemini-3-flash-preview","src":"https://web.lmarena.ai/leaderboard","v":1383.12},{"b":"webdev_arena","hi":1379.98,"lo":1367.53,"m":"grok-4-20","metrics":{"votes":14571},"oid":"webdev_arena-3fcaf7f3f29bec5a","rd":"2026-02-17","sid":"recJM2PnLkEFRQj7g","sm":"grok-4-20","src":"https://web.lmarena.ai/leaderboard","v":1373.76},{"b":"webdev_arena","hi":1381.39,"lo":1358.34,"m":"gpt-5-3-codex","metrics":{"votes":3491},"oid":"webdev_arena-7d5dc8ea9381c16d","rd":"2026-02-05","sid":"recWzul4ypZ1wsTQr","sm":"gpt-5.3-codex","src":"https://web.lmarena.ai/leaderboard","v":1369.86},{"b":"webdev_arena","cfg":{"effort":"thinking"},"e":"thinking","hi":1387,"lo":1348,"m":"deepseek-v3-2-exp","metrics":{"votes":955},"oid":"webdev_arena-d810c74a6d8a2935","rd":"2025-09-29","sid":"reciI0uZdChhyGrUw","sm":"DeepSeek-V3.2-Exp_thinking","src":"https://web.lmarena.ai/leaderboard","v":1367},{"b":"webdev_arena","hi":1371.18,"lo":1357.08,"m":"gemma-4-31b-it","metrics":{"votes":10139},"oid":"webdev_arena-41315611db8b8aa4","rd":"2026-04-02","sid":"recHaglQ3fCswDCo6","sm":"gemma-4-31b-it","src":"https://web.lmarena.ai/leaderboard","v":1364.13},{"b":"webdev_arena","hi":1378.63,"lo":1344.16,"m":"gemma-4-26b-a4b","metrics":{"votes":1442},"oid":"webdev_arena-96efc6e2046b1f6c","rd":"2026-04-02","sid":"recMiFTjUcXp6kvjc","sm":"gemma-4-26b-a4b","src":"https://web.lmarena.ai/leaderboard","v":1361.4},{"b":"webdev_arena","cfg":{"effort":"low"},"e":"low","hi":1369,"lo":1349,"m":"gpt-5-1-2025-11-13","metrics":{"votes":6540},"oid":"webdev_arena-c17714a095597bdb","rd":"2025-11-13","sid":"recGfqSb7DIX51ffo","sm":"gpt-5.1-2025-11-13_low","src":"https://web.lmarena.ai/leaderboard","v":1359},{"b":"webdev_arena","hi":1365.55,"lo":1349.2,"m":"qwen3-5-27b","metrics":{"votes":7427},"oid":"webdev_arena-3cddd767854ee6e0","rd":"2026-02-24","sid":"rec9UAZ0xg5glh6F9","sm":"qwen3.5-27B","src":"https://web.lmarena.ai/leaderboard","v":1357.38},{"b":"webdev_arena","hi":1363.56,"lo":1350.78,"m":"grok-4-3","metrics":{"votes":14192},"oid":"webdev_arena-67f4a8f7efdc7ef6","rd":"2026-04-17","sid":"rectm2PjgBLYazB5n","sm":"grok-4.3_unknown","src":"https://web.lmarena.ai/leaderboard","v":1357.17},{"b":"webdev_arena","hi":1362,"lo":1344,"m":"claude-sonnet-4-5-20250929-32k","metrics":{"votes":7544},"oid":"webdev_arena-8e4cfa682ce07521","rd":"2025-09-29","sid":"recKmKjyv8AiR9k1T","sm":"claude-sonnet-4-5-20250929_32K","src":"https://web.lmarena.ai/leaderboard","v":1353},{"b":"webdev_arena","hi":1357.05,"lo":1337.92,"m":"laguna-m-1","metrics":{"votes":5062},"oid":"webdev_arena-e4845ee65855ab6e","rd":"2026-04-28","sid":"recInkSCJqwnymPR8","sm":"laguna-m.1","src":"https://web.lmarena.ai/leaderboard","v":1347.49},{"b":"webdev_arena","hi":1349.46,"lo":1332.34,"m":"gpt-5-1-2025-11-13","metrics":{"votes":10326},"oid":"webdev_arena-b4f94c0ac6d02033","rd":"2025-11-13","sid":"recqQAnpZjzh0yuYv","sm":"gpt-5.1-2025-11-13_unknown","src":"https://web.lmarena.ai/leaderboard","v":1340.9},{"b":"webdev_arena","hi":1351.49,"lo":1329.71,"m":"glm-4-6","metrics":{"votes":6600},"oid":"webdev_arena-7f043e2637b7ce1b","rd":"2025-09-30","sid":"recT1lwdoUHQkrBMV","sm":"glm-4.6","src":"https://web.lmarena.ai/leaderboard","v":1340.6},{"b":"webdev_arena","hi":1347.73,"lo":1329.67,"m":"gpt-5-2-codex","metrics":{"votes":6402},"oid":"webdev_arena-a8a44dd321124965","rd":"2025-12-18","sid":"rec9X8U1ig36tPXhy","sm":"gpt-5.2-codex","src":"https://web.lmarena.ai/leaderboard","v":1338.7},{"b":"webdev_arena","hi":1346,"lo":1328,"m":"kimi-k2-thinking-turbo","metrics":{"votes":7336},"oid":"webdev_arena-4baf91e9ef812918","rd":"2025-11-06","sid":"recumTXyHTALFh1iH","sm":"kimi-k2-thinking-turbo","src":"https://web.lmarena.ai/leaderboard","v":1337},{"b":"webdev_arena","hi":1348.01,"lo":1324.55,"m":"gpt-5-1-codex","metrics":{"votes":4861},"oid":"webdev_arena-e91c18a6b8454b9b","rd":"2025-11-12","sid":"recSXM0s3Vsny5oZP","sm":"gpt-5.1-codex","src":"https://web.lmarena.ai/leaderboard","v":1336.28},{"b":"webdev_arena","hi":1340.6,"lo":1320.68,"m":"mimo-v2-flash","metrics":{"votes":5401},"oid":"webdev_arena-d2b73019544cb654","sid":"recz0HuV9vzoIpgOi","sm":"mimo-v2-flash","src":"https://web.lmarena.ai/leaderboard","v":1330.64},{"b":"webdev_arena","hi":1334.29,"lo":1324.33,"m":"claude-haiku-4-5-2025-10-01","metrics":{"votes":27785},"oid":"webdev_arena-f3d80f96770cb82f","rd":"2025-10-15","sid":"recCPUmkyIVkx6rmv","sm":"claude-haiku-4-5-20251001_unknown","src":"https://web.lmarena.ai/leaderboard","v":1329.31},{"b":"webdev_arena","hi":1332.76,"lo":1316.57,"m":"deepseekdeepseek-v3-2","metrics":{"votes":8966},"oid":"webdev_arena-cba179d207f90fab","rd":"2025-12-01","sid":"recBICRjNsv9JEKQt","sm":"deepseek/deepseek-v3.2","src":"https://web.lmarena.ai/leaderboard","v":1324.67},{"b":"webdev_arena","hi":1313.51,"lo":1291.3,"m":"laguna-xs-2","metrics":{"votes":3934},"oid":"webdev_arena-200155cb9418a5bb","rd":"2026-04-28","sid":"recSfF9Gf6DhWdjTB","sm":"laguna-xs.2","src":"https://web.lmarena.ai/leaderboard","v":1302.4},{"b":"webdev_arena","hi":1308.83,"lo":1286.73,"m":"minimax-m2","metrics":{"votes":6539},"oid":"webdev_arena-07fb10ab075de7e4","rd":"2025-10-27","sid":"recZU91DVcezHiw1W","sm":"MiniMax-M2","src":"https://web.lmarena.ai/leaderboard","v":1297.78},{"b":"webdev_arena","hi":1295,"lo":1277,"m":"claude-haiku-4-5-2025-10-01","metrics":{"votes":8276},"oid":"webdev_arena-e4ffcbb97e0ea833","rd":"2025-10-15","sid":"rec4L4znR9E7Yeb7E","sm":"claude-haiku-4-5-20251001","src":"https://web.lmarena.ai/leaderboard","v":1286},{"b":"webdev_arena","hi":1281.64,"lo":1265.87,"m":"qwen3-coder-480b-a35b-instruct","metrics":{"votes":12265},"oid":"webdev_arena-e07bfe69888716a6","rd":"2025-07-31","sid":"recb5Q2Ez7Shxvl2o","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://web.lmarena.ai/leaderboard","v":1273.75},{"b":"webdev_arena","hi":1285.38,"lo":1258.11,"m":"deepseek-v3-2-exp","metrics":{"votes":3865},"oid":"webdev_arena-f747838de0eaf2ed","rd":"2025-09-29","sid":"recoUouxwJTO6jCv1","sm":"DeepSeek-V3.2-Exp","src":"https://web.lmarena.ai/leaderboard","v":1271.75},{"b":"webdev_arena","hi":1280.24,"lo":1249.59,"m":"mistral-medium-2604","metrics":{"votes":2154},"oid":"webdev_arena-7938357834b37b3a","rd":"2026-04-28","sid":"recWpRsMIcDfKiWR8","sm":"mistral-medium-2604","src":"https://web.lmarena.ai/leaderboard","v":1264.92},{"b":"webdev_arena","hi":1261.24,"lo":1247.29,"m":"gemini-3-1-flash-lite","metrics":{"votes":13634},"oid":"webdev_arena-a1aa73a2caa42a0c","rd":"2026-03-03","sid":"recJCbQHLFFNCdoMh","sm":"gemini-3.1-flash-lite","src":"https://web.lmarena.ai/leaderboard","v":1254.27},{"b":"webdev_arena","hi":1268.05,"lo":1232.25,"m":"qwen3-6-35b-a3b","metrics":{"votes":1535},"oid":"webdev_arena-e06aa8a11d49517e","rd":"2026-04-14","sid":"recOYbT19AfMLPU6Z","sm":"qwen3.6-35b-a3b","src":"https://web.lmarena.ai/leaderboard","v":1250.15},{"b":"webdev_arena","hi":1264,"lo":1230,"m":"gpt-5-1-codex-mini","metrics":{"votes":1566},"oid":"webdev_arena-4da8a0d552df4c49","rd":"2025-11-12","sid":"recPRratljctS351K","sm":"gpt-5.1-codex-mini","src":"https://web.lmarena.ai/leaderboard","v":1247},{"b":"webdev_arena","hi":1258.29,"lo":1218.06,"m":"qwen3-5-flash","metrics":{"votes":1270},"oid":"webdev_arena-b62a8b44db5be6fa","rd":"2026-02-25","sid":"recXaGYNzvxoD25Gj","sm":"qwen3.5-flash","src":"https://web.lmarena.ai/leaderboard","v":1238.17},{"b":"webdev_arena","hi":1257.12,"lo":1216.09,"m":"trinity-large-thinking","metrics":{"votes":1303},"oid":"webdev_arena-bb48855b4c429e03","rd":"2026-04-01","sid":"recOdmWvMBKCrAR2K","sm":"trinity-large-thinking","src":"https://web.lmarena.ai/leaderboard","v":1236.61},{"b":"webdev_arena","hi":1255.47,"lo":1203.57,"m":"mistral-large-2512","metrics":{"votes":812},"oid":"webdev_arena-db1e9bfb80c5660d","rd":"2025-12-02","sid":"rec75ePp1KlTVhbON","sm":"mistral-large-2512","src":"https://web.lmarena.ai/leaderboard","v":1229.52},{"b":"webdev_arena","hi":1241.74,"lo":1209.73,"m":"gemini-2-5-pro","metrics":{"votes":2667},"oid":"webdev_arena-e8695ab3314e14b8","rd":"2025-06-17","sid":"recmBglMHLXb6m6s8","sm":"gemini-2.5-pro","src":"https://web.lmarena.ai/leaderboard","v":1225.73},{"b":"webdev_arena","hi":1236,"lo":1210,"m":"grok-4-1-fast-reasoning","metrics":{"votes":3721},"oid":"webdev_arena-73dda33dc8b5e4fd","rd":"2025-11-19","sid":"recgHWkkshK8nYtfz","sm":"grok-4-1-fast-reasoning","src":"https://web.lmarena.ai/leaderboard","v":1223},{"b":"webdev_arena","hi":1236.05,"lo":1185.84,"m":"grok-4-1","metrics":{"votes":939},"oid":"webdev_arena-47c4ce272d51d938","rd":"2025-11-17","sid":"recmY1Sf15dhJzzON","sm":"grok-4-1","src":"https://web.lmarena.ai/leaderboard","v":1210.95},{"b":"webdev_arena","hi":1191.78,"lo":1141.41,"m":"mercury-2","metrics":{"votes":904},"oid":"webdev_arena-f371dfe20b37d103","sid":"rechVEiDFpARu63zz","sm":"mercury-2","src":"https://web.lmarena.ai/leaderboard","v":1166.59},{"b":"webdev_arena","hi":1193.32,"lo":1136.85,"m":"grok-code-fast-1","metrics":{"votes":783},"oid":"webdev_arena-055e0812d4b352e3","rd":"2025-08-28","sid":"recT6aoJlcWcMufso","sm":"grok-code-fast-1","src":"https://web.lmarena.ai/leaderboard","v":1165.08},{"b":"webdev_arena","hi":1172,"lo":1126,"m":"grok-4-fast","metrics":{"votes":945},"oid":"webdev_arena-f445938c500bf96e","rd":"2025-09-19","sid":"rec8Eg2KdgjeEWepd","sm":"grok-4-fast","src":"https://web.lmarena.ai/leaderboard","v":1149},{"b":"weirdml","c":29.305755666666663,"cfg":{"effort":"promax"},"e":"promax","m":"gpt-6-astra","oid":"weirdml-8b6edb72977f6ee2","rd":"2026-09-03","se":0.000026473963528335825,"sid":"recEy3l0e3fb7Ia3k","sm":"gpt-6-astra_promax","src":"https://htihle.github.io/weirdml.html","v":0.9357},{"b":"weirdml","c":7.748507900000001,"cfg":{"effort":"max"},"e":"max","m":"gpt-6-astra","oid":"weirdml-633cf2e54b0110f2","rd":"2026-09-03","se":0.000015446475430902452,"sid":"rec133XekkfngA36k","sm":"gpt-6-astra_max","src":"https://htihle.github.io/weirdml.html","v":0.9328000000000001},{"b":"weirdml","c":11.686583529411763,"cfg":{"effort":"max"},"e":"max","m":"claude-fable-5-1","oid":"weirdml-f09a25366fc932d0","rd":"2026-09-01","se":0.00001280965433748779,"sid":"recrm7c71JFig2bUs","sm":"claude-fable-5-1_max","src":"https://htihle.github.io/weirdml.html","v":0.929},{"b":"weirdml","c":2.975405612244898,"cfg":{"effort":"high"},"e":"high","m":"gpt-6-astra","oid":"weirdml-025259e837a26991","rd":"2026-09-03","se":0.00001648119724718335,"sid":"rect2MKJz3YKOcoxs","sm":"gpt-6-astra_high","src":"https://htihle.github.io/weirdml.html","v":0.9287000000000001},{"b":"weirdml","c":3.2013411235955056,"cfg":{"effort":"high"},"e":"high","m":"claude-fable-5-1","oid":"weirdml-2f6e612c1d280c1a","rd":"2026-09-01","se":0.00001376337560603623,"sid":"recfo5ksX2HG4Gkay","sm":"claude-fable-5-1_high","src":"https://htihle.github.io/weirdml.html","v":0.923},{"b":"weirdml","c":8.424409705882354,"cfg":{"effort":"max"},"e":"max","m":"claude-fable-5","oid":"weirdml-3f7740ace2b21ce8","rd":"2026-06-09","se":0.000024186536765080326,"sid":"recFU4S6sD5dKVLFF","sm":"claude-fable-5_max","src":"https://htihle.github.io/weirdml.html","v":0.9194},{"b":"weirdml","c":4.7419545588235295,"cfg":{"effort":"max"},"e":"max","m":"claude-opus-5","oid":"weirdml-db5f98a8d8ae37b3","rd":"2026-07-24","se":0.00003098361045836742,"sid":"recWwk0RBjkPAjios","sm":"claude-opus-5_max","src":"https://htihle.github.io/weirdml.html","v":0.9178000000000001},{"b":"weirdml","c":2.1933491176470588,"cfg":{"effort":"high"},"e":"high","m":"claude-opus-5","oid":"weirdml-bb938cc3b27eb8e3","rd":"2026-07-24","se":0.000017927171677920716,"sid":"recG8bVdbUmuVccNX","sm":"claude-opus-5_high","src":"https://htihle.github.io/weirdml.html","v":0.9159},{"b":"weirdml","c":3.8707419117647057,"cfg":{"effort":"promax"},"e":"promax","m":"gpt-5-6-sol","oid":"weirdml-f6e1666c070ad3fa","rd":"2026-07-09","se":0.00006068246016949213,"sid":"recKCrWh6fGtCumC9","sm":"gpt-5.6-sol_promax","src":"https://htihle.github.io/weirdml.html","v":0.8943000000000001},{"b":"weirdml","c":1.25479875,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-6-sol","oid":"weirdml-24dda704472e9fb2","rd":"2026-07-09","se":0.0000323705550671775,"sid":"recDXSQrmsktQrxr3","sm":"gpt-5.6-sol_high","src":"https://htihle.github.io/weirdml.html","v":0.8876000000000001},{"b":"weirdml","c":3.1068966666666666,"cfg":{"effort":"high"},"e":"high","m":"claude-fable-5","oid":"weirdml-6709b72debbe4434","rd":"2026-06-09","se":0.00008026931867056432,"sid":"recm1YL5ROE3e3a8v","sm":"claude-fable-5_high","v":0.8785},{"b":"weirdml","c":2.3122075294117646,"cfg":{"effort":"max"},"e":"max","m":"gpt-5-6-sol","oid":"weirdml-a6b3b8619d3516d7","rd":"2026-07-09","se":0.00008443334076782342,"sid":"rec7UbzIyPljXighC","sm":"gpt-5.6-sol_max","src":"https://htihle.github.io/weirdml.html","v":0.8697},{"b":"weirdml","c":1.419572117647059,"m":"claude-opus-5","oid":"weirdml-3f358956063cfae0","rd":"2026-07-24","se":0.0001453204125517409,"sid":"recVNrTRYPu0FgJv9","sm":"claude-opus-5","src":"https://htihle.github.io/weirdml.html","v":0.8634000000000001},{"b":"weirdml","c":2.568345317647059,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-5","oid":"weirdml-68b808d5b58865a6","rd":"2026-04-23","se":0.0001091535447188698,"sid":"recNJLmrITg65HnLP","sm":"gpt-5.5_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.8491},{"b":"weirdml","c":0.5072680071428571,"cfg":{"effort":"high"},"e":"high","m":"gemini-3-8-flash","oid":"weirdml-e0b98fd35947b36b","rd":"2026-09-02","se":0.00010115853370778513,"sid":"recRoiC6r1xCmmciK","sm":"gemini-3.8-flash_high","src":"https://htihle.github.io/weirdml.html","v":0.8481000000000001},{"b":"weirdml","c":2.221576529411765,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-5","oid":"weirdml-67d64596e399ffce","rd":"2026-04-23","se":0.00010857064112429189,"sid":"rece9m5QDHJk0kphX","sm":"gpt-5.5_high","src":"https://htihle.github.io/weirdml.html","v":0.8390000000000001},{"b":"weirdml","c":2.3498122772277226,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"claude-opus-4-8","oid":"weirdml-f7662b6192c07adb","rd":"2026-05-28","se":0.00009958824837467327,"sid":"recIQsIEKMmBYxvWF","sm":"claude-opus-4-8_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.8289},{"b":"weirdml","c":2.0536589833333334,"cfg":{"effort":"max"},"e":"max","m":"kimi-k3","oid":"weirdml-62750154ca2428fd","rd":"2026-07-16","se":0.00014431240190401943,"sid":"recQcmwjtb4CiWGMH","sm":"kimi-k3_max","src":"https://htihle.github.io/weirdml.html","v":0.8257},{"b":"weirdml","c":2.35,"m":"gpt-5-3-codex","oid":"weirdml-86cfc3a264baee04","rd":"2026-02-05","sid":"recQoGmOgTINEyg46","sm":"gpt-5.3-codex","v":0.793},{"b":"weirdml","c":0.3119056164215686,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-6-terra","oid":"weirdml-5dc15dd2c9e30d00","rd":"2026-07-09","se":0.0001445138804172179,"sid":"recmnP6rlIVdQcuyG","sm":"gpt-5.6-terra_high","src":"https://htihle.github.io/weirdml.html","v":0.7827},{"b":"weirdml","c":5.779370588235294,"cfg":{"effort":"high"},"e":"high","m":"claude-opus-4-6","oid":"weirdml-05b5cef31ea88665","rd":"2026-02-05","se":0.00023465628049314672,"sid":"recAZOaIZoeMECTb2","sm":"claude-opus-4-6_high","src":"https://htihle.github.io/weirdml.html","v":0.7795000000000001},{"b":"weirdml","c":5.78,"m":"claude-opus-4-6","notes":"Adaptive thinking","oid":"weirdml-0dd77d2b6fd8e963","rd":"2026-02-05","sid":"recSlHl7gDvgN7Iv6","sm":"claude-opus-4-6_unknown","v":0.779},{"b":"weirdml","c":2.452577288823529,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-3-codex","oid":"weirdml-213c228bd9fca125","rd":"2026-02-05","se":0.00012201261999680532,"sid":"recSL5zN2vlWKsInd","sm":"gpt-5.3-codex_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.779},{"b":"weirdml","c":5.719939852941176,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-4-2026-03-05","oid":"weirdml-9a315f17c26766a7","rd":"2026-03-05","se":0.00015646383228370685,"sid":"recxpHhBx0NDfrFVq","sm":"gpt-5.4-2026-03-05_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.777},{"b":"weirdml","c":1.8207113690476193,"cfg":{"effort":"high"},"e":"high","m":"claude-opus-4-7","oid":"weirdml-b0891e3904504e25","rd":"2026-04-16","se":0.00017860295681347224,"sid":"recIku5vHDX9VoDi9","sm":"claude-opus-4-7_high","src":"https://htihle.github.io/weirdml.html","v":0.7644},{"b":"weirdml","c":0.6895180000000002,"m":"claude-opus-4-7","oid":"weirdml-b79094d36d71d69b","rd":"2026-04-16","se":0.00017246892139005817,"sid":"recuoBayQAlGA5M8a","sm":"claude-opus-4-7","src":"https://htihle.github.io/weirdml.html","v":0.7640000000000001},{"b":"weirdml","c":0.6895,"m":"claude-opus-4-7","oid":"weirdml-5050c64fb105e184","rd":"2026-04-16","sid":"recIyMQSWffJ5JAmX","sm":"claude-opus-4-7_unknown","v":0.764},{"b":"weirdml","c":0.8472938235294117,"cfg":{"effort":"medium"},"e":"medium","m":"claude-opus-4-8","oid":"weirdml-12c7ee0238ca68be","rd":"2026-05-28","se":0.0001646215855604908,"sid":"reciU71sdJ0Xc773s","sm":"claude-opus-4-8_medium","src":"https://htihle.github.io/weirdml.html","v":0.7604000000000001},{"b":"weirdml","c":7.952621764705882,"cfg":{"effort":"max"},"e":"max","m":"claude-opus-4-7","oid":"weirdml-54cfa6c2fca8a8b7","rd":"2026-04-16","se":0.0002512541407541041,"sid":"recExT6mKHp0IGQ4Q","sm":"claude-opus-4-7_max","src":"https://htihle.github.io/weirdml.html","v":0.7545000000000001},{"b":"weirdml","c":0.5351449411428572,"cfg":{"effort":"max"},"e":"max","m":"glm-5-3","oid":"weirdml-2cf15aad5e3838a4","rd":"2026-08-14","se":0.0001665915023174048,"sid":"rec1RsAHf189UoIIA","sm":"glm-5.3_max","src":"https://htihle.github.io/weirdml.html","v":0.7540000000000001},{"b":"weirdml","c":2.0473807117647063,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"gpt-5-2-2025-12-11","oid":"weirdml-bca777c1a6a1cd34","rd":"2025-12-11","se":0.00019191659635557604,"sid":"recBg3iB5UB07x2hT","sm":"gpt-5.2-2025-12-11_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.7219},{"b":"weirdml","c":1.3601870235294122,"m":"gemini-3-1-pro-preview","oid":"weirdml-f129676fc20bdb74","rd":"2026-02-19","se":0.00017632957273620633,"sid":"recGmSE5Oa6sPNY3I","sm":"gemini-3.1-pro-preview","src":"https://htihle.github.io/weirdml.html","v":0.7206999999999999},{"b":"weirdml","c":0.4752000990099009,"cfg":{"effort":"none"},"e":"none","m":"claude-opus-4-8","oid":"weirdml-0ea9a0e256854b0c","rd":"2026-05-28","se":0.00013620999100200573,"sid":"rectzf70zaXWJLj63","sm":"claude-opus-4-8_none","src":"https://htihle.github.io/weirdml.html","v":0.7045},{"b":"weirdml","c":0.5072874972,"cfg":{"effort":"max"},"e":"max","m":"glm-5-2","oid":"weirdml-dba52fdf599e56eb","rd":"2026-06-16","se":0.00016924669523230864,"sid":"recSQFD35vSn14YQh","sm":"glm-5.2_max","src":"https://htihle.github.io/weirdml.html","v":0.7012},{"b":"weirdml","c":0.5257711076923077,"m":"gemini-3-pro-preview","oid":"weirdml-125bed6a4c867063","rd":"2025-11-18","se":0.00012658927784506301,"sid":"recMwExVNzQFOuRPy","sm":"gemini-3-pro-preview","src":"https://htihle.github.io/weirdml.html","v":0.6993},{"b":"weirdml","c":0.895463294117647,"cfg":{"effort":"high"},"e":"high","m":"claude-sonnet-5","oid":"weirdml-4ebf576c9e838c00","rd":"2026-06-30","se":0.00017696329549524788,"sid":"recwWCD4jlF5lW7rt","sm":"claude-sonnet-5_high","src":"https://htihle.github.io/weirdml.html","v":0.6878000000000001},{"b":"weirdml","c":0.28333104480874316,"cfg":{"effort":"high"},"e":"high","m":"glm-5-2","oid":"weirdml-c6b3e33bca3683d2","rd":"2026-06-16","se":0.00014915387714757675,"sid":"reccjeovOo5ppZ5Fh","sm":"glm-5.2_high","src":"https://htihle.github.io/weirdml.html","v":0.6731},{"b":"weirdml","c":0.33483234117647054,"cfg":{"effort":"high"},"e":"high","m":"grok-4-6","oid":"weirdml-26ade003de698e34","rd":"2026-08-12","se":0.0001783973695549794,"sid":"recjPHayQp5hGuMXS","sm":"grok-4.6_high","src":"https://htihle.github.io/weirdml.html","v":0.6729},{"b":"weirdml","c":0.5389514705882353,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-5","oid":"weirdml-721977fde92afa5f","rd":"2026-04-23","se":0.00017253756008593507,"sid":"recB9k0CpkaxiR6xD","sm":"gpt-5.5_none","src":"https://htihle.github.io/weirdml.html","v":0.6715000000000001},{"b":"weirdml","c":0.19103203769491522,"cfg":{"effort":"max"},"e":"max","m":"deepseek-v4-pro-0813","oid":"weirdml-3bb506f056a89581","rd":"2026-08-13","se":0.0001638262150844637,"sid":"rec3A9S3gzPC8h4kX","sm":"deepseek-v4-pro-0813_max","src":"https://htihle.github.io/weirdml.html","v":0.6622},{"b":"weirdml","c":2.8593499285714286,"cfg":{"effort":"medium"},"e":"medium","m":"claude-sonnet-4-6","oid":"weirdml-ba50b658543b2eb5","rd":"2026-02-17","se":0.0002980618554382716,"sid":"recKgAUWa5eUZxXQM","sm":"claude-sonnet-4-6_medium","v":0.6607},{"b":"weirdml","c":0.7355440860215054,"m":"claude-opus-4-6","oid":"weirdml-bf1c4215c3aef817","rd":"2026-02-05","se":0.0002043551326871128,"sid":"recffS4JAWtKqn3Ku","sm":"claude-opus-4-6","src":"https://htihle.github.io/weirdml.html","v":0.6587000000000001},{"b":"weirdml","c":0.7383040804597701,"m":"claude-opus-4-5-20251101-16k","oid":"weirdml-d66bc521b235244c","rd":"2025-11-24","se":0.00016646342828811227,"sid":"reci9ur1kRcoJsSdF","sm":"claude-opus-4-5-20251101_16K","src":"https://htihle.github.io/weirdml.html","v":0.6374000000000001},{"b":"weirdml","c":0.4159760980392157,"cfg":{"effort":"medium"},"e":"medium","m":"gpt-5-2-2025-12-11","oid":"weirdml-e8a29d37858fcd79","rd":"2025-12-11","se":0.0001438769820982512,"sid":"recpsACaI9bWFMg53","sm":"gpt-5.2-2025-12-11_medium","src":"https://htihle.github.io/weirdml.html","v":0.6344},{"b":"weirdml","c":0.028578264316831685,"cfg":{"effort":"max"},"e":"max","m":"deepseek-v4-flash-0731","oid":"weirdml-0bbbbcd9ec30e28d","rd":"2026-07-31","se":0.0001825242529226991,"sid":"recHz3VON5MYuSgCS","sm":"deepseek-v4-flash-0731_max","src":"https://htihle.github.io/weirdml.html","v":0.6296},{"b":"weirdml","c":0.7560409661764705,"cfg":{"effort":"high"},"e":"high","m":"gemini-3-5-flash","oid":"weirdml-bef524ab38927623","rd":"2026-05-19","se":0.00024091415851094065,"sid":"recrIPcpOeEj60VKj","sm":"gemini-3.5-flash_high","src":"https://htihle.github.io/weirdml.html","v":0.6264000000000001},{"b":"weirdml","c":0.22158831288659797,"m":"gemini-3-flash-preview","oid":"weirdml-9df93954fe6208eb","rd":"2025-12-17","se":0.00015113939132131612,"sid":"recORWiDWDuE6ne5i","sm":"gemini-3-flash-preview","src":"https://htihle.github.io/weirdml.html","v":0.616},{"b":"weirdml","c":0.19999024607843144,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-6-luna","oid":"weirdml-4301ae6c41653284","rd":"2026-07-09","se":0.00018295213556011064,"sid":"recCwfTTdk592JqP0","sm":"gpt-5.6-luna_high","src":"https://htihle.github.io/weirdml.html","v":0.6086},{"b":"weirdml","c":0.6913346831683168,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-1-2025-11-13","oid":"weirdml-64f1db1d0f75ba57","rd":"2025-11-13","se":0.00014219673026109514,"sid":"reci36t9Q8We2ARmf","sm":"gpt-5.1-2025-11-13_high","src":"https://htihle.github.io/weirdml.html","v":0.6077},{"b":"weirdml","c":0.5931804044117648,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-2025-08-07","oid":"weirdml-3a5879f91ba4ff6a","rd":"2025-08-07","se":0.00016777026323197355,"sid":"rec4qZr44BtF62vqq","sm":"gpt-5-2025-08-07_high","src":"https://htihle.github.io/weirdml.html","v":0.6070000000000001},{"b":"weirdml","c":7.647644117647059,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-pro-2025-10-06","oid":"weirdml-f3e7dfe0e311a6bc","rd":"2025-10-07","se":0.0002475,"sid":"recX2OmQjv53BIY3l","sm":"gpt-5-pro-2025-10-06_high","src":"https://htihle.github.io/weirdml.html","v":0.6039},{"b":"weirdml","c":1.2910174676470587,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-4-mini-2026-03-17","oid":"weirdml-9ac7b2eb960987c0","rd":"2026-03-17","se":0.0001686737161666318,"sid":"recX7Ablwb2UkN5s3","sm":"gpt-5.4-mini-2026-03-17_high","src":"https://htihle.github.io/weirdml.html","v":0.603},{"b":"weirdml","c":0.38010874811320755,"cfg":{"effort":"xhigh"},"e":"xhigh","m":"muse-spark-1-2","oid":"weirdml-e18a3384562ce8b0","rd":"2026-08-05","se":0.00022385111686989455,"sid":"recPAu5qSXms7oxpX","sm":"muse-spark-1.2_xhigh","src":"https://htihle.github.io/weirdml.html","v":0.603},{"b":"weirdml","c":2.540382702702703,"cfg":{"effort":"high"},"e":"high","m":"o3-pro-2025-06-10","oid":"weirdml-d93cb9eb93c2eaf4","rd":"2025-06-10","se":0.00019073618306924184,"sid":"recrKJicVBBBp5oLq","sm":"o3-pro-2025-06-10_high","src":"https://htihle.github.io/weirdml.html","v":0.5821000000000001},{"b":"weirdml","c":0.26790913235294117,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-4-2026-03-05","oid":"weirdml-48feb2b9353f288b","rd":"2026-03-05","se":0.0001493713713906097,"sid":"reczzKMLHOv2ZMWkk","sm":"gpt-5.4-2026-03-05_none","v":0.5744},{"b":"weirdml","c":0.26790913235294117,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-4-pro-2026-03-05","oid":"weirdml-6040a641c38eeb0f","rd":"2026-03-05","se":0.0001493713713906097,"sid":"recShobEBXmUWOQsU","sm":"gpt-5.4-pro-2026-03-05_none","src":"https://htihle.github.io/weirdml.html","v":0.5744},{"b":"weirdml","c":0.3175119256097561,"m":"glm-5-1","oid":"weirdml-7f2b67af4f4ca4cf","rd":"2026-04-07","se":0.0001756716185413227,"sid":"rec5NQhvAEkM6TdnT","sm":"glm-5.1","src":"https://htihle.github.io/weirdml.html","v":0.5710000000000001},{"b":"weirdml","c":0.025342430526912183,"cfg":{"effort":"high"},"e":"high","m":"deepseek-v4-flash-0731","oid":"weirdml-fc93917aabe03ecf","rd":"2026-07-31","se":0.00012292173470515065,"sid":"recobqFXuYggQJT8p","sm":"deepseek-v4-flash-0731_high","src":"https://htihle.github.io/weirdml.html","v":0.5705},{"b":"weirdml","c":0.6227120676470587,"cfg":{"effort":"high"},"e":"high","m":"gemini-3-6-flash","oid":"weirdml-ed7d740d424c0164","rd":"2026-07-21","se":0.00027228615189695254,"sid":"recMmGYltihm9blZq","sm":"gemini-3.6-flash_high","src":"https://htihle.github.io/weirdml.html","v":0.561},{"b":"weirdml","c":0.35035446042016805,"m":"kimi-k2-6","oid":"weirdml-bf2557c4a8deefde","rd":"2026-04-20","se":0.00016578737450769108,"sid":"recn6OmUksaPCOjI9","sm":"kimi-k2.6","src":"https://htihle.github.io/weirdml.html","v":0.5586},{"b":"weirdml","c":1.095613721,"m":"gpt-5-codex","oid":"weirdml-9b453a0b45152553","rd":"2025-09-15","sid":"recn9o1kxCulEzqBH","sm":"gpt-5-codex","src":"https://htihle.github.io/weirdml.html","v":0.5453},{"b":"weirdml","c":1.0956137205882353,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-codex","oid":"weirdml-63c8c5a107ed9b85","rd":"2025-09-15","se":0.0001337443874877935,"sid":"rechNhHxFdftISgRr","sm":"gpt-5-codex_high","v":0.5453},{"b":"weirdml","c":0.23163436923809524,"m":"kimi-k2-7-code","oid":"weirdml-ea7e5346e9daaf96","rd":"2026-06-12","se":0.00016836483062585453,"sid":"recmdjDJrTcPOyMUD","sm":"kimi-k2.7-code","src":"https://htihle.github.io/weirdml.html","v":0.5412},{"b":"weirdml","c":0.4096601321,"m":"gemini-2-5-pro-16k","oid":"weirdml-2a8aae9c9d5f0344","rd":"2025-06-17","sid":"recHsTdNwgdKqNyYc","sm":"gemini-2.5-pro_16K","src":"https://htihle.github.io/weirdml.html","v":0.5403},{"b":"weirdml","c":0.10976651117647059,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-mini-2025-08-07","oid":"weirdml-cbdddc3d0ef06dab","rd":"2025-08-07","se":0.00014642503026075516,"sid":"recSCZGZmZVpfZcHB","sm":"gpt-5-mini-2025-08-07_high","src":"https://htihle.github.io/weirdml.html","v":0.5267000000000001},{"b":"weirdml","c":0.19568510000000003,"cfg":{"effort":"high"},"e":"high","m":"o4-mini-2025-04-16","oid":"weirdml-f80747824e8fdb25","rd":"2025-04-16","se":0.00018323764208750046,"sid":"recGzRI7z9Ript5yD","sm":"o4-mini-2025-04-16_high","src":"https://htihle.github.io/weirdml.html","v":0.5256000000000001},{"b":"weirdml","c":0.22606690322580647,"cfg":{"effort":"high"},"e":"high","m":"o3-2025-04-16","oid":"weirdml-56abeb6b973834c7","rd":"2025-04-16","se":0.00015441931633636616,"sid":"recSHUB127PDQxkfv","sm":"o3-2025-04-16_high","src":"https://htihle.github.io/weirdml.html","v":0.5242},{"b":"weirdml","c":0.1262829254901961,"m":"grok-4-20","oid":"weirdml-79fc5c5d77549d7e","rd":"2026-02-17","se":0.00010422137973438821,"sid":"recVUJyYaIs5yx2NE","sm":"grok-4-20","src":"https://htihle.github.io/weirdml.html","v":0.5226},{"b":"weirdml","c":0.013264671000000004,"m":"gemma-4-31b-it","oid":"weirdml-6148c3cd33ac077b","rd":"2026-04-02","se":0.00010633985942245199,"sid":"recp4KhtVysoQijD0","sm":"gemma-4-31b-it","src":"https://htihle.github.io/weirdml.html","v":0.5226},{"b":"weirdml","c":0.0664239769607843,"m":"gemini-3-1-flash-lite","oid":"weirdml-0ebbb0e58055e1c0","rd":"2026-03-03","se":0.00015603967429857592,"sid":"rec6ZiITCMpWQVmg1","sm":"gemini-3.1-flash-lite","v":0.5219},{"b":"weirdml","c":0.09031993548387096,"m":"grok-4-3","oid":"weirdml-311d6b28875ca74d","rd":"2026-04-17","se":0.0001461008341580845,"sid":"recWMMOgio9JyMvaZ","sm":"grok-4.3_unknown","src":"https://htihle.github.io/weirdml.html","v":0.4989},{"b":"weirdml","c":0.21462569999999997,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-2-2025-12-11","oid":"weirdml-87dfe71bb68d001b","rd":"2025-12-11","se":0.00014542225995169673,"sid":"rec2TRMqFG2udu8Rz","sm":"gpt-5.2-2025-12-11_none","v":0.49600000000000005},{"b":"weirdml","c":0.215,"cfg":{"effort":"low"},"e":"low","m":"gpt-5-2-2025-12-11","oid":"weirdml-054b14db345364ed","rd":"2025-12-11","sid":"reclZgTEmoOmWH7fs","sm":"gpt-5.2-2025-12-11_low","src":"https://htihle.github.io/weirdml.html","v":0.496},{"b":"weirdml","c":0.047747187745098035,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-4-nano-2026-03-17","oid":"weirdml-4f26c1f6c7b7a78f","rd":"2026-03-17","se":0.0001592769681566432,"sid":"rectXu0v9GEDSKcu7","sm":"gpt-5.4-nano-2026-03-17_high","src":"https://htihle.github.io/weirdml.html","v":0.49229999999999996},{"b":"weirdml","c":0.34030045166666667,"cfg":{"effort":"max"},"e":"max","m":"deepseek-v4-pro","oid":"weirdml-9b08302b2d12c206","rd":"2026-04-24","se":0.00017538854527333745,"sid":"recfRm5fINdrqgGIA","sm":"deepseek-v4-pro_max","src":"https://htihle.github.io/weirdml.html","v":0.489},{"b":"weirdml","c":0.033483015,"cfg":{"effort":"high"},"e":"high","m":"gpt-oss-120b","oid":"weirdml-1e53737b1219946b","rd":"2025-08-05","sid":"recWRKD0ymts02CqB","sm":"gpt-oss-120b_high","src":"https://htihle.github.io/weirdml.html","v":0.4817},{"b":"weirdml","c":0.03348301499999999,"cfg":{"effort":"high"},"e":"high","m":"openaigpt-oss-120b","oid":"weirdml-27ebe973b2546ceb","rd":"2025-08-05","se":0.00017207317818265387,"sid":"recbVp3QhWPOFJDDS","sm":"openai/gpt-oss-120b_high","v":0.4817},{"b":"weirdml","c":0.07874515247524752,"m":"glm-5","oid":"weirdml-3b126cfd302be4ed","rd":"2026-02-11","se":0.0001654346503350198,"sid":"receopdhR3orpxf0k","sm":"glm-5","v":0.4817},{"b":"weirdml","c":0.3343779683544304,"m":"claude-sonnet-4-5-20250929-16k","oid":"weirdml-5cd2576d5cc73e3a","rd":"2025-09-29","se":0.00010683114543195189,"sid":"recnLgZJKzq0pWJ9p","sm":"claude-sonnet-4-5-20250929_16K","src":"https://htihle.github.io/weirdml.html","v":0.4771},{"b":"weirdml","c":1.8487597058823528,"m":"o1-preview-2024-09-12","oid":"weirdml-354b853af08d5262","rd":"2024-09-12","se":0.00022664784170150546,"sid":"recJyT9ShE1YMRU74","sm":"o1-preview-2024-09-12","src":"https://htihle.github.io/weirdml.html","v":0.4756},{"b":"weirdml","c":0.0352105525490196,"m":"deepseek-v3-2-speciale","oid":"weirdml-48fa7a25130a5c17","rd":"2025-12-01","se":0.00016725773307150208,"sid":"rec61Nw717glfIbZL","sm":"DeepSeek-V3.2-Speciale","v":0.4673},{"b":"weirdml","c":0.2796832978723404,"m":"claude-sonnet-4-5-2025-09-29","oid":"weirdml-861aa0128dee1001","rd":"2025-09-29","se":0.00012670310629511955,"sid":"recWzey65DpSgKtm8","sm":"claude-sonnet-4-5-20250929","src":"https://htihle.github.io/weirdml.html","v":0.467},{"b":"weirdml","c":0.12994430126582276,"cfg":{"effort":"high"},"e":"high","m":"deepseek-v4-pro","oid":"weirdml-8de2778eb768d573","rd":"2026-04-24","se":0.00010879277129526622,"sid":"recQgbRlFbpfdoonZ","sm":"deepseek-v4-pro_high","src":"https://htihle.github.io/weirdml.html","v":0.46530000000000005},{"b":"weirdml","c":0.08046285647058822,"m":"grok-4-5","oid":"weirdml-55aa5579200b3052","rd":"2026-07-08","se":0.00017141187211715435,"sid":"recS0MGYDnpqvXhzT","sm":"grok-4.5_unknown","src":"https://htihle.github.io/weirdml.html","v":0.4643},{"b":"weirdml","c":0.33598588636363635,"m":"claude-sonnet-4-20250514-16k","oid":"weirdml-9a257014d25cc266","rd":"2025-05-22","se":0.00014712163795807962,"sid":"recuIpEzqzmBXSuu5","sm":"claude-sonnet-4-20250514_16K","src":"https://htihle.github.io/weirdml.html","v":0.4611},{"b":"weirdml","c":2.0286688235294115,"cfg":{"effort":"high"},"e":"high","m":"o1-2024-12-17","oid":"weirdml-aa44769e56a46768","rd":"2024-12-17","se":0.00010408467555535694,"sid":"reczHopD946ynh9x0","sm":"o1-2024-12-17_high","src":"https://htihle.github.io/weirdml.html","v":0.4608},{"b":"weirdml","c":1.755552705882353,"m":"claude-opus-4-1-20250805-16k","oid":"weirdml-d1c959001c89a482","rd":"2025-08-05","se":0.00014370461180194928,"sid":"reckSuCYFJ6a1ksrw","sm":"claude-opus-4-1-20250805_16K","src":"https://htihle.github.io/weirdml.html","v":0.4586},{"b":"weirdml","c":0.4517731019417476,"m":"grok-4-0709","oid":"weirdml-1f88eb71ee0249d4","rd":"2025-07-09","se":0.0001305177304939186,"sid":"recGNHgU4woBudrxm","sm":"grok-4-0709","src":"https://htihle.github.io/weirdml.html","v":0.4573},{"b":"weirdml","c":0.014616010092615771,"cfg":{"effort":"max"},"e":"max","m":"deepseek-v4-flash","oid":"weirdml-bbe8adb2ace70153","rd":"2026-04-24","se":0.00005552935999080915,"sid":"recgsOG5WTI3HAflE","sm":"deepseek-v4-flash_max","src":"https://htihle.github.io/weirdml.html","v":0.45630000000000004},{"b":"weirdml","c":0.052612491000000004,"m":"kimi-k2-5","oid":"weirdml-b0ba679fed384e03","rd":"2026-01-27","se":0.00014982905478957542,"sid":"recrvTpYhCK0tATZT","sm":"kimi-k2.5","src":"https://htihle.github.io/weirdml.html","v":0.456},{"b":"weirdml","c":0.08938744000000001,"m":"claude-haiku-4-5-2025-10-01","oid":"weirdml-223f61dd895f72b3","rd":"2025-10-15","se":0.00011362920132010245,"sid":"recbC3Pk2XJrtS6jq","sm":"claude-haiku-4-5-20251001","src":"https://htihle.github.io/weirdml.html","v":0.454},{"b":"weirdml","c":0.12278560784313723,"m":"claude-haiku-4-5-20251001-16k","oid":"weirdml-a3812f333daa8dc2","rd":"2025-10-15","se":0.00010276696227397038,"sid":"recde5VLEdMBKj0Gz","sm":"claude-haiku-4-5-20251001_16K","src":"https://htihle.github.io/weirdml.html","v":0.4415},{"b":"weirdml","c":0.30805649999999996,"m":"claude-sonnet-4-2025-05-14","oid":"weirdml-e728fbe98bd91587","rd":"2025-05-22","se":0.00014137609333575993,"sid":"recKsN4DgMnrJJqHR","sm":"claude-sonnet-4-20250514","src":"https://htihle.github.io/weirdml.html","v":0.4388},{"b":"weirdml","c":0.007279538164705883,"cfg":{"effort":"high"},"e":"high","m":"deepseek-v4-flash","oid":"weirdml-fe67116688838379","rd":"2026-04-24","se":0.00009118142469320882,"sid":"recKxCdGxVPD9xKOh","sm":"deepseek-v4-flash_high","src":"https://htihle.github.io/weirdml.html","v":0.4376},{"b":"weirdml","c":1.7352728823529409,"m":"claude-opus-4-20250514-16k","oid":"weirdml-918a95a6a9877958","rd":"2025-05-22","se":0.00014096613593228162,"sid":"recQV87PGo9MozwhO","sm":"claude-opus-4-20250514_16K","src":"https://htihle.github.io/weirdml.html","v":0.4372},{"b":"weirdml","c":0.278465325,"m":"mistral-medium-2604","oid":"weirdml-b322c5c3a2be99a7","rd":"2026-04-28","se":0.00010624798303565262,"sid":"recfqFx05Y465G4nA","sm":"mistral-medium-2604","src":"https://htihle.github.io/weirdml.html","v":0.43710000000000004},{"b":"weirdml","c":0.17446746058823528,"cfg":{"effort":"high"},"e":"high","m":"o3-mini-2025-01-31","oid":"weirdml-6f7f35ff0e382fa2","rd":"2025-01-31","se":0.00015164624724508388,"sid":"recb7bIHY4oRuegzM","sm":"o3-mini-2025-01-31_high","src":"https://htihle.github.io/weirdml.html","v":0.43700000000000006},{"b":"weirdml","c":0.0772020119760479,"m":"nemotron-3-ultra","oid":"weirdml-06cc209fa399f69e","rd":"2026-06-04","se":0.00008448567640489385,"sid":"recgNfeirwt2to5GK","sm":"nemotron-3-ultra","src":"https://htihle.github.io/weirdml.html","v":0.43450000000000005},{"b":"weirdml","c":0.02977649828431373,"m":"mercury-2","oid":"weirdml-add5433aaa1b5231","se":0.00012700550676006744,"sid":"recLpCV9prcxoluoO","sm":"mercury-2","src":"https://htihle.github.io/weirdml.html","v":0.43200000000000005},{"b":"weirdml","c":0.012726230392156862,"m":"grok-4-fast","oid":"weirdml-c98e433e36106c86","rd":"2025-09-19","se":0.00015484043858714642,"sid":"recgozq9bI4nBC6MI","sm":"grok-4-fast","src":"https://htihle.github.io/weirdml.html","v":0.4286},{"b":"weirdml","c":0.08093773510638298,"m":"kimi-k2-thinking","oid":"weirdml-b704138d4e144ee2","rd":"2025-11-06","se":0.00015491065069442097,"sid":"recgbBevs5M9TnP36","sm":"kimi-k2-thinking","src":"https://htihle.github.io/weirdml.html","v":0.4279},{"b":"weirdml","c":0.01739341941,"cfg":{"effort":"high"},"e":"high","m":"grok-3-mini","oid":"weirdml-9d7aa97d6ebedcb9","rd":"2025-06-24","sid":"recGEblb3SNnepRe1","sm":"grok-3-mini_high","src":"https://htihle.github.io/weirdml.html","v":0.4258},{"b":"weirdml","c":0.01739341941176471,"cfg":{"effort":"high"},"e":"high","m":"grok-3-mini-beta","oid":"weirdml-def4648ccf454eef","rd":"2025-04-09","se":0.0001421263532118665,"sid":"recc4ZZmpYmjwzUju","sm":"grok-3-mini-beta_high","v":0.4258},{"b":"weirdml","c":0.07324502983193276,"m":"gemini-2-5-flash-preview-09-2025-16k","oid":"weirdml-51baa8232ea83d95","rd":"2025-09-25","se":0.0001030795435843648,"sid":"recSenSiVOzoyrhM7","sm":"gemini-2.5-flash-preview-09-2025_16k","src":"https://htihle.github.io/weirdml.html","v":0.4191},{"b":"weirdml","c":0.011485011176470588,"cfg":{"effort":"medium"},"e":"medium","m":"gpt-oss-120b","oid":"weirdml-e1aafd97b891c1ad","rd":"2025-08-05","se":0.0001318182235022303,"sid":"recwbp07k8ku0pbh5","sm":"gpt-oss-120b_medium","src":"https://htihle.github.io/weirdml.html","v":0.41880000000000006},{"b":"weirdml","c":0.07106829941176471,"m":"deepseek-r1-0528","oid":"weirdml-c67fe8bc08eb9a61","rd":"2025-05-28","se":0.00015420663260519693,"sid":"recLmo6NcvCM4Ye7a","sm":"DeepSeek-R1-0528","src":"https://htihle.github.io/weirdml.html","v":0.41630000000000006},{"b":"weirdml","c":0.07259235294117648,"m":"qwen3-coder-480b-a35b-instruct","oid":"weirdml-616d994e641ad0f6","rd":"2025-07-31","se":0.00009552690607848418,"sid":"recA6D3LkM2OuG2TH","sm":"Qwen3-Coder-480B-A35B-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.4117},{"b":"weirdml","c":0.01989473141,"m":"qwen3-235b-a22b-thinking-2507","oid":"weirdml-eb8b8a8f45f4f8a4","rd":"2025-07-25","sid":"rec1XWGoYSV4OlC8J","sm":"Qwen3-235B-A22B-Thinking-2507","src":"https://htihle.github.io/weirdml.html","v":0.4104},{"b":"weirdml","c":0.10585353676470587,"m":"gemini-2-5-flash-preview-04-17-16k-thinking","oid":"weirdml-26df611a34835a8d","rd":"2025-04-17","se":0.0001555094264147071,"sid":"recgBmYMNvK9ubOYp","sm":"gemini-2.5-flash-preview-04-17 (16K thinking)","v":0.40950000000000003},{"b":"weirdml","c":0.1058535368,"m":"gemini-2-5-flash-preview-05-20-16k","oid":"weirdml-d27e478c096854ac","rd":"2025-05-20","sid":"reci9lthvBxTe1qFc","sm":"gemini-2.5-flash-preview-05-20_16K","src":"https://htihle.github.io/weirdml.html","v":0.4095},{"b":"weirdml","c":0.010800342499999999,"cfg":{"effort":"high"},"e":"high","m":"gpt-oss-20b","oid":"weirdml-32204ed6462f3470","rd":"2025-08-05","se":0.0000913519232666839,"sid":"rectdVmbAMbhHQ5K3","sm":"gpt-oss-20b_high","src":"https://htihle.github.io/weirdml.html","v":0.4093},{"b":"weirdml","c":0.07348009059999999,"cfg":{"effort":"thinking"},"e":"thinking","m":"glm-4-5","oid":"weirdml-399cd7738a30dad1","rd":"2025-08-05","se":0.00020058268766219312,"sid":"rec7uPDokMq3CWQDy","sm":"glm-4.5_thinking","src":"https://htihle.github.io/weirdml.html","v":0.4058},{"b":"weirdml","c":0.24832379999999996,"m":"claude-3-5-sonnet-2024-10-22","oid":"weirdml-d5f678450f19f548","rd":"2024-10-22","se":0.00019381599109007252,"sid":"rec8W7RlpDQoWgvzo","sm":"claude-3-5-sonnet-20241022","src":"https://htihle.github.io/weirdml.html","v":0.3997},{"b":"weirdml","c":0.0785561838235294,"m":"gpt-5-chat","oid":"weirdml-eaf5bdb18e7b63b5","se":0.00010348849096364595,"sid":"rec1QFsYDTNUf8UB0","sm":"gpt-5-chat","src":"https://htihle.github.io/weirdml.html","v":0.39770000000000005},{"b":"weirdml","c":0.040395000000000014,"m":"qwen3-5-27b","oid":"weirdml-35a5e17f948f9dc8","rd":"2026-02-24","se":0.00018188559541985888,"sid":"recwg0Ywd7OOXMftD","sm":"qwen3.5-27B","src":"https://htihle.github.io/weirdml.html","v":0.39530000000000004},{"b":"weirdml","c":0.01772299089108911,"cfg":{"effort":"thinking"},"e":"thinking","m":"deepseek-v3-2-exp","oid":"weirdml-3982903238211c96","rd":"2025-09-29","se":0.00010339508761863283,"sid":"recTOeDgbKEbXih3E","sm":"DeepSeek-V3.2-Exp_thinking","src":"https://htihle.github.io/weirdml.html","v":0.3948},{"b":"weirdml","c":1.575002205882353,"m":"gpt-4-5-preview-2025-02-27","oid":"weirdml-219d99816847e777","rd":"2025-02-27","se":0.000185360729158954,"sid":"recwqjqGOkm0f51Io","sm":"gpt-4.5-preview-2025-02-27","src":"https://htihle.github.io/weirdml.html","v":0.3937},{"b":"weirdml","c":0.03429974068,"m":"kimi-k2-instruct","oid":"weirdml-b2f806165dc0f35d","rd":"2025-07-12","sid":"recOD22bjdiSPqTnP","sm":"Kimi-K2-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.3936},{"b":"weirdml","c":0.03429974068181818,"m":"kimi-k2-0711-preview","oid":"weirdml-e936a7d665763283","rd":"2025-07-12","se":0.00012841657406203382,"sid":"rec2wisOkQlcRMXxP","sm":"kimi-k2-0711-preview","v":0.3936},{"b":"weirdml","c":0.09702961904761905,"m":"gpt-4-1-2025-04-14","oid":"weirdml-a2e3da8cc01458b6","rd":"2025-04-14","se":0.00010299257007316828,"sid":"reculBSjwckG4bYHD","sm":"gpt-4.1-2025-04-14","src":"https://htihle.github.io/weirdml.html","v":0.3904},{"b":"weirdml","c":0.04675567557894736,"cfg":{"effort":"high"},"e":"high","m":"gemini-3-5-flash-lite","oid":"weirdml-322015c5c6c08b86","rd":"2026-07-21","se":0.00017156164116084946,"sid":"recwspdjXSGDnHdX0","sm":"gemini-3.5-flash-lite_high","src":"https://htihle.github.io/weirdml.html","v":0.39},{"b":"weirdml","c":0.01340708465,"m":"qwen3-235b-a22b-instruct-2507","oid":"weirdml-72a1c9959b2df61d","rd":"2025-07-25","sid":"recJkoZfVe1YtOBro","sm":"Qwen3-235B-A22B-Instruct-2507","src":"https://htihle.github.io/weirdml.html","v":0.387},{"b":"weirdml","c":0.013407084653465346,"m":"qwenqwen3-235b-a22b-thinking-2507","oid":"weirdml-ae616618129a6a28","rd":"2025-07-25","se":0.0000957710919039418,"sid":"recnlRdiKD49ooOXW","sm":"qwen/qwen3-235b-a22b-thinking-2507","v":0.387},{"b":"weirdml","c":0.0311958091954023,"cfg":{"effort":"thinking"},"e":"thinking","m":"deepseek-v3-1","oid":"weirdml-40c6521c92bab011","rd":"2025-08-21","se":0.00012050699265006076,"sid":"recpYiDZJiwfMbLKa","sm":"DeepSeek-V3.1_thinking","src":"https://htihle.github.io/weirdml.html","v":0.3836},{"b":"weirdml","c":0.026220323529411768,"cfg":{"effort":"high"},"e":"high","m":"gpt-5-nano-2025-08-07","oid":"weirdml-7e182e282aff4d20","rd":"2025-08-07","se":0.00013414257782068856,"sid":"recYTpsoV61tqQPyp","sm":"gpt-5-nano-2025-08-07_high","src":"https://htihle.github.io/weirdml.html","v":0.38060000000000005},{"b":"weirdml","c":0.012169454999999997,"m":"nemotron-3-super","oid":"weirdml-7cabd6e274118037","rd":"2026-03-11","se":0.00012324635701145654,"sid":"reczzWNlMCsX5rCJb","sm":"nemotron-3-super","src":"https://htihle.github.io/weirdml.html","v":0.3801},{"b":"weirdml","c":0.019951820588235296,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-4-nano-2026-03-17","oid":"weirdml-d5b9dd0e50ca1362","rd":"2026-03-17","se":0.00011700544258476443,"sid":"recCmJV0yAWMo0juE","sm":"gpt-5.4-nano-2026-03-17_none","src":"https://htihle.github.io/weirdml.html","v":0.3796},{"b":"weirdml","c":0.07285873235294117,"cfg":{"effort":"none"},"e":"none","m":"gpt-5-4-mini-2026-03-17","oid":"weirdml-6926eedaea72afc6","rd":"2026-03-17","se":0.00009932670299645241,"sid":"rec32pxBQyrHurEJr","sm":"gpt-5.4-mini-2026-03-17_none","src":"https://htihle.github.io/weirdml.html","v":0.379},{"b":"weirdml","c":0.016606658823529415,"m":"gpt-4-1-mini-2025-04-14","oid":"weirdml-93802c39c820e278","rd":"2025-04-14","se":0.0001109914843500551,"sid":"recl9hxiaWkqe40FN","sm":"gpt-4.1-mini-2025-04-14","src":"https://htihle.github.io/weirdml.html","v":0.3761},{"b":"weirdml","c":0.019730824705882354,"m":"deepseek-v3-1","oid":"weirdml-9b514bfcd85b8171","rd":"2025-08-21","se":0.0001137810779787712,"sid":"recSE9wLRGC5yRt2x","sm":"DeepSeek-V3.1","src":"https://htihle.github.io/weirdml.html","v":0.375},{"b":"weirdml","c":0.02087528789,"m":"qwen3-235b-a22b-thinking-2507","oid":"weirdml-b6359c54d0f48a80","rd":"2025-07-25","sid":"recQxexQ8ZXyXQfY3","sm":"Qwen3-235B-A22B-Thinking-2507","src":"https://htihle.github.io/weirdml.html","v":0.3728},{"b":"weirdml","c":0.020875287894736843,"m":"qwen3-235b-a22b","oid":"weirdml-a8d3c3428f8f99e6","rd":"2025-04-29","se":0.00009299174150022437,"sid":"recWOQqT8ls0zAHYo","sm":"Qwen3-235B-A22B","v":0.3728},{"b":"weirdml","c":0.24772464411764714,"m":"grok-3","oid":"weirdml-1b2cc230c9736428","rd":"2025-04-09","se":0.00011776232968866098,"sid":"rec1A4AJeDxbjisAk","sm":"grok-3","src":"https://htihle.github.io/weirdml.html","v":0.3724},{"b":"weirdml","c":0.03490697524752475,"m":"minimax-m2-7","oid":"weirdml-34e54f7f496c9b98","rd":"2026-03-18","se":0.00014318297014256491,"sid":"recr7AW8WaCqW0Akc","sm":"MiniMax-M2.7","src":"https://htihle.github.io/weirdml.html","v":0.36950000000000005},{"b":"weirdml","c":0.0038062152941176477,"cfg":{"effort":"medium"},"e":"medium","m":"gpt-oss-20b","oid":"weirdml-26adcdce84f3b850","rd":"2025-08-05","se":0.00013480181366483192,"sid":"recfATNb3nYbAeRBC","sm":"gpt-oss-20b_medium","src":"https://htihle.github.io/weirdml.html","v":0.3683},{"b":"weirdml","c":0.04181978350515465,"m":"kimi-k2-instruct-0905","oid":"weirdml-0922bb086e33b0a4","rd":"2025-09-05","se":0.00015131549735094337,"sid":"rec5rZXinvvAnv0LQ","sm":"Kimi-K2-Instruct-0905","src":"https://htihle.github.io/weirdml.html","v":0.3668},{"b":"weirdml","c":0.07850552000000001,"m":"deepseek-r1","oid":"weirdml-4f776a79914f9e2a","rd":"2025-01-20","se":0.00010907363270026519,"sid":"recWpIUcWCiiJjsuv","sm":"DeepSeek-R1","src":"https://htihle.github.io/weirdml.html","v":0.3649},{"b":"weirdml","c":0.11127495238095236,"cfg":{"effort":"medium"},"e":"medium","m":"o1-mini-2024-09-12","oid":"weirdml-68115d9aaf85001e","rd":"2024-09-12","se":0.00014606121859779935,"sid":"rec9nHByk40IO0MOR","sm":"o1-mini-2024-09-12_medium","src":"https://htihle.github.io/weirdml.html","v":0.3632},{"b":"weirdml","c":0.013903999058823524,"m":"deepseek-v3-0324","oid":"weirdml-2cdc62df1b15693f","rd":"2025-03-24","se":0.00018047638488570057,"sid":"recRV03WT6QZrAKTk","sm":"DeepSeek-V3-0324","src":"https://htihle.github.io/weirdml.html","v":0.3608},{"b":"weirdml","c":0.03025316353,"m":"gemini-2-5-flash-lite-preview-06-17-16k","oid":"weirdml-9d1fcd2232825f3c","rd":"2025-06-17","sid":"recCOfjHgyxAboN8H","sm":"gemini-2.5-flash-lite-preview-06-17_16K","src":"https://htihle.github.io/weirdml.html","v":0.3522},{"b":"weirdml","c":0.030253163529411764,"m":"gemini-2-5-flash-lite-preview-06-17-thinking","oid":"weirdml-e26fa46888e12f63","rd":"2025-06-17","se":0.00018253871445538628,"sid":"rectFGVtIctxXIHpZ","sm":"gemini-2.5-flash-lite-preview-06-17-thinking","v":0.3522},{"b":"weirdml","c":0.020793745,"m":"gemma-4-26b-a4b","oid":"weirdml-0b473194f126a94a","rd":"2026-04-02","se":0.00010492592726399106,"sid":"recBIBEDeXoCbo1WJ","sm":"gemma-4-26b-a4b","src":"https://htihle.github.io/weirdml.html","v":0.3517},{"b":"weirdml","c":0.023491821454545456,"m":"grok-code-fast-1","oid":"weirdml-b8878e47dcd6a619","rd":"2025-08-28","se":0.00014404097485279613,"sid":"recNqAyRo9Bchbr4t","sm":"grok-code-fast-1","src":"https://htihle.github.io/weirdml.html","v":0.3506},{"b":"weirdml","c":0.05961612000000001,"m":"qwen3-6-35b-a3b","oid":"weirdml-274b4f0a469cbfdb","rd":"2026-04-14","se":0.00014262488975378608,"sid":"rec8PK1T6HNd0I4pj","sm":"qwen3.6-35b-a3b","src":"https://htihle.github.io/weirdml.html","v":0.34490000000000004},{"b":"weirdml","c":0.032830421568627456,"m":"qwen3-coder-next","oid":"weirdml-6b01eb18df19e626","rd":"2026-02-02","se":0.00015170115908041672,"sid":"recSrZj9sCvfB6GAm","sm":"qwen3-coder-next","v":0.344},{"b":"weirdml","c":0.040179830588235294,"m":"mistral-medium-2508","oid":"weirdml-658a90f245dc92c3","rd":"2025-08-12","se":0.00012017965528137121,"sid":"recaGKXTNqQWwgcGX","sm":"mistral-medium-2508","src":"https://htihle.github.io/weirdml.html","v":0.33130000000000004},{"b":"weirdml","c":0.12554279170588237,"cfg":{"effort":"high"},"e":"high","m":"inkling","oid":"weirdml-952d60320f4871e7","rd":"2026-07-15","se":0.00017455866904238094,"sid":"recvwP0nt7dqhRuLF","sm":"Inkling_high","src":"https://htihle.github.io/weirdml.html","v":0.3229},{"b":"weirdml","c":0.24560110588235293,"m":"claude-3-5-sonnet-2024-06-20","oid":"weirdml-4a1ee6f0efaf637f","rd":"2024-06-20","se":0.00009722443701120124,"sid":"recZ2Fnhy1oCABquH","sm":"claude-3-5-sonnet-20240620","src":"https://htihle.github.io/weirdml.html","v":0.3097},{"b":"weirdml","c":0.05748017882352941,"m":"claude-3-5-haiku-2024-10-22","oid":"weirdml-c76801ab0692dad8","rd":"2024-10-22","se":0.00013622502133312476,"sid":"reccOpUe11CUnfz3D","sm":"claude-3-5-haiku-20241022","src":"https://htihle.github.io/weirdml.html","v":0.3073},{"b":"weirdml","c":0.008148141553,"m":"qwen3-30b-a3b","oid":"weirdml-399d95cccac6baf7","rd":"2025-04-29","sid":"reccesPAGx7VM9isY","sm":"Qwen3-30B-A3B","src":"https://htihle.github.io/weirdml.html","v":0.2975},{"b":"weirdml","c":0.004474679117647059,"cfg":{"effort":"low"},"e":"low","m":"gpt-5-nano-2025-08-07","oid":"weirdml-96506d9fe5484ae4","rd":"2025-08-07","se":0.0000860594758851737,"sid":"recrxrcJRqQrfigzs","sm":"gpt-5-nano-2025-08-07_low","src":"https://htihle.github.io/weirdml.html","v":0.2589},{"b":"weirdml","c":0.006734424117647059,"m":"gemini-2-0-flash-001","oid":"weirdml-09cf46bdad37956d","rd":"2025-02-05","se":0.00011148983505136389,"sid":"recnyEQFFHcxoiWZd","sm":"gemini-2.0-flash-001","src":"https://htihle.github.io/weirdml.html","v":0.2577},{"b":"weirdml","c":0.0835560588235294,"m":"gpt-4o-2024-11-20","oid":"weirdml-977011aff25accee","rd":"2024-11-20","se":0.0001312140616428375,"sid":"recH4eRsQ6SLuBw9B","sm":"gpt-4o-2024-11-20","src":"https://htihle.github.io/weirdml.html","v":0.25120000000000003},{"b":"weirdml","c":0.0027842108823529408,"m":"gemini-1-5-flash-002","oid":"weirdml-540b68255444acbd","rd":"2024-09-24","se":0.00009350927706881112,"sid":"recOirGXtvoehxH10","sm":"gemini-1.5-flash-002","src":"https://htihle.github.io/weirdml.html","v":0.2487},{"b":"weirdml","c":0.006132220588235293,"m":"llama-4-maverick-17b-128e-instruct","oid":"weirdml-68cd1817d0c9a88b","rd":"2025-04-05","se":0.00011000248403000925,"sid":"recjgmcFQfC0Kfg2a","sm":"Llama-4-Maverick-17B-128E-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.2447},{"b":"weirdml","c":0.11412654117647059,"m":"grok-2-1212","oid":"weirdml-29f5db5596e5287c","rd":"2024-12-12","se":0.00014919612352069363,"sid":"recDbZKRnS6z5OZTF","sm":"grok-2-1212","src":"https://htihle.github.io/weirdml.html","v":0.2224},{"b":"weirdml","c":0.05045860294117647,"m":"gemini-1-5-pro-002","oid":"weirdml-7528f585044d9968","rd":"2024-09-24","se":0.0001324555730288255,"sid":"recppqHhl200vYPWu","sm":"gemini-1.5-pro-002","src":"https://htihle.github.io/weirdml.html","v":0.222},{"b":"weirdml","c":0.02029766588235294,"m":"llama-3-1-405b-instruct","oid":"weirdml-e40d649d81130126","rd":"2024-07-23","se":0.0001471030627450674,"sid":"recnDucX915Gq1w1w","sm":"Llama-3.1-405B-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.2138},{"b":"weirdml","c":0.856788,"m":"claude-3-opus-2024-02-29","oid":"weirdml-fe5360e8c55ea281","rd":"2024-02-29","se":0.00012705431504968014,"sid":"recWD9aNKNFHjknWF","sm":"claude-3-opus-20240229","src":"https://htihle.github.io/weirdml.html","v":0.19219999999999998},{"b":"weirdml","c":0.0025817174757281555,"m":"gpt-4-1-nano-2025-04-14","oid":"weirdml-cf4909a5ba871353","rd":"2025-04-14","se":0.00011759249970392629,"sid":"recUJYFZqBsEuykUe","sm":"gpt-4.1-nano-2025-04-14","src":"https://htihle.github.io/weirdml.html","v":0.1898},{"b":"weirdml","c":0.32102783132530116,"m":"gpt-4-turbo-2024-04-09","oid":"weirdml-16ff664f87700937","rd":"2024-04-09","se":0.00014098474583975408,"sid":"recteY2LQE2gKMz1y","sm":"gpt-4-turbo-2024-04-09","src":"https://htihle.github.io/weirdml.html","v":0.1801},{"b":"weirdml","c":0.011327742138364781,"m":"qwen2-5-72b-instruct","oid":"weirdml-8c465a98853ed725","rd":"2024-09-19","se":0.0000844050021308925,"sid":"recX1yJGXQJH8sZXs","sm":"qwen2.5-72b-instruct","v":0.1597},{"b":"weirdml","c":0.0014694893882352942,"m":"llama-3-3-70b-instruct","oid":"weirdml-4f2b1e856a258d69","rd":"2024-12-06","se":0.00011523640365438144,"sid":"rec47nnjaiMGa2Hvi","sm":"Llama-3.3-70B-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.1444},{"b":"weirdml","c":0.6376224705882353,"m":"gpt-4-0613","oid":"weirdml-60c1a6d92381c27b","rd":"2023-06-13","se":0.00017596677892746298,"sid":"recUY5OtXWCsKXibx","sm":"gpt-4-0613","src":"https://htihle.github.io/weirdml.html","v":0.1236},{"b":"weirdml","c":0.005260762500000001,"m":"gpt-4o-mini-2024-07-18","oid":"weirdml-1bba1437bbe3ae0c","rd":"2024-07-18","se":0.00013958141397025036,"sid":"recUUJ0BGuNnLDeXK","sm":"gpt-4o-mini-2024-07-18","src":"https://htihle.github.io/weirdml.html","v":0.1176},{"b":"weirdml","c":0.010191000000000006,"m":"qwen2-72b-instruct","oid":"weirdml-0ffabc45f825bd83","rd":"2024-06-07","se":0.0001139030639865106,"sid":"recwNn5qNiY4HZMn3","sm":"qwen2-72b-instruct","src":"https://htihle.github.io/weirdml.html","v":0.113},{"b":"weirdml","c":0.17829060714285716,"m":"claude-3-sonnet-2024-02-29","oid":"weirdml-1d51ae6eb0744599","rd":"2024-02-29","se":0.0001521221735455649,"sid":"recr58UTdDueiBsM9","sm":"claude-3-sonnet-20240229","src":"https://htihle.github.io/weirdml.html","v":0.10160000000000001},{"b":"weirdml","c":0.014075108823529416,"m":"claude-3-haiku-2024-03-07","oid":"weirdml-94ef3ecbab0d3d8e","rd":"2024-03-07","se":0.00011264697856541451,"sid":"recgnkWUBffrHJLJ3","sm":"claude-3-haiku-20240307","src":"https://htihle.github.io/weirdml.html","v":0.0984},{"b":"weirdml","c":0.003222669882352941,"m":"llama-3-1-70b-instruct","oid":"weirdml-2200938d97c86261","rd":"2024-07-23","se":0.00013647196826037332,"sid":"recFJHpXHYhXFptDA","sm":"Llama-3.1-70B-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.0897},{"b":"weirdml","c":0.1724719012345679,"m":"claude-2-1","oid":"weirdml-0f28364049eaf522","rd":"2023-11-21","se":0.00009088852126102397,"sid":"recMLC0xkEkCfcRb8","sm":"claude-2.1","src":"https://htihle.github.io/weirdml.html","v":0.0706},{"b":"weirdml","c":0.04237081395348837,"m":"gpt-3-5-turbo-0125","oid":"weirdml-b72f3052ea93c384","rd":"2024-01-25","se":0.00006328795727774941,"sid":"recZRRnQERBJbnqYb","sm":"gpt-3.5-turbo-0125","src":"https://htihle.github.io/weirdml.html","v":0.0348},{"b":"weirdml","c":0.03903073333333333,"m":"mixtral-8x22b-instruct-v0-1","oid":"weirdml-9f179ff14f554f3a","rd":"2024-04-17","se":0.00007155156980290171,"sid":"recZSSo7RkyUGkJFX","sm":"Mixtral-8x22B-Instruct-v0.1","src":"https://htihle.github.io/weirdml.html","v":0.0317},{"b":"weirdml","c":0.00041039552941176473,"m":"llama-3-1-8b-instruct","oid":"weirdml-2c52db8af613ff3e","rd":"2024-07-23","se":0.000055259673993004597,"sid":"rech2mtGGqFrKRxQO","sm":"Llama-3.1-8B-Instruct","src":"https://htihle.github.io/weirdml.html","v":0.0173},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"PaLM 2-L","m":"palm-2-l","oid":"wino_grande-e70d55bc9674e3ce","rd":"2023-05-17","sid":"rec0Dj1umKnw4t9yi","sm":"PaLM 2-L","src":"https://arxiv.org/pdf/2305.10403","v":0.83},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"GPT-4","m":"gpt-4-0314","oid":"wino_grande-253040d7996196d1","rd":"2023-03-14","sid":"recaR4Pf3RkCiAANz","sm":"gpt-4-0314","src":"https://arxiv.org/pdf/2303.08774","v":0.875},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"5-shot PaLM","m":"palm-540b","oid":"wino_grande-cc581f64eded1ca0","rd":"2022-04-04","sid":"recAD31ZnGeO3eOwm","sm":"PaLM 540B","src":"https://arxiv.org/pdf/2303.08774","v":0.851},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"GPT-3.5","m":"text-davinci-002","oid":"wino_grande-9b620993b9486101","rd":"2022-03-15","sid":"recXkgGBeKRwNlOKi","sm":"text-davinci-002","src":"https://arxiv.org/pdf/2303.08774","v":0.816},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"PaLM 540B","m":"palm-540b","oid":"wino_grande-dfc5fb9a7cffa499","rd":"2022-04-04","sid":"recWVCWC5EMRDNSgL","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.811},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"PaLM 540B","m":"palm-540b","oid":"wino_grande-142b4ff918b144e3","rd":"2022-04-04","sid":"rec55d9FyJAq5fQhK","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.837},{"b":"wino_grande","cfg":{"shots":"few"},"dn":"PaLM 540B","m":"palm-540b","oid":"wino_grande-8ffa2533b4136bbc","rd":"2022-04-04","sid":"recx099eSrOmwL3QU","sm":"PaLM 540B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.851},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"PaLM 2-M","m":"palm-2-m","oid":"wino_grande-e36a08d0f4d7100c","rd":"2023-05-17","sid":"recFe6bxFdt4wLRkZ","sm":"PaLM 2-M","src":"https://arxiv.org/pdf/2305.10403","v":0.792},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"PaLM 2-S","m":"palm-2-s","oid":"wino_grande-616cc119b2fe5aca","rd":"2023-05-17","sid":"recuJFdgWw2tZaQoH","sm":"PaLM 2-S","src":"https://arxiv.org/pdf/2305.10403","v":0.779},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"PaLM 62B","m":"palm-62b","oid":"wino_grande-e998c137587a6a6a","rd":"2022-04-04","sid":"recsfrlaaHsWLi9qx","sm":"PaLM 62B","src":"https://www.jmlr.org/papers/volume24/22-1144/22-1144.pdf","v":0.77},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-3 175B","m":"text-davinci-001","oid":"wino_grande-d94337e72cbc09c9","rd":"2022-01-27","sid":"rec8u6KWQWeBGN0Km","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2302.13971","v":0.702},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gopher 280B","m":"gopher-280b","oid":"wino_grande-77664f77193295ab","rd":"2021-12-08","sid":"recDkEgLhPRjy4v8h","sm":"Gopher (280B)","src":"https://arxiv.org/pdf/2302.13971","v":0.701},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"wino_grande-78fa4239b401f41f","rd":"2022-03-29","sid":"recGnjvDzN0ewjxiG","sm":"Chinchilla (70B)","src":"https://arxiv.org/pdf/2302.13971","v":0.749},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"PaLM 62B","m":"palm-62b","oid":"wino_grande-026cc1c63a3e354f","rd":"2022-04-04","sid":"reclQUJcaj2JrjyI8","sm":"PaLM 62B","src":"https://arxiv.org/pdf/2302.13971","v":0.77},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"PaLM 540B","m":"palm-540b","oid":"wino_grande-4a12a7ec874a7509","rd":"2022-04-04","sid":"recI8uxkEPA3KQdbm","sm":"PaLM 540B","src":"https://arxiv.org/pdf/2302.13971","v":0.811},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"wino_grande-89b2da75f4547826","rd":"2023-02-24","sid":"recIMvkFXTo2hwuhs","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2302.13971","v":0.701},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"wino_grande-297e777172a5c147","rd":"2023-02-24","sid":"rechN7puQcuTcXy3a","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2302.13971","v":0.73},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 65B","m":"llama-65b","oid":"wino_grande-a53bb65b916de202","rd":"2023-02-24","sid":"recKKmmLkXD2Vjn2e","sm":"LLaMA-65B","src":"https://arxiv.org/pdf/2302.13971","v":0.77},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 33B","m":"llama-33b","oid":"wino_grande-a01edc72755812c8","rd":"2023-02-24","sid":"recBt1SpsffLcrMS9","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2302.13971","v":0.76},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"wino_grande-41a021ed91400530","rd":"2023-09-27","sid":"recfxiFrvSCnxbjsX","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/abs/2203.15556","v":0.753},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Chinchilla 70B","m":"chinchilla-70b","oid":"wino_grande-c958c15048c6b23e","rd":"2022-03-29","sid":"recZr3ktMwbyI4XeS","sm":"Chinchilla (70B)","src":"https://arxiv.org/abs/2203.15556","v":0.749},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Claude 3 Sonnet","m":"claude-3-sonnet-2024-02-29","oid":"wino_grande-7b295ab51cea9b20","rd":"2024-02-29","sid":"recfo636SsgKz6bre","sm":"claude-3-sonnet-20240229","src":"https://assets.anthropic.com/m/61e7d27f8c8f5919/original/Claude-3-Model-Card.pdf","v":0.751},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Claude 3 Haiku","m":"claude-3-haiku-2024-03-07","oid":"wino_grande-7088fb7f062e2f58","rd":"2024-03-07","sid":"rec48LdyvDEsxod6W","sm":"claude-3-haiku-20240307","src":"https://assets.anthropic.com/m/61e7d27f8c8f5919/original/Claude-3-Model-Card.pdf","v":0.742},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Claude 3 Opus","m":"claude-3-opus-2024-02-29","oid":"wino_grande-4b2667dc5fa50814","rd":"2024-02-29","sid":"recwhfZU1uHwUbOtB","sm":"claude-3-opus-20240229","src":"https://assets.anthropic.com/m/61e7d27f8c8f5919/original/Claude-3-Model-Card.pdf","v":0.885},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 13B","m":"llama-13b","oid":"wino_grande-c926e0892c56756f","rd":"2023-02-24","sid":"recpYeKQZjkeJmFjK","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2302.13971","v":0.73},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-e97baecdd879e00d","rd":"2022-01-27","sid":"recuE1AFI2Xi9TRZy","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.702},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-f432e3fd77d0b9a8","rd":"2022-01-27","sid":"recFDEwWa9MDvdGp7","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.732},{"b":"wino_grande","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-344962a04fb798a8","rd":"2022-01-27","sid":"reczqA3lUwnnCXwU9","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2005.14165v4","v":0.777},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gopher 280B","m":"gopher-280b","oid":"wino_grande-65d7bfc3d7d6e26e","rd":"2021-12-08","sid":"recCO5IfFJGtcuttU","sm":"Gopher (280B)","src":"https://arxiv.org/abs/2203.15556","v":0.701},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA 7B","m":"llama-7b","oid":"wino_grande-3b65a1fd1a733fdf","rd":"2023-02-24","sid":"recmWn1dhJq25abBB","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2302.13971","v":0.701},{"b":"wino_grande","dn":"GPT-2-XL 1.5B","m":"gpt2-xl","oid":"wino_grande-fb95fd97ad0f36eb","rd":"2019-11-05","sid":"rec9CyRHN8LPTzTOx","sm":"gpt2-xl","v":0.583},{"b":"wino_grande","dn":"MPT 7B","m":"mpt-7b","oid":"wino_grande-42b1ec7515e30c69","rd":"2023-05-05","sid":"recktbU0qGqP0imC6","sm":"mpt-7b","src":"https://arxiv.org/pdf/2307.09288","v":0.683},{"b":"wino_grande","dn":"MPT 30B","m":"mpt-30b","oid":"wino_grande-970b114ae1067aff","rd":"2023-06-22","sid":"recT5IXUPTJfXiCp2","sm":"mpt-30b","src":"https://arxiv.org/pdf/2307.09288","v":0.71},{"b":"wino_grande","dn":"Falcon 7B","m":"falcon-7b","oid":"wino_grande-13a04688e3cdc4c4","rd":"2023-04-24","sid":"recSntxSuHYG1VyHo","sm":"falcon-7b","src":"https://arxiv.org/pdf/2307.09288","v":0.663},{"b":"wino_grande","dn":"Falcon 40B","m":"falcon-40b","oid":"wino_grande-ed3a4e7ebebce10b","rd":"2023-03-15","sid":"recgYrmCHlDlyq9yx","sm":"falcon-40b","src":"https://arxiv.org/pdf/2307.09288","v":0.769},{"b":"wino_grande","dn":"LLAMA 1 7B","m":"llama-7b","oid":"wino_grande-d25b684d57338ca5","rd":"2023-02-24","sid":"rec7Ph7OFjE8lEJHt","sm":"LLaMA-7B","src":"https://arxiv.org/pdf/2307.09288","v":0.701},{"b":"wino_grande","dn":"LLAMA 1 13B","m":"llama-13b","oid":"wino_grande-adfdac09d4baed0b","rd":"2023-02-24","sid":"recT2f8OQfNtIgwue","sm":"LLaMA-13B","src":"https://arxiv.org/pdf/2307.09288","v":0.73},{"b":"wino_grande","dn":"LLAMA 1 33B","m":"llama-33b","oid":"wino_grande-1b55346ab12cdecb","rd":"2023-02-24","sid":"recJAkorJg8DBb8w6","sm":"LLaMA-33B","src":"https://arxiv.org/pdf/2307.09288","v":0.76},{"b":"wino_grande","dn":"LLAMA 1 65B","m":"llama-65b","oid":"wino_grande-ae3908cd846f4733","rd":"2023-02-24","sid":"rec7MyukTwpdy8CSL","sm":"LLaMA-65B","src":"https://arxiv.org/pdf/2307.09288","v":0.77},{"b":"wino_grande","dn":"LLaMA 2 7B","m":"llama-2-7b","oid":"wino_grande-48b053b186f06d60","rd":"2023-07-18","sid":"recEkDlzbxAaHusuf","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2307.09288","v":0.692},{"b":"wino_grande","dn":"LLaMA 2 13B","m":"llama-2-13b","oid":"wino_grande-33d8a4f06c1f63d7","rd":"2023-07-18","sid":"recdNf2o3JtJPXmKH","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2307.09288","v":0.728},{"b":"wino_grande","dn":"LLaMA 2 34B","m":"llama-2-34b","oid":"wino_grande-3f3192a10a444c3d","rd":"2023-07-18","sid":"recd4W7iTigIgk5d5","sm":"Llama-2-34b","src":"https://arxiv.org/pdf/2307.09288","v":0.767},{"b":"wino_grande","dn":"LLaMA 2 70B","m":"llama-2-70b-hf","oid":"wino_grande-271bce05cd38eeda","rd":"2023-07-18","sid":"recaurFL0h5T9Whbj","sm":"Llama-2-70b-hf","src":"https://arxiv.org/pdf/2307.09288","v":0.802},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"LLaMA 3 8B","m":"meta-llama-3-8b","notes":"\"For pre-trained models, we use a choice based setup for evaluation where we fill in the missing blank with the two possible choices and then compute log-likelihood over the suffix. We use a 5-shot config. We use the MMLU setup where we provide all the choices in the prompt and calculate likelihood over choice characters.\"","oid":"wino_grande-6654a00c98e2cc68","rd":"2024-04-18","sid":"recJOWqocH7c9al6M","sm":"Meta-Llama-3-8B","src":"https://arxiv.org/pdf/2407.21783","v":0.757},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"LLaMA 3 70B","m":"meta-llama-3-70b","notes":"\"For pre-trained models, we use a choice based setup for evaluation where we fill in the missing blank with the two possible choices and then compute log-likelihood over the suffix. We use a 5-shot config. We use the MMLU setup where we provide all the choices in the prompt and calculate likelihood over choice characters.\"","oid":"wino_grande-c9baacfa17986b85","rd":"2024-04-18","sid":"rec876vd5CO6jC2xx","sm":"Meta-Llama-3-70B","src":"https://arxiv.org/pdf/2407.21783","v":0.835},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"LLaMA 3 405B","m":"llama-3-1-405b-instruct","notes":"\"For pre-trained models, we use a choice based setup for evaluation where we fill in the missing blank with the two possible choices and then compute log-likelihood over the suffix. We use a 5-shot config. We use the MMLU setup where we provide all the choices in the prompt and calculate likelihood over choice characters.\"","oid":"wino_grande-acf636fe54d99bdc","rd":"2024-07-23","sid":"rectAF9Y0lcDhgjKK","sm":"Llama-3.1-405B-Instruct","src":"https://arxiv.org/pdf/2407.21783","v":0.822},{"b":"wino_grande","dn":"Qwen2.5-Coder-0.5B","m":"qwen2-5-coder-0-5b","oid":"wino_grande-ed13bf26493349dd","rd":"2024-09-18","sid":"recUhmmH8qA5xRclp","sm":"Qwen2.5-Coder-0.5B","src":"https://arxiv.org/pdf/2409.12186","v":0.548},{"b":"wino_grande","dn":"Qwen2.5-Coder-1.5B","m":"qwen2-5-coder-1-5b","oid":"wino_grande-9c4e2f65d652dd83","rd":"2024-09-18","sid":"recO1g2UPif64BQXO","sm":"Qwen2.5-Coder-1.5B","src":"https://arxiv.org/pdf/2409.12186","v":0.607},{"b":"wino_grande","dn":"Qwen2.5-Coder-3B","m":"qwen2-5-coder-3b","oid":"wino_grande-7a204dbf3ba12dfe","rd":"2024-09-18","sid":"recPKH8NBSdyoeNF7","sm":"Qwen2.5-Coder-3B","src":"https://arxiv.org/pdf/2409.12186","v":0.674},{"b":"wino_grande","dn":"Qwen2.5-Coder-7B","m":"qwen2-5-coder-7b","oid":"wino_grande-25f8e4663f6e4097","rd":"2024-09-18","sid":"recsRQvpUY7AtRA2v","sm":"Qwen2.5-Coder-7B","src":"https://arxiv.org/pdf/2409.12186","v":0.729},{"b":"wino_grande","dn":"Qwen2.5-Coder-14B","m":"qwen2-5-coder-14b","oid":"wino_grande-678d807062fc4a8a","rd":"2024-09-18","sid":"recWCbZW93Xb7mgLW","sm":"Qwen2.5-Coder-14B","src":"https://arxiv.org/pdf/2409.12186","v":0.768},{"b":"wino_grande","dn":"Qwen2.5-Coder-32B","m":"qwen2-5-coder-32b","oid":"wino_grande-72d5a86348d35a32","rd":"2024-09-18","sid":"rec8gc150DTSOhEKr","sm":"Qwen2.5-Coder-32B","src":"https://arxiv.org/pdf/2409.12186","v":0.808},{"b":"wino_grande","dn":"DS-Coder-1.3B-Base","m":"deepseek-coder-1-3b-base","oid":"wino_grande-36fb6aecf46d974e","rd":"2023-11-02","sid":"recVHFqbsrGnj2Vx2","sm":"deepseek-coder-1.3b-base","src":"https://arxiv.org/pdf/2409.12186","v":0.533},{"b":"wino_grande","dn":"StarCoder2-3B","m":"starcoder2-3b","oid":"wino_grande-6fa700926a9fc05e","rd":"2024-02-22","sid":"recyselPDxHqK6S7m","sm":"starcoder2-3b","src":"https://arxiv.org/pdf/2409.12186","v":0.571},{"b":"wino_grande","dn":"StarCoder2-7B","m":"starcoder2-7b","oid":"wino_grande-a9da4890f31026a0","rd":"2024-02-20","sid":"recXQnGi8m8ri82zZ","sm":"starcoder2-7b","src":"https://arxiv.org/pdf/2409.12186","v":0.571},{"b":"wino_grande","dn":"DS-Coder-6.7B-Base","m":"deepseek-coder-6-7b-base","oid":"wino_grande-77d897c99e4a396c","rd":"2023-11-02","sid":"recjz7QSFeeZcphoR","sm":"deepseek-coder-6.7b-base","src":"https://arxiv.org/pdf/2409.12186","v":0.576},{"b":"wino_grande","dn":"DS-Coder-V2-Lite-Base","m":"deepseek-coder-v2-lite-base","oid":"wino_grande-716374b13b7e3c63","rd":"2024-06-13","sid":"rec8sY5AZGjTEHSIb","sm":"DeepSeek-Coder-V2-Lite-Base","src":"https://arxiv.org/pdf/2409.12186","v":0.729},{"b":"wino_grande","dn":"CodeQwen1.5-7B","m":"codeqwen1-5-7b","oid":"wino_grande-9210fbdbc3221719","rd":"2024-04-15","sid":"recZQWDgmY6X1bV95","sm":"CodeQwen1.5-7B","src":"https://arxiv.org/pdf/2409.12186","v":0.598},{"b":"wino_grande","dn":"StarCoder2-15B","m":"starcoder2-15b","oid":"wino_grande-e07608a2df1f9666","rd":"2024-02-20","sid":"recr8hrjiAzMZKfoe","sm":"starcoder2-15b","src":"https://arxiv.org/pdf/2409.12186","v":0.643},{"b":"wino_grande","dn":"DS-Coder-33B-Base","m":"deepseek-coder-33b-base","oid":"wino_grande-cc08103afd961ee6","rd":"2023-11-02","sid":"recU2oocllLjEXaxq","sm":"deepseek-coder-33b-base","src":"https://arxiv.org/pdf/2409.12186","v":0.62},{"b":"wino_grande","dn":"DS-Coder-V2-Base","m":"deepseek-coder-v2-base","oid":"wino_grande-a7c1f798f68481ba","rd":"2024-06-17","sid":"rec7J2f61YnAY2lHm","sm":"DeepSeek-Coder-V2-Base","src":"https://arxiv.org/pdf/2409.12186","v":0.837},{"b":"wino_grande","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"wino_grande-0ac51cbbf5929887","rd":"2023-09-27","sid":"rec5MLRl0r4y0sPhr","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.742},{"b":"wino_grande","dn":"Mixtral 8x7B","m":"mixtral-8x7b-v0-1","oid":"wino_grande-3cc886d2eb884391","rd":"2023-12-11","sid":"recOiKNC7k9QoMUyx","sm":"Mixtral-8x7B-v0.1","src":"https://arxiv.org/pdf/2401.04088","v":0.772},{"b":"wino_grande","dn":"Gemma 7B","m":"gemma-7b","notes":"Run with HuggingFace H6 suite","oid":"wino_grande-47a85f5404981c02","rd":"2024-02-21","sid":"recVCwFxC4Uz2gxlu","sm":"gemma-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.79},{"b":"wino_grande","dn":"LLaMA-2 7B","m":"llama-2-7b","oid":"wino_grande-a0ae322fe6829304","rd":"2023-07-18","sid":"rechilCTigqP9uXxA","sm":"Llama-2-7b","src":"https://arxiv.org/pdf/2403.08295","v":0.692},{"b":"wino_grande","dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"wino_grande-25d33234f498cbcb","rd":"2023-07-18","sid":"reczujVPa7Ol17Ep3","sm":"Llama-2-13b","src":"https://arxiv.org/pdf/2403.08295","v":0.728},{"b":"wino_grande","dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"wino_grande-fe55d11f89ffc618","rd":"2023-09-27","sid":"recdZaaWYWrX1u3Si","sm":"Mistral-7B-v0.1","src":"https://arxiv.org/pdf/2403.08295","v":0.742},{"b":"wino_grande","dn":"Gemma 2B","m":"gemma-2b","oid":"wino_grande-bc90435526598a12","rd":"2024-02-21","sid":"recAROl6cvkyIz2XU","sm":"gemma-2b","src":"https://arxiv.org/pdf/2403.08295","v":0.654},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"DeepSeek-V2 Base","m":"deepseek-v2","oid":"wino_grande-5f76bc37760618bc","rd":"2024-05-07","sid":"recv143kxseDbH93b","sm":"DeepSeek-V2","src":"https://arxiv.org/pdf/2412.19437","v":0.863},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"DeepSeek-V3 Base","m":"deepseek-v3","oid":"wino_grande-0ae43b10a3bbb2e7","rd":"2024-12-26","sid":"recRvIn2aFK6GuRSa","sm":"DeepSeek-V3","src":"https://arxiv.org/pdf/2412.19437","v":0.852},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"LLaMA-3.1 405B Base","m":"llama-3-1-405b","oid":"wino_grande-e363679805beeeb0","rd":"2024-07-23","sid":"rec6ChHalhgDSzRNN","sm":"Llama-3.1-405B","src":"https://arxiv.org/pdf/2412.19437","v":0.892},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Qwen2.5 72B Base","m":"qwen2-5-72b","oid":"wino_grande-6e5f9d1ea066f501","rd":"2024-09-19","sid":"recFEIHaal9FCmhYN","sm":"Qwen2.5-72B","src":"https://arxiv.org/pdf/2412.19437","v":0.823},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"phi-1.5","m":"phi-1-5","oid":"wino_grande-ad8198e0af11562e","rd":"2023-09-11","sid":"rechsjFFmjkjr59Th","sm":"phi-1_5","src":"http://arxiv.org/abs/2309.05463","v":0.734},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Vicuna-13B (v1.1)","m":"vicuna-13b-v1-1","oid":"wino_grande-fec8f03c4f755e20","rd":"2023-04-12","sid":"rec1zWhRyar1a1ya7","sm":"vicuna-13b-v1.1","src":"http://arxiv.org/abs/2309.05463","v":0.708},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Llama2-7B","m":"llama-2-7b","oid":"wino_grande-d4ac13da4a2fb505","rd":"2023-07-18","sid":"rec6mSfCZRnko6k95","sm":"Llama-2-7b","src":"http://arxiv.org/abs/2309.05463","v":0.691},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Llama-7B","m":"llama-7b","oid":"wino_grande-73cb5b359fd2d650","rd":"2023-02-24","sid":"recnremIrBRActHkF","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.05463","v":0.669},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"wino_grande-9c6121a098d50c92","rd":"2023-05-05","sid":"recIj2gakuqj9fTET","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.05463","v":0.68},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"wino_grande-f4d91867c4ff399c","rd":"2023-04-24","sid":"recMpIBB0KkHPY3G4","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.05463","v":0.662},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"OPT-1.3B","m":"opt-1-3b","oid":"wino_grande-307f72a0e16debcd","rd":"2022-05-11","sid":"recfHebaHDX7FJeud","sm":"opt-1.3b","src":"http://arxiv.org/abs/2309.05463","v":0.61},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT2-XL-1.5B","m":"gpt2-xl","oid":"wino_grande-c5b860a0e0bc639c","rd":"2019-11-05","sid":"rec4vKSLgu8gPF3BP","sm":"gpt2-xl","src":"http://arxiv.org/abs/2309.05463","v":0.583},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Yi-6B","m":"yi-6b","oid":"wino_grande-7bb368682d31b852","rd":"2023-11-02","sid":"recHcfsJvgD1v3Za1","sm":"Yi-6B","src":"http://arxiv.org/abs/2403.04652","v":0.713},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Yi-9B","m":"yi-9b","oid":"wino_grande-4c9346de22b44a18","rd":"2024-03-01","sid":"rec2uwtoTBL3OG07a","sm":"Yi-9B","src":"http://arxiv.org/abs/2403.04652","v":0.73},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"wino_grande-ef250a4f0e51e225","rd":"2023-04-24","sid":"rec0bXWNApugy6PsT","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.6717},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"wino_grande-1d5000ea6634d66d","rd":"2023-03-15","sid":"recCsPlKCS5V7sQ3u","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.764},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"wino_grande-a3d5d54a0e201e84","rd":"2024-05-09","sid":"rec07tuBa7aAoZtf3","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.783},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon-180B","m":"falcon-180b","oid":"wino_grande-84c0225faa1cc8c1","rd":"2023-09-06","sid":"recqDqOI7R6EJDWXm","sm":"falcon-180B","src":"http://arxiv.org/abs/2311.16867","v":0.871},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"GPT-3.5","m":"gpt-3-5-turbo-0613","oid":"wino_grande-a89190f4610f61e6","rd":"2023-06-13","sid":"recVLx9PCXRj3W5o5","sm":"gpt-3.5-turbo-0613","src":"http://arxiv.org/abs/2311.16867","v":0.816},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"GPT-4","m":"gpt-4-32k-0314","oid":"wino_grande-16c30718fd6eb5d4","rd":"2023-03-14","sid":"recHX5MxEY42Quo60","sm":"gpt-4-32k-0314","src":"http://arxiv.org/abs/2311.16867","v":0.875},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Nemotron-4 15B","m":"nemotron-4-15b","oid":"wino_grande-1ca58d512ebae565","rd":"2024-02-26","sid":"rechEkVlfpVzd5dGw","sm":"Nemotron-4 15B","src":"http://arxiv.org/abs/2402.16819","v":0.78},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA-2 13B","m":"llama-2-13b","oid":"wino_grande-9b8e2bb4d415e20a","rd":"2023-07-18","sid":"recqBhxwgsUhCNmVj","sm":"Llama-2-13b","src":"http://arxiv.org/abs/2402.16819","v":0.728},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA-2 34B","m":"llama-2-34b","oid":"wino_grande-ec46aa2264429323","rd":"2023-07-18","sid":"recqPFwpVlEXRAuAZ","sm":"Llama-2-34b","src":"http://arxiv.org/abs/2402.16819","v":0.767},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Mistral 7B","m":"mistral-7b-v0-1","oid":"wino_grande-1b4efeead9441bb1","rd":"2023-09-27","sid":"recIbFwTUgH5IpHS5","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2402.16819","v":0.753},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gemma 7B","m":"gemma-7b","oid":"wino_grande-0599161ae59c0d59","rd":"2024-02-21","sid":"recK1nMPSyPQhO9AS","sm":"gemma-7b","src":"http://arxiv.org/abs/2402.16819","v":0.723},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"wino_grande-816bacb025baa396","rd":"2022-01-27","sid":"recijvxN8UXEh3SBp","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.702},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GLaM (MoE)","m":"glam-moe","oid":"wino_grande-1633683a513b91cf","rd":"2021-12-13","sid":"recqVwM8SVe4yfcGZ","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.735},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"wino_grande-f8bc142aebfa96d5","rd":"2022-01-27","sid":"recl9X5JuCNXMgSCh","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.732},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"GLaM (MoE)","m":"glam-moe","oid":"wino_grande-01b32cc1df2729b3","rd":"2021-12-13","sid":"recu2bgRPUptWpqiE","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.73},{"b":"wino_grande","cfg":{"shots":"16"},"dn":"GPT-3 (175B)","m":"text-davinci-001","oid":"wino_grande-0da2c497976bf54b","rd":"2022-01-27","sid":"recQ2RjlzBCPbvsAn","sm":"text-davinci-001","src":"http://arxiv.org/abs/2112.06905","v":0.777},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gopher (280B)","m":"gopher-280b","oid":"wino_grande-e941f1e17e06df53","rd":"2021-12-08","sid":"recMniA3bIw2cxmTk","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.06905","v":0.701},{"b":"wino_grande","dn":"Megatron-NLG (530B)","m":"megatron-turing-nlg-530b","oid":"wino_grande-9278313e505fbdcd","rd":"2021-10-11","sid":"recgrfBPvZUqv8bhZ","sm":"Megatron-Turing NLG 530B","src":"http://arxiv.org/abs/2112.06905","v":0.789},{"b":"wino_grande","cfg":{"shots":"8"},"dn":"GLaM","m":"glam-moe","oid":"wino_grande-059188212e9086c9","rd":"2021-12-13","sid":"reclLFFps8D3lgnnZ","sm":"GLaM (MoE)","src":"http://arxiv.org/abs/2112.06905","v":0.792},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-3 Zero-Shot","m":"text-davinci-001","oid":"wino_grande-284760914cca6032","rd":"2022-01-27","sid":"recBRuKvfAnXUpJGz","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.702},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"GPT-3 One-Shot","m":"text-davinci-001","oid":"wino_grande-96c0bb55ddb9e427","rd":"2022-01-27","sid":"recBbxInhFByx7qHT","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.732},{"b":"wino_grande","cfg":{"shots":"few"},"dn":"GPT-3 Few-Shot","m":"text-davinci-001","oid":"wino_grande-c162af11494766ac","rd":"2022-01-27","sid":"recLMdsVONh2oFds1","sm":"text-davinci-001","src":"http://arxiv.org/abs/2005.14165","v":0.777},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"wino_grande-91c5f966a7fa2320","rd":"2021-10-11","sid":"rec0czb9iRGEYELDL","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.7301},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"wino_grande-3f4ef4eb9d03844a","rd":"2021-10-11","sid":"recdxz6xNRRDR3Ect","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.7372},{"b":"wino_grande","cfg":{"shots":"few"},"dn":"MT-NLG","m":"megatron-turing-nlg-530b","oid":"wino_grande-d7361232fe6e4033","rd":"2021-10-11","sid":"reciL6h4SYzWnjYZA","sm":"Megatron-Turing NLG 530B","src":"https://arxiv.org/pdf/2201.11990","v":0.7301},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-549ad21403bd6f6f","rd":"2022-01-27","sid":"recMIaObYJI2WngO3","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.702},{"b":"wino_grande","cfg":{"shots":"1"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-ec9ca02bef7896df","rd":"2022-01-27","sid":"recuZIZGrqaZmTQHx","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.732},{"b":"wino_grande","cfg":{"shots":"few"},"dn":"GPT-3","m":"text-davinci-001","oid":"wino_grande-246b46ee0f1c84ce","rd":"2022-01-27","sid":"reczOX5oz5FcrxCks","sm":"text-davinci-001","src":"https://arxiv.org/pdf/2201.11990","v":0.777},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gopher","m":"gopher-280b","oid":"wino_grande-c5f1743ba845ae73","rd":"2021-12-08","sid":"recsE9YoNgrdm5sJC","sm":"Gopher (280B)","src":"https://arxiv.org/pdf/2201.11990","v":0.702},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Phi-3-mini 3.8b","m":"phi-3-mini-4k-instruct","oid":"wino_grande-784a1b3af1e33d5f","rd":"2024-04-23","sid":"rec8NmM7FImqk915G","sm":"Phi-3-mini-4k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.708},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Phi-3-small 7b","m":"phi-3-small-8k-instruct","oid":"wino_grande-a3931589efa16645","rd":"2024-04-23","sid":"rectBk3iRc4LVIqKf","sm":"Phi-3-small-8k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.815},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Phi-3-medium 14b","m":"phi-3-medium-128k-instruct","oid":"wino_grande-7670c637a964026c","rd":"2024-04-23","sid":"recawVi9MBcu9YVNA","sm":"Phi-3-medium-128k-instruct","src":"http://arxiv.org/abs/2404.14219","v":0.815},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Phi-2 2.7b","m":"phi-2","oid":"wino_grande-3e89d355ba9c3870","rd":"2023-12-12","sid":"recj9UKpqcNorm2DD","sm":"phi-2","src":"http://arxiv.org/abs/2404.14219","v":0.547},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Mistral 7b","m":"mistral-7b-v0-1","oid":"wino_grande-fc3088785e83008d","rd":"2023-09-27","sid":"recxXMjf8MjkinJeQ","sm":"Mistral-7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.542},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Gemma 7b","m":"gemma-7b","oid":"wino_grande-286c90e915106307","rd":"2024-02-21","sid":"recQOKrVhA7MxaYqV","sm":"gemma-7b","src":"http://arxiv.org/abs/2404.14219","v":0.556},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Llama-3-In 8b","m":"meta-llama-3-8b-instruct","oid":"wino_grande-a5d872b109837bfd","rd":"2024-04-18","sid":"rec71xWl1TzaoATvj","sm":"Meta-Llama-3-8B-Instruct","src":"http://arxiv.org/abs/2404.14219","v":0.65},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Mixtral 8x7b","m":"mixtral-8x7b-v0-1","oid":"wino_grande-20e5d3ed8942a474","rd":"2023-12-11","sid":"rectsfybsCEwcCVin","sm":"Mixtral-8x7B-v0.1","src":"http://arxiv.org/abs/2404.14219","v":0.62},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"GPT-3.5 version 1106","m":"gpt-3-5-turbo-1106","oid":"wino_grande-99688d59f2842d01","rd":"2023-11-06","sid":"rec3A3yupKXU0nX4B","sm":"gpt-3.5-turbo-1106","src":"http://arxiv.org/abs/2404.14219","v":0.688},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Gopher (k-Shot)","m":"gopher-280b","oid":"wino_grande-d29b1cffce008ec6","rd":"2021-12-08","sid":"recsLsEiFg3I9Ntcq","sm":"Gopher (280B)","src":"http://arxiv.org/abs/2112.11446","v":0.701},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon 7B","m":"falcon-7b","oid":"wino_grande-10d911efddaf97d1","rd":"2023-04-24","sid":"reck2rBf1aBKCdo96","sm":"falcon-7b","src":"http://arxiv.org/abs/2407.14885","v":0.6717},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon 40B","m":"falcon-40b","oid":"wino_grande-f59a78b784afc14d","rd":"2023-03-15","sid":"recc4PwFMQbVu2672","sm":"falcon-40b","src":"http://arxiv.org/abs/2407.14885","v":0.764},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"Falcon2-11B stage 4","m":"falcon-11b","oid":"wino_grande-8d650bad43b9ed51","rd":"2024-05-09","sid":"recXN0upU7mukMw2r","sm":"falcon-11b","src":"http://arxiv.org/abs/2407.14885","v":0.783},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"XGen-7B","m":"xgen-7b-8k-base","oid":"wino_grande-61cf2160c78cf549","rd":"2023-06-27","sid":"recRH0L7wxPtAYPUF","sm":"xgen-7b-8k-base","src":"http://arxiv.org/abs/2309.03450","v":0.649},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"LLaMA-7B","m":"llama-7b","oid":"wino_grande-cbd8edfd8f6ab463","rd":"2023-02-24","sid":"recHZxb3yRFkZMu5a","sm":"LLaMA-7B","src":"http://arxiv.org/abs/2309.03450","v":0.696},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Falcon-7B","m":"falcon-7b","oid":"wino_grande-4af938c412b5ae92","rd":"2023-04-24","sid":"recBPubyTMjeIazf2","sm":"falcon-7b","src":"http://arxiv.org/abs/2309.03450","v":0.672},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"MPT-7B","m":"mpt-7b","oid":"wino_grande-26f918ebd04f0a04","rd":"2023-05-05","sid":"recv1knMvkcKfwZYE","sm":"mpt-7b","src":"http://arxiv.org/abs/2309.03450","v":0.686},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"OpenLLaMA-7B","m":"open-llama-7b","oid":"wino_grande-328100f0f07e7869","rd":"2023-06-07","sid":"recfcPcVYmxdywnKe","sm":"open_llama_7b","src":"http://arxiv.org/abs/2309.03450","v":0.67},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Redpajama-7B","m":"redpajama-incite-7b-base","oid":"wino_grande-32e1505208b9dfe0","rd":"2023-05-04","sid":"recvO9MD0YBHFshpN","sm":"RedPajama-INCITE-7B-Base","src":"http://arxiv.org/abs/2309.03450","v":0.638},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-neox-20B","m":"gpt-neox-20b","oid":"wino_grande-b061b24cccf94caa","rd":"2022-04-07","sid":"rec00370DghOGVzXq","sm":"gpt-neox-20b","src":"http://arxiv.org/abs/2309.03450","v":0.661},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"OPT-13B","m":"opt-13b","oid":"wino_grande-07277b1c05ac93af","rd":"2022-05-11","sid":"rec1gcVGZXPvpvkCm","sm":"opt-13b","src":"http://arxiv.org/abs/2309.03450","v":0.647},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"GPT-J-6B","m":"gpt-j-6b","oid":"wino_grande-ffda0ddc9f9d0b7c","rd":"2021-08-05","sid":"recqm0WP49ptSUceV","sm":"gpt-j-6b","src":"http://arxiv.org/abs/2309.03450","v":0.645},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Dolly-v2-12B","m":"dolly-v2-12b","oid":"wino_grande-80ee9e71c673a7fd","rd":"2023-04-11","sid":"recorForiZEuS4nQF","sm":"dolly-v2-12b","src":"http://arxiv.org/abs/2309.03450","v":0.618},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"Cerebras-GPT-13B","m":"cerebras-gpt-13b","oid":"wino_grande-6b103dabf70a6a84","rd":"2023-03-20","sid":"recluVRCBoH49In4Y","sm":"Cerebras-GPT-13B","src":"http://arxiv.org/abs/2309.03450","v":0.608},{"b":"wino_grande","cfg":{"shots":"0"},"dn":"StableLM-alpha-7B","m":"stablelm-tuned-alpha-7b","oid":"wino_grande-70997ddcb396349c","rd":"2023-04-19","sid":"recfH4Zt9mBnsKlXy","sm":"stablelm-tuned-alpha-7b","src":"http://arxiv.org/abs/2309.03450","v":0.515},{"b":"wino_grande","cfg":{"shots":"5"},"dn":"INTELLECT-1-Instruct","m":"intellect-1-instruct","oid":"wino_grande-5115b2e1cd59dac6","rd":"2024-11-29","sid":"recmV2BWZecWzixYc","sm":"INTELLECT-1-Instruct","src":"https://arxiv.org/abs/2412.01152","v":0.6582}]}