{ "schema_version": 3, "updated_at": "2026-10-02T21:29:18+00:00", "dataset_n": 2035, "dataset_sha256": "e22afaac1db009fb05ab0fef183ed1fc66e7999960ad26235b7c2d70e4f0913a", "results": [ { "name": "DeepSeek V4 Flash", "provider": "DeepSeek", "access": "open", "release_date": "2026-04-24", "release_note": "Preview release, matching the June evaluation. Later revisions of this API alias are not evaluated here.", "parameters_b": 284, "active_parameters_b": 13, "release_source": "https://api-docs.deepseek.com/news/news260424/", "parameters_source": "https://api-docs.deepseek.com/news/news260424/", "id": "deepseek-v4-flash-20260611", "model_key": "deepseek-v4-flash", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/deepseek-v4-flash/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/deepseek-v4-flash/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 78.3, "renderable": 73.29, "non_renderable": 84.03, "render_rate": 93.66, "t_generation": 91.94, "t_conversion": 81.21, "v_generation": 71.9, "v_conversion": 75.38 }, "artifact_url": "./static/data/artifacts/deepseek-v4-flash-20260611.json" }, { "name": "Qwen3.7 Plus", "provider": "Qwen", "access": "closed", "release_date": "2026-06-01", "release_note": "Vendor-listed Global/International availability; May 26 is the snapshot date.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://www.alibabacloud.com/help/en/model-studio/newly-released-models", "parameters_source": null, "id": "qwen3.7-plus-20260611", "model_key": "qwen3.7-plus", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/qwen3.7-plus/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/qwen3.7-plus/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 76.8, "renderable": 70.22, "non_renderable": 84.31, "render_rate": 91.15, "t_generation": 92.21, "t_conversion": 81.49, "v_generation": 66.92, "v_conversion": 75.17 }, "artifact_url": "./static/data/artifacts/qwen3.7-plus-20260611.json" }, { "name": "GPT-4.1 mini", "provider": "OpenAI", "access": "closed", "release_date": "2025-04-14", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "gpt41mini-20260611", "model_key": "GPT-4.1-mini", "evaluation": "experiment_results/gpt41mini_openrouter_reproduce/full-rerender-fixed-20260611T054618Z/evaluation.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/gpt41mini_openrouter_reproduce/full-rerender-fixed-20260611T054618Z/evaluate.log", "notes": "Latest completed rerender run with renderer fixes.", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 76.06, "renderable": 72.2, "non_renderable": 80.47, "render_rate": 92.53, "t_generation": 93.43, "t_conversion": 75.84, "v_generation": 69.94, "v_conversion": 75.58 }, "artifact_url": "./static/data/artifacts/gpt41mini-20260611.json" }, { "name": "GPT-4o", "provider": "OpenAI", "access": "closed", "release_date": "2024-05-13", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-0", "model_key": "GPT-4o", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 76.02, "t_generation": 91.52, "t_conversion": 73.95, "v_generation": 65.39, "v_conversion": 73.2 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "o1-mini", "provider": "OpenAI", "access": "closed", "release_date": "2024-09-12", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-1", "model_key": "o1-mini", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 75.58, "t_generation": 88.12, "t_conversion": 81.82, "v_generation": 61.98, "v_conversion": 70.4 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "GPT-4o mini", "provider": "OpenAI", "access": "closed", "release_date": "2024-07-18", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "paper-3", "model_key": "GPT-4o-mini", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 73.19, "t_generation": 79.86, "t_conversion": 75.57, "v_generation": 60.77, "v_conversion": 76.54 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Gemini 1.5 Pro", "provider": "Google", "access": "closed", "release_date": "2024-02-15", "release_note": "Initial preview availability; evaluated snapshot was not recorded in the original table.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/products/google-gemini-next-generation-model-february-2024/", "parameters_source": null, "id": "paper-4", "model_key": "Gemini-1.5-pro", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 71.75, "t_generation": 88.07, "t_conversion": 74.24, "v_generation": 58.11, "v_conversion": 66.59 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "GPT-5.4 nano", "provider": "OpenAI", "access": "closed", "release_date": "2026-03-17", "release_note": "", "parameters_b": null, "active_parameters_b": null, "release_source": "https://developers.openai.com/api/docs/changelog", "parameters_source": null, "id": "gpt-5.4-nano-20260611", "model_key": "gpt-5.4-nano", "evaluation": "experiment_results/openrouter_full_20260611T064211Z/gpt-5.4-nano/evaluation_eval.json", "evaluated_on": "2026-06-11", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/openrouter_full_20260611T064211Z/gpt-5.4-nano/evaluate_eval.log", "notes": "", "score_aggregation": "Mean over evaluated examples", "complete": true, "n": 2035, "expected_n": 2035, "subset_counts": { "v_generation": 650, "v_conversion": 435, "t_generation": 250, "t_conversion": 700 }, "metrics": { "overall": 69.75, "renderable": 62.12, "non_renderable": 78.46, "render_rate": 85.9, "t_generation": 81.79, "t_conversion": 77.28, "v_generation": 60.03, "v_conversion": 65.24 }, "artifact_url": "./static/data/artifacts/gpt-5.4-nano-20260611.json" }, { "name": "Qwen3 4B", "provider": "Qwen", "access": "open", "release_date": "2025-04-29", "release_note": "", "parameters_b": 4, "active_parameters_b": null, "release_source": "https://qwenlm.github.io/blog/qwen3/", "parameters_source": "https://huggingface.co/Qwen/Qwen3-4B", "id": "paper-5", "model_key": "Qwen3-4B", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 67.04, "t_generation": 64.95, "t_conversion": 81.13, "v_generation": 57.0, "v_conversion": 65.08 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Gemini 2.0 Flash", "provider": "Google", "access": "closed", "release_date": "2025-02-05", "release_note": "General availability of the stable model; experimental preview was released earlier.", "parameters_b": null, "active_parameters_b": null, "release_source": "https://blog.google/innovation-and-ai/models-and-research/google-deepmind/gemini-model-updates-february-2025/", "parameters_source": null, "id": "paper-7", "model_key": "Gemini-2.0-flash", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 62.55, "t_generation": 72.42, "t_conversion": 72.2, "v_generation": 53.62, "v_conversion": 51.97 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Llama 3.1 8B Instruct", "provider": "Meta", "access": "open", "release_date": "2024-07-23", "release_note": "Vendor-reported 8B size class.", "parameters_b": 8, "active_parameters_b": null, "release_source": "https://ai.meta.com/blog/meta-llama-3-1/", "parameters_source": "https://ai.meta.com/blog/meta-llama-3-1/", "id": "paper-6", "model_key": "Llama-3.1-8B-Instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 61.77, "t_generation": 60.22, "t_conversion": 71.26, "v_generation": 54.44, "v_conversion": 61.15 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Qwen2.5 7B Instruct", "provider": "Qwen", "access": "open", "release_date": "2024-09-19", "release_note": "", "parameters_b": 7.61, "active_parameters_b": null, "release_source": "https://qwenlm.github.io/blog/qwen2.5/", "parameters_source": "https://huggingface.co/Qwen/Qwen2.5-7B-Instruct", "id": "paper-8", "model_key": "Qwen2.5-7B-Instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 59.03, "t_generation": 59.21, "t_conversion": 62.18, "v_generation": 53.28, "v_conversion": 61.43 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Phi-4 mini", "provider": "Microsoft", "access": "open", "release_date": "2025-02-26", "release_note": "", "parameters_b": 3.8, "active_parameters_b": null, "release_source": "https://azure.microsoft.com/en-us/blog/empowering-innovation-the-next-generation-of-the-phi-family/", "parameters_source": "https://azure.microsoft.com/en-us/blog/empowering-innovation-the-next-generation-of-the-phi-family/", "id": "paper-9", "model_key": "Phi-4-mini-instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 56.97, "t_generation": 51.38, "t_conversion": 72.39, "v_generation": 51.62, "v_conversion": 52.48 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Llama 3 8B Instruct", "provider": "Meta", "access": "open", "release_date": "2024-04-18", "release_note": "Vendor-reported 8B size class.", "parameters_b": 8, "active_parameters_b": null, "release_source": "https://ai.meta.com/blog/meta-llama-3/", "parameters_source": "https://ai.meta.com/blog/meta-llama-3/", "id": "paper-10", "model_key": "Meta-Llama-3-8B-Instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 51.59, "t_generation": 49.18, "t_conversion": 53.65, "v_generation": 46.61, "v_conversion": 56.91 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Phi-3 mini 128K", "provider": "Microsoft", "access": "open", "release_date": "2024-04-23", "release_note": "", "parameters_b": 3.8, "active_parameters_b": null, "release_source": "https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential/", "parameters_source": "https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential/", "id": "paper-11", "model_key": "Phi-3-mini-128k-instruct", "n": 2035, "expected_n": 2035, "complete": true, "score_aggregation": "Reported average of four subset scores", "evaluated_on": null, "judge": "Published paper protocol", "metrics": { "overall": 40.79, "t_generation": 47.39, "t_conversion": 29.78, "v_generation": 44.77, "v_conversion": 41.23 }, "artifact_url": "./static/data/paper-baseline.json", "notes": "Original website paper results; exact evaluation date and model snapshot not recorded in that table." }, { "name": "Nemotron 3 Ultra", "provider": "NVIDIA", "access": "open", "release_date": "2026-06-04", "release_note": "", "parameters_b": 550, "active_parameters_b": 55, "release_source": "https://developer.nvidia.com/blog/nvidia-nemotron-3-ultra-powers-faster-more-efficient-reasoning-for-long-running-agents/", "parameters_source": "https://developer.nvidia.com/blog/nvidia-nemotron-3-ultra-powers-faster-more-efficient-reasoning-for-long-running-agents/", "id": "nemotron-ultra-gpt41mini-20261001", "model_key": "nemotron-3-ultra", "evaluation": "experiment_results/nemotron_ultra_free_full_20261001/evaluation_gpt41mini.json", "evaluated_on": "2026-10-01", "judge": "GPT-4.1 mini", "protocol_source": "experiment_results/nemotron_ultra_free_full_20261001/run.json", "notes": "Two judges evaluate the same generated outputs. Only tasks with completed scoring are included; incomplete runs are excluded from full-benchmark rankings.", "score_aggregation": "Mean over evaluated examples", "complete": false, "n": 912, "expected_n": 2035, "subset_counts": { "v_generation": 626, "v_conversion": 286 }, "metrics": { "overall": 67.2, "renderable": 67.2, "non_renderable": null, "render_rate": 79.5, "t_generation": null, "t_conversion": null, "v_generation": 62.29, "v_conversion": 77.95 }, "artifact_url": "./static/data/artifacts/nemotron-ultra-gpt41mini-20261001.json" } ] }