{ "meta": { "date": "2026-10-03", "tokenizer": "tiktoken cl100k_base", "secondary_tokenizer": "tiktoken o200k_base (median only, per benchmark under meta.benchmarks)", "stats_method": "numpy; percentiles linear-interpolated; pct_under_N = % with tokens < N", "script": "scripts/measure_problems.py", "benchmarks": { "SWE-bench Verified": { "field": "problem_statement", "source": "hf:SWE-bench/SWE-bench_Verified (test)", "revision": "78f471bf655a3137b2e8a75af1501690ec009ec3", "notes": "Control re-measurement of the 2026-05 note's figure (median 294.5).", "o200k_median": 297.5, "count": 500 }, "SWE-bench Multilingual": { "field": "problem_statement", "source": "hf:SWE-bench/SWE-bench_Multilingual (test)", "revision": "846e647b9f33c0b51b739d005d13d85493c9af09", "notes": "9 languages, 42 repos.", "o200k_median": 372.0, "count": 300 }, "SWE-bench Multimodal": { "field": "problem_statement (text only)", "source": "hf:SWE-bench/SWE-bench_Multimodal (test)", "revision": "4e6662d51c48e475f7f346e4fa09a6f8b31fcaa5", "notes": "Issue images (image_assets) are also given to the model and are NOT counted here, so this understates input.", "o200k_median": 249.5, "count": 480 }, "SWE-bench Pro (V2, public)": { "field": "v2/tasks//instruction.md (= wrapper + problem_statement + requirements + interface + constraints)", "source": "github:scaleapi/SWE-bench_Pro-os v2/tasks", "revision": "66f92766bba642462d4bbe5479e83f91f9211862", "notes": "Full agent-facing instruction. V2 released 2026-09-22 (642 tasks).", "o200k_median": 710.0, "count": 642 }, "SWE-bench Pro (V2, problem_statement only)": { "field": "problem_statement", "source": "hf:ScaleAI/SWE-bench_Pro (default=V2, test)", "revision": "2d52cb3df914a3fcf80c7f66738b3a88ae37fc50", "notes": "Issue text alone, comparable to the SWE-bench problem_statement field; excludes requirements/interface.", "o200k_median": 183.0, "count": 642 }, "Multi-SWE-bench": { "field": "resolved_issues[*].title + body", "source": "hf:ByteDance-Seed/Multi-SWE-bench (7 original languages: c,cpp,go,java,js,ts,rust)", "revision": "56ff018c04a38e27ada1e9d0a6d5839a51f88f0d", "notes": "Repo also carries kotlin (105) and python (500) files added later; excluded to match the published 7-language set.", "o200k_median": 307.0, "count": 1632 }, "SWE-PolyBench": { "field": "problem_statement", "source": "hf:AmazonScience/SWE-PolyBench (test.csv)", "revision": "d56445f9940eae4e9d2974ec66820c2f1d7754e6", "notes": "Java/JS/TS/Python.", "o200k_median": 306.0, "count": 2110 }, "SWE-rebench (leaderboard, all HF months)": { "field": "problem_statement", "source": "hf:nebius/SWE-rebench-leaderboard (test = 2025-01..2026-03)", "revision": "34d5a58864acf91613740a09ec5d205228dcfa39", "notes": "The current leaderboard window (2026-05-15..07-01, 111 tasks) is distributed only via Harbor Hub (ibragim-badertdinov/swe-rebench-07-2026) and was not measured.", "o200k_median": 348.5, "count": 860 }, "SWE-rebench (2026-01..03, latest on HF)": { "field": "problem_statement", "source": "hf:nebius/SWE-rebench-leaderboard splits 2026_01,2026_02,2026_03", "revision": "34d5a58864acf91613740a09ec5d205228dcfa39", "notes": null, "o200k_median": 456.0, "count": 215 }, "SWE-Lancer Diamond (IC SWE)": { "field": "title + description", "source": "github:openai/preparedness project/swelancer/all_swelancer_tasks.csv", "revision": "51052cede8cc608f95bb00346635e03759013e5a", "notes": "Literal '\\n' escapes in the CSV converted to newlines.", "o200k_median": 659.0, "count": 198 }, "SWE-Lancer Diamond (SWE Manager)": { "field": "title + description + proposals", "source": "github:openai/preparedness project/swelancer/all_swelancer_tasks.csv", "revision": "51052cede8cc608f95bb00346635e03759013e5a", "notes": "Manager tasks include the competing proposals the model must choose between.", "o200k_median": 2135.0, "count": 265 }, "Terminal-Bench 2.0": { "field": "instruction.md", "source": "github:harbor-framework/terminal-bench-2", "revision": "2fd12b88aafdd04a52c298e3940bcb189f9766d6", "notes": null, "o200k_median": 165.0, "count": 89 }, "Terminal-Bench 4.0": { "field": "instruction.md", "source": "github:harbor-framework/terminal-bench tag v4.0.0", "revision": "452bf305c6daa62fc59061d22133a7cbc7c1572e", "notes": "Released 2026-08-26; cited in the Claude Opus 5.5 launch (2026-09-22).", "o200k_median": 388.5, "count": 66 }, "DeepSWE": { "field": "instruction.md", "source": "github:datacurve-ai/deep-swe", "revision": "0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea", "notes": "arXiv 2607.07946 (2026-07-08).", "o200k_median": 438.0, "count": 113 }, "LiveCodeBench (release_v6)": { "field": "question_content", "source": "hf:livecodebench/code_generation_lite test.jsonl..test6.jsonl", "revision": "0fe84c3912ea0c4d4a78037083943e8f0c4dd505", "notes": "release_v6 (May 2023 - Apr 2025) is still the latest release on HF as of 2026-10-03.", "o200k_median": 415.0, "count": 1055 }, "LiveCodeBench (release_v1, control)": { "field": "question_content", "source": "hf:livecodebench/code_generation_lite test.jsonl", "revision": "0fe84c3912ea0c4d4a78037083943e8f0c4dd505", "notes": "Control: the 400-problem set the 2026-05 note appears to have used (median 421).", "o200k_median": 420.0, "count": 400 }, "Aider Polyglot": { "field": ".docs/introduction.md + instructions.md + instructions.append.md", "source": "github:Aider-AI/polyglot-benchmark", "revision": "7e0611e77b54e2dea774cdc0aa00cf9f7ed6144f", "notes": "Excludes Aider's fixed addendum and the stub source files it adds to the chat.", "o200k_median": 432.0, "count": 225 } }, "not_measured": { "SWE-bench Pro Verified (arXiv 2609.08149)": "No public dataset link found (hf:ScaleAI/SWE-bench_Pro_Verified does not exist / not public).", "FrontierCode v1.1 (Cognition)": "Cited in the Claude Opus 5.5 launch; task texts not public.", "CursorBench 4.0": "Cited in the Claude Opus 5.5 launch; internal to Cursor, not public.", "SWE-rebench 2026-05..07 window": "Harbor Hub only (ibragim-badertdinov/swe-rebench-07-2026); harbor CLI not used.", "Terminal-Bench 2.1 / 3.0": "2.1 referenced by third parties; 3.0 superseded by 4.0 (same repo, tag v3.0.0); not measured separately." } }, "SWE-bench Verified": { "benchmark": "SWE-bench Verified", "status": "ok", "stats": { "count": 500, "min": 24, "max": 6939, "mean": 447.716, "median": 294.5, "p25": 163.0, "p75": 539.25, "p95": 1266.2999999999997, "p99": 2513.989999999999, "pct_under_1024": 91.8, "pct_under_2048": 98.0, "pct_under_4096": 99.6, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 322, 684, 469, 1230, 2162, 1418, 894, 243, 508, 1272, 444, 688, 759, 436, 748, 737, 35, 384, 273, 502, 193, 288, 560, 812, 55, 209, 77, 216, 406, 2407, 85, 165, 80, 182, 890, 176, 181, 95, 62, 154, 214, 267, 1034, 254, 74, 737, 318, 188, 135, 441, 44, 122, 674, 1260, 162, 78, 114, 220, 478, 272, 114, 354, 98, 158, 261, 287, 151, 500, 142, 287, 66, 360, 351, 179, 155, 264, 226, 218, 146, 122, 97, 186, 264, 156, 735, 234, 59, 78, 183, 277, 208, 670, 326, 112, 666, 1028, 396, 409, 456, 292, 77, 257, 560, 92, 116, 354, 58, 241, 289, 142, 382, 283, 786, 338, 251, 1755, 110, 288, 139, 123, 163, 503, 192, 301, 224, 203, 153, 60, 373, 96, 473, 433, 146, 363, 884, 268, 372, 283, 424, 356, 309, 718, 68, 120, 280, 109, 425, 676, 165, 155, 564, 2355, 48, 245, 78, 36, 356, 63, 369, 109, 122, 383, 136, 299, 1734, 82, 633, 129, 299, 175, 422, 209, 251, 55, 282, 120, 726, 88, 1108, 55, 654, 100, 357, 326, 693, 292, 594, 186, 321, 461, 430, 92, 185, 52, 497, 1117, 314, 73, 440, 154, 359, 566, 309, 255, 143, 83, 738, 148, 189, 367, 173, 181, 1508, 33, 220, 57, 53, 165, 577, 346, 163, 387, 567, 121, 317, 131, 261, 474, 64, 219, 490, 125, 159, 404, 361, 198, 181, 67, 289, 142, 424, 146, 24, 97, 169, 292, 985, 359, 236, 171, 262, 107, 91, 438, 329, 520, 385, 676, 636, 374, 2513, 398, 389, 470, 232, 575, 1074, 604, 881, 510, 388, 134, 532, 347, 421, 288, 539, 206, 248, 1731, 153, 220, 2636, 612, 984, 152, 419, 262, 377, 48, 78, 2067, 205, 166, 214, 79, 426, 375, 655, 540, 718, 596, 401, 212, 719, 568, 204, 865, 1297, 1266, 1234, 267, 1602, 2649, 440, 919, 405, 918, 529, 763, 240, 202, 221, 147, 217, 553, 817, 6939, 282, 947, 422, 702, 1216, 1177, 626, 796, 268, 176, 862, 657, 1137, 693, 130, 207, 4184, 1074, 120, 125, 244, 386, 580, 490, 282, 1766, 323, 129, 397, 2612, 1542, 975, 1339, 374, 325, 142, 821, 670, 35, 585, 513, 1639, 567, 411, 606, 556, 322, 1673, 819, 460, 317, 841, 936, 780, 770, 1042, 796, 783, 340, 527, 1484, 257, 751, 340, 158, 85, 195, 311, 318, 445, 160, 447, 482, 509, 316, 346, 176, 94, 543, 175, 186, 537, 694, 144, 984, 362, 125, 211, 292, 138, 529, 468, 354, 755, 336, 236, 393, 74, 176, 333, 90, 262, 291, 430, 354, 152, 204, 66, 657, 148, 228, 576, 1319, 877, 370, 186, 224, 267, 241, 288, 74, 227, 575, 81, 142, 381, 248, 85, 151, 470, 48, 636, 870, 1076, 487, 308, 96, 192, 215, 207, 385, 146, 625, 65, 96, 308, 116, 1238, 431, 184, 161, 74, 466, 261, 246, 297, 192, 203, 75, 472, 169, 282, 124, 174, 561, 218, 147, 265, 209, 225, 79, 100 ] }, "SWE-bench Multilingual": { "benchmark": "SWE-bench Multilingual", "status": "ok", "stats": { "count": 300, "min": 33, "max": 22492, "mean": 614.383, "median": 374.0, "p25": 217.75, "p75": 604.0, "p95": 1525.1000000000017, "p99": 4862.5099999999975, "pct_under_1024": 90.0, "pct_under_2048": 97.0, "pct_under_4096": 98.67, "pct_under_8192": 99.67, "pct_under_16384": 99.67, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 183, 261, 268, 378, 559, 87, 221, 293, 906, 185, 375, 421, 571, 399, 82, 301, 599, 258, 377, 286, 226, 387, 586, 374, 414, 189, 833, 231, 777, 1517, 332, 850, 381, 638, 811, 33, 730, 755, 818, 808, 330, 872, 1328, 524, 412, 1078, 1209, 206, 70, 475, 309, 134, 291, 232, 82, 562, 212, 104, 680, 790, 462, 363, 470, 265, 180, 22492, 4860, 2992, 3019, 590, 2563, 5111, 2244, 1789, 2940, 188, 545, 563, 892, 1056, 290, 326, 433, 247, 203, 140, 175, 218, 1173, 107, 312, 185, 99, 153, 81, 177, 179, 562, 567, 374, 118, 778, 362, 669, 242, 162, 437, 427, 242, 250, 232, 209, 320, 206, 36, 430, 242, 383, 297, 211, 588, 980, 230, 1134, 422, 193, 216, 734, 284, 472, 562, 301, 466, 146, 101, 119, 305, 697, 176, 87, 151, 216, 1679, 362, 203, 397, 317, 222, 194, 720, 407, 496, 488, 270, 409, 794, 203, 219, 312, 1142, 183, 175, 325, 225, 98, 716, 467, 410, 142, 499, 852, 121, 173, 132, 419, 220, 505, 271, 306, 409, 442, 438, 994, 376, 697, 632, 432, 226, 227, 400, 531, 261, 446, 124, 118, 250, 632, 395, 566, 323, 43, 1900, 315, 262, 656, 469, 353, 202, 582, 173, 672, 338, 177, 217, 2015, 283, 289, 375, 201, 55, 206, 752, 1267, 721, 230, 610, 108, 5908, 530, 594, 400, 357, 108, 861, 644, 225, 423, 265, 369, 748, 176, 158, 319, 1906, 1918, 148, 432, 229, 1323, 1004, 380, 254, 350, 400, 629, 1102, 429, 222, 126, 468, 388, 408, 123, 814, 838, 183, 1400, 786, 349, 150, 257, 179, 417, 602, 136, 448, 407, 871, 367, 285, 279, 437, 312, 450, 485, 721, 35, 136, 346, 96, 139, 363, 296, 1244, 1488, 981, 1438, 736, 871, 168 ] }, "SWE-bench Multimodal": { "benchmark": "SWE-bench Multimodal", "status": "ok", "stats": { "count": 480, "min": 52, "max": 60719, "mean": 441.156, "median": 251.5, "p25": 164.0, "p75": 380.0, "p95": 794.3499999999988, "p99": 1376.499999999999, "pct_under_1024": 97.71, "pct_under_2048": 99.38, "pct_under_4096": 99.79, "pct_under_8192": 99.79, "pct_under_16384": 99.79, "pct_under_32768": 99.79 }, "per_problem_tokens": [ 451, 151, 426, 472, 236, 205, 182, 839, 304, 134, 283, 92, 154, 441, 103, 330, 495, 109, 97, 81, 55, 82, 199, 108, 110, 151, 74, 154, 258, 87, 215, 386, 157, 295, 472, 255, 206, 256, 67, 478, 276, 374, 178, 171, 282, 455, 213, 52, 66, 66, 264, 497, 377, 147, 158, 215, 118, 69, 438, 156, 155, 640, 233, 239, 273, 291, 108, 126, 263, 270, 390, 68, 60719, 501, 507, 593, 91, 790, 349, 258, 183, 148, 278, 548, 189, 239, 162, 225, 792, 288, 317, 165, 213, 86, 86, 120, 213, 144, 145, 249, 233, 175, 75, 90, 637, 166, 85, 248, 172, 95, 269, 81, 86, 89, 162, 79, 167, 105, 64, 160, 124, 111, 106, 156, 81, 108, 213, 60, 143, 252, 232, 283, 287, 263, 272, 269, 230, 229, 303, 89, 112, 107, 205, 175, 163, 155, 143, 110, 273, 490, 422, 390, 610, 207, 157, 198, 158, 159, 885, 563, 254, 125, 210, 544, 252, 197, 399, 230, 239, 167, 167, 298, 184, 318, 156, 292, 401, 143, 222, 174, 462, 534, 565, 128, 292, 341, 91, 146, 280, 363, 336, 319, 386, 220, 354, 205, 75, 325, 484, 392, 299, 184, 185, 217, 181, 182, 179, 181, 182, 472, 183, 842, 417, 216, 98, 1008, 502, 375, 166, 409, 527, 386, 566, 1008, 177, 119, 195, 201, 230, 143, 551, 164, 108, 434, 271, 108, 395, 271, 142, 416, 271, 155, 195, 159, 152, 377, 108, 245, 439, 158, 583, 215, 225, 315, 218, 247, 333, 284, 102, 125, 453, 363, 379, 123, 373, 239, 290, 370, 257, 342, 109, 273, 290, 283, 498, 477, 96, 355, 133, 680, 447, 266, 75, 823, 165, 269, 374, 749, 314, 642, 419, 95, 295, 289, 208, 374, 72, 1098, 175, 510, 664, 92, 447, 511, 319, 100, 920, 363, 488, 293, 310, 118, 357, 285, 2261, 1366, 1366, 523, 201, 222, 1092, 1510, 367, 1416, 844, 176, 206, 81, 202, 239, 297, 308, 523, 258, 416, 187, 307, 283, 182, 277, 226, 211, 301, 189, 380, 378, 711, 342, 744, 210, 915, 473, 332, 158, 330, 188, 182, 270, 225, 386, 384, 740, 274, 517, 215, 301, 250, 302, 340, 222, 380, 541, 300, 262, 265, 610, 292, 367, 414, 793, 163, 232, 224, 336, 250, 229, 134, 121, 245, 289, 288, 562, 122, 236, 138, 82, 437, 169, 160, 820, 289, 195, 324, 380, 269, 402, 199, 296, 127, 201, 366, 192, 306, 671, 251, 116, 98, 237, 149, 307, 143, 650, 268, 78, 317, 193, 232, 148, 285, 176, 205, 104, 388, 216, 1249, 485, 375, 191, 375, 164, 97, 97, 223, 473, 389, 333, 189, 78, 201, 336, 241, 220, 223, 565, 252, 245, 412, 203, 266, 208, 326, 766, 727, 630, 560, 440, 2383, 868, 836, 1130, 425, 502, 224, 574, 902, 494, 185, 232, 157, 277 ] }, "SWE-bench Pro (V2, public)": { "benchmark": "SWE-bench Pro (V2, public)", "status": "ok", "stats": { "count": 642, "min": 194, "max": 5649, "mean": 840.766, "median": 707.0, "p25": 513.25, "p75": 1006.0, "p95": 1884.349999999999, "p99": 2640.230000000003, "pct_under_1024": 75.86, "pct_under_2048": 96.57, "pct_under_4096": 99.84, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 807, 700, 496, 551, 570, 490, 645, 441, 446, 411, 507, 406, 803, 422, 567, 594, 709, 341, 1872, 1141, 391, 513, 574, 621, 482, 1250, 656, 404, 721, 762, 566, 662, 701, 723, 1108, 594, 935, 296, 581, 397, 623, 434, 506, 340, 429, 1934, 761, 1218, 568, 1044, 450, 651, 1912, 1162, 416, 916, 570, 5649, 976, 1527, 475, 567, 343, 526, 692, 848, 1146, 636, 1176, 505, 1040, 822, 1205, 1936, 1468, 321, 350, 533, 459, 603, 739, 280, 588, 1116, 522, 447, 509, 556, 707, 1118, 525, 768, 595, 745, 2164, 547, 1069, 637, 1885, 736, 1022, 1914, 586, 969, 2328, 663, 1008, 993, 403, 1098, 1895, 1183, 1176, 564, 438, 1232, 264, 1452, 867, 810, 799, 765, 450, 453, 995, 1275, 541, 757, 529, 535, 477, 1217, 1479, 472, 364, 430, 941, 870, 773, 532, 1656, 568, 684, 486, 434, 1553, 312, 972, 742, 468, 1023, 561, 1532, 1603, 1285, 2332, 867, 445, 470, 856, 1130, 1212, 1166, 489, 703, 819, 511, 581, 592, 291, 831, 2027, 1196, 659, 787, 1330, 618, 611, 371, 426, 482, 333, 1441, 792, 552, 415, 894, 2277, 992, 1156, 836, 733, 548, 1731, 738, 654, 732, 532, 1378, 1040, 770, 1003, 518, 911, 380, 1438, 464, 1995, 874, 360, 717, 222, 554, 832, 336, 594, 474, 798, 541, 655, 798, 763, 925, 515, 419, 857, 2058, 401, 809, 1070, 1468, 1399, 588, 415, 676, 345, 840, 1378, 2089, 858, 1300, 716, 1272, 1249, 375, 1075, 1797, 1113, 1132, 1783, 641, 918, 670, 1231, 2185, 638, 701, 306, 1166, 1195, 879, 610, 707, 675, 621, 899, 1700, 727, 1450, 2944, 1085, 789, 1112, 638, 194, 1187, 1316, 411, 1025, 297, 485, 568, 437, 717, 1370, 310, 505, 423, 331, 1583, 430, 1888, 1103, 644, 574, 490, 1322, 851, 707, 422, 954, 420, 2528, 700, 562, 1087, 404, 401, 2241, 676, 354, 483, 480, 632, 485, 617, 1256, 2680, 237, 460, 786, 1742, 496, 654, 737, 801, 730, 457, 870, 732, 488, 658, 1120, 569, 568, 1586, 1576, 2724, 795, 740, 1826, 503, 444, 3488, 578, 526, 528, 873, 502, 361, 477, 795, 715, 422, 288, 454, 471, 1080, 656, 1854, 417, 1051, 539, 1165, 445, 2768, 1183, 264, 885, 708, 783, 783, 1489, 1895, 555, 1221, 915, 595, 528, 1081, 2171, 892, 855, 870, 588, 602, 699, 1095, 1396, 383, 321, 719, 1280, 2093, 537, 408, 854, 788, 297, 514, 534, 509, 673, 525, 836, 401, 830, 1186, 888, 740, 464, 3205, 476, 869, 776, 462, 497, 1250, 607, 918, 1375, 1267, 2583, 826, 656, 531, 1308, 624, 667, 496, 1320, 710, 760, 671, 1052, 433, 2447, 765, 499, 374, 1251, 418, 523, 756, 989, 840, 575, 625, 731, 1233, 767, 290, 620, 975, 1068, 913, 500, 371, 773, 668, 313, 660, 1261, 1016, 592, 723, 554, 481, 1421, 527, 515, 604, 680, 591, 1033, 790, 781, 870, 575, 641, 849, 740, 567, 719, 904, 920, 589, 1609, 814, 845, 472, 1500, 622, 569, 637, 996, 489, 927, 584, 414, 828, 444, 525, 857, 774, 1458, 455, 438, 1100, 911, 762, 699, 267, 957, 423, 485, 467, 724, 1055, 964, 1446, 456, 269, 349, 812, 730, 899, 813, 1576, 1373, 2061, 2052, 465, 596, 1353, 519, 971, 698, 624, 980, 445, 372, 762, 547, 784, 864, 369, 405, 773, 572, 827, 1036, 392, 719, 542, 662, 963, 871, 1893, 1211, 632, 964, 591, 526, 601, 481, 256, 575, 332, 915, 1339, 899, 830, 604, 386, 496, 487, 808, 1162, 531, 453, 528, 1809, 771, 1767, 481, 565, 535, 1401, 847, 374, 434, 506, 1193, 357, 501, 1007, 550, 486, 1163, 310, 757, 942, 597, 493, 652, 661, 771, 744, 358, 957, 666, 676, 540, 729, 577, 994, 791, 689, 541, 1571, 1387, 589, 1007, 639, 515, 454, 1378, 959, 686, 403, 1179, 967, 597, 1001, 273, 769, 657, 1390, 388, 1329, 1337 ] }, "SWE-bench Pro (V2, problem_statement only)": { "benchmark": "SWE-bench Pro (V2, problem_statement only)", "status": "ok", "stats": { "count": 642, "min": 34, "max": 2432, "mean": 200.963, "median": 182.0, "p25": 117.0, "p75": 253.0, "p95": 371.8499999999998, "p99": 567.4200000000012, "pct_under_1024": 99.53, "pct_under_2048": 99.84, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 79, 50, 208, 84, 211, 155, 196, 182, 44, 186, 155, 135, 302, 99, 228, 282, 117, 148, 121, 465, 181, 76, 128, 39, 259, 120, 111, 166, 232, 122, 306, 53, 191, 276, 149, 96, 98, 133, 306, 98, 89, 67, 160, 94, 196, 222, 88, 539, 292, 498, 99, 308, 57, 341, 48, 638, 239, 226, 273, 212, 110, 236, 108, 292, 218, 307, 297, 368, 280, 78, 274, 284, 512, 65, 216, 113, 63, 110, 169, 58, 153, 72, 220, 526, 89, 183, 308, 179, 360, 222, 136, 363, 215, 168, 137, 232, 165, 226, 112, 248, 124, 100, 282, 355, 148, 248, 230, 46, 155, 655, 1462, 481, 536, 54, 126, 356, 112, 215, 248, 385, 117, 116, 89, 151, 303, 290, 273, 193, 82, 109, 193, 348, 158, 132, 85, 97, 163, 49, 135, 112, 139, 215, 304, 194, 86, 78, 102, 188, 406, 122, 122, 80, 196, 204, 266, 96, 285, 75, 190, 244, 53, 300, 162, 178, 175, 254, 166, 162, 320, 89, 321, 55, 212, 153, 285, 78, 179, 311, 113, 195, 204, 60, 433, 107, 247, 121, 353, 156, 251, 246, 287, 277, 112, 79, 267, 335, 294, 221, 87, 340, 446, 119, 55, 281, 88, 589, 80, 111, 101, 146, 378, 62, 306, 116, 109, 253, 183, 136, 228, 155, 250, 267, 242, 341, 211, 255, 85, 84, 68, 150, 115, 141, 187, 178, 52, 175, 234, 150, 126, 244, 223, 193, 242, 48, 142, 355, 197, 48, 154, 164, 215, 115, 151, 289, 91, 278, 403, 112, 239, 361, 217, 83, 305, 111, 186, 190, 1096, 141, 83, 150, 286, 132, 308, 52, 44, 348, 162, 189, 545, 132, 184, 276, 165, 300, 35, 142, 107, 87, 154, 335, 242, 471, 428, 197, 235, 280, 41, 293, 266, 214, 151, 196, 203, 234, 210, 80, 52, 144, 251, 361, 135, 118, 142, 282, 193, 311, 331, 152, 53, 194, 261, 60, 224, 284, 217, 253, 180, 210, 208, 289, 233, 235, 80, 244, 281, 125, 141, 2432, 131, 239, 57, 138, 76, 167, 149, 85, 132, 58, 136, 59, 99, 237, 203, 224, 60, 60, 86, 300, 67, 124, 48, 388, 220, 290, 146, 87, 346, 75, 77, 208, 250, 163, 162, 159, 37, 153, 205, 198, 110, 94, 167, 161, 165, 163, 192, 327, 350, 280, 267, 134, 105, 236, 103, 89, 63, 109, 107, 203, 66, 254, 200, 174, 197, 127, 216, 172, 253, 157, 291, 72, 158, 140, 188, 34, 268, 131, 159, 192, 202, 104, 285, 112, 128, 412, 209, 165, 113, 95, 182, 216, 130, 201, 245, 56, 273, 240, 307, 288, 213, 99, 169, 163, 178, 152, 526, 109, 203, 213, 151, 94, 59, 170, 163, 182, 123, 249, 222, 179, 260, 178, 178, 229, 583, 512, 149, 47, 142, 178, 141, 225, 67, 146, 117, 193, 167, 179, 74, 50, 274, 195, 276, 150, 265, 70, 56, 339, 125, 242, 367, 275, 130, 299, 262, 147, 195, 113, 183, 108, 277, 134, 303, 220, 192, 342, 199, 80, 228, 125, 411, 136, 420, 315, 95, 86, 121, 145, 185, 203, 86, 175, 57, 162, 109, 195, 312, 268, 221, 249, 63, 142, 111, 62, 184, 206, 98, 185, 419, 308, 212, 261, 226, 114, 192, 254, 123, 226, 195, 103, 288, 301, 340, 121, 80, 85, 147, 247, 175, 234, 79, 134, 253, 346, 177, 286, 206, 193, 132, 332, 84, 372, 124, 186, 171, 210, 171, 130, 119, 132, 189, 294, 73, 175, 255, 210, 281, 209, 208, 217, 175, 155, 195, 112, 211, 349, 183, 160, 327, 268, 163, 369, 94, 217, 282, 231, 203, 261, 228, 289, 306, 160, 234, 97, 169, 245, 200, 75, 331, 284, 314, 90, 210, 432, 260, 83, 239, 165, 169, 161, 235, 270, 98, 120, 269, 119, 237, 81, 405, 225, 100, 182, 100, 174 ] }, "Multi-SWE-bench": { "benchmark": "Multi-SWE-bench", "status": "ok", "stats": { "count": 1632, "min": 15, "max": 11400, "mean": 489.915, "median": 306.5, "p25": 174.0, "p75": 558.5, "p95": 1458.0, "p99": 3188.900000000006, "pct_under_1024": 91.79, "pct_under_2048": 97.37, "pct_under_4096": 99.45, "pct_under_8192": 99.82, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 416, 61, 253, 399, 273, 865, 501, 296, 106, 118, 118, 102, 35, 257, 79, 203, 74, 744, 4147, 364, 253, 547, 80, 111, 1019, 200, 321, 624, 317, 419, 273, 161, 361, 164, 1678, 180, 84, 151, 506, 176, 211, 773, 237, 120, 555, 193, 830, 451, 187, 574, 94, 1559, 131, 250, 951, 1380, 651, 405, 58, 686, 1047, 324, 258, 63, 713, 194, 127, 720, 799, 130, 394, 164, 239, 120, 548, 74, 3002, 592, 287, 117, 917, 455, 125, 345, 772, 178, 166, 71, 40, 141, 47, 101, 145, 787, 163, 673, 176, 835, 696, 51, 413, 121, 1174, 265, 530, 304, 708, 806, 65, 368, 580, 301, 688, 57, 84, 274, 49, 224, 136, 58, 142, 80, 82, 159, 139, 153, 201, 316, 206, 745, 199, 257, 124, 627, 212, 479, 182, 319, 197, 263, 644, 506, 108, 192, 111, 111, 152, 101, 236, 99, 323, 1748, 184, 932, 169, 139, 174, 814, 42, 312, 331, 255, 106, 1173, 114, 70, 1116, 174, 139, 352, 139, 141, 235, 126, 67, 97, 104, 549, 121, 53, 1730, 565, 345, 661, 241, 255, 281, 311, 489, 2302, 184, 434, 599, 441, 1716, 303, 7947, 602, 72, 10578, 533, 395, 110, 247, 625, 946, 282, 474, 736, 292, 596, 172, 274, 79, 1146, 647, 682, 154, 964, 1971, 90, 539, 771, 212, 355, 303, 493, 540, 256, 584, 238, 191, 145, 86, 57, 165, 349, 308, 214, 326, 428, 1455, 15, 340, 295, 38, 231, 93, 318, 174, 272, 148, 229, 173, 502, 57, 223, 1491, 279, 392, 1035, 331, 223, 627, 673, 869, 253, 230, 1493, 748, 321, 321, 407, 317, 101, 802, 466, 117, 391, 348, 300, 432, 200, 400, 443, 1827, 252, 248, 412, 226, 342, 230, 252, 433, 272, 211, 401, 665, 504, 781, 331, 251, 252, 322, 224, 240, 128, 204, 219, 216, 98, 90, 578, 404, 141, 75, 266, 896, 2266, 118, 469, 208, 365, 90, 291, 291, 115, 341, 222, 8500, 123, 76, 363, 325, 308, 212, 271, 229, 75, 171, 402, 426, 164, 148, 166, 240, 114, 69, 260, 268, 208, 171, 377, 277, 178, 349, 89, 261, 193, 116, 338, 333, 468, 74, 265, 55, 195, 260, 311, 408, 175, 161, 113, 168, 554, 386, 202, 240, 746, 96, 143, 160, 721, 278, 92, 79, 221, 486, 191, 811, 152, 264, 184, 195, 108, 120, 407, 451, 1375, 79, 154, 2712, 79, 96, 167, 168, 290, 78, 277, 503, 135, 76, 472, 291, 265, 285, 138, 156, 234, 206, 111, 115, 201, 213, 294, 149, 359, 277, 49, 233, 183, 527, 221, 446, 392, 428, 144, 384, 80, 56, 1122, 147, 147, 73, 101, 434, 250, 287, 282, 307, 209, 330, 273, 108, 84, 158, 116, 386, 227, 506, 237, 269, 113, 64, 22, 323, 353, 3352, 411, 154, 237, 354, 360, 313, 174, 170, 668, 116, 395, 344, 312, 429, 637, 238, 143, 357, 142, 37, 37, 471, 421, 421, 124, 257, 135, 300, 121, 125, 96, 259, 516, 284, 177, 138, 272, 457, 85, 72, 140, 476, 169, 239, 198, 140, 141, 125, 91, 203, 526, 327, 258, 122, 315, 177, 195, 49, 49, 593, 226, 55, 55, 142, 142, 270, 281, 47, 242, 254, 430, 211, 392, 1728, 97, 168, 345, 49, 68, 168, 79, 248, 204, 53, 219, 226, 57, 380, 333, 201, 261, 175, 152, 145, 125, 142, 171, 113, 111, 106, 95, 125, 179, 178, 201, 142, 126, 102, 142, 250, 138, 134, 31, 249, 356, 219, 67, 178, 86, 400, 500, 138, 52, 60, 88, 27, 1728, 131, 177, 779, 143, 123, 303, 116, 97, 86, 232, 248, 174, 174, 131, 42, 145, 55, 195, 287, 22, 244, 129, 26, 144, 233, 91, 324, 140, 59, 149, 318, 164, 225, 238, 406, 147, 200, 29, 263, 519, 267, 81, 103, 274, 148, 113, 105, 78, 89, 527, 570, 386, 184, 1538, 941, 120, 120, 259, 53, 92, 484, 24, 1006, 392, 93, 378, 75, 317, 91, 290, 110, 51, 27, 59, 396, 157, 218, 46, 142, 803, 54, 231, 90, 44, 189, 154, 344, 33, 142, 64, 74, 787, 601, 303, 823, 54, 1317, 394, 252, 1458, 1458, 1458, 1458, 11400, 1019, 1019, 1019, 185, 473, 2703, 473, 563, 329, 329, 151, 151, 2612, 2612, 2316, 2316, 1103, 1103, 2316, 2316, 2316, 554, 134, 69, 318, 69, 114, 282, 282, 300, 282, 300, 3777, 231, 88, 132, 83, 461, 125, 80, 100, 306, 184, 529, 744, 196, 129, 601, 148, 109, 198, 589, 208, 119, 510, 678, 278, 415, 206, 782, 93, 604, 1040, 637, 840, 290, 823, 715, 393, 879, 184, 538, 141, 77, 1216, 296, 548, 650, 239, 171, 476, 758, 539, 285, 174, 381, 308, 219, 455, 941, 396, 704, 691, 1739, 198, 453, 341, 194, 288, 344, 95, 252, 209, 243, 287, 303, 1913, 1997, 596, 134, 570, 358, 87, 269, 635, 76, 183, 119, 465, 119, 271, 572, 360, 360, 103, 117, 213, 246, 167, 147, 1233, 107, 19, 24, 632, 472, 84, 560, 89, 315, 136, 128, 655, 254, 655, 126, 103, 405, 320, 338, 323, 266, 129, 171, 206, 307, 41, 374, 20, 171, 266, 284, 208, 276, 75, 175, 293, 377, 218, 83, 223, 123, 661, 134, 238, 104, 359, 121, 156, 147, 238, 179, 143, 496, 672, 105, 343, 28, 520, 227, 343, 983, 983, 368, 149, 61, 113, 260, 590, 421, 280, 243, 497, 205, 204, 454, 151, 741, 156, 175, 367, 315, 681, 179, 359, 388, 280, 414, 564, 997, 356, 490, 282, 274, 247, 722, 280, 719, 271, 505, 322, 235, 174, 335, 2351, 628, 1802, 998, 739, 1018, 295, 370, 235, 403, 438, 330, 248, 363, 477, 496, 962, 66, 308, 1233, 359, 711, 405, 641, 434, 274, 664, 250, 306, 68, 508, 336, 556, 488, 481, 524, 1019, 623, 118, 424, 347, 818, 680, 139, 291, 320, 66, 380, 3856, 442, 282, 141, 440, 314, 546, 438, 394, 1043, 388, 993, 968, 237, 414, 544, 627, 847, 311, 885, 447, 585, 518, 298, 545, 71, 503, 452, 110, 791, 505, 608, 558, 1356, 407, 535, 231, 454, 907, 303, 442, 179, 266, 1887, 179, 676, 1086, 497, 287, 2265, 181, 714, 153, 153, 894, 134, 548, 532, 757, 385, 496, 888, 663, 569, 1122, 277, 714, 393, 247, 273, 246, 204, 211, 896, 228, 303, 531, 1407, 697, 4111, 656, 482, 415, 296, 674, 502, 378, 369, 104, 848, 580, 714, 1294, 528, 366, 202, 726, 263, 483, 452, 339, 329, 148, 261, 192, 471, 336, 1328, 729, 354, 649, 1579, 2704, 507, 251, 917, 402, 502, 898, 840, 143, 2116, 1038, 1850, 668, 484, 532, 230, 624, 1005, 288, 491, 205, 1139, 515, 90, 750, 325, 682, 359, 529, 753, 509, 445, 636, 637, 183, 193, 465, 359, 513, 1235, 188, 358, 916, 276, 605, 350, 830, 538, 298, 2094, 447, 177, 507, 695, 138, 257, 338, 925, 327, 317, 301, 323, 1792, 510, 700, 725, 711, 231, 466, 382, 230, 233, 766, 381, 282, 326, 582, 40, 523, 422, 1231, 1327, 236, 526, 912, 124, 194, 60, 327, 595, 5351, 1625, 893, 2720, 3032, 4813, 5484, 365, 1189, 470, 614, 969, 724, 895, 523, 439, 153, 620, 2611, 636, 1672, 189, 620, 204, 165, 447, 216, 2023, 264, 210, 240, 68, 122, 111, 287, 899, 3376, 3376, 244, 136, 769, 341, 46, 600, 3363, 204, 469, 205, 159, 215, 489, 1898, 567, 745, 679, 268, 322, 153, 818, 302, 266, 3223, 966, 344, 3113, 571, 486, 1282, 300, 153, 366, 623, 455, 340, 448, 699, 767, 891, 381, 1571, 2016, 256, 260, 815, 237, 1084, 333, 203, 279, 491, 1097, 404, 68, 252, 230, 1510, 1518, 469, 566, 608, 720, 227, 205, 1036, 123, 1651, 159, 557, 227, 3825, 2266, 1117, 643, 291, 2619, 1541, 1486, 354, 177, 65, 2043, 1492, 279, 180, 636, 144, 121, 295, 498, 141, 260, 301, 468, 2112, 343, 238, 342, 114, 452, 234, 430, 820, 344, 820, 1507, 1507, 128, 153, 227, 78, 851, 348, 266, 1725, 131, 752, 174, 568, 198, 823, 330, 376, 265, 133, 226, 91, 87, 41, 205, 41, 54, 84, 41, 205, 300, 76, 119, 190, 190, 311, 321, 201, 382, 267, 177, 962, 209, 577, 220, 164, 343, 643, 1021, 298, 152, 950, 357, 282, 426, 533, 243, 303, 2673, 1201, 303, 303, 540, 209, 262, 445, 645, 445, 151, 103, 1319, 512, 492, 527, 619, 424, 424, 527, 101, 494, 298, 915, 439, 773, 104, 216, 589, 175, 278, 278, 404, 481, 253, 847, 398, 207, 333, 845, 346, 912, 181, 195, 105, 356, 360, 357, 155, 298, 151, 195, 568, 232, 751, 268, 464, 754, 587, 938, 126, 470, 1168, 132, 208, 242, 834, 328, 379, 656, 856, 162, 237, 247, 147, 331, 1118, 367, 1296, 259, 224, 728, 541, 656, 1136, 242, 791, 626, 529, 707, 237, 335, 64, 162, 81, 184, 184, 273, 709, 89, 257, 615, 86, 186, 650, 596, 1010, 264, 155, 1130, 162, 1058, 162, 259, 776, 266, 187, 37, 358, 147, 311, 160, 356, 649, 133, 632, 491, 410, 630, 1094, 263, 382, 794, 774, 522, 251, 316, 545, 363, 397, 409, 688, 515, 770, 233, 1199, 609, 479, 371, 509, 463, 393, 100, 211, 710, 437, 157, 862, 248, 324, 890, 175, 265, 237, 859, 524, 736, 384, 688, 111, 317, 574, 1887, 870, 334, 443, 208, 135, 429, 550, 968, 525, 507, 507, 705, 259, 871, 269, 193, 886, 148, 229, 747, 237, 525, 475, 459, 389, 254, 871, 602, 587, 2173, 661, 1336, 669, 255, 673, 984, 318, 1149, 537, 550, 424, 812, 717, 575, 672, 465, 929, 668, 588, 608, 623, 532, 420, 943, 889, 219, 299, 522, 1050, 253, 400, 517, 614, 498, 636, 166, 1204, 775, 238, 1345, 394, 638, 437, 341 ] }, "SWE-PolyBench": { "benchmark": "SWE-PolyBench", "status": "ok", "stats": { "count": 2110, "min": 7, "max": 16222, "mean": 452.066, "median": 306.5, "p25": 158.0, "p75": 554.0, "p95": 1210.55, "p99": 2527.5999999999767, "pct_under_1024": 92.56, "pct_under_2048": 98.39, "pct_under_4096": 99.67, "pct_under_8192": 99.95, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 504, 176, 521, 118, 256, 11, 230, 440, 556, 212, 282, 225, 258, 190, 429, 118, 897, 125, 77, 375, 607, 240, 344, 272, 187, 870, 294, 716, 296, 1108, 183, 210, 1508, 285, 296, 241, 276, 376, 139, 396, 781, 259, 551, 36, 683, 61, 1095, 319, 1516, 296, 126, 326, 759, 547, 15, 228, 132, 93, 496, 687, 318, 94, 143, 242, 216, 236, 183, 66, 31, 579, 55, 316, 1169, 179, 586, 120, 116, 3321, 778, 869, 366, 245, 1317, 1332, 250, 183, 869, 390, 44, 15, 319, 44, 56, 245, 533, 250, 41, 458, 1064, 237, 1415, 112, 370, 318, 51, 3186, 54, 384, 554, 574, 473, 169, 554, 65, 94, 548, 563, 1999, 59, 79, 129, 54, 48, 394, 225, 71, 91, 26, 141, 224, 109, 147, 1830, 437, 102, 3674, 252, 876, 451, 267, 124, 53, 436, 159, 89, 714, 313, 990, 1155, 112, 1083, 327, 43, 39, 173, 883, 166, 111, 69, 77, 128, 650, 90, 50, 13, 122, 179, 472, 306, 75, 305, 309, 254, 236, 101, 57, 52, 73, 269, 59, 169, 1486, 179, 438, 175, 56, 77, 44, 43, 99, 120, 225, 196, 323, 558, 195, 218, 280, 236, 522, 124, 62, 164, 98, 165, 1809, 121, 76, 149, 149, 315, 122, 158, 83, 372, 112, 80, 147, 288, 106, 175, 145, 260, 852, 65, 58, 75, 130, 26, 294, 58, 304, 128, 102, 252, 42, 278, 219, 143, 128, 91, 471, 347, 73, 125, 97, 172, 168, 150, 170, 26, 242, 130, 73, 474, 96, 109, 68, 206, 266, 69, 338, 167, 138, 98, 219, 143, 79, 246, 99, 207, 124, 282, 95, 66, 594, 798, 160, 325, 248, 740, 295, 101, 49, 166, 71, 266, 101, 148, 107, 148, 266, 21, 87, 216, 102, 112, 281, 156, 482, 226, 109, 102, 94, 74, 65, 54, 1140, 246, 336, 200, 101, 209, 172, 224, 224, 336, 278, 105, 230, 160, 107, 128, 200, 73, 232, 115, 324, 339, 179, 142, 196, 194, 224, 296, 212, 315, 156, 846, 194, 252, 102, 399, 208, 183, 642, 216, 249, 98, 82, 77, 516, 186, 331, 76, 189, 85, 187, 68, 75, 95, 192, 100, 115, 102, 66, 323, 76, 75, 114, 55, 338, 77, 69, 69, 244, 52, 63, 349, 81, 63, 87, 142, 56, 65, 89, 126, 188, 34, 1122, 120, 169, 37, 82, 122, 220, 327, 193, 142, 135, 50, 135, 180, 178, 271, 30, 630, 214, 135, 173, 150, 86, 220, 285, 74, 64, 102, 81, 210, 88, 251, 131, 496, 140, 140, 93, 91, 229, 57, 115, 110, 111, 315, 186, 80, 249, 90, 175, 179, 572, 48, 418, 323, 67, 110, 66, 102, 70, 222, 423, 123, 286, 388, 88, 98, 203, 509, 61, 256, 126, 365, 211, 219, 159, 506, 150, 92, 97, 273, 220, 74, 106, 686, 75, 139, 722, 307, 785, 191, 262, 53, 116, 155, 308, 115, 246, 179, 145, 88, 260, 123, 157, 273, 398, 227, 171, 319, 191, 217, 185, 398, 285, 218, 879, 107, 453, 147, 188, 181, 267, 213, 164, 150, 58, 310, 361, 229, 203, 1896, 135, 328, 171, 290, 280, 194, 445, 281, 138, 252, 345, 329, 220, 300, 230, 337, 246, 260, 715, 48, 211, 98, 125, 179, 190, 124, 516, 265, 158, 618, 204, 832, 306, 190, 632, 501, 237, 190, 304, 239, 382, 116, 49, 414, 389, 522, 87, 142, 186, 396, 122, 227, 723, 303, 127, 99, 167, 206, 168, 708, 641, 417, 273, 230, 156, 632, 755, 128, 216, 309, 651, 393, 408, 370, 515, 79, 280, 230, 109, 723, 110, 606, 337, 37, 103, 238, 378, 20, 412, 142, 153, 555, 253, 171, 343, 245, 331, 221, 527, 186, 192, 549, 152, 221, 116, 251, 146, 121, 1833, 194, 476, 167, 130, 863, 436, 374, 374, 282, 552, 67, 111, 263, 116, 702, 680, 67, 138, 457, 286, 298, 362, 360, 341, 280, 444, 165, 194, 184, 241, 129, 615, 167, 309, 747, 467, 956, 270, 230, 376, 594, 369, 367, 214, 103, 389, 519, 318, 495, 284, 237, 1050, 409, 963, 47, 1137, 746, 541, 26, 858, 827, 1488, 716, 666, 396, 636, 335, 766, 541, 2062, 100, 677, 201, 672, 114, 1010, 1271, 502, 545, 808, 590, 622, 1130, 107, 315, 575, 892, 653, 469, 521, 505, 298, 676, 351, 985, 69, 565, 636, 1029, 201, 587, 141, 666, 820, 1823, 462, 668, 682, 622, 1723, 612, 719, 1704, 574, 623, 840, 921, 418, 610, 871, 743, 704, 523, 1060, 1025, 1097, 411, 451, 386, 117, 505, 479, 508, 502, 652, 502, 301, 435, 800, 190, 810, 521, 644, 511, 488, 1218, 57, 1386, 130, 96, 430, 830, 1479, 237, 496, 448, 942, 583, 470, 619, 608, 419, 477, 131, 842, 364, 383, 516, 782, 1783, 475, 1478, 462, 492, 159, 463, 150, 138, 413, 444, 526, 963, 499, 764, 530, 251, 201, 140, 80, 37, 314, 264, 118, 569, 193, 199, 334, 445, 372, 1908, 208, 487, 376, 593, 92, 835, 178, 363, 495, 196, 626, 595, 455, 494, 318, 199, 42, 142, 897, 56, 478, 672, 139, 282, 1211, 99, 1055, 211, 256, 248, 446, 291, 142, 375, 415, 66, 519, 63, 237, 229, 181, 95, 483, 220, 249, 488, 232, 520, 469, 2002, 1060, 67, 3007, 179, 230, 189, 6080, 171, 260, 291, 251, 170, 430, 104, 474, 256, 519, 552, 1853, 117, 482, 464, 535, 240, 295, 506, 307, 545, 222, 294, 229, 187, 164, 461, 833, 376, 1592, 94, 652, 1107, 516, 161, 278, 278, 1716, 227, 78, 398, 98, 94, 941, 2317, 821, 1720, 818, 1324, 1290, 44, 1116, 5155, 16222, 533, 319, 304, 184, 529, 197, 589, 470, 208, 86, 343, 255, 162, 183, 1132, 419, 393, 880, 115, 737, 658, 678, 529, 417, 810, 576, 562, 1144, 164, 219, 364, 1416, 796, 429, 476, 744, 295, 475, 207, 540, 1564, 556, 1356, 29, 414, 90, 343, 107, 205, 1132, 255, 229, 254, 797, 108, 303, 242, 677, 475, 406, 1154, 182, 128, 55, 176, 860, 448, 743, 547, 305, 430, 979, 517, 83, 460, 961, 476, 244, 812, 100, 230, 223, 214, 456, 52, 516, 191, 326, 523, 415, 337, 306, 330, 380, 299, 197, 861, 674, 177, 542, 527, 271, 536, 358, 518, 374, 507, 337, 376, 301, 2382, 194, 161, 1016, 550, 542, 278, 618, 437, 1759, 426, 3128, 496, 280, 1903, 475, 399, 113, 1210, 337, 502, 343, 197, 175, 162, 151, 347, 274, 155, 373, 718, 121, 889, 101, 1122, 219, 182, 557, 2674, 107, 446, 161, 355, 153, 267, 328, 221, 1354, 132, 80, 107, 277, 572, 208, 151, 61, 298, 61, 132, 767, 1999, 438, 344, 147, 371, 762, 250, 452, 104, 273, 132, 488, 157, 592, 344, 83, 286, 335, 487, 1200, 114, 41, 48, 1119, 998, 830, 822, 564, 167, 86, 2112, 584, 548, 104, 112, 378, 913, 781, 449, 81, 117, 63, 1143, 432, 472, 401, 312, 566, 279, 584, 80, 726, 76, 112, 81, 825, 562, 460, 305, 300, 213, 222, 372, 87, 141, 102, 411, 189, 488, 79, 495, 111, 619, 649, 370, 461, 1332, 76, 189, 159, 159, 349, 632, 323, 291, 462, 378, 518, 380, 549, 358, 722, 292, 381, 302, 822, 92, 206, 444, 232, 908, 1694, 884, 314, 121, 478, 819, 135, 22, 836, 318, 246, 224, 765, 302, 44, 86, 10, 1100, 1465, 1304, 810, 324, 310, 1282, 1246, 577, 475, 717, 77, 115, 45, 60, 1613, 160, 14, 166, 4246, 1304, 162, 193, 129, 178, 1086, 58, 86, 193, 519, 206, 109, 109, 147, 403, 107, 77, 178, 500, 473, 117, 60, 340, 135, 108, 514, 210, 72, 975, 330, 188, 100, 161, 194, 90, 273, 10, 86, 62, 424, 66, 162, 896, 90, 273, 104, 474, 75, 1323, 333, 478, 90, 73, 70, 149, 428, 551, 71, 155, 606, 864, 69, 493, 165, 898, 134, 121, 442, 950, 2186, 7, 618, 1747, 497, 186, 78, 439, 40, 76, 587, 80, 352, 127, 632, 402, 597, 1120, 492, 116, 89, 102, 134, 268, 421, 288, 96, 2752, 2752, 685, 579, 627, 627, 1539, 1365, 847, 325, 51, 782, 500, 174, 554, 493, 55, 35, 289, 211, 512, 75, 4190, 537, 227, 65, 82, 35, 263, 260, 56, 140, 57, 127, 84, 24, 177, 151, 363, 463, 640, 326, 151, 47, 666, 772, 154, 278, 221, 646, 622, 641, 182, 129, 271, 413, 97, 406, 267, 289, 175, 288, 431, 288, 323, 143, 467, 331, 96, 432, 354, 439, 195, 288, 376, 193, 641, 211, 397, 266, 107, 134, 173, 538, 91, 142, 350, 202, 178, 375, 429, 478, 574, 576, 474, 587, 203, 408, 116, 565, 480, 330, 1184, 486, 270, 59, 623, 225, 869, 469, 473, 104, 488, 617, 306, 829, 267, 237, 275, 800, 424, 170, 100, 548, 245, 251, 306, 571, 116, 381, 259, 80, 278, 610, 2371, 255, 180, 528, 347, 353, 301, 345, 87, 790, 665, 665, 450, 114, 687, 736, 967, 233, 158, 575, 622, 494, 516, 474, 521, 537, 679, 200, 560, 479, 14, 202, 420, 498, 257, 614, 576, 657, 539, 562, 327, 561, 427, 491, 399, 250, 50, 791, 326, 1212, 240, 631, 326, 725, 78, 245, 431, 590, 101, 545, 347, 40, 586, 106, 229, 271, 431, 170, 228, 279, 667, 573, 598, 345, 539, 492, 646, 154, 816, 89, 575, 251, 390, 998, 254, 265, 609, 61, 197, 258, 591, 291, 306, 344, 273, 320, 316, 191, 413, 827, 101, 432, 211, 156, 739, 480, 462, 95, 546, 529, 151, 349, 1064, 360, 302, 855, 350, 177, 50, 558, 242, 425, 404, 624, 591, 313, 253, 1284, 261, 44, 69, 1466, 200, 1240, 940, 402, 843, 133, 443, 729, 187, 612, 72, 560, 280, 336, 126, 787, 551, 778, 586, 310, 209, 532, 245, 85, 287, 277, 108, 417, 203, 426, 440, 541, 392, 246, 197, 151, 612, 645, 388, 112, 695, 478, 825, 777, 229, 183, 476, 570, 485, 369, 275, 349, 552, 63, 229, 674, 421, 221, 354, 388, 576, 181, 630, 407, 945, 123, 309, 433, 331, 736, 191, 374, 179, 590, 319, 74, 792, 543, 345, 471, 546, 545, 620, 204, 390, 248, 460, 476, 525, 238, 1027, 748, 314, 143, 230, 149, 887, 70, 193, 643, 561, 270, 264, 872, 260, 507, 833, 192, 526, 84, 430, 135, 114, 209, 444, 160, 270, 96, 262, 871, 530, 1887, 575, 317, 112, 286, 672, 737, 525, 860, 238, 151, 266, 419, 175, 891, 325, 249, 767, 799, 303, 265, 298, 173, 432, 423, 199, 74, 257, 314, 199, 725, 446, 440, 40, 703, 594, 332, 1014, 414, 1062, 763, 871, 101, 696, 464, 510, 371, 479, 234, 515, 364, 546, 391, 251, 443, 790, 523, 775, 675, 1095, 631, 743, 633, 430, 358, 455, 187, 538, 1118, 931, 260, 162, 1059, 162, 1131, 155, 264, 1011, 651, 929, 842, 981, 86, 616, 257, 710, 274, 184, 81, 336, 238, 708, 524, 530, 627, 792, 1169, 225, 363, 1297, 368, 1118, 331, 148, 857, 657, 379, 328, 835, 242, 209, 1169, 126, 755, 1163, 465, 232, 569, 226, 195, 157, 275, 298, 198, 594, 360, 433, 356, 105, 182, 346, 605, 846, 333, 847, 63, 405, 175, 590, 104, 773, 761, 795, 226, 329, 2542, 567, 968, 78, 204, 100, 145, 74, 141, 104, 110, 234, 175, 48, 81, 126, 232, 181, 360, 169, 57, 147, 387, 691, 578, 60, 60, 446, 193, 60, 102, 81, 127, 312, 393, 194, 4058, 442, 636, 457, 565, 299, 1370, 81, 693, 625, 848, 198, 225, 390, 979, 3140, 300, 1620, 1114, 92, 291, 138, 658, 188, 1068, 180, 114, 1960, 369, 213, 337, 335, 892, 569, 590, 1390, 980, 1309, 518, 134, 281, 5531, 169, 741, 1016, 635, 74, 169, 953, 441, 120, 622, 1001, 62, 532, 569, 333, 1044, 2046, 539, 864, 656, 570, 1292, 481, 561, 530, 416, 793, 923, 312, 708, 409, 1098, 349, 375, 311, 1429, 43, 2204, 3752, 948, 849, 947, 898, 109, 578, 630, 576, 714, 143, 422, 703, 2286, 506, 138, 581, 586, 538, 338, 1602, 723, 1138, 73, 495, 993, 430, 1477, 783, 826, 332, 108, 376, 1704, 1127, 2237, 323, 464, 297, 404, 328, 109, 2278, 76, 863, 402, 420, 192, 146, 460, 467, 237, 237, 544, 470, 363, 178, 569, 554, 846, 384, 502, 390, 291, 660, 276, 2616, 354, 650, 767, 220, 3358, 7743, 2152, 2622, 1995, 1053, 1488, 2300, 1400, 66, 128, 97, 109, 171, 192, 241, 74, 708, 316, 260, 724, 25, 474, 279, 23, 950, 205, 351, 152, 1318, 183, 164, 793, 176, 209, 492, 255, 404, 1144, 85, 171, 992, 399, 776, 138, 456, 1585, 1794 ] }, "SWE-rebench (leaderboard, all HF months)": { "benchmark": "SWE-rebench (leaderboard, all HF months)", "status": "ok", "stats": { "count": 860, "min": 15, "max": 15154, "mean": 571.934, "median": 350.0, "p25": 173.5, "p75": 640.0, "p95": 1574.2999999999997, "p99": 4234.619999999997, "pct_under_1024": 88.14, "pct_under_2048": 96.98, "pct_under_4096": 98.72, "pct_under_8192": 99.65, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 222, 583, 225, 90, 43, 214, 174, 779, 865, 143, 207, 156, 837, 121, 1200, 337, 579, 446, 254, 251, 795, 1304, 153, 171, 248, 350, 233, 424, 232, 198, 16, 114, 199, 274, 828, 103, 415, 688, 85, 1670, 236, 866, 1411, 318, 776, 303, 626, 1328, 294, 387, 1709, 508, 269, 381, 275, 448, 142, 88, 635, 208, 596, 188, 1178, 1146, 80, 141, 321, 485, 739, 437, 436, 1446, 105, 770, 545, 54, 117, 88, 243, 253, 597, 1807, 102, 71, 84, 34, 83, 752, 1101, 999, 233, 197, 1168, 50, 678, 206, 108, 163, 885, 642, 58, 94, 297, 889, 256, 617, 309, 37, 475, 2051, 146, 513, 425, 28, 254, 99, 46, 532, 1008, 41, 401, 259, 333, 35, 1323, 697, 180, 164, 72, 237, 269, 80, 1580, 247, 96, 148, 59, 727, 1008, 70, 221, 101, 99, 315, 766, 1115, 53, 170, 178, 394, 274, 62, 329, 59, 336, 386, 342, 529, 49, 171, 891, 2886, 963, 132, 135, 187, 112, 626, 178, 1342, 427, 245, 58, 150, 340, 649, 111, 1375, 363, 448, 92, 103, 138, 588, 210, 226, 479, 1085, 317, 412, 407, 951, 714, 1448, 635, 101, 325, 305, 49, 201, 214, 409, 67, 2419, 648, 126, 281, 808, 603, 258, 73, 647, 394, 110, 966, 546, 472, 37, 312, 131, 49, 141, 61, 674, 187, 796, 200, 412, 113, 630, 184, 1467, 264, 544, 504, 95, 183, 290, 177, 467, 310, 664, 719, 442, 72, 256, 491, 168, 388, 273, 216, 127, 402, 648, 277, 179, 986, 482, 146, 976, 1158, 323, 717, 37, 345, 1044, 766, 244, 1722, 442, 127, 1805, 300, 51, 790, 875, 157, 958, 397, 171, 707, 2950, 274, 45, 370, 282, 361, 131, 155, 144, 783, 553, 912, 480, 198, 1810, 252, 35, 456, 671, 319, 123, 445, 618, 467, 463, 950, 967, 349, 614, 217, 52, 166, 183, 68, 107, 82, 44, 178, 76, 168, 396, 189, 369, 370, 343, 1224, 153, 174, 117, 138, 343, 117, 159, 158, 43, 972, 408, 260, 329, 149, 223, 102, 245, 170, 168, 78, 43, 498, 304, 59, 124, 94, 312, 123, 160, 966, 292, 100, 68, 50, 147, 165, 1037, 88, 5829, 127, 3467, 340, 346, 1350, 105, 269, 301, 434, 458, 2346, 204, 533, 186, 256, 1123, 95, 616, 168, 392, 815, 158, 64, 33, 199, 44, 521, 117, 155, 515, 146, 58, 163, 612, 688, 100, 133, 522, 358, 405, 200, 133, 130, 61, 571, 425, 1122, 136, 180, 124, 250, 9189, 1741, 793, 999, 168, 107, 431, 216, 50, 183, 274, 429, 1018, 910, 164, 149, 371, 71, 6698, 63, 909, 360, 375, 252, 34, 454, 304, 559, 409, 190, 105, 131, 532, 321, 494, 494, 478, 318, 30, 288, 135, 80, 488, 156, 382, 386, 764, 86, 470, 75, 1856, 196, 1920, 481, 49, 522, 216, 175, 284, 283, 544, 161, 59, 188, 205, 505, 515, 854, 339, 639, 245, 377, 258, 553, 690, 86, 64, 61, 34, 865, 1188, 177, 132, 115, 397, 993, 180, 42, 781, 275, 249, 1116, 1955, 578, 792, 168, 813, 294, 258, 249, 314, 368, 250, 265, 771, 1434, 2563, 1920, 308, 334, 257, 1574, 928, 70, 395, 135, 929, 2746, 540, 270, 232, 172, 1688, 513, 397, 136, 156, 149, 576, 224, 97, 730, 519, 645, 360, 345, 387, 207, 665, 424, 478, 351, 406, 184, 193, 197, 148, 492, 121, 1023, 1385, 1012, 337, 640, 578, 497, 82, 262, 1618, 2423, 102, 363, 314, 764, 387, 676, 339, 57, 208, 542, 409, 1025, 45, 431, 162, 129, 355, 32, 151, 226, 60, 337, 668, 238, 234, 52, 394, 774, 82, 1252, 725, 332, 443, 113, 339, 1253, 368, 774, 370, 363, 180, 161, 15, 474, 367, 837, 126, 595, 210, 519, 90, 132, 136, 399, 579, 40, 525, 1226, 77, 568, 140, 36, 138, 201, 137, 734, 70, 935, 1009, 237, 4201, 265, 1388, 268, 13085, 287, 567, 1219, 404, 206, 95, 179, 925, 892, 1614, 332, 314, 232, 154, 175, 950, 1069, 400, 275, 806, 395, 413, 3203, 685, 391, 74, 4573, 613, 526, 70, 102, 914, 610, 353, 191, 4099, 412, 69, 1259, 241, 181, 1036, 1178, 234, 334, 346, 253, 451, 430, 177, 452, 451, 658, 953, 1210, 416, 697, 397, 425, 350, 1309, 485, 79, 966, 2405, 1053, 500, 290, 499, 462, 288, 408, 1328, 1312, 746, 796, 2875, 387, 326, 229, 4639, 1418, 718, 2390, 1164, 475, 50, 287, 446, 793, 382, 264, 567, 880, 409, 267, 449, 352, 885, 15154, 1431, 392, 763, 220, 371, 1426, 297, 397, 412, 279, 351, 664, 347, 2114, 520, 605, 370, 511, 241, 1037, 4283, 766, 1418, 559, 568, 72, 269, 546, 918, 447, 1183, 572, 274, 323, 463, 253, 350, 204, 251, 1064, 552, 95, 261, 111, 481, 775, 426, 812, 202, 1149, 479, 583, 538, 197, 640, 558, 431, 297, 470, 83, 213, 436, 692, 237, 77, 433, 314, 397, 3371, 754, 1206, 690, 1189, 1430, 210, 336, 208, 431, 242, 309, 509, 476, 1941, 652, 1333, 353, 1343, 406, 518, 518, 6874, 262, 813, 911, 441, 515, 925, 188, 472, 190, 423, 516, 1424, 345, 404, 1123, 1656, 150 ] }, "SWE-rebench (2026-01..03, latest on HF)": { "benchmark": "SWE-rebench (2026-01..03, latest on HF)", "status": "ok", "stats": { "count": 215, "min": 50, "max": 15154, "mean": 863.391, "median": 452.0, "p25": 293.5, "p75": 888.5, "p95": 2545.999999999992, "p99": 6561.099999999967, "pct_under_1024": 79.53, "pct_under_2048": 93.49, "pct_under_4096": 96.28, "pct_under_8192": 99.07, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 1009, 237, 4201, 265, 1388, 268, 13085, 287, 567, 1219, 404, 206, 95, 179, 925, 892, 1614, 332, 314, 232, 154, 175, 950, 1069, 400, 275, 806, 395, 413, 3203, 685, 391, 74, 4573, 613, 526, 70, 102, 914, 610, 353, 191, 4099, 412, 69, 1259, 241, 181, 1036, 1178, 234, 334, 346, 253, 451, 430, 177, 452, 451, 658, 953, 1210, 416, 697, 397, 425, 350, 1309, 485, 79, 966, 2405, 1053, 500, 290, 499, 462, 288, 408, 1328, 1312, 746, 796, 2875, 387, 326, 229, 4639, 1418, 718, 2390, 1164, 475, 50, 287, 446, 793, 382, 264, 567, 880, 409, 267, 449, 352, 885, 15154, 1431, 392, 763, 220, 371, 1426, 297, 397, 412, 279, 351, 664, 347, 2114, 520, 605, 370, 511, 241, 1037, 4283, 766, 1418, 559, 568, 72, 269, 546, 918, 447, 1183, 572, 274, 323, 463, 253, 350, 204, 251, 1064, 552, 95, 261, 111, 481, 775, 426, 812, 202, 1149, 479, 583, 538, 197, 640, 558, 431, 297, 470, 83, 213, 436, 692, 237, 77, 433, 314, 397, 3371, 754, 1206, 690, 1189, 1430, 210, 336, 208, 431, 242, 309, 509, 476, 1941, 652, 1333, 353, 1343, 406, 518, 518, 6874, 262, 813, 911, 441, 515, 925, 188, 472, 190, 423, 516, 1424, 345, 404, 1123, 1656, 150 ] }, "SWE-Lancer Diamond (IC SWE)": { "benchmark": "SWE-Lancer Diamond (IC SWE)", "status": "ok", "stats": { "count": 198, "min": 229, "max": 1653, "mean": 656.025, "median": 653.0, "p25": 589.0, "p75": 716.0, "p95": 877.7499999999999, "p99": 1018.6300000000001, "pct_under_1024": 98.99, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 736, 1653, 765, 723, 653, 688, 369, 651, 543, 677, 657, 643, 576, 525, 589, 405, 844, 618, 596, 708, 643, 720, 548, 656, 691, 655, 718, 537, 531, 292, 682, 539, 710, 655, 723, 760, 565, 630, 557, 586, 612, 623, 599, 522, 694, 675, 671, 644, 649, 797, 721, 686, 680, 735, 412, 564, 600, 636, 630, 649, 522, 556, 633, 677, 653, 665, 634, 598, 877, 556, 739, 297, 768, 583, 505, 781, 713, 561, 644, 650, 945, 590, 607, 559, 654, 677, 641, 580, 535, 787, 683, 634, 651, 704, 1004, 589, 860, 785, 571, 640, 619, 770, 826, 705, 572, 622, 882, 672, 667, 394, 434, 752, 702, 823, 774, 385, 597, 576, 701, 658, 229, 531, 1015, 460, 712, 788, 387, 996, 703, 589, 838, 618, 667, 910, 667, 842, 560, 725, 709, 723, 726, 657, 707, 758, 802, 640, 666, 555, 726, 619, 771, 308, 641, 654, 662, 616, 994, 653, 620, 607, 667, 674, 580, 624, 696, 735, 669, 781, 668, 551, 584, 607, 656, 614, 657, 272, 717, 1136, 750, 747, 601, 622, 648, 589, 852, 763, 651, 589, 371, 661, 659, 676, 244, 519, 640, 629, 598, 890 ] }, "SWE-Lancer Diamond (SWE Manager)": { "benchmark": "SWE-Lancer Diamond (SWE Manager)", "status": "ok", "stats": { "count": 265, "min": 642, "max": 7610, "mean": 2296.698, "median": 2128.0, "p25": 1686.0, "p75": 2770.0, "p95": 3889.9999999999986, "p99": 5296.840000000015, "pct_under_1024": 2.26, "pct_under_2048": 47.17, "pct_under_4096": 96.23, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 3087, 3228, 2748, 3134, 2957, 5981, 1535, 1881, 2978, 1919, 1623, 2493, 1453, 3569, 3346, 1935, 1678, 2236, 1939, 3539, 1710, 2150, 1270, 2367, 2030, 1168, 2286, 2796, 4412, 1979, 1757, 1498, 1191, 1724, 2189, 2131, 1613, 1256, 1327, 747, 1587, 3822, 1452, 1416, 1295, 1313, 2566, 2495, 1447, 1452, 2667, 2672, 3510, 2372, 3070, 3364, 3003, 1215, 1988, 3693, 2399, 1277, 1802, 3116, 2183, 1846, 2404, 2082, 1825, 2831, 1248, 2868, 2687, 3928, 1431, 1363, 1809, 1515, 1756, 2448, 2361, 1217, 2028, 1998, 1741, 2095, 1973, 1972, 3656, 4912, 2109, 1884, 2870, 2525, 1390, 3016, 1850, 2128, 2936, 3132, 2542, 2303, 1820, 2440, 2208, 3677, 2197, 2827, 1804, 1925, 2018, 1875, 1649, 2201, 1775, 3320, 2859, 1523, 943, 3024, 1803, 1686, 1659, 1856, 3955, 1312, 2352, 3181, 1301, 1121, 2736, 3659, 6033, 3929, 1631, 1729, 1731, 2770, 2048, 2515, 1311, 1961, 2733, 1709, 2381, 1740, 1939, 1980, 1261, 2005, 1512, 3324, 1507, 2176, 2295, 3232, 1743, 2299, 4097, 1805, 1604, 1980, 1538, 1689, 7610, 1933, 1490, 1942, 1779, 2610, 2528, 3394, 2320, 1785, 2805, 1057, 2453, 3041, 2115, 2009, 1566, 3144, 1593, 2728, 3291, 3726, 1783, 1696, 2969, 2819, 2327, 1527, 1911, 2701, 2016, 1807, 2558, 3907, 2219, 1595, 1284, 2071, 1912, 2377, 1343, 2727, 2680, 4286, 3813, 2196, 2028, 1689, 2956, 2184, 4368, 3202, 1210, 3480, 3004, 1170, 2421, 3107, 2291, 1360, 1018, 1635, 3163, 1463, 1486, 2660, 2512, 1853, 1223, 2975, 1226, 1260, 3229, 4364, 2172, 2273, 983, 2325, 4195, 1530, 2494, 3313, 2164, 1686, 2373, 2818, 2538, 1286, 2380, 2636, 2266, 2526, 2485, 976, 2106, 642, 1634, 2846, 1942, 2531, 1897 ] }, "Terminal-Bench 2.0": { "benchmark": "Terminal-Bench 2.0", "status": "ok", "stats": { "count": 89, "min": 34, "max": 1148, "mean": 231.067, "median": 167.0, "p25": 116.0, "p75": 303.0, "p95": 575.1999999999998, "p99": 906.8800000000012, "pct_under_1024": 98.88, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 562, 311, 117, 394, 135, 142, 132, 122, 70, 262, 319, 70, 72, 117, 584, 103, 34, 262, 179, 286, 446, 254, 156, 70, 126, 146, 123, 250, 874, 34, 99, 57, 113, 237, 122, 151, 305, 209, 274, 304, 167, 1148, 343, 221, 116, 105, 351, 341, 142, 290, 80, 100, 446, 421, 303, 73, 134, 121, 190, 128, 147, 201, 619, 127, 196, 113, 313, 77, 81, 121, 155, 516, 230, 291, 433, 593, 173, 284, 281, 73, 50, 395, 276, 64, 145, 315, 89, 276, 88 ] }, "Terminal-Bench 4.0": { "benchmark": "Terminal-Bench 4.0", "status": "ok", "stats": { "count": 66, "min": 89, "max": 1972, "mean": 455.682, "median": 393.5, "p25": 228.5, "p75": 583.5, "p95": 945.0, "p99": 1493.5999999999958, "pct_under_1024": 96.97, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 957, 826, 1236, 552, 89, 496, 254, 296, 122, 251, 602, 184, 909, 324, 337, 755, 349, 181, 665, 367, 513, 610, 141, 819, 201, 457, 363, 420, 990, 254, 338, 570, 588, 140, 224, 432, 237, 304, 218, 203, 226, 838, 112, 496, 443, 654, 236, 730, 563, 476, 471, 501, 542, 217, 201, 181, 435, 318, 420, 316, 243, 609, 119, 213, 769, 1972 ] }, "DeepSWE": { "benchmark": "DeepSWE", "status": "ok", "stats": { "count": 113, "min": 117, "max": 1302, "mean": 485.168, "median": 439.0, "p25": 299.0, "p75": 619.0, "p95": 917.3999999999997, "p99": 1119.3199999999997, "pct_under_1024": 96.46, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 520, 781, 444, 219, 384, 169, 381, 1302, 439, 416, 381, 286, 561, 831, 288, 619, 261, 383, 592, 316, 746, 486, 243, 273, 469, 278, 1124, 321, 740, 362, 814, 161, 412, 351, 424, 420, 294, 143, 319, 563, 233, 724, 466, 573, 299, 220, 885, 167, 546, 766, 533, 907, 991, 309, 201, 526, 192, 177, 630, 389, 117, 296, 700, 603, 509, 225, 393, 575, 490, 778, 565, 672, 190, 662, 141, 699, 568, 237, 381, 299, 593, 807, 362, 513, 290, 637, 408, 193, 777, 386, 640, 563, 933, 576, 230, 214, 387, 512, 450, 374, 333, 425, 529, 615, 1060, 309, 1085, 503, 775, 708, 302, 343, 642 ] }, "LiveCodeBench (release_v6)": { "benchmark": "LiveCodeBench (release_v6)", "status": "ok", "stats": { "count": 1055, "min": 114, "max": 1521, "mean": 447.891, "median": 416.0, "p25": 298.5, "p75": 553.5, "p95": 825.1999999999998, "p99": 1092.1200000000008, "pct_under_1024": 98.67, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 404, 290, 507, 695, 688, 231, 703, 435, 692, 287, 295, 253, 262, 292, 249, 275, 619, 508, 446, 417, 268, 520, 386, 243, 360, 358, 407, 199, 287, 332, 375, 124, 242, 508, 512, 334, 580, 313, 433, 555, 317, 661, 378, 357, 383, 204, 469, 313, 525, 484, 217, 279, 436, 224, 342, 465, 334, 333, 233, 294, 353, 286, 498, 381, 283, 342, 328, 354, 327, 637, 377, 564, 355, 228, 603, 508, 573, 462, 438, 295, 427, 312, 465, 277, 351, 398, 205, 239, 361, 393, 395, 284, 361, 481, 525, 533, 390, 239, 325, 235, 311, 591, 201, 404, 546, 473, 326, 330, 305, 349, 504, 385, 416, 573, 454, 335, 351, 478, 478, 276, 450, 315, 382, 474, 495, 373, 621, 395, 427, 235, 294, 353, 258, 299, 883, 312, 260, 412, 498, 375, 304, 220, 529, 549, 272, 264, 965, 422, 311, 328, 514, 630, 292, 445, 708, 408, 653, 275, 379, 517, 188, 477, 789, 418, 388, 385, 300, 466, 589, 461, 390, 405, 270, 441, 304, 458, 811, 456, 277, 345, 253, 471, 372, 366, 504, 287, 224, 393, 356, 510, 312, 466, 361, 347, 730, 330, 186, 963, 337, 523, 607, 700, 404, 558, 811, 557, 563, 423, 392, 530, 753, 758, 295, 295, 602, 822, 757, 190, 439, 457, 884, 649, 402, 421, 1062, 311, 311, 299, 391, 392, 601, 467, 421, 189, 557, 618, 689, 500, 319, 1000, 439, 460, 822, 516, 278, 429, 607, 700, 646, 186, 770, 343, 285, 697, 525, 259, 607, 365, 283, 712, 610, 653, 1369, 1214, 189, 458, 448, 512, 625, 557, 286, 397, 501, 538, 319, 389, 927, 621, 815, 160, 184, 634, 513, 783, 381, 645, 314, 562, 650, 275, 364, 447, 1104, 572, 249, 423, 700, 491, 580, 268, 254, 740, 512, 490, 759, 253, 454, 545, 511, 630, 596, 218, 196, 372, 631, 555, 223, 189, 929, 538, 451, 275, 437, 644, 346, 816, 174, 265, 288, 633, 359, 563, 324, 340, 185, 520, 597, 371, 230, 583, 739, 714, 523, 504, 693, 1195, 115, 286, 209, 534, 1022, 197, 424, 491, 440, 671, 203, 547, 448, 603, 167, 233, 228, 485, 395, 467, 667, 700, 960, 232, 230, 683, 852, 352, 628, 200, 457, 326, 575, 232, 403, 556, 533, 417, 138, 658, 894, 292, 702, 910, 169, 466, 492, 299, 1521, 226, 527, 300, 733, 1082, 556, 175, 445, 306, 634, 460, 349, 445, 327, 747, 254, 306, 393, 298, 539, 167, 408, 441, 192, 787, 524, 309, 514, 268, 810, 642, 289, 634, 485, 456, 450, 602, 634, 680, 347, 828, 673, 683, 337, 622, 492, 553, 318, 674, 292, 325, 378, 632, 400, 447, 495, 432, 907, 353, 435, 265, 287, 255, 464, 351, 493, 184, 154, 339, 872, 130, 259, 520, 686, 165, 157, 390, 302, 331, 308, 274, 184, 352, 577, 182, 569, 440, 512, 229, 633, 359, 236, 634, 147, 275, 169, 452, 229, 332, 157, 644, 293, 354, 481, 354, 285, 187, 239, 548, 245, 283, 388, 260, 411, 181, 309, 313, 551, 430, 794, 363, 380, 416, 247, 235, 438, 477, 476, 675, 477, 199, 609, 312, 640, 314, 435, 432, 342, 400, 254, 591, 486, 418, 286, 172, 319, 401, 757, 404, 432, 281, 247, 356, 274, 432, 374, 169, 789, 882, 547, 544, 535, 434, 274, 222, 312, 466, 424, 491, 428, 263, 309, 448, 336, 262, 213, 245, 467, 432, 201, 528, 229, 266, 446, 224, 140, 192, 371, 189, 515, 214, 329, 262, 334, 224, 203, 316, 386, 437, 479, 329, 307, 155, 600, 201, 144, 283, 501, 190, 543, 205, 224, 254, 369, 340, 242, 471, 469, 457, 353, 1074, 281, 592, 642, 484, 624, 344, 687, 224, 659, 661, 643, 319, 295, 347, 512, 361, 806, 569, 784, 441, 320, 515, 948, 291, 256, 638, 893, 509, 424, 671, 437, 485, 857, 817, 1401, 309, 336, 859, 519, 421, 677, 685, 160, 318, 318, 408, 669, 294, 611, 656, 854, 566, 641, 743, 524, 1104, 439, 528, 446, 565, 845, 354, 238, 310, 187, 336, 268, 207, 427, 206, 612, 248, 303, 206, 684, 346, 321, 329, 366, 339, 367, 275, 551, 409, 239, 356, 201, 613, 203, 236, 245, 185, 410, 192, 242, 388, 281, 477, 590, 428, 945, 460, 173, 649, 500, 534, 824, 244, 258, 641, 651, 807, 429, 650, 491, 691, 642, 473, 680, 527, 184, 445, 689, 526, 376, 413, 567, 438, 371, 390, 254, 454, 313, 284, 467, 585, 413, 638, 247, 367, 676, 579, 499, 217, 596, 300, 401, 600, 305, 447, 460, 536, 446, 488, 1276, 627, 651, 465, 545, 796, 597, 114, 738, 549, 583, 249, 896, 324, 850, 480, 226, 337, 774, 472, 1009, 634, 328, 554, 385, 290, 359, 339, 505, 605, 387, 584, 487, 601, 612, 904, 395, 364, 952, 524, 980, 509, 899, 568, 522, 1023, 656, 512, 481, 543, 324, 179, 233, 509, 221, 385, 507, 199, 397, 271, 284, 238, 345, 693, 371, 308, 402, 449, 425, 539, 449, 304, 201, 383, 162, 252, 164, 716, 404, 711, 127, 311, 520, 394, 246, 326, 479, 296, 285, 370, 742, 219, 231, 331, 217, 266, 216, 590, 391, 167, 684, 470, 384, 338, 398, 297, 442, 216, 334, 436, 284, 322, 145, 221, 518, 361, 942, 518, 182, 464, 467, 249, 178, 181, 495, 832, 303, 532, 378, 292, 294, 661, 510, 600, 678, 854, 196, 518, 412, 505, 239, 319, 375, 467, 401, 387, 342, 572, 474, 410, 237, 488, 242, 490, 176, 1282, 482, 699, 326, 666, 623, 329, 821, 549, 396, 429, 568, 726, 692, 399, 261, 445, 460, 422, 414, 318, 310, 159, 318, 688, 128, 327, 560, 400, 663, 258, 340, 339, 611, 239, 1171, 172, 250, 359, 897, 1350, 778, 765, 586, 435, 356, 461, 627, 529, 778, 713, 938, 828, 755, 403, 640, 519, 381, 697, 524, 780, 526, 366, 884, 566, 330, 855, 280, 517, 553, 433, 303, 355, 598, 299, 339, 690, 271, 235, 462, 260, 342, 427, 278, 463, 248, 455, 284, 595, 368, 326, 447, 577, 324, 262, 403, 483, 389, 423, 289, 381, 755, 156, 228, 325, 522, 518, 424, 271, 474, 226, 504, 157, 442, 372, 239, 437, 463, 228, 381, 266, 284, 385, 419, 421, 288, 467, 258, 458, 570 ] }, "LiveCodeBench (release_v1, control)": { "benchmark": "LiveCodeBench (release_v1, control)", "status": "ok", "stats": { "count": 400, "min": 115, "max": 1521, "mean": 456.51, "median": 421.0, "p25": 304.75, "p75": 557.0, "p95": 822.0, "p99": 1104.9099999999992, "pct_under_1024": 98.25, "pct_under_2048": 100.0, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 404, 290, 507, 695, 688, 231, 703, 435, 692, 287, 295, 253, 262, 292, 249, 275, 619, 508, 446, 417, 268, 520, 386, 243, 360, 358, 407, 199, 287, 332, 375, 124, 242, 508, 512, 334, 580, 313, 433, 555, 317, 661, 378, 357, 383, 204, 469, 313, 525, 484, 217, 279, 436, 224, 342, 465, 334, 333, 233, 294, 353, 286, 498, 381, 283, 342, 328, 354, 327, 637, 377, 564, 355, 228, 603, 508, 573, 462, 438, 295, 427, 312, 465, 277, 351, 398, 205, 239, 361, 393, 395, 284, 361, 481, 525, 533, 390, 239, 325, 235, 311, 591, 201, 404, 546, 473, 326, 330, 305, 349, 504, 385, 416, 573, 454, 335, 351, 478, 478, 276, 450, 315, 382, 474, 495, 373, 621, 395, 427, 235, 294, 353, 258, 299, 883, 312, 260, 412, 498, 375, 304, 220, 529, 549, 272, 264, 965, 422, 311, 328, 514, 630, 292, 445, 708, 408, 653, 275, 379, 517, 188, 477, 789, 418, 388, 385, 300, 466, 589, 461, 390, 405, 270, 441, 304, 458, 811, 456, 277, 345, 253, 471, 372, 366, 504, 287, 224, 393, 356, 510, 312, 466, 361, 347, 730, 330, 186, 963, 337, 523, 607, 700, 404, 558, 811, 557, 563, 423, 392, 530, 753, 758, 295, 295, 602, 822, 757, 190, 439, 457, 884, 649, 402, 421, 1062, 311, 311, 299, 391, 392, 601, 467, 421, 189, 557, 618, 689, 500, 319, 1000, 439, 460, 822, 516, 278, 429, 607, 700, 646, 186, 770, 343, 285, 697, 525, 259, 607, 365, 283, 712, 610, 653, 1369, 1214, 189, 458, 448, 512, 625, 557, 286, 397, 501, 538, 319, 389, 927, 621, 815, 160, 184, 634, 513, 783, 381, 645, 314, 562, 650, 275, 364, 447, 1104, 572, 249, 423, 700, 491, 580, 268, 254, 740, 512, 490, 759, 253, 454, 545, 511, 630, 596, 218, 196, 372, 631, 555, 223, 189, 929, 538, 451, 275, 437, 644, 346, 816, 174, 265, 288, 633, 359, 563, 324, 340, 185, 520, 597, 371, 230, 583, 739, 714, 523, 504, 693, 1195, 115, 286, 209, 534, 1022, 197, 424, 491, 440, 671, 203, 547, 448, 603, 167, 233, 228, 485, 395, 467, 667, 700, 960, 232, 230, 683, 852, 352, 628, 200, 457, 326, 575, 232, 403, 556, 533, 417, 138, 658, 894, 292, 702, 910, 169, 466, 492, 299, 1521, 226, 527, 300, 733, 1082 ] }, "Aider Polyglot": { "benchmark": "Aider Polyglot", "status": "ok", "stats": { "count": 225, "min": 38, "max": 3270, "mean": 483.444, "median": 434.0, "p25": 268.0, "p75": 598.0, "p95": 915.9999999999995, "p99": 2853.319999999989, "pct_under_1024": 96.44, "pct_under_2048": 98.67, "pct_under_4096": 100.0, "pct_under_8192": 100.0, "pct_under_16384": 100.0, "pct_under_32768": 100.0 }, "per_problem_tokens": [ 380, 436, 403, 434, 466, 38, 505, 516, 310, 697, 462, 268, 452, 311, 672, 642, 520, 442, 303, 297, 132, 589, 269, 586, 662, 466, 429, 3270, 665, 393, 548, 283, 199, 516, 697, 517, 569, 598, 211, 89, 452, 139, 149, 312, 347, 88, 406, 625, 134, 715, 465, 135, 876, 331, 921, 223, 586, 215, 277, 236, 896, 365, 143, 276, 598, 874, 380, 179, 666, 572, 393, 548, 171, 466, 283, 57, 255, 598, 211, 282, 552, 731, 452, 139, 760, 398, 406, 303, 626, 51, 930, 462, 139, 297, 837, 135, 580, 510, 223, 129, 873, 223, 529, 215, 277, 706, 542, 365, 143, 276, 466, 346, 831, 179, 3143, 282, 644, 393, 548, 505, 283, 598, 211, 632, 268, 686, 731, 1240, 139, 343, 711, 398, 406, 1077, 303, 626, 51, 477, 510, 837, 135, 175, 580, 640, 132, 318, 921, 223, 589, 529, 383, 485, 215, 222, 706, 652, 365, 143, 276, 604, 346, 1058, 3143, 572, 393, 809, 283, 255, 707, 598, 672, 518, 336, 292, 444, 301, 88, 797, 626, 51, 632, 318, 135, 510, 132, 921, 1200, 1936, 215, 476, 766, 604, 832, 466, 346, 209, 122, 179, 572, 548, 368, 443, 727, 215, 261, 513, 268, 655, 125, 554, 217, 398, 286, 625, 51, 135, 205, 496, 490, 770, 542, 365, 413, 305, 887 ] } }