{ "name": "Brainsystems Multi-GPU Scaling Benchmarks", "published": "2026-07-18", "hardware": "NVIDIA RTX PRO 6000 Blackwell Max-Q, 96 GB GDDR7 ECC je GPU", "method": { "activeGpus": [ 1, 2, 4 ], "concurrentRequests": [ 1, 2, 4, 8, 16, 32 ], "durationPerPointSeconds": 300, "contextTokens": 8192, "generatedTokens": 256, "workerArchitecture": "one complete model instance per GPU" }, "totalCompletedRequests": 28950, "totalFailedRequests": 0, "overviewNoThinkingMean": [ { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 1, "mean_p95_response_start_seconds": 1.881917, "mean_p95_256_token_duration_seconds": 7.52175, "mean_aggregate_output_tokens_per_second": 72.327667, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 2, "mean_p95_response_start_seconds": 8.414333, "mean_p95_256_token_duration_seconds": 14.052417, "mean_aggregate_output_tokens_per_second": 87.361917, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 4, "mean_p95_response_start_seconds": 22.479917, "mean_p95_256_token_duration_seconds": 28.134333, "mean_aggregate_output_tokens_per_second": 87.11425, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 8, "mean_p95_response_start_seconds": 50.562667, "mean_p95_256_token_duration_seconds": 56.215583, "mean_aggregate_output_tokens_per_second": 88.434667, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 16, "mean_p95_response_start_seconds": 106.738667, "mean_p95_256_token_duration_seconds": 112.387667, "mean_aggregate_output_tokens_per_second": 87.46275, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 1, "concurrent_requests": 32, "mean_p95_response_start_seconds": 219.092917, "mean_p95_256_token_duration_seconds": 224.740917, "mean_aggregate_output_tokens_per_second": 86.96575, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 1, "mean_p95_response_start_seconds": 1.862083, "mean_p95_256_token_duration_seconds": 7.33, "mean_aggregate_output_tokens_per_second": 73.013917, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 2, "mean_p95_response_start_seconds": 1.890667, "mean_p95_256_token_duration_seconds": 7.586167, "mean_aggregate_output_tokens_per_second": 143.838167, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 4, "mean_p95_response_start_seconds": 10.34575, "mean_p95_256_token_duration_seconds": 16.051333, "mean_aggregate_output_tokens_per_second": 174.035417, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 8, "mean_p95_response_start_seconds": 24.618667, "mean_p95_256_token_duration_seconds": 30.322417, "mean_aggregate_output_tokens_per_second": 174.23475, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 16, "mean_p95_response_start_seconds": 52.976917, "mean_p95_256_token_duration_seconds": 58.686167, "mean_aggregate_output_tokens_per_second": 172.81175, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 2, "concurrent_requests": 32, "mean_p95_response_start_seconds": 109.595167, "mean_p95_256_token_duration_seconds": 115.299417, "mean_aggregate_output_tokens_per_second": 172.418, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 1, "mean_p95_response_start_seconds": 1.848417, "mean_p95_256_token_duration_seconds": 7.208333, "mean_aggregate_output_tokens_per_second": 72.904417, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 2, "mean_p95_response_start_seconds": 1.867167, "mean_p95_256_token_duration_seconds": 7.35125, "mean_aggregate_output_tokens_per_second": 145.889917, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 4, "mean_p95_response_start_seconds": 1.907, "mean_p95_256_token_duration_seconds": 7.648917, "mean_aggregate_output_tokens_per_second": 289.852583, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 8, "mean_p95_response_start_seconds": 13.253583, "mean_p95_256_token_duration_seconds": 19.006833, "mean_aggregate_output_tokens_per_second": 342.748583, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 16, "mean_p95_response_start_seconds": 27.917, "mean_p95_256_token_duration_seconds": 33.678917, "mean_aggregate_output_tokens_per_second": 346.3625, "models_included": 4 }, { "dataset": "no_thinking_mean", "active_gpus": 4, "concurrent_requests": 32, "mean_p95_response_start_seconds": 56.25875, "mean_p95_256_token_duration_seconds": 62.017, "mean_aggregate_output_tokens_per_second": 344.41225, "models_included": 4 } ], "systemOrientation": [ { "system": "Brainsystems X1", "active_gpus": 1, "orientation_concurrent_requests": 5, "team_size_orientation": 10, "interpolated_mean_p95_start_seconds": 29.501 }, { "system": "Brainsystems X2", "active_gpus": 2, "orientation_concurrent_requests": 10, "team_size_orientation": 20, "interpolated_mean_p95_start_seconds": 31.708 }, { "system": "Brainsystems X4", "active_gpus": 4, "orientation_concurrent_requests": 20, "team_size_orientation": 40, "interpolated_mean_p95_start_seconds": 35.002 } ], "models": [ { "id": "qwen30", "name": "Qwen 3 30B", "ollamaTag": "qwen3:30b", "thinkingMode": "false", "startMetric": "visible_response_start", "rounds": 1, "completedRequests": 8000, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.98809, "gpu4": 3.967462 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 0.708, "p95_256_token_duration_seconds": 1.659, "aggregate_output_tokens_per_second": 153.597, "completed_requests": 186, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 1.641, "p95_256_token_duration_seconds": 2.582, "aggregate_output_tokens_per_second": 193.235, "completed_requests": 234, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 4.282, "p95_256_token_duration_seconds": 5.236, "aggregate_output_tokens_per_second": 194.06, "completed_requests": 235, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 9.42, "p95_256_token_duration_seconds": 10.374, "aggregate_output_tokens_per_second": 198.188, "completed_requests": 240, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 19.719, "p95_256_token_duration_seconds": 20.661, "aggregate_output_tokens_per_second": 192.382, "completed_requests": 248, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 40.71, "p95_256_token_duration_seconds": 41.655, "aggregate_output_tokens_per_second": 192.358, "completed_requests": 263, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 0.699, "p95_256_token_duration_seconds": 1.637, "aggregate_output_tokens_per_second": 155.249, "completed_requests": 188, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 0.703, "p95_256_token_duration_seconds": 1.642, "aggregate_output_tokens_per_second": 308.021, "completed_requests": 373, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 2.486, "p95_256_token_duration_seconds": 3.432, "aggregate_output_tokens_per_second": 385.644, "completed_requests": 467, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 5.242, "p95_256_token_duration_seconds": 6.191, "aggregate_output_tokens_per_second": 388.946, "completed_requests": 471, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 10.664, "p95_256_token_duration_seconds": 11.619, "aggregate_output_tokens_per_second": 383.188, "completed_requests": 479, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 21.044, "p95_256_token_duration_seconds": 21.989, "aggregate_output_tokens_per_second": 382.425, "completed_requests": 493, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 0.737, "p95_256_token_duration_seconds": 1.687, "aggregate_output_tokens_per_second": 152.772, "completed_requests": 185, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 0.701, "p95_256_token_duration_seconds": 1.645, "aggregate_output_tokens_per_second": 308.847, "completed_requests": 374, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 0.698, "p95_256_token_duration_seconds": 1.641, "aggregate_output_tokens_per_second": 616.867, "completed_requests": 747, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 3.071, "p95_256_token_duration_seconds": 4.021, "aggregate_output_tokens_per_second": 765.505, "completed_requests": 927, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 5.657, "p95_256_token_duration_seconds": 6.611, "aggregate_output_tokens_per_second": 772.931, "completed_requests": 936, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 10.969, "p95_256_token_duration_seconds": 11.921, "aggregate_output_tokens_per_second": 763.173, "completed_requests": 954, "failed_requests": 0 } ] }, { "id": "qwen122", "name": "Qwen 3.5 122B", "ollamaTag": "qwen3.5:122b-a10b-q4_K_M", "thinkingMode": "false", "startMetric": "visible_response_start", "rounds": 1, "completedRequests": 3408, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.979509, "gpu4": 4.003381 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 1.691, "p95_256_token_duration_seconds": 3.801, "aggregate_output_tokens_per_second": 66.889, "completed_requests": 81, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 4.245, "p95_256_token_duration_seconds": 6.366, "aggregate_output_tokens_per_second": 79.276, "completed_requests": 96, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 10.602, "p95_256_token_duration_seconds": 12.723, "aggregate_output_tokens_per_second": 78.398, "completed_requests": 98, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 23.36, "p95_256_token_duration_seconds": 25.48, "aggregate_output_tokens_per_second": 79.125, "completed_requests": 102, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 48.786, "p95_256_token_duration_seconds": 50.904, "aggregate_output_tokens_per_second": 80.455, "completed_requests": 110, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 99.573, "p95_256_token_duration_seconds": 101.694, "aggregate_output_tokens_per_second": 78.67, "completed_requests": 126, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 1.717, "p95_256_token_duration_seconds": 3.813, "aggregate_output_tokens_per_second": 66.889, "completed_requests": 81, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 1.703, "p95_256_token_duration_seconds": 3.833, "aggregate_output_tokens_per_second": 132.127, "completed_requests": 160, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 6.105, "p95_256_token_duration_seconds": 8.249, "aggregate_output_tokens_per_second": 157.726, "completed_requests": 191, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 12.56, "p95_256_token_duration_seconds": 14.701, "aggregate_output_tokens_per_second": 155.997, "completed_requests": 195, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 25.448, "p95_256_token_duration_seconds": 27.588, "aggregate_output_tokens_per_second": 157.475, "completed_requests": 203, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 51.044, "p95_256_token_duration_seconds": 53.182, "aggregate_output_tokens_per_second": 155.728, "completed_requests": 219, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 1.686, "p95_256_token_duration_seconds": 3.735, "aggregate_output_tokens_per_second": 67.715, "completed_requests": 82, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 1.69, "p95_256_token_duration_seconds": 3.766, "aggregate_output_tokens_per_second": 134.604, "completed_requests": 163, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 1.704, "p95_256_token_duration_seconds": 3.824, "aggregate_output_tokens_per_second": 266.729, "completed_requests": 323, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 7.843, "p95_256_token_duration_seconds": 9.994, "aggregate_output_tokens_per_second": 304.792, "completed_requests": 381, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 14.59, "p95_256_token_duration_seconds": 16.743, "aggregate_output_tokens_per_second": 312.791, "completed_requests": 391, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 27.159, "p95_256_token_duration_seconds": 29.318, "aggregate_output_tokens_per_second": 314.946, "completed_requests": 406, "failed_requests": 0 } ] }, { "id": "llamaScout", "name": "Llama 4 Scout", "ollamaTag": "llama4:17b-scout-16e-instruct-q4_K_M", "thinkingMode": "false", "startMetric": "visible_response_start", "rounds": 3, "completedRequests": 8417, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.970641, "gpu4": 3.911953 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 1.610667, "p95_256_token_duration_seconds": 4.539, "aggregate_output_tokens_per_second": 56.154, "completed_requests": 204, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 4.974667, "p95_256_token_duration_seconds": 7.925667, "aggregate_output_tokens_per_second": 64.136667, "completed_requests": 233, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 12.925333, "p95_256_token_duration_seconds": 15.874667, "aggregate_output_tokens_per_second": 63.199, "completed_requests": 237, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 28.690667, "p95_256_token_duration_seconds": 31.637667, "aggregate_output_tokens_per_second": 63.625667, "completed_requests": 251, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 60.311667, "p95_256_token_duration_seconds": 63.257, "aggregate_output_tokens_per_second": 64.004333, "completed_requests": 275, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 123.563667, "p95_256_token_duration_seconds": 126.509333, "aggregate_output_tokens_per_second": 63.898, "completed_requests": 322, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 1.609667, "p95_256_token_duration_seconds": 4.487333, "aggregate_output_tokens_per_second": 56.704667, "completed_requests": 206, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 1.607, "p95_256_token_duration_seconds": 4.547, "aggregate_output_tokens_per_second": 110.656, "completed_requests": 402, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 6.978667, "p95_256_token_duration_seconds": 9.962667, "aggregate_output_tokens_per_second": 127.171667, "completed_requests": 462, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 15.292, "p95_256_token_duration_seconds": 18.274667, "aggregate_output_tokens_per_second": 126.396667, "completed_requests": 474, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 31.044667, "p95_256_token_duration_seconds": 34.028333, "aggregate_output_tokens_per_second": 124.985, "completed_requests": 498, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 62.926, "p95_256_token_duration_seconds": 65.914, "aggregate_output_tokens_per_second": 125.92, "completed_requests": 546, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 1.577667, "p95_256_token_duration_seconds": 4.426, "aggregate_output_tokens_per_second": 57.530667, "completed_requests": 209, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 1.611, "p95_256_token_duration_seconds": 4.474, "aggregate_output_tokens_per_second": 113.683667, "completed_requests": 413, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 1.613333, "p95_256_token_duration_seconds": 4.578, "aggregate_output_tokens_per_second": 224.615333, "completed_requests": 816, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 9.153, "p95_256_token_duration_seconds": 12.152667, "aggregate_output_tokens_per_second": 251.692333, "completed_requests": 924, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 17.297, "p95_256_token_duration_seconds": 20.305333, "aggregate_output_tokens_per_second": 250.498333, "completed_requests": 949, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 33.41, "p95_256_token_duration_seconds": 36.419667, "aggregate_output_tokens_per_second": 249.966, "completed_requests": 996, "failed_requests": 0 } ] }, { "id": "mistral128", "name": "Mistral Medium 3.5 128B", "ollamaTag": "mistral-medium-3.5:128b-q4_K_M", "thinkingMode": "false", "startMetric": "visible_response_start", "rounds": 3, "completedRequests": 2178, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.978743, "gpu4": 3.831182 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 3.518, "p95_256_token_duration_seconds": 20.088, "aggregate_output_tokens_per_second": 12.670667, "completed_requests": 48, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 22.796667, "p95_256_token_duration_seconds": 39.336, "aggregate_output_tokens_per_second": 12.8, "completed_requests": 51, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 62.110333, "p95_256_token_duration_seconds": 78.703667, "aggregate_output_tokens_per_second": 12.8, "completed_requests": 57, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 140.78, "p95_256_token_duration_seconds": 157.370667, "aggregate_output_tokens_per_second": 12.8, "completed_requests": 69, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 298.138, "p95_256_token_duration_seconds": 314.728667, "aggregate_output_tokens_per_second": 13.009667, "completed_requests": 93, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 612.525, "p95_256_token_duration_seconds": 629.105333, "aggregate_output_tokens_per_second": 12.937, "completed_requests": 141, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 3.422667, "p95_256_token_duration_seconds": 19.382667, "aggregate_output_tokens_per_second": 13.213, "completed_requests": 48, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 3.549667, "p95_256_token_duration_seconds": 20.322667, "aggregate_output_tokens_per_second": 24.548667, "completed_requests": 93, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 25.813333, "p95_256_token_duration_seconds": 42.561667, "aggregate_output_tokens_per_second": 25.6, "completed_requests": 102, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 65.380667, "p95_256_token_duration_seconds": 82.123, "aggregate_output_tokens_per_second": 25.599333, "completed_requests": 114, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 144.751, "p95_256_token_duration_seconds": 161.509333, "aggregate_output_tokens_per_second": 25.599, "completed_requests": 138, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 303.366667, "p95_256_token_duration_seconds": 320.112667, "aggregate_output_tokens_per_second": 25.599, "completed_requests": 186, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 3.393, "p95_256_token_duration_seconds": 18.985333, "aggregate_output_tokens_per_second": 13.6, "completed_requests": 51, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 3.466667, "p95_256_token_duration_seconds": 19.52, "aggregate_output_tokens_per_second": 26.425, "completed_requests": 96, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 3.612667, "p95_256_token_duration_seconds": 20.552667, "aggregate_output_tokens_per_second": 51.199, "completed_requests": 192, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 32.947333, "p95_256_token_duration_seconds": 49.859667, "aggregate_output_tokens_per_second": 49.005, "completed_requests": 201, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 74.124, "p95_256_token_duration_seconds": 91.056333, "aggregate_output_tokens_per_second": 49.229667, "completed_requests": 225, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 153.497, "p95_256_token_duration_seconds": 170.409333, "aggregate_output_tokens_per_second": 49.564, "completed_requests": 273, "failed_requests": 0 } ] }, { "id": "deepseek70", "name": "DeepSeek R1 70B", "ollamaTag": "deepseek-r1:70b", "thinkingMode": "true", "startMetric": "first_generated_token_including_reasoning", "rounds": 1, "completedRequests": 1096, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.967077, "gpu4": 3.813068 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 2.668, "p95_256_token_duration_seconds": 12.079, "aggregate_output_tokens_per_second": 21.47, "completed_requests": 26, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 13.857, "p95_256_token_duration_seconds": 23.293, "aggregate_output_tokens_per_second": 21.6, "completed_requests": 27, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 37.174, "p95_256_token_duration_seconds": 46.692, "aggregate_output_tokens_per_second": 21.834, "completed_requests": 29, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 83.673, "p95_256_token_duration_seconds": 93.196, "aggregate_output_tokens_per_second": 21.661, "completed_requests": 33, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 177.094, "p95_256_token_duration_seconds": 186.605, "aggregate_output_tokens_per_second": 21.866, "completed_requests": 41, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 362.985, "p95_256_token_duration_seconds": 372.479, "aggregate_output_tokens_per_second": 21.778, "completed_requests": 57, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 2.603, "p95_256_token_duration_seconds": 11.766, "aggregate_output_tokens_per_second": 21.6, "completed_requests": 27, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 2.562, "p95_256_token_duration_seconds": 12.155, "aggregate_output_tokens_per_second": 41.599, "completed_requests": 52, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 17.449, "p95_256_token_duration_seconds": 27.058, "aggregate_output_tokens_per_second": 43.199, "completed_requests": 54, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 41.581, "p95_256_token_duration_seconds": 51.19, "aggregate_output_tokens_per_second": 42.422, "completed_requests": 58, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 88.161, "p95_256_token_duration_seconds": 97.767, "aggregate_output_tokens_per_second": 43.322, "completed_requests": 66, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 182.241, "p95_256_token_duration_seconds": 191.841, "aggregate_output_tokens_per_second": 42.839, "completed_requests": 82, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 2.424, "p95_256_token_duration_seconds": 11.325, "aggregate_output_tokens_per_second": 22.296, "completed_requests": 27, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 2.502, "p95_256_token_duration_seconds": 11.691, "aggregate_output_tokens_per_second": 44.593, "completed_requests": 54, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 2.541, "p95_256_token_duration_seconds": 12.25, "aggregate_output_tokens_per_second": 85.882, "completed_requests": 104, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 23.332, "p95_256_token_duration_seconds": 33.046, "aggregate_output_tokens_per_second": 82.069, "completed_requests": 109, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 48.069, "p95_256_token_duration_seconds": 57.779, "aggregate_output_tokens_per_second": 83.198, "completed_requests": 117, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 95.22, "p95_256_token_duration_seconds": 104.924, "aggregate_output_tokens_per_second": 83.041, "completed_requests": 133, "failed_requests": 0 } ] }, { "id": "gptoss120", "name": "GPT-OSS 120B", "ollamaTag": "gpt-oss:120b", "thinkingMode": "low", "startMetric": "first_generated_token_including_reasoning", "rounds": 1, "completedRequests": 5851, "failedRequests": 0, "scalingAt32Requests": { "gpu1": 1.0, "gpu2": 1.981982, "gpu4": 3.965406 }, "points": [ { "active_gpus": 1, "concurrency": 1, "p95_start_seconds": 1.096, "p95_256_token_duration_seconds": 2.517, "aggregate_output_tokens_per_second": 104.873, "completed_requests": 127, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 2, "p95_start_seconds": 2.083, "p95_256_token_duration_seconds": 3.518, "aggregate_output_tokens_per_second": 142.862, "completed_requests": 173, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 4, "p95_start_seconds": 5.599, "p95_256_token_duration_seconds": 7.034, "aggregate_output_tokens_per_second": 144.513, "completed_requests": 175, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 8, "p95_start_seconds": 12.637, "p95_256_token_duration_seconds": 14.074, "aggregate_output_tokens_per_second": 142.396, "completed_requests": 178, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 16, "p95_start_seconds": 26.721, "p95_256_token_duration_seconds": 28.157, "aggregate_output_tokens_per_second": 144.286, "completed_requests": 186, "failed_requests": 0 }, { "active_gpus": 1, "concurrency": 32, "p95_start_seconds": 54.865, "p95_256_token_duration_seconds": 56.3, "aggregate_output_tokens_per_second": 143.638, "completed_requests": 202, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 1, "p95_start_seconds": 0.932, "p95_256_token_duration_seconds": 2.329, "aggregate_output_tokens_per_second": 109.005, "completed_requests": 132, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 2, "p95_start_seconds": 0.932, "p95_256_token_duration_seconds": 2.353, "aggregate_output_tokens_per_second": 216.358, "completed_requests": 262, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 4, "p95_start_seconds": 3.287, "p95_256_token_duration_seconds": 4.743, "aggregate_output_tokens_per_second": 279.929, "completed_requests": 339, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 8, "p95_start_seconds": 7.395, "p95_256_token_duration_seconds": 8.846, "aggregate_output_tokens_per_second": 284.884, "completed_requests": 345, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 16, "p95_start_seconds": 14.715, "p95_256_token_duration_seconds": 16.168, "aggregate_output_tokens_per_second": 282.389, "completed_requests": 353, "failed_requests": 0 }, { "active_gpus": 2, "concurrency": 32, "p95_start_seconds": 29.131, "p95_256_token_duration_seconds": 30.604, "aggregate_output_tokens_per_second": 284.688, "completed_requests": 367, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 1, "p95_start_seconds": 0.964, "p95_256_token_duration_seconds": 2.341, "aggregate_output_tokens_per_second": 108.179, "completed_requests": 131, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 2, "p95_start_seconds": 0.928, "p95_256_token_duration_seconds": 2.322, "aggregate_output_tokens_per_second": 219.661, "completed_requests": 266, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 4, "p95_start_seconds": 0.943, "p95_256_token_duration_seconds": 2.372, "aggregate_output_tokens_per_second": 432.715, "completed_requests": 524, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 8, "p95_start_seconds": 5.195, "p95_256_token_duration_seconds": 6.657, "aggregate_output_tokens_per_second": 564.842, "completed_requests": 684, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 16, "p95_start_seconds": 9.732, "p95_256_token_duration_seconds": 11.192, "aggregate_output_tokens_per_second": 555.987, "completed_requests": 695, "failed_requests": 0 }, { "active_gpus": 4, "concurrency": 32, "p95_start_seconds": 17.221, "p95_256_token_duration_seconds": 18.68, "aggregate_output_tokens_per_second": 569.583, "completed_requests": 712, "failed_requests": 0 } ] } ] }