{ "summary": [ { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "per_endpoint_concurrency": 2, "rounds": 3, "requests": 12, "input_tokens_min": 4125, "input_tokens_max": 4129, "prefill_tok_s": 1227.718591348095, "decode_tok_s": 39.85863317951881, "ttft_p50_s": 3.7439319999999996, "ttft_p95_s": 4.06203, "queue_mean_s": 1.729358370994305e-05, "cluster_output_tok_s": 102.61589162892484, "cluster_output_min_tok_s": 73.69954356555616, "cluster_output_max_tok_s": 130.38770132549442, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 16.988998413085938, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 59.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 16.948501586914062, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 55.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "per_endpoint_concurrency": 4, "rounds": 3, "requests": 24, "input_tokens_min": 4124, "input_tokens_max": 4129, "prefill_tok_s": 1002.9888001468144, "decode_tok_s": 33.43841955419695, "ttft_p50_s": 5.770987, "ttft_p95_s": 8.098073000000001, "queue_mean_s": 1.398664349416227, "cluster_output_tok_s": 184.94684078839785, "cluster_output_min_tok_s": 179.91285139430903, "cluster_output_max_tok_s": 189.37477072994642, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 17.005046844482422, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 61.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 16.938369750976562, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 54.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "per_endpoint_concurrency": 2, "rounds": 3, "requests": 12, "input_tokens_min": 32797, "input_tokens_max": 32801, "prefill_tok_s": 2018.5701776205333, "decode_tok_s": 27.834302286389775, "ttft_p50_s": 16.564867999999997, "ttft_p95_s": 30.026297, "queue_mean_s": 6.319600381083546, "cluster_output_tok_s": 48.35337369395338, "cluster_output_min_tok_s": 47.92957336674718, "cluster_output_max_tok_s": 48.63807369384488, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 17.00326919555664, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 67.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 16.929946899414062, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 63.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "per_endpoint_concurrency": 4, "rounds": 3, "requests": 24, "input_tokens_min": 32796, "input_tokens_max": 32801, "prefill_tok_s": 1903.6724650404713, "decode_tok_s": 15.371388333454126, "ttft_p50_s": 31.607294999999997, "ttft_p95_s": 60.557783, "queue_mean_s": 20.691433597666883, "cluster_output_tok_s": 55.124090156407604, "cluster_output_min_tok_s": 54.57253206765009, "cluster_output_max_tok_s": 55.84482566835486, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 17.00228500366211, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 69.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 16.912338256835938, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 64.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 1, "per_endpoint_concurrency": 1, "rounds": 3, "requests": 3, "input_tokens_min": 4126, "input_tokens_max": 4126, "prefill_tok_s": 2590.282170427339, "decode_tok_s": 32.13133000306693, "ttft_p50_s": 1.525979, "ttft_p95_s": 1.866663, "queue_mean_s": 1.9413666602000983e-05, "cluster_output_tok_s": 29.21646116888693, "cluster_output_min_tok_s": 17.965964499000563, "cluster_output_max_tok_s": 42.54334740500345, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.8281021118164062, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 63.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 6.208595275878906, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 57.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 2, "per_endpoint_concurrency": 2, "rounds": 3, "requests": 6, "input_tokens_min": 4125, "input_tokens_max": 4129, "prefill_tok_s": 2402.7312318073705, "decode_tok_s": 43.38624886303924, "ttft_p50_s": 1.787595, "ttft_p95_s": 3.8862460000000003, "queue_mean_s": 1.8773666700629594e-05, "cluster_output_tok_s": 70.07951835832216, "cluster_output_min_tok_s": 67.62216483452502, "cluster_output_max_tok_s": 72.05083408744038, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.8168678283691406, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 63.0, "node_a_memory_psi_some_avg10_max": 0.36, "node_b_min_mem_available_gib": 6.234554290771484, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 57.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 4, "per_endpoint_concurrency": 4, "rounds": 3, "requests": 12, "input_tokens_min": 4125, "input_tokens_max": 4129, "prefill_tok_s": 1090.2015171387309, "decode_tok_s": 28.9493856412062, "ttft_p50_s": 5.884027, "ttft_p95_s": 6.410818, "queue_mean_s": 0.03627799158330921, "cluster_output_tok_s": 77.84120280961216, "cluster_output_min_tok_s": 53.60119561902798, "cluster_output_max_tok_s": 102.60250539548161, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.8083114624023438, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 65.0, "node_a_memory_psi_some_avg10_max": 0.78, "node_b_min_mem_available_gib": 6.223834991455078, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 57.0, "node_b_memory_psi_some_avg10_max": 0.18 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 8, "per_endpoint_concurrency": 8, "rounds": 3, "requests": 24, "input_tokens_min": 4124, "input_tokens_max": 4129, "prefill_tok_s": 597.3979294968179, "decode_tok_s": 23.040226070777603, "ttft_p50_s": 11.13959, "ttft_p95_s": 12.863458000000001, "queue_mean_s": 1.4775439082917312, "cluster_output_tok_s": 90.24841287528122, "cluster_output_min_tok_s": 86.8477665200433, "cluster_output_max_tok_s": 94.46622735174199, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.7652053833007812, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 64.0, "node_a_memory_psi_some_avg10_max": 0.28, "node_b_min_mem_available_gib": 6.212638854980469, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 60.0, "node_b_memory_psi_some_avg10_max": 8.17 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 1, "per_endpoint_concurrency": 1, "rounds": 3, "requests": 3, "input_tokens_min": 32796, "input_tokens_max": 32800, "prefill_tok_s": 2832.2227900891207, "decode_tok_s": 55.25001193315479, "ttft_p50_s": 11.737114, "ttft_p95_s": 11.930352999999998, "queue_mean_s": 2.781866654307426e-05, "cluster_output_tok_s": 24.477706522993525, "cluster_output_min_tok_s": 24.274962096859046, "cluster_output_max_tok_s": 24.668471573973505, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.7960968017578125, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 67.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 6.212272644042969, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 62.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 2, "per_endpoint_concurrency": 2, "rounds": 3, "requests": 6, "input_tokens_min": 32797, "input_tokens_max": 32801, "prefill_tok_s": 2071.9231228507747, "decode_tok_s": 37.71617004640549, "ttft_p50_s": 17.119398, "ttft_p95_s": 23.342043, "queue_mean_s": 2.862762203166843, "cluster_output_tok_s": 30.350054727324483, "cluster_output_min_tok_s": 29.64485098700298, "cluster_output_max_tok_s": 30.721506152942236, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.8059234619140625, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 69.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 6.217098236083984, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 63.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 4, "per_endpoint_concurrency": 4, "rounds": 3, "requests": 12, "input_tokens_min": 32797, "input_tokens_max": 32801, "prefill_tok_s": 1800.3281669322744, "decode_tok_s": 17.125307225918814, "ttft_p50_s": 28.852493, "ttft_p95_s": 46.955934, "queue_mean_s": 12.949136609999831, "cluster_output_tok_s": 29.3596534669551, "cluster_output_min_tok_s": 27.07541770036161, "cluster_output_max_tok_s": 34.076305967014875, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.7830238342285156, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 69.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 6.182071685791016, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 64.0, "node_b_memory_psi_some_avg10_max": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 8, "per_endpoint_concurrency": 8, "rounds": 3, "requests": 24, "input_tokens_min": 32796, "input_tokens_max": 32801, "prefill_tok_s": 1687.0249421597573, "decode_tok_s": 9.682298984165207, "ttft_p50_s": 52.374592, "ttft_p95_s": 94.25911500000001, "queue_mean_s": 35.53127950304164, "cluster_output_tok_s": 34.99821303155239, "cluster_output_min_tok_s": 31.90320646472334, "cluster_output_max_tok_s": 36.79174173900627, "full_running_rounds": 3, "preemptions": 0.0, "node_a_min_mem_available_gib": 2.775318145751953, "node_a_loaded_clock_min_mhz": 1976.0, "node_a_loaded_clock_max_mhz": 1982.0, "node_a_max_gpu_temp_c": 69.0, "node_a_memory_psi_some_avg10_max": 0.0, "node_b_min_mem_available_gib": 6.187477111816406, "node_b_loaded_clock_min_mhz": 1976.0, "node_b_loaded_clock_max_mhz": 1982.0, "node_b_max_gpu_temp_c": 64.0, "node_b_memory_psi_some_avg10_max": 0.0 } ], "batches": [ { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "round": 0, "requests": 4, "elapsed_s": 16.37826108932495, "cluster_output_tok_s": 125.04379975569255, "ttft_p50_s": 3.7221819999999997, "ttft_p95_s": 4.002457, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "round": 0, "requests": 8, "elapsed_s": 22.766578197479248, "cluster_output_tok_s": 179.91285139430903, "ttft_p50_s": 5.770987, "ttft_p95_s": 8.156623999999999, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "round": 1, "requests": 8, "elapsed_s": 21.629069089889526, "cluster_output_tok_s": 189.37477072994642, "ttft_p50_s": 5.661555, "ttft_p95_s": 8.001693, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "round": 1, "requests": 4, "elapsed_s": 15.707002878189087, "cluster_output_tok_s": 130.38770132549442, "ttft_p50_s": 3.7439319999999996, "ttft_p95_s": 4.06203, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "round": 2, "requests": 4, "elapsed_s": 27.788503170013428, "cluster_output_tok_s": 73.69954356555616, "ttft_p50_s": 3.707874, "ttft_p95_s": 4.007137, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "round": 2, "requests": 8, "elapsed_s": 22.045065879821777, "cluster_output_tok_s": 185.8012138557108, "ttft_p50_s": 5.692112, "ttft_p95_s": 8.098073000000001, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "round": 0, "requests": 4, "elapsed_s": 42.22826600074768, "cluster_output_tok_s": 48.498321005265495, "ttft_p50_s": 16.521252, "ttft_p95_s": 29.872294, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "round": 0, "requests": 8, "elapsed_s": 73.34609699249268, "cluster_output_tok_s": 55.84482566835486, "ttft_p50_s": 31.504069, "ttft_p95_s": 60.557783, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "round": 1, "requests": 8, "elapsed_s": 75.05607390403748, "cluster_output_tok_s": 54.57253206765009, "ttft_p50_s": 31.607294999999997, "ttft_p95_s": 60.41997, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "round": 1, "requests": 4, "elapsed_s": 42.72936010360718, "cluster_output_tok_s": 47.92957336674718, "ttft_p50_s": 16.564867999999997, "ttft_p95_s": 29.860362000000002, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "round": 2, "requests": 4, "elapsed_s": 42.10693073272705, "cluster_output_tok_s": 48.63807369384488, "ttft_p50_s": 16.557077, "ttft_p95_s": 30.026297, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "round": 2, "requests": 8, "elapsed_s": 74.5130729675293, "cluster_output_tok_s": 54.97021981344027, "ttft_p50_s": 31.513866, "ttft_p95_s": 60.624672, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 1, "round": 0, "requests": 1, "elapsed_s": 12.034784078598022, "cluster_output_tok_s": 42.54334740500345, "ttft_p50_s": 1.866663, "ttft_p95_s": 1.866663, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 2, "round": 0, "requests": 2, "elapsed_s": 15.142963886260986, "cluster_output_tok_s": 67.62216483452502, "ttft_p50_s": 1.787595, "ttft_p95_s": 3.8862460000000003, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 4, "round": 0, "requests": 4, "elapsed_s": 38.20810294151306, "cluster_output_tok_s": 53.60119561902798, "ttft_p50_s": 6.171999, "ttft_p95_s": 6.410818, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 8, "round": 0, "requests": 8, "elapsed_s": 47.162986040115356, "cluster_output_tok_s": 86.8477665200433, "ttft_p50_s": 11.13959, "ttft_p95_s": 12.86355, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 8, "round": 1, "requests": 8, "elapsed_s": 45.63512182235718, "cluster_output_tok_s": 89.75543038855923, "ttft_p50_s": 9.928896, "ttft_p95_s": 11.739524, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 4, "round": 1, "requests": 4, "elapsed_s": 19.96052622795105, "cluster_output_tok_s": 102.60250539548161, "ttft_p50_s": 5.763537, "ttft_p95_s": 5.885232, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 2, "round": 1, "requests": 2, "elapsed_s": 14.212188005447388, "cluster_output_tok_s": 72.05083408744038, "ttft_p50_s": 1.612636, "ttft_p95_s": 3.121584, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 1, "round": 1, "requests": 1, "elapsed_s": 12.039987325668335, "cluster_output_tok_s": 42.52496170892598, "ttft_p50_s": 1.5073910000000001, "ttft_p95_s": 1.5073910000000001, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 1, "round": 2, "requests": 1, "elapsed_s": 28.49833083152771, "cluster_output_tok_s": 17.965964499000563, "ttft_p50_s": 1.525979, "ttft_p95_s": 1.525979, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 2, "round": 2, "requests": 2, "elapsed_s": 14.48076581954956, "cluster_output_tok_s": 70.71449208974589, "ttft_p50_s": 1.523039, "ttft_p95_s": 3.0837730000000003, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 4, "round": 2, "requests": 4, "elapsed_s": 20.761291980743408, "cluster_output_tok_s": 98.64511331469971, "ttft_p50_s": 5.733073, "ttft_p95_s": 5.88447, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 8, "round": 2, "requests": 8, "elapsed_s": 43.35941123962402, "cluster_output_tok_s": 94.46622735174199, "ttft_p50_s": 10.093817999999999, "ttft_p95_s": 11.979562, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 1, "round": 0, "requests": 1, "elapsed_s": 20.75523805618286, "cluster_output_tok_s": 24.668471573973505, "ttft_p50_s": 11.930352999999998, "ttft_p95_s": 11.930352999999998, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 2, "round": 0, "requests": 2, "elapsed_s": 34.54225492477417, "cluster_output_tok_s": 29.64485098700298, "ttft_p50_s": 17.119398, "ttft_p95_s": 23.342043, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 4, "round": 0, "requests": 4, "elapsed_s": 75.64056897163391, "cluster_output_tok_s": 27.07541770036161, "ttft_p50_s": 28.730521, "ttft_p95_s": 46.778841, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 8, "round": 0, "requests": 8, "elapsed_s": 111.32933115959167, "cluster_output_tok_s": 36.79174173900627, "ttft_p50_s": 52.374592, "ttft_p95_s": 94.25911500000001, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 8, "round": 1, "requests": 8, "elapsed_s": 111.38596034049988, "cluster_output_tok_s": 36.773036632972286, "ttft_p50_s": 52.028567, "ttft_p95_s": 94.29217600000001, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 4, "round": 1, "requests": 4, "elapsed_s": 73.52579498291016, "cluster_output_tok_s": 27.854170097392668, "ttft_p50_s": 28.794699, "ttft_p95_s": 46.955934, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 2, "round": 1, "requests": 2, "elapsed_s": 33.33169913291931, "cluster_output_tok_s": 30.721506152942236, "ttft_p50_s": 17.056321, "ttft_p95_s": 23.221071, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 1, "round": 1, "requests": 1, "elapsed_s": 21.09169101715088, "cluster_output_tok_s": 24.274962096859046, "ttft_p50_s": 11.737114, "ttft_p95_s": 11.737114, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 1, "round": 2, "requests": 1, "elapsed_s": 20.904047966003418, "cluster_output_tok_s": 24.492863814351825, "ttft_p50_s": 11.595906000000001, "ttft_p95_s": 11.595906000000001, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 2, "round": 2, "requests": 2, "elapsed_s": 33.34497380256653, "cluster_output_tok_s": 30.709275888565365, "ttft_p50_s": 16.828394, "ttft_p95_s": 23.014348000000002, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 4, "round": 2, "requests": 4, "elapsed_s": 60.10041117668152, "cluster_output_tok_s": 34.076305967014875, "ttft_p50_s": 28.852493, "ttft_p95_s": 46.651948, "full_running_observed": true, "preemptions": 0.0 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 8, "round": 2, "requests": 8, "elapsed_s": 128.38834881782532, "cluster_output_tok_s": 31.90320646472334, "ttft_p50_s": 52.061821, "ttft_p95_s": 93.885428, "full_running_observed": true, "preemptions": 0.0 } ], "endpoints": [ { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "endpoint": "node_a", "requests": 6, "prefill_tok_s": 1234.0377814686972, "decode_tok_s": 37.3581047577187, "ttft_p50_s": 3.677276, "ttft_p95_s": 3.9871260000000004 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 4, "endpoint": "node_b", "requests": 6, "prefill_tok_s": 1221.4637894071784, "decode_tok_s": 42.71791530972217, "ttft_p50_s": 3.7439319999999996, "ttft_p95_s": 4.06203 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "endpoint": "node_a", "requests": 12, "prefill_tok_s": 1012.0042978681292, "decode_tok_s": 33.13876965993198, "ttft_p50_s": 5.7436229999999995, "ttft_p95_s": 8.030245 }, { "configuration": "tp1", "input_target": 4096, "total_concurrency": 8, "endpoint": "node_b", "requests": 12, "prefill_tok_s": 994.1323369573639, "decode_tok_s": 33.74353792994216, "ttft_p50_s": 5.770987, "ttft_p95_s": 8.156623999999999 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "endpoint": "node_a", "requests": 6, "prefill_tok_s": 2026.7380577907534, "decode_tok_s": 27.39166263012083, "ttft_p50_s": 16.521252, "ttft_p95_s": 29.766068999999998 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 4, "endpoint": "node_b", "requests": 6, "prefill_tok_s": 2010.467990343453, "decode_tok_s": 28.29148271955382, "ttft_p50_s": 16.564867999999997, "ttft_p95_s": 30.026297 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "endpoint": "node_a", "requests": 12, "prefill_tok_s": 1907.8588808539716, "decode_tok_s": 15.15676501974784, "ttft_p50_s": 31.420054, "ttft_p95_s": 60.397594 }, { "configuration": "tp1", "input_target": 32768, "total_concurrency": 8, "endpoint": "node_b", "requests": 12, "prefill_tok_s": 1899.5044660438116, "decode_tok_s": 15.592177184563614, "ttft_p50_s": 31.607294999999997, "ttft_p95_s": 60.624672 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 1, "endpoint": "tp2_cluster", "requests": 3, "prefill_tok_s": 2590.282170427339, "decode_tok_s": 32.13133000306693, "ttft_p50_s": 1.525979, "ttft_p95_s": 1.866663 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 2, "endpoint": "tp2_cluster", "requests": 6, "prefill_tok_s": 2402.7312318073705, "decode_tok_s": 43.38624886303924, "ttft_p50_s": 1.787595, "ttft_p95_s": 3.8862460000000003 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 4, "endpoint": "tp2_cluster", "requests": 12, "prefill_tok_s": 1090.2015171387309, "decode_tok_s": 28.9493856412062, "ttft_p50_s": 5.884027, "ttft_p95_s": 6.410818 }, { "configuration": "tp2", "input_target": 4096, "total_concurrency": 8, "endpoint": "tp2_cluster", "requests": 24, "prefill_tok_s": 597.3979294968179, "decode_tok_s": 23.040226070777603, "ttft_p50_s": 11.13959, "ttft_p95_s": 12.863458000000001 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 1, "endpoint": "tp2_cluster", "requests": 3, "prefill_tok_s": 2832.2227900891207, "decode_tok_s": 55.25001193315479, "ttft_p50_s": 11.737114, "ttft_p95_s": 11.930352999999998 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 2, "endpoint": "tp2_cluster", "requests": 6, "prefill_tok_s": 2071.9231228507747, "decode_tok_s": 37.71617004640549, "ttft_p50_s": 17.119398, "ttft_p95_s": 23.342043 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 4, "endpoint": "tp2_cluster", "requests": 12, "prefill_tok_s": 1800.3281669322744, "decode_tok_s": 17.125307225918814, "ttft_p50_s": 28.852493, "ttft_p95_s": 46.955934 }, { "configuration": "tp2", "input_target": 32768, "total_concurrency": 8, "endpoint": "tp2_cluster", "requests": 24, "prefill_tok_s": 1687.0249421597573, "decode_tok_s": 9.682298984165207, "ttft_p50_s": 52.374592, "ttft_p95_s": 94.25911500000001 } ], "matched_input_checks": [ { "input_target": 4096, "total_concurrency": 4, "round": 0, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 4096, "total_concurrency": 8, "round": 0, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] }, { "input_target": 4096, "total_concurrency": 8, "round": 1, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] }, { "input_target": 4096, "total_concurrency": 4, "round": 1, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 4096, "total_concurrency": 4, "round": 2, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 4096, "total_concurrency": 8, "round": 2, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 4, "round": 0, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 8, "round": 0, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 8, "round": 1, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 4, "round": 1, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 4, "round": 2, "matched_unique_requests": 4, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0 ] }, { "input_target": 32768, "total_concurrency": 8, "round": 2, "matched_unique_requests": 8, "token_counts_equal": true, "token_count_deltas": [ 0, 0, 0, 0, 0, 0, 0, 0 ] } ] }