Upload microsoft_phi-4_2025-05-27-09-33-47.json

server executed with ./llama-server \
-m ./phi-4-Q8_0.gguf \
-c 16384 \
-np 64 \
-ngl 99 \
-fa \
-t 8 \
--host 0.0.0.0 --port 8000
and benchmarked via sudo docker run --network host -e HF_TOKEN=$HF_TOKEN -v ~/inference-benchmarker-results:/opt/inference-benchmarker/results inference_benchmarker inference-benchmarker --prompt-options "num_tokens=8000,max_tokens=8020,min_tokens=7980,variance=10" --decode-options "num_tokens=8000,max_tokens=8020,min_tokens=7980,variance=10" --url $URL --rates 1.0 --rates 10.0 --rates 30.0 --rates 100.0 --max-vus 800 --duration 120s --warmup 30s --benchmark-kind rate --model-name "phi-4" --tokenizer-name "microsoft/phi-4"

results/microsoft_phi-4_2025-05-27-09-33-47.json

@@ -0,0 +1,296 @@
+{
+	"config": {
+		"max_vus": 800,
+		"duration_secs": 120,
+		"benchmark_kind": "Rate",
+		"warmup_duration_secs": 30,
+		"rates": [
+			1.0,
+			10.0,
+			30.0,
+			100.0
+		],
+		"num_rates": 10,
+		"prompt_options": {
+			"num_tokens": 8000,
+			"min_tokens": 7980,
+			"max_tokens": 8020,
+			"variance": 10
+		},
+		"decode_options": {
+			"num_tokens": 8000,
+			"min_tokens": 7980,
+			"max_tokens": 8020,
+			"variance": 10
+		},
+		"tokenizer": "microsoft/phi-4",
+		"model_name": "phi-4",
+		"profile": null,
+		"meta": null,
+		"run_id": "llama.cpp -np 64 -fa: unsloth/phi-4-GGUF:Q8_0 (8000 tokens)"
+	},
+	"results": [
+		{
+			"id": "warmup",
+			"executor_type": "ConstantVUs",
+			"config": {
+				"max_vus": 1,
+				"duration_secs": 30,
+				"rate": null
+			},
+			"total_requests": 27,
+			"total_tokens": 9718,
+			"token_throughput_secs": 81.74082189646838,
+			"duration_ms": 118887,
+			"time_to_first_token_ms": {
+				"p50": 48.61,
+				"p60": 48.891,
+				"p70": 49.168,
+				"p80": 49.498,
+				"p90": 50.034,
+				"p95": 51.088,
+				"p99": 261.014,
+				"avg": 59.311
+			},
+			"inter_token_latency_ms": {
+				"p50": 12.017,
+				"p60": 12.026,
+				"p70": 12.064,
+				"p80": 12.082,
+				"p90": 12.093,
+				"p95": 12.101,
+				"p99": 12.253,
+				"avg": 12.029
+			},
+			"failed_requests": 0,
+			"successful_requests": 27,
+			"request_rate": 0.22710456793626735,
+			"total_tokens_sent": 216000,
+			"e2e_latency_ms": {
+				"p50": 757.477,
+				"p60": 758.668,
+				"p70": 760.92,
+				"p80": 762.799,
+				"p90": 1499.47,
+				"p95": 1531.676,
+				"p99": 71879.405,
+				"avg": 4403.016
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 800,
+				"duration_secs": 120,
+				"rate": 1.0
+			},
+			"total_requests": 106,
+			"total_tokens": 8313,
+			"token_throughput_secs": 70.87113803161708,
+			"duration_ms": 117297,
+			"time_to_first_token_ms": {
+				"p50": 65.923,
+				"p60": 68.563,
+				"p70": 70.138,
+				"p80": 72.142,
+				"p90": 75.565,
+				"p95": 86.811,
+				"p99": 185.545,
+				"avg": 71.059
+			},
+			"inter_token_latency_ms": {
+				"p50": 23.858,
+				"p60": 24.409,
+				"p70": 25.174,
+				"p80": 26.634,
+				"p90": 30.208,
+				"p95": 32.664,
+				"p99": 34.869,
+				"avg": 23.811
+			},
+			"failed_requests": 0,
+			"successful_requests": 106,
+			"request_rate": 0.9036858692832203,
+			"total_tokens_sent": 848000,
+			"e2e_latency_ms": {
+				"p50": 1513.038,
+				"p60": 1568.321,
+				"p70": 1768.852,
+				"p80": 1985.17,
+				"p90": 2476.386,
+				"p95": 2851.038,
+				"p99": 4129.591,
+				"avg": 1925.59
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 800,
+				"duration_secs": 120,
+				"rate": 10.0
+			},
+			"total_requests": 446,
+			"total_tokens": 30228,
+			"token_throughput_secs": 252.13178505255695,
+			"duration_ms": 119889,
+			"time_to_first_token_ms": {
+				"p50": 17218.753,
+				"p60": 24217.22,
+				"p70": 31635.918,
+				"p80": 40705.096,
+				"p90": 50007.449,
+				"p95": 55325.95,
+				"p99": 60904.405,
+				"avg": 21707.867
+			},
+			"inter_token_latency_ms": {
+				"p50": 147.701,
+				"p60": 149.864,
+				"p70": 152.119,
+				"p80": 154.926,
+				"p90": 160.571,
+				"p95": 165.906,
+				"p99": 172.958,
+				"avg": 147.392
+			},
+			"failed_requests": 9,
+			"successful_requests": 437,
+			"request_rate": 3.645017535661221,
+			"total_tokens_sent": 3496000,
+			"e2e_latency_ms": {
+				"p50": 28135.612,
+				"p60": 34510.797,
+				"p70": 41724.159,
+				"p80": 51495.322,
+				"p90": 60757.905,
+				"p95": 66146.289,
+				"p99": 71105.223,
+				"avg": 31938.065
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 800,
+				"duration_secs": 120,
+				"rate": 30.0
+			},
+			"total_requests": 448,
+			"total_tokens": 30912,
+			"token_throughput_secs": 258.2609449225948,
+			"duration_ms": 119692,
+			"time_to_first_token_ms": {
+				"p50": 25290.398,
+				"p60": 35623.865,
+				"p70": 46048.766,
+				"p80": 60693.417,
+				"p90": 75994.066,
+				"p95": 84006.657,
+				"p99": 94917.151,
+				"avg": 33043.743
+			},
+			"inter_token_latency_ms": {
+				"p50": 121.857,
+				"p60": 125.877,
+				"p70": 130.608,
+				"p80": 136.086,
+				"p90": 142.114,
+				"p95": 144.885,
+				"p99": 150.811,
+				"avg": 122.097
+			},
+			"failed_requests": 9,
+			"successful_requests": 439,
+			"request_rate": 3.6677198117565704,
+			"total_tokens_sent": 3512000,
+			"e2e_latency_ms": {
+				"p50": 36013.727,
+				"p60": 45412.83,
+				"p70": 56300.618,
+				"p80": 70423.796,
+				"p90": 83725.896,
+				"p95": 94968.614,
+				"p99": 102032.047,
+				"avg": 41844.204
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 800,
+				"duration_secs": 120,
+				"rate": 100.0
+			},
+			"total_requests": 466,
+			"total_tokens": 31108,
+			"token_throughput_secs": 259.8608345964365,
+			"duration_ms": 119710,
+			"time_to_first_token_ms": {
+				"p50": 34434.388,
+				"p60": 44191.564,
+				"p70": 56627.842,
+				"p80": 71990.192,
+				"p90": 87110.437,
+				"p95": 97402.223,
+				"p99": 105147.119,
+				"avg": 40335.484
+			},
+			"inter_token_latency_ms": {
+				"p50": 132.627,
+				"p60": 136.975,
+				"p70": 140.653,
+				"p80": 144.214,
+				"p90": 147.981,
+				"p95": 150.702,
+				"p99": 160.429,
+				"avg": 129.4
+			},
+			"failed_requests": 26,
+			"successful_requests": 440,
+			"request_rate": 3.6755422149425248,
+			"total_tokens_sent": 3520000,
+			"e2e_latency_ms": {
+				"p50": 43840.122,
+				"p60": 55318.742,
+				"p70": 69480.146,
+				"p80": 81453.487,
+				"p90": 97388.642,
+				"p95": 106561.334,
+				"p99": 113609.244,
+				"avg": 49651.897
+			}
+		}
+	],
+	"start_time": "2025-05-27T09:16:48.866142727+00:00",
+	"end_time": "2025-05-27T09:33:47.431269495+00:00",
+	"system": {
+		"cpu": [
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
+		],
+		"memory": "83.47 GB",
+		"os_name": "Debian GNU/Linux",
+		"os_version": "11",
+		"kernel": "5.15.167.4-microsoft-standard-WSL2",
+		"hostname": "computer"
+	}
+}