Upload microsoft_phi-4_2025-05-28-14-33-07.json

ik llama cpp server: ./llama-server -m ./phi-4-Q8_0.gguf -c 16384 -ngl 99 -np 8 -fa --cont-batching --host 0.0.0.0 --port 8000
benchmark: sudo docker run --network host -e HF_TOKEN=$HF_TOKEN -v ~/inference-benchmarker-results:/opt/inference-benchmarker/results inference_benchmarker inference-benchmarker --no-console --prompt-options "num_tokens=200,max_tokens=220,min_tokens=180,variance=10" --decode-options "num_tokens=200,max_tokens=220,min_tokens=180,variance=10" --url $URL --rates 1.0 --rates 10.0 --rates 30.0 --rates 100.0 --max-vus 8 --duration 120s --warmup 30s --benchmark-kind rate --model-name "phi-4" --tokenizer-name "microsoft/phi-4"

microsoft_phi-4_2025-05-28-14-33-07.json

@@ -0,0 +1,296 @@
+{
+	"config": {
+		"max_vus": 8,
+		"duration_secs": 120,
+		"benchmark_kind": "Rate",
+		"warmup_duration_secs": 30,
+		"rates": [
+			1.0,
+			10.0,
+			30.0,
+			100.0
+		],
+		"num_rates": 10,
+		"prompt_options": {
+			"num_tokens": 200,
+			"min_tokens": 180,
+			"max_tokens": 220,
+			"variance": 10
+		},
+		"decode_options": {
+			"num_tokens": 200,
+			"min_tokens": 180,
+			"max_tokens": 220,
+			"variance": 10
+		},
+		"tokenizer": "microsoft/phi-4",
+		"model_name": "phi-4",
+		"profile": null,
+		"meta": null,
+		"run_id": "ik_llama.cpp -np 8 -fa, max-vus 8: unsloth/phi-4-GGUF:Q8_0 (200 tokens)"
+	},
+	"results": [
+		{
+			"id": "warmup",
+			"executor_type": "ConstantVUs",
+			"config": {
+				"max_vus": 1,
+				"duration_secs": 30,
+				"rate": null
+			},
+			"total_requests": 12,
+			"total_tokens": 2308,
+			"token_throughput_secs": 82.4664278690674,
+			"duration_ms": 27987,
+			"time_to_first_token_ms": {
+				"p50": 49.879,
+				"p60": 53.321,
+				"p70": 55.046,
+				"p80": 56.736,
+				"p90": 59.507,
+				"p95": 108.881,
+				"p99": 156.897,
+				"avg": 59.258
+			},
+			"inter_token_latency_ms": {
+				"p50": 11.875,
+				"p60": 12.034,
+				"p70": 12.086,
+				"p80": 12.097,
+				"p90": 12.165,
+				"p95": 12.349,
+				"p99": 12.521,
+				"avg": 11.87
+			},
+			"failed_requests": 0,
+			"successful_requests": 12,
+			"request_rate": 0.4287682558183747,
+			"total_tokens_sent": 2400,
+			"e2e_latency_ms": {
+				"p50": 2391.624,
+				"p60": 2468.535,
+				"p70": 2503.242,
+				"p80": 2548.516,
+				"p90": 2559.961,
+				"p95": 2594.089,
+				"p99": 2627.326,
+				"avg": 2332.105
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 8,
+				"duration_secs": 120,
+				"rate": 1.0
+			},
+			"total_requests": 116,
+			"total_tokens": 21863,
+			"token_throughput_secs": 184.67660248254646,
+			"duration_ms": 118385,
+			"time_to_first_token_ms": {
+				"p50": 49.419,
+				"p60": 51.239,
+				"p70": 52.777,
+				"p80": 53.873,
+				"p90": 56.373,
+				"p95": 60.17,
+				"p99": 63.231,
+				"avg": 49.82
+			},
+			"inter_token_latency_ms": {
+				"p50": 17.018,
+				"p60": 17.173,
+				"p70": 17.372,
+				"p80": 17.563,
+				"p90": 17.768,
+				"p95": 17.968,
+				"p99": 18.132,
+				"avg": 17.029
+			},
+			"failed_requests": 0,
+			"successful_requests": 116,
+			"request_rate": 0.9798511589432095,
+			"total_tokens_sent": 23200,
+			"e2e_latency_ms": {
+				"p50": 3393.24,
+				"p60": 3458.146,
+				"p70": 3525.546,
+				"p80": 3601.401,
+				"p90": 3672.722,
+				"p95": 3775.723,
+				"p99": 3899.947,
+				"avg": 3247.9
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 8,
+				"duration_secs": 120,
+				"rate": 10.0
+			},
+			"total_requests": 235,
+			"total_tokens": 45584,
+			"token_throughput_secs": 381.12691659339066,
+			"duration_ms": 119603,
+			"time_to_first_token_ms": {
+				"p50": 51.704,
+				"p60": 53.373,
+				"p70": 54.916,
+				"p80": 56.524,
+				"p90": 59.428,
+				"p95": 60.845,
+				"p99": 63.778,
+				"avg": 51.745
+			},
+			"inter_token_latency_ms": {
+				"p50": 19.964,
+				"p60": 20.132,
+				"p70": 20.354,
+				"p80": 20.523,
+				"p90": 20.799,
+				"p95": 21.086,
+				"p99": 21.468,
+				"avg": 20.033
+			},
+			"failed_requests": 0,
+			"successful_requests": 235,
+			"request_rate": 1.9648303220306862,
+			"total_tokens_sent": 47000,
+			"e2e_latency_ms": {
+				"p50": 4003.139,
+				"p60": 4062.659,
+				"p70": 4121.389,
+				"p80": 4195.217,
+				"p90": 4296.563,
+				"p95": 4344.704,
+				"p99": 4380.68,
+				"avg": 3919.803
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 8,
+				"duration_secs": 120,
+				"rate": 30.0
+			},
+			"total_requests": 241,
+			"total_tokens": 46298,
+			"token_throughput_secs": 387.99671398196426,
+			"duration_ms": 119325,
+			"time_to_first_token_ms": {
+				"p50": 54.681,
+				"p60": 56.362,
+				"p70": 57.553,
+				"p80": 59.078,
+				"p90": 62.431,
+				"p95": 68.074,
+				"p99": 83.383,
+				"avg": 55.154
+			},
+			"inter_token_latency_ms": {
+				"p50": 20.023,
+				"p60": 20.144,
+				"p70": 20.312,
+				"p80": 20.486,
+				"p90": 20.72,
+				"p95": 20.918,
+				"p99": 21.329,
+				"avg": 20.037
+			},
+			"failed_requests": 0,
+			"successful_requests": 241,
+			"request_rate": 2.019681370030096,
+			"total_tokens_sent": 48200,
+			"e2e_latency_ms": {
+				"p50": 4015.72,
+				"p60": 4072.497,
+				"p70": 4146.357,
+				"p80": 4223.948,
+				"p90": 4319.526,
+				"p95": 4456.223,
+				"p99": 4535.434,
+				"avg": 3889.656
+			}
+		},
+		{
+			"id": "[email protected]/s",
+			"executor_type": "ConstantArrivalRate",
+			"config": {
+				"max_vus": 8,
+				"duration_secs": 120,
+				"rate": 100.0
+			},
+			"total_requests": 260,
+			"total_tokens": 46146,
+			"token_throughput_secs": 386.21664733150055,
+			"duration_ms": 119482,
+			"time_to_first_token_ms": {
+				"p50": 56.289,
+				"p60": 57.47,
+				"p70": 59.33,
+				"p80": 62.238,
+				"p90": 71.092,
+				"p95": 79.577,
+				"p99": 189.253,
+				"avg": 61.294
+			},
+			"inter_token_latency_ms": {
+				"p50": 20.116,
+				"p60": 20.338,
+				"p70": 20.55,
+				"p80": 20.781,
+				"p90": 21.174,
+				"p95": 21.285,
+				"p99": 22.105,
+				"avg": 20.219
+			},
+			"failed_requests": 0,
+			"successful_requests": 260,
+			"request_rate": 2.1760570429980963,
+			"total_tokens_sent": 52000,
+			"e2e_latency_ms": {
+				"p50": 3999.759,
+				"p60": 4067.239,
+				"p70": 4150.176,
+				"p80": 4236.082,
+				"p90": 4376.27,
+				"p95": 4443.535,
+				"p99": 4578.458,
+				"avg": 3632.546
+			}
+		}
+	],
+	"start_time": "2025-05-28T14:24:39.396108668+00:00",
+	"end_time": "2025-05-28T14:33:07.834005623+00:00",
+	"system": {
+		"cpu": [
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
+			"AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
+		],
+		"memory": "83.47 GB",
+		"os_name": "Debian GNU/Linux",
+		"os_version": "11",
+		"kernel": "5.15.167.4-microsoft-standard-WSL2",
+		"hostname": "computer"
+	}
+}