Hugging Face's logo

Spaces:
textgeflecht
/
inference-benchmarking-results-phi4-200-tokens
Running

App Files Community
textgeflecht commited on
Commit
0eec002
·
verified ·
1 Parent(s): fdf2fa8

Upload microsoft_phi-4_2025-05-28-14-09-35.json

Browse files

ik llama run with ./llama-server -m ./phi-4-Q8_0.gguf -c 16384 -ngl 99 -np 16 --cont-batching --host 0.0.0.0 --port 8000
benchmarked with sudo docker run --network host -e HF_TOKEN=$HF_TOKEN -v ~/inference-benchmarker-results:/opt/inference-benchmarker/results inference_benchmarker inference-benchmarker --no-console --prompt-options "num_tokens=200,max_tokens=220,min_tokens=180,variance=10" --decode-options "num_tokens=200,max_tokens=220,min_tokens=180,variance=10" --url $URL --rates 1.0 --rates 10.0 --rates 30.0 --rates 100.0 --max-vus 8 --duration 120s --warmup 30s --benchmark-kind rate --model-name "phi-4" --tokenizer-name "microsoft/phi-4"

Files changed (1) hide show
  1. results/microsoft_phi-4_2025-05-28-14-09-35.json +296 -0
results/microsoft_phi-4_2025-05-28-14-09-35.json ADDED
@@ -0,0 +1,296 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "max_vus": 8,
4
+ "duration_secs": 120,
5
+ "benchmark_kind": "Rate",
6
+ "warmup_duration_secs": 30,
7
+ "rates": [
8
+ 1.0,
9
+ 10.0,
10
+ 30.0,
11
+ 100.0
12
+ ],
13
+ "num_rates": 10,
14
+ "prompt_options": {
15
+ "num_tokens": 200,
16
+ "min_tokens": 180,
17
+ "max_tokens": 220,
18
+ "variance": 10
19
+ },
20
+ "decode_options": {
21
+ "num_tokens": 200,
22
+ "min_tokens": 180,
23
+ "max_tokens": 220,
24
+ "variance": 10
25
+ },
26
+ "tokenizer": "microsoft/phi-4",
27
+ "model_name": "phi-4",
28
+ "profile": null,
29
+ "meta": null,
30
+ "run_id": "ik_llama.cpp -np 16, max-vus 8: unsloth/phi-4-GGUF:Q8_0 (200 tokens)"
31
+ },
32
+ "results": [
33
+ {
34
+ "id": "warmup",
35
+ "executor_type": "ConstantVUs",
36
+ "config": {
37
+ "max_vus": 1,
38
+ "duration_secs": 30,
39
+ "rate": null
40
+ },
41
+ "total_requests": 11,
42
+ "total_tokens": 2042,
43
+ "token_throughput_secs": 69.33786408424527,
44
+ "duration_ms": 29449,
45
+ "time_to_first_token_ms": {
46
+ "p50": 60.733,
47
+ "p60": 62.158,
48
+ "p70": 65.977,
49
+ "p80": 69.027,
50
+ "p90": 72.715,
51
+ "p95": 73.763,
52
+ "p99": 74.601,
53
+ "avg": 61.471
54
+ },
55
+ "inter_token_latency_ms": {
56
+ "p50": 14.12,
57
+ "p60": 14.369,
58
+ "p70": 14.541,
59
+ "p80": 14.79,
60
+ "p90": 15.038,
61
+ "p95": 15.163,
62
+ "p99": 15.263,
63
+ "avg": 13.957
64
+ },
65
+ "failed_requests": 0,
66
+ "successful_requests": 11,
67
+ "request_rate": 0.3735144490336425,
68
+ "total_tokens_sent": 2200,
69
+ "e2e_latency_ms": {
70
+ "p50": 2975.459,
71
+ "p60": 3009.442,
72
+ "p70": 3024.936,
73
+ "p80": 3054.238,
74
+ "p90": 3092.173,
75
+ "p95": 3125.964,
76
+ "p99": 3152.996,
77
+ "avg": 2677.072
78
+ }
79
+ },
80
+ {
81
+ "id": "[email protected]/s",
82
+ "executor_type": "ConstantArrivalRate",
83
+ "config": {
84
+ "max_vus": 8,
85
+ "duration_secs": 120,
86
+ "rate": 1.0
87
+ },
88
+ "total_requests": 115,
89
+ "total_tokens": 21668,
90
+ "token_throughput_secs": 182.3320833907104,
91
+ "duration_ms": 118838,
92
+ "time_to_first_token_ms": {
93
+ "p50": 104.757,
94
+ "p60": 106.656,
95
+ "p70": 109.218,
96
+ "p80": 111.488,
97
+ "p90": 114.362,
98
+ "p95": 117.768,
99
+ "p99": 121.927,
100
+ "avg": 105.067
101
+ },
102
+ "inter_token_latency_ms": {
103
+ "p50": 22.886,
104
+ "p60": 23.016,
105
+ "p70": 23.167,
106
+ "p80": 23.27,
107
+ "p90": 23.402,
108
+ "p95": 23.502,
109
+ "p99": 23.78,
110
+ "avg": 22.916
111
+ },
112
+ "failed_requests": 0,
113
+ "successful_requests": 115,
114
+ "request_rate": 0.9677030455017397,
115
+ "total_tokens_sent": 23000,
116
+ "e2e_latency_ms": {
117
+ "p50": 4553.56,
118
+ "p60": 4622.268,
119
+ "p70": 4682.461,
120
+ "p80": 4762.847,
121
+ "p90": 4849.725,
122
+ "p95": 4981.403,
123
+ "p99": 5146.162,
124
+ "avg": 4403.649
125
+ }
126
+ },
127
+ {
128
+ "id": "[email protected]/s",
129
+ "executor_type": "ConstantArrivalRate",
130
+ "config": {
131
+ "max_vus": 8,
132
+ "duration_secs": 120,
133
+ "rate": 10.0
134
+ },
135
+ "total_requests": 180,
136
+ "total_tokens": 34169,
137
+ "token_throughput_secs": 287.4269957193569,
138
+ "duration_ms": 118878,
139
+ "time_to_first_token_ms": {
140
+ "p50": 107.982,
141
+ "p60": 110.085,
142
+ "p70": 112.718,
143
+ "p80": 114.259,
144
+ "p90": 116.621,
145
+ "p95": 118.596,
146
+ "p99": 126.345,
147
+ "avg": 108.063
148
+ },
149
+ "inter_token_latency_ms": {
150
+ "p50": 26.586,
151
+ "p60": 26.723,
152
+ "p70": 26.832,
153
+ "p80": 26.954,
154
+ "p90": 27.378,
155
+ "p95": 27.765,
156
+ "p99": 28.187,
157
+ "avg": 26.571
158
+ },
159
+ "failed_requests": 0,
160
+ "successful_requests": 180,
161
+ "request_rate": 1.514146133322141,
162
+ "total_tokens_sent": 36000,
163
+ "e2e_latency_ms": {
164
+ "p50": 5340.395,
165
+ "p60": 5423.095,
166
+ "p70": 5515.212,
167
+ "p80": 5624.709,
168
+ "p90": 5733.59,
169
+ "p95": 5808.81,
170
+ "p99": 6051.086,
171
+ "avg": 5140.977
172
+ }
173
+ },
174
+ {
175
+ "id": "[email protected]/s",
176
+ "executor_type": "ConstantArrivalRate",
177
+ "config": {
178
+ "max_vus": 8,
179
+ "duration_secs": 120,
180
+ "rate": 30.0
181
+ },
182
+ "total_requests": 177,
183
+ "total_tokens": 33800,
184
+ "token_throughput_secs": 285.37390254245355,
185
+ "duration_ms": 118441,
186
+ "time_to_first_token_ms": {
187
+ "p50": 114.476,
188
+ "p60": 116.168,
189
+ "p70": 119.348,
190
+ "p80": 122.294,
191
+ "p90": 129.873,
192
+ "p95": 182.17,
193
+ "p99": 550.519,
194
+ "avg": 128.85
195
+ },
196
+ "inter_token_latency_ms": {
197
+ "p50": 26.808,
198
+ "p60": 26.923,
199
+ "p70": 27.109,
200
+ "p80": 27.26,
201
+ "p90": 27.46,
202
+ "p95": 27.966,
203
+ "p99": 29.407,
204
+ "avg": 26.861
205
+ },
206
+ "failed_requests": 0,
207
+ "successful_requests": 177,
208
+ "request_rate": 1.4944136316572272,
209
+ "total_tokens_sent": 35400,
210
+ "e2e_latency_ms": {
211
+ "p50": 5407.958,
212
+ "p60": 5455.363,
213
+ "p70": 5538.56,
214
+ "p80": 5674.894,
215
+ "p90": 5823.38,
216
+ "p95": 5868.327,
217
+ "p99": 6157.756,
218
+ "avg": 5240.665
219
+ }
220
+ },
221
+ {
222
+ "id": "[email protected]/s",
223
+ "executor_type": "ConstantArrivalRate",
224
+ "config": {
225
+ "max_vus": 8,
226
+ "duration_secs": 120,
227
+ "rate": 100.0
228
+ },
229
+ "total_requests": 182,
230
+ "total_tokens": 33812,
231
+ "token_throughput_secs": 285.72660230737114,
232
+ "duration_ms": 118336,
233
+ "time_to_first_token_ms": {
234
+ "p50": 116.911,
235
+ "p60": 118.312,
236
+ "p70": 119.797,
237
+ "p80": 122.559,
238
+ "p90": 179.079,
239
+ "p95": 196.698,
240
+ "p99": 649.72,
241
+ "avg": 141.58
242
+ },
243
+ "inter_token_latency_ms": {
244
+ "p50": 26.804,
245
+ "p60": 26.937,
246
+ "p70": 27.148,
247
+ "p80": 27.363,
248
+ "p90": 27.66,
249
+ "p95": 27.979,
250
+ "p99": 30.34,
251
+ "avg": 26.903
252
+ },
253
+ "failed_requests": 0,
254
+ "successful_requests": 182,
255
+ "request_rate": 1.5379818295262495,
256
+ "total_tokens_sent": 36400,
257
+ "e2e_latency_ms": {
258
+ "p50": 5386.275,
259
+ "p60": 5437.527,
260
+ "p70": 5544.241,
261
+ "p80": 5702.711,
262
+ "p90": 5834.793,
263
+ "p95": 5923.803,
264
+ "p99": 6090.788,
265
+ "avg": 5111.129
266
+ }
267
+ }
268
+ ],
269
+ "start_time": "2025-05-28T14:01:37.984694850+00:00",
270
+ "end_time": "2025-05-28T14:09:35.158748747+00:00",
271
+ "system": {
272
+ "cpu": [
273
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
+ ],
290
+ "memory": "83.47 GB",
291
+ "os_name": "Debian GNU/Linux",
292
+ "os_version": "11",
293
+ "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
+ "hostname": "computer"
295
+ }
296
+ }