Hugging Face's logo

Spaces:
textgeflecht
/
inference-benchmarking-results-phi4-8000-tokens
Sleeping

App Files Community
textgeflecht commited on
Commit
4b27b88
·
verified ·
1 Parent(s): 67465f4

Update results/microsoft_phi-4_2025-05-26-13-38-07.json

Browse files
Files changed (1) hide show
  1. results/microsoft_phi-4_2025-05-26-13-38-07.json +295 -295
results/microsoft_phi-4_2025-05-26-13-38-07.json CHANGED
@@ -1,296 +1,296 @@
1
- {
2
- "config": {
3
- "max_vus": 800,
4
- "duration_secs": 120,
5
- "benchmark_kind": "Rate",
6
- "warmup_duration_secs": 30,
7
- "rates": [
8
- 1.0,
9
- 10.0,
10
- 30.0,
11
- 100.0
12
- ],
13
- "num_rates": 10,
14
- "prompt_options": {
15
- "num_tokens": 8000,
16
- "min_tokens": 7980,
17
- "max_tokens": 8020,
18
- "variance": 10
19
- },
20
- "decode_options": {
21
- "num_tokens": 8000,
22
- "min_tokens": 7980,
23
- "max_tokens": 8020,
24
- "variance": 10
25
- },
26
- "tokenizer": "microsoft/phi-4",
27
- "model_name": "phi-4",
28
- "profile": null,
29
- "meta": null,
30
- "run_id": "llama.cpp -np 64: lmstudio-community/phi-4-GGUF:Q8_0 (8000 tokens)"
31
- },
32
- "results": [
33
- {
34
- "id": "warmup",
35
- "executor_type": "ConstantVUs",
36
- "config": {
37
- "max_vus": 1,
38
- "duration_secs": 30,
39
- "rate": null
40
- },
41
- "total_requests": 18,
42
- "total_tokens": 9298,
43
- "token_throughput_secs": 76.42258729185906,
44
- "duration_ms": 121665,
45
- "time_to_first_token_ms": {
46
- "p50": 55.505,
47
- "p60": 55.95,
48
- "p70": 56.524,
49
- "p80": 57.802,
50
- "p90": 59.35,
51
- "p95": 94.303,
52
- "p99": 244.796,
53
- "avg": 68.022
54
- },
55
- "inter_token_latency_ms": {
56
- "p50": 12.802,
57
- "p60": 12.853,
58
- "p70": 12.895,
59
- "p80": 12.941,
60
- "p90": 12.989,
61
- "p95": 13.01,
62
- "p99": 13.018,
63
- "avg": 12.796
64
- },
65
- "failed_requests": 0,
66
- "successful_requests": 18,
67
- "request_rate": 0.1479465015329601,
68
- "total_tokens_sent": 144000,
69
- "e2e_latency_ms": {
70
- "p50": 819.638,
71
- "p60": 1078.152,
72
- "p70": 1334.123,
73
- "p80": 1352.935,
74
- "p90": 1655.858,
75
- "p95": 17057.743,
76
- "p99": 86498.29,
77
- "avg": 6758.932
78
- }
79
- },
80
- {
81
- "id": "[email protected]/s",
82
- "executor_type": "ConstantArrivalRate",
83
- "config": {
84
- "max_vus": 800,
85
- "duration_secs": 120,
86
- "rate": 1.0
87
- },
88
- "total_requests": 105,
89
- "total_tokens": 8350,
90
- "token_throughput_secs": 70.25975615790085,
91
- "duration_ms": 118844,
92
- "time_to_first_token_ms": {
93
- "p50": 113.845,
94
- "p60": 117.419,
95
- "p70": 123.233,
96
- "p80": 131.152,
97
- "p90": 139.356,
98
- "p95": 163.611,
99
- "p99": 303.98,
100
- "avg": 115.135
101
- },
102
- "inter_token_latency_ms": {
103
- "p50": 27.844,
104
- "p60": 34.642,
105
- "p70": 37.916,
106
- "p80": 39.484,
107
- "p90": 42.372,
108
- "p95": 44.597,
109
- "p99": 46.044,
110
- "avg": 28.069
111
- },
112
- "failed_requests": 0,
113
- "successful_requests": 105,
114
- "request_rate": 0.8835059157580347,
115
- "total_tokens_sent": 840000,
116
- "e2e_latency_ms": {
117
- "p50": 2151.603,
118
- "p60": 2383.017,
119
- "p70": 2485.018,
120
- "p80": 2699.796,
121
- "p90": 4168.034,
122
- "p95": 4897.888,
123
- "p99": 17451.561,
124
- "avg": 2432.795
125
- }
126
- },
127
- {
128
- "id": "[email protected]/s",
129
- "executor_type": "ConstantArrivalRate",
130
- "config": {
131
- "max_vus": 800,
132
- "duration_secs": 120,
133
- "rate": 10.0
134
- },
135
- "total_requests": 349,
136
- "total_tokens": 23362,
137
- "token_throughput_secs": 197.6289807084138,
138
- "duration_ms": 118211,
139
- "time_to_first_token_ms": {
140
- "p50": 22515.167,
141
- "p60": 30082.349,
142
- "p70": 37884.696,
143
- "p80": 46757.755,
144
- "p90": 56546.774,
145
- "p95": 62668.271,
146
- "p99": 67641.607,
147
- "avg": 26254.679
148
- },
149
- "inter_token_latency_ms": {
150
- "p50": 197.021,
151
- "p60": 201.532,
152
- "p70": 204.049,
153
- "p80": 208.887,
154
- "p90": 213.693,
155
- "p95": 217.329,
156
- "p99": 226.676,
157
- "avg": 191.854
158
- },
159
- "failed_requests": 3,
160
- "successful_requests": 346,
161
- "request_rate": 2.9269594780032175,
162
- "total_tokens_sent": 2768000,
163
- "e2e_latency_ms": {
164
- "p50": 35443.794,
165
- "p60": 42658.714,
166
- "p70": 51099.945,
167
- "p80": 61016.267,
168
- "p90": 70551.296,
169
- "p95": 75827.879,
170
- "p99": 79399.983,
171
- "avg": 39475.17
172
- }
173
- },
174
- {
175
- "id": "[email protected]/s",
176
- "executor_type": "ConstantArrivalRate",
177
- "config": {
178
- "max_vus": 800,
179
- "duration_secs": 120,
180
- "rate": 30.0
181
- },
182
- "total_requests": 319,
183
- "total_tokens": 22250,
184
- "token_throughput_secs": 186.10828266528932,
185
- "duration_ms": 119554,
186
- "time_to_first_token_ms": {
187
- "p50": 35826.543,
188
- "p60": 44793.932,
189
- "p70": 55421.67,
190
- "p80": 66980.202,
191
- "p90": 79127.239,
192
- "p95": 86131.439,
193
- "p99": 93035.177,
194
- "avg": 38400.307
195
- },
196
- "inter_token_latency_ms": {
197
- "p50": 207.85,
198
- "p60": 210.726,
199
- "p70": 214.375,
200
- "p80": 219.303,
201
- "p90": 227.545,
202
- "p95": 234.41,
203
- "p99": 251.788,
204
- "avg": 208.423
205
- },
206
- "failed_requests": 3,
207
- "successful_requests": 316,
208
- "request_rate": 2.6431558347070303,
209
- "total_tokens_sent": 2528000,
210
- "e2e_latency_ms": {
211
- "p50": 50255.589,
212
- "p60": 59231.58,
213
- "p70": 70559.844,
214
- "p80": 81153.331,
215
- "p90": 94581.636,
216
- "p95": 101789.257,
217
- "p99": 106946.694,
218
- "avg": 53237.226
219
- }
220
- },
221
- {
222
- "id": "[email protected]/s",
223
- "executor_type": "ConstantArrivalRate",
224
- "config": {
225
- "max_vus": 800,
226
- "duration_secs": 120,
227
- "rate": 100.0
228
- },
229
- "total_requests": 324,
230
- "total_tokens": 21410,
231
- "token_throughput_secs": 178.83031727780752,
232
- "duration_ms": 119722,
233
- "time_to_first_token_ms": {
234
- "p50": 38021.987,
235
- "p60": 47866.104,
236
- "p70": 60184.398,
237
- "p80": 72081.38,
238
- "p90": 87777.712,
239
- "p95": 94512.192,
240
- "p99": 103950.72,
241
- "avg": 42264.382
242
- },
243
- "inter_token_latency_ms": {
244
- "p50": 180.663,
245
- "p60": 183.606,
246
- "p70": 187.656,
247
- "p80": 193.252,
248
- "p90": 201.638,
249
- "p95": 213.235,
250
- "p99": 221.288,
251
- "avg": 180.335
252
- },
253
- "failed_requests": 14,
254
- "successful_requests": 310,
255
- "request_rate": 2.589322669599268,
256
- "total_tokens_sent": 2480000,
257
- "e2e_latency_ms": {
258
- "p50": 49768.497,
259
- "p60": 60765.234,
260
- "p70": 72000.927,
261
- "p80": 87011.812,
262
- "p90": 99250.179,
263
- "p95": 106760.715,
264
- "p99": 114837.294,
265
- "avg": 54998.304
266
- }
267
- }
268
- ],
269
- "start_time": "2025-05-26T13:21:23.924182518+00:00",
270
- "end_time": "2025-05-26T13:38:07.308687438+00:00",
271
- "system": {
272
- "cpu": [
273
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
- "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
- ],
290
- "memory": "83.47 GB",
291
- "os_name": "Debian GNU/Linux",
292
- "os_version": "11",
293
- "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
- "hostname": "computer"
295
- }
296
  }
 
1
+ {
2
+ "config": {
3
+ "max_vus": 800,
4
+ "duration_secs": 120,
5
+ "benchmark_kind": "Rate",
6
+ "warmup_duration_secs": 30,
7
+ "rates": [
8
+ 1.0,
9
+ 10.0,
10
+ 30.0,
11
+ 100.0
12
+ ],
13
+ "num_rates": 10,
14
+ "prompt_options": {
15
+ "num_tokens": 8000,
16
+ "min_tokens": 7980,
17
+ "max_tokens": 8020,
18
+ "variance": 10
19
+ },
20
+ "decode_options": {
21
+ "num_tokens": 8000,
22
+ "min_tokens": 7980,
23
+ "max_tokens": 8020,
24
+ "variance": 10
25
+ },
26
+ "tokenizer": "microsoft/phi-4",
27
+ "model_name": "phi-4",
28
+ "profile": null,
29
+ "meta": null,
30
+ "run_id": "llama.cpp -np 64: unsloth/phi-4-GGUF:Q8_0 (8000 tokens)"
31
+ },
32
+ "results": [
33
+ {
34
+ "id": "warmup",
35
+ "executor_type": "ConstantVUs",
36
+ "config": {
37
+ "max_vus": 1,
38
+ "duration_secs": 30,
39
+ "rate": null
40
+ },
41
+ "total_requests": 18,
42
+ "total_tokens": 9298,
43
+ "token_throughput_secs": 76.42258729185906,
44
+ "duration_ms": 121665,
45
+ "time_to_first_token_ms": {
46
+ "p50": 55.505,
47
+ "p60": 55.95,
48
+ "p70": 56.524,
49
+ "p80": 57.802,
50
+ "p90": 59.35,
51
+ "p95": 94.303,
52
+ "p99": 244.796,
53
+ "avg": 68.022
54
+ },
55
+ "inter_token_latency_ms": {
56
+ "p50": 12.802,
57
+ "p60": 12.853,
58
+ "p70": 12.895,
59
+ "p80": 12.941,
60
+ "p90": 12.989,
61
+ "p95": 13.01,
62
+ "p99": 13.018,
63
+ "avg": 12.796
64
+ },
65
+ "failed_requests": 0,
66
+ "successful_requests": 18,
67
+ "request_rate": 0.1479465015329601,
68
+ "total_tokens_sent": 144000,
69
+ "e2e_latency_ms": {
70
+ "p50": 819.638,
71
+ "p60": 1078.152,
72
+ "p70": 1334.123,
73
+ "p80": 1352.935,
74
+ "p90": 1655.858,
75
+ "p95": 17057.743,
76
+ "p99": 86498.29,
77
+ "avg": 6758.932
78
+ }
79
+ },
80
+ {
81
+ "id": "[email protected]/s",
82
+ "executor_type": "ConstantArrivalRate",
83
+ "config": {
84
+ "max_vus": 800,
85
+ "duration_secs": 120,
86
+ "rate": 1.0
87
+ },
88
+ "total_requests": 105,
89
+ "total_tokens": 8350,
90
+ "token_throughput_secs": 70.25975615790085,
91
+ "duration_ms": 118844,
92
+ "time_to_first_token_ms": {
93
+ "p50": 113.845,
94
+ "p60": 117.419,
95
+ "p70": 123.233,
96
+ "p80": 131.152,
97
+ "p90": 139.356,
98
+ "p95": 163.611,
99
+ "p99": 303.98,
100
+ "avg": 115.135
101
+ },
102
+ "inter_token_latency_ms": {
103
+ "p50": 27.844,
104
+ "p60": 34.642,
105
+ "p70": 37.916,
106
+ "p80": 39.484,
107
+ "p90": 42.372,
108
+ "p95": 44.597,
109
+ "p99": 46.044,
110
+ "avg": 28.069
111
+ },
112
+ "failed_requests": 0,
113
+ "successful_requests": 105,
114
+ "request_rate": 0.8835059157580347,
115
+ "total_tokens_sent": 840000,
116
+ "e2e_latency_ms": {
117
+ "p50": 2151.603,
118
+ "p60": 2383.017,
119
+ "p70": 2485.018,
120
+ "p80": 2699.796,
121
+ "p90": 4168.034,
122
+ "p95": 4897.888,
123
+ "p99": 17451.561,
124
+ "avg": 2432.795
125
+ }
126
+ },
127
+ {
128
+ "id": "[email protected]/s",
129
+ "executor_type": "ConstantArrivalRate",
130
+ "config": {
131
+ "max_vus": 800,
132
+ "duration_secs": 120,
133
+ "rate": 10.0
134
+ },
135
+ "total_requests": 349,
136
+ "total_tokens": 23362,
137
+ "token_throughput_secs": 197.6289807084138,
138
+ "duration_ms": 118211,
139
+ "time_to_first_token_ms": {
140
+ "p50": 22515.167,
141
+ "p60": 30082.349,
142
+ "p70": 37884.696,
143
+ "p80": 46757.755,
144
+ "p90": 56546.774,
145
+ "p95": 62668.271,
146
+ "p99": 67641.607,
147
+ "avg": 26254.679
148
+ },
149
+ "inter_token_latency_ms": {
150
+ "p50": 197.021,
151
+ "p60": 201.532,
152
+ "p70": 204.049,
153
+ "p80": 208.887,
154
+ "p90": 213.693,
155
+ "p95": 217.329,
156
+ "p99": 226.676,
157
+ "avg": 191.854
158
+ },
159
+ "failed_requests": 3,
160
+ "successful_requests": 346,
161
+ "request_rate": 2.9269594780032175,
162
+ "total_tokens_sent": 2768000,
163
+ "e2e_latency_ms": {
164
+ "p50": 35443.794,
165
+ "p60": 42658.714,
166
+ "p70": 51099.945,
167
+ "p80": 61016.267,
168
+ "p90": 70551.296,
169
+ "p95": 75827.879,
170
+ "p99": 79399.983,
171
+ "avg": 39475.17
172
+ }
173
+ },
174
+ {
175
+ "id": "[email protected]/s",
176
+ "executor_type": "ConstantArrivalRate",
177
+ "config": {
178
+ "max_vus": 800,
179
+ "duration_secs": 120,
180
+ "rate": 30.0
181
+ },
182
+ "total_requests": 319,
183
+ "total_tokens": 22250,
184
+ "token_throughput_secs": 186.10828266528932,
185
+ "duration_ms": 119554,
186
+ "time_to_first_token_ms": {
187
+ "p50": 35826.543,
188
+ "p60": 44793.932,
189
+ "p70": 55421.67,
190
+ "p80": 66980.202,
191
+ "p90": 79127.239,
192
+ "p95": 86131.439,
193
+ "p99": 93035.177,
194
+ "avg": 38400.307
195
+ },
196
+ "inter_token_latency_ms": {
197
+ "p50": 207.85,
198
+ "p60": 210.726,
199
+ "p70": 214.375,
200
+ "p80": 219.303,
201
+ "p90": 227.545,
202
+ "p95": 234.41,
203
+ "p99": 251.788,
204
+ "avg": 208.423
205
+ },
206
+ "failed_requests": 3,
207
+ "successful_requests": 316,
208
+ "request_rate": 2.6431558347070303,
209
+ "total_tokens_sent": 2528000,
210
+ "e2e_latency_ms": {
211
+ "p50": 50255.589,
212
+ "p60": 59231.58,
213
+ "p70": 70559.844,
214
+ "p80": 81153.331,
215
+ "p90": 94581.636,
216
+ "p95": 101789.257,
217
+ "p99": 106946.694,
218
+ "avg": 53237.226
219
+ }
220
+ },
221
+ {
222
+ "id": "[email protected]/s",
223
+ "executor_type": "ConstantArrivalRate",
224
+ "config": {
225
+ "max_vus": 800,
226
+ "duration_secs": 120,
227
+ "rate": 100.0
228
+ },
229
+ "total_requests": 324,
230
+ "total_tokens": 21410,
231
+ "token_throughput_secs": 178.83031727780752,
232
+ "duration_ms": 119722,
233
+ "time_to_first_token_ms": {
234
+ "p50": 38021.987,
235
+ "p60": 47866.104,
236
+ "p70": 60184.398,
237
+ "p80": 72081.38,
238
+ "p90": 87777.712,
239
+ "p95": 94512.192,
240
+ "p99": 103950.72,
241
+ "avg": 42264.382
242
+ },
243
+ "inter_token_latency_ms": {
244
+ "p50": 180.663,
245
+ "p60": 183.606,
246
+ "p70": 187.656,
247
+ "p80": 193.252,
248
+ "p90": 201.638,
249
+ "p95": 213.235,
250
+ "p99": 221.288,
251
+ "avg": 180.335
252
+ },
253
+ "failed_requests": 14,
254
+ "successful_requests": 310,
255
+ "request_rate": 2.589322669599268,
256
+ "total_tokens_sent": 2480000,
257
+ "e2e_latency_ms": {
258
+ "p50": 49768.497,
259
+ "p60": 60765.234,
260
+ "p70": 72000.927,
261
+ "p80": 87011.812,
262
+ "p90": 99250.179,
263
+ "p95": 106760.715,
264
+ "p99": 114837.294,
265
+ "avg": 54998.304
266
+ }
267
+ }
268
+ ],
269
+ "start_time": "2025-05-26T13:21:23.924182518+00:00",
270
+ "end_time": "2025-05-26T13:38:07.308687438+00:00",
271
+ "system": {
272
+ "cpu": [
273
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu0@4699MHz",
274
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu1@4699MHz",
275
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu2@4699MHz",
276
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu3@4699MHz",
277
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu4@4699MHz",
278
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu5@4699MHz",
279
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu6@4699MHz",
280
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu7@4699MHz",
281
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu8@4699MHz",
282
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu9@4699MHz",
283
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu10@4699MHz",
284
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu11@4699MHz",
285
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu12@4699MHz",
286
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu13@4699MHz",
287
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu14@4699MHz",
288
+ "AMD Ryzen 7 9800X3D 8-Core Processor cpu15@4699MHz"
289
+ ],
290
+ "memory": "83.47 GB",
291
+ "os_name": "Debian GNU/Linux",
292
+ "os_version": "11",
293
+ "kernel": "5.15.167.4-microsoft-standard-WSL2",
294
+ "hostname": "computer"
295
+ }
296
  }