Hugging Face's logo

bwang0911
/
jev2-legal

Sentence Similarity
sentence-transformers
Safetensors
English
mpnet
feature-extraction
Generated from Trainer
dataset_size:53224
loss:MultipleNegativesRankingLoss
Eval Results
Inference Endpoints
Model card Files Community
bwang0911 commited on
Commit
9e5cf6d
·
verified ·
1 Parent(s): cf5f861

Add new SentenceTransformer model

Browse files
Files changed (2) hide show
  1. README.md +98 -110
  2. model.safetensors +1 -1
README.md CHANGED
@@ -230,19 +230,19 @@ model-index:
230
  type: mteb/AILA_casedocs
231
  metrics:
232
  - type: cosine_accuracy@1
233
- value: 0.32
234
  name: Cosine Accuracy@1
235
  - type: cosine_accuracy@3
236
  value: 0.36
237
  name: Cosine Accuracy@3
238
  - type: cosine_accuracy@5
239
- value: 0.36
240
  name: Cosine Accuracy@5
241
  - type: cosine_accuracy@10
242
- value: 0.52
243
  name: Cosine Accuracy@10
244
  - type: cosine_precision@1
245
- value: 0.32
246
  name: Cosine Precision@1
247
  - type: cosine_precision@3
248
  value: 0.2
@@ -251,28 +251,28 @@ model-index:
251
  value: 0.14
252
  name: Cosine Precision@5
253
  - type: cosine_precision@10
254
- value: 0.09799999999999998
255
  name: Cosine Precision@10
256
  - type: cosine_recall@1
257
- value: 0.10344755244755245
258
  name: Cosine Recall@1
259
  - type: cosine_recall@3
260
- value: 0.173986013986014
261
  name: Cosine Recall@3
262
  - type: cosine_recall@5
263
- value: 0.19522843822843824
264
  name: Cosine Recall@5
265
  - type: cosine_recall@10
266
- value: 0.28000932400932405
267
  name: Cosine Recall@10
268
  - type: cosine_ndcg@10
269
- value: 0.2554010429460017
270
  name: Cosine Ndcg@10
271
  - type: cosine_mrr@10
272
- value: 0.35526984126984135
273
  name: Cosine Mrr@10
274
  - type: cosine_map@100
275
- value: 0.2279369510758354
276
  name: Cosine Map@100
277
  - task:
278
  type: information-retrieval
@@ -282,49 +282,49 @@ model-index:
282
  type: mteb/AILA_statutes
283
  metrics:
284
  - type: cosine_accuracy@1
285
- value: 0.24
286
  name: Cosine Accuracy@1
287
  - type: cosine_accuracy@3
288
- value: 0.4
289
  name: Cosine Accuracy@3
290
  - type: cosine_accuracy@5
291
- value: 0.5
292
  name: Cosine Accuracy@5
293
  - type: cosine_accuracy@10
294
- value: 0.66
295
  name: Cosine Accuracy@10
296
  - type: cosine_precision@1
297
- value: 0.24
298
  name: Cosine Precision@1
299
  - type: cosine_precision@3
300
- value: 0.15333333333333332
301
  name: Cosine Precision@3
302
  - type: cosine_precision@5
303
- value: 0.124
304
  name: Cosine Precision@5
305
  - type: cosine_precision@10
306
- value: 0.10399999999999998
307
  name: Cosine Precision@10
308
  - type: cosine_recall@1
309
- value: 0.066
310
  name: Cosine Recall@1
311
  - type: cosine_recall@3
312
- value: 0.115
313
  name: Cosine Recall@3
314
  - type: cosine_recall@5
315
- value: 0.15600000000000003
316
  name: Cosine Recall@5
317
  - type: cosine_recall@10
318
- value: 0.2506666666666667
319
  name: Cosine Recall@10
320
  - type: cosine_ndcg@10
321
- value: 0.21795221118394462
322
  name: Cosine Ndcg@10
323
  - type: cosine_mrr@10
324
- value: 0.34710317460317447
325
  name: Cosine Mrr@10
326
  - type: cosine_map@100
327
- value: 0.1907807681666918
328
  name: Cosine Map@100
329
  - task:
330
  type: information-retrieval
@@ -334,49 +334,49 @@ model-index:
334
  type: mteb/legalbench_consumer_contracts_qa
335
  metrics:
336
  - type: cosine_accuracy@1
337
- value: 0.45454545454545453
338
  name: Cosine Accuracy@1
339
  - type: cosine_accuracy@3
340
- value: 0.6944444444444444
341
  name: Cosine Accuracy@3
342
  - type: cosine_accuracy@5
343
- value: 0.7828282828282829
344
  name: Cosine Accuracy@5
345
  - type: cosine_accuracy@10
346
- value: 0.8787878787878788
347
  name: Cosine Accuracy@10
348
  - type: cosine_precision@1
349
- value: 0.45454545454545453
350
  name: Cosine Precision@1
351
  - type: cosine_precision@3
352
- value: 0.23148148148148145
353
  name: Cosine Precision@3
354
  - type: cosine_precision@5
355
- value: 0.15656565656565652
356
  name: Cosine Precision@5
357
  - type: cosine_precision@10
358
- value: 0.08787878787878788
359
  name: Cosine Precision@10
360
  - type: cosine_recall@1
361
- value: 0.45454545454545453
362
  name: Cosine Recall@1
363
  - type: cosine_recall@3
364
- value: 0.6944444444444444
365
  name: Cosine Recall@3
366
  - type: cosine_recall@5
367
- value: 0.7828282828282829
368
  name: Cosine Recall@5
369
  - type: cosine_recall@10
370
- value: 0.8787878787878788
371
  name: Cosine Recall@10
372
  - type: cosine_ndcg@10
373
- value: 0.6629903557959665
374
  name: Cosine Ndcg@10
375
  - type: cosine_mrr@10
376
- value: 0.5942259900593235
377
  name: Cosine Mrr@10
378
  - type: cosine_map@100
379
- value: 0.5996552027282283
380
  name: Cosine Map@100
381
  - task:
382
  type: information-retrieval
@@ -386,49 +386,49 @@ model-index:
386
  type: mteb/legalbench_corporate_lobbying
387
  metrics:
388
  - type: cosine_accuracy@1
389
- value: 0.7588235294117647
390
  name: Cosine Accuracy@1
391
  - type: cosine_accuracy@3
392
- value: 0.9029411764705882
393
  name: Cosine Accuracy@3
394
  - type: cosine_accuracy@5
395
- value: 0.9294117647058824
396
  name: Cosine Accuracy@5
397
  - type: cosine_accuracy@10
398
- value: 0.9647058823529412
399
  name: Cosine Accuracy@10
400
  - type: cosine_precision@1
401
- value: 0.7588235294117647
402
  name: Cosine Precision@1
403
  - type: cosine_precision@3
404
- value: 0.3009803921568628
405
  name: Cosine Precision@3
406
  - type: cosine_precision@5
407
- value: 0.18588235294117644
408
  name: Cosine Precision@5
409
  - type: cosine_precision@10
410
- value: 0.09647058823529411
411
  name: Cosine Precision@10
412
  - type: cosine_recall@1
413
- value: 0.7588235294117647
414
  name: Cosine Recall@1
415
  - type: cosine_recall@3
416
- value: 0.9029411764705882
417
  name: Cosine Recall@3
418
  - type: cosine_recall@5
419
- value: 0.9294117647058824
420
  name: Cosine Recall@5
421
  - type: cosine_recall@10
422
- value: 0.9647058823529412
423
  name: Cosine Recall@10
424
  - type: cosine_ndcg@10
425
- value: 0.8685058033071409
426
  name: Cosine Ndcg@10
427
  - type: cosine_mrr@10
428
- value: 0.8369257703081232
429
  name: Cosine Mrr@10
430
  - type: cosine_map@100
431
- value: 0.8383005473609861
432
  name: Cosine Map@100
433
  - task:
434
  type: information-retrieval
@@ -438,49 +438,49 @@ model-index:
438
  type: mteb/legal_summarization
439
  metrics:
440
  - type: cosine_accuracy@1
441
- value: 0.5
442
  name: Cosine Accuracy@1
443
  - type: cosine_accuracy@3
444
- value: 0.6549295774647887
445
  name: Cosine Accuracy@3
446
  - type: cosine_accuracy@5
447
- value: 0.7288732394366197
448
  name: Cosine Accuracy@5
449
  - type: cosine_accuracy@10
450
- value: 0.8028169014084507
451
  name: Cosine Accuracy@10
452
  - type: cosine_precision@1
453
- value: 0.5
454
  name: Cosine Precision@1
455
  - type: cosine_precision@3
456
- value: 0.24295774647887325
457
  name: Cosine Precision@3
458
  - type: cosine_precision@5
459
- value: 0.16901408450704225
460
  name: Cosine Precision@5
461
  - type: cosine_precision@10
462
- value: 0.09929577464788732
463
  name: Cosine Precision@10
464
  - type: cosine_recall@1
465
- value: 0.4474502316931894
466
  name: Cosine Recall@1
467
  - type: cosine_recall@3
468
- value: 0.5799413907688555
469
  name: Cosine Recall@3
470
  - type: cosine_recall@5
471
- value: 0.6543419608560453
472
  name: Cosine Recall@5
473
  - type: cosine_recall@10
474
- value: 0.7329290134747881
475
  name: Cosine Recall@10
476
  - type: cosine_ndcg@10
477
- value: 0.6075214960447544
478
  name: Cosine Ndcg@10
479
  - type: cosine_mrr@10
480
- value: 0.5952380952380955
481
  name: Cosine Mrr@10
482
  - type: cosine_map@100
483
- value: 0.5669161422906482
484
  name: Cosine Map@100
485
  ---
486
 
@@ -591,21 +591,21 @@ You can finetune this model on your own dataset.
591
 
592
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
593
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
594
- | cosine_accuracy@1 | 0.32 | 0.24 | 0.4545 | 0.7588 | 0.5 |
595
- | cosine_accuracy@3 | 0.36 | 0.4 | 0.6944 | 0.9029 | 0.6549 |
596
- | cosine_accuracy@5 | 0.36 | 0.5 | 0.7828 | 0.9294 | 0.7289 |
597
- | cosine_accuracy@10 | 0.52 | 0.66 | 0.8788 | 0.9647 | 0.8028 |
598
- | cosine_precision@1 | 0.32 | 0.24 | 0.4545 | 0.7588 | 0.5 |
599
- | cosine_precision@3 | 0.2 | 0.1533 | 0.2315 | 0.301 | 0.243 |
600
- | cosine_precision@5 | 0.14 | 0.124 | 0.1566 | 0.1859 | 0.169 |
601
- | cosine_precision@10 | 0.098 | 0.104 | 0.0879 | 0.0965 | 0.0993 |
602
- | cosine_recall@1 | 0.1034 | 0.066 | 0.4545 | 0.7588 | 0.4475 |
603
- | cosine_recall@3 | 0.174 | 0.115 | 0.6944 | 0.9029 | 0.5799 |
604
- | cosine_recall@5 | 0.1952 | 0.156 | 0.7828 | 0.9294 | 0.6543 |
605
- | cosine_recall@10 | 0.28 | 0.2507 | 0.8788 | 0.9647 | 0.7329 |
606
- | **cosine_ndcg@10** | **0.2554** | **0.218** | **0.663** | **0.8685** | **0.6075** |
607
- | cosine_mrr@10 | 0.3553 | 0.3471 | 0.5942 | 0.8369 | 0.5952 |
608
- | cosine_map@100 | 0.2279 | 0.1908 | 0.5997 | 0.8383 | 0.5669 |
609
 
610
  <!--
611
  ## Bias, Risks and Limitations
@@ -819,8 +819,8 @@ You can finetune this model on your own dataset.
819
  #### Non-Default Hyperparameters
820
 
821
  - `eval_strategy`: steps
822
- - `per_device_train_batch_size`: 64
823
- - `learning_rate`: 1e-06
824
  - `num_train_epochs`: 2
825
  - `warmup_ratio`: 0.1
826
  - `fp16`: True
@@ -833,14 +833,14 @@ You can finetune this model on your own dataset.
833
  - `do_predict`: False
834
  - `eval_strategy`: steps
835
  - `prediction_loss_only`: True
836
- - `per_device_train_batch_size`: 64
837
  - `per_device_eval_batch_size`: 8
838
  - `per_gpu_train_batch_size`: None
839
  - `per_gpu_eval_batch_size`: None
840
  - `gradient_accumulation_steps`: 1
841
  - `eval_accumulation_steps`: None
842
  - `torch_empty_cache_steps`: None
843
- - `learning_rate`: 1e-06
844
  - `weight_decay`: 0.0
845
  - `adam_beta1`: 0.9
846
  - `adam_beta2`: 0.999
@@ -946,25 +946,13 @@ You can finetune this model on your own dataset.
946
  </details>
947
 
948
  ### Training Logs
949
- | Epoch | Step | Training Loss | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
950
- |:------:|:----:|:-------------:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
951
- | 0 | 0 | - | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
952
- | 0.1196 | 100 | - | 0.2034 | 0.2108 | 0.6541 | 0.8663 | 0.5957 |
953
- | 0.2392 | 200 | - | 0.2087 | 0.2178 | 0.6579 | 0.8680 | 0.6008 |
954
- | 0.3589 | 300 | - | 0.2099 | 0.2195 | 0.6567 | 0.8701 | 0.6053 |
955
- | 0.4785 | 400 | - | 0.2162 | 0.2196 | 0.6600 | 0.8700 | 0.6091 |
956
- | 0.5981 | 500 | 3.0903 | 0.2129 | 0.2158 | 0.6599 | 0.8709 | 0.6078 |
957
- | 0.7177 | 600 | - | 0.2283 | 0.2203 | 0.6623 | 0.8674 | 0.6118 |
958
- | 0.8373 | 700 | - | 0.2264 | 0.2217 | 0.6643 | 0.8687 | 0.6136 |
959
- | 0.9569 | 800 | - | 0.2363 | 0.2198 | 0.6618 | 0.8716 | 0.6113 |
960
- | 1.0766 | 900 | - | 0.2448 | 0.2177 | 0.6633 | 0.8728 | 0.6139 |
961
- | 1.1962 | 1000 | 2.1076 | 0.2570 | 0.2204 | 0.6634 | 0.8730 | 0.6111 |
962
- | 1.3158 | 1100 | - | 0.2589 | 0.2214 | 0.6655 | 0.8726 | 0.6124 |
963
- | 1.4354 | 1200 | - | 0.2572 | 0.2213 | 0.6648 | 0.8691 | 0.6101 |
964
- | 1.5550 | 1300 | - | 0.2552 | 0.2186 | 0.6644 | 0.8685 | 0.6096 |
965
- | 1.6746 | 1400 | - | 0.2553 | 0.2198 | 0.6634 | 0.8673 | 0.6098 |
966
- | 1.7943 | 1500 | 2.2031 | 0.2554 | 0.2205 | 0.6629 | 0.8684 | 0.6081 |
967
- | 1.9139 | 1600 | - | 0.2554 | 0.2180 | 0.6630 | 0.8685 | 0.6075 |
968
 
969
 
970
  ### Framework Versions
 
230
  type: mteb/AILA_casedocs
231
  metrics:
232
  - type: cosine_accuracy@1
233
+ value: 0.26
234
  name: Cosine Accuracy@1
235
  - type: cosine_accuracy@3
236
  value: 0.36
237
  name: Cosine Accuracy@3
238
  - type: cosine_accuracy@5
239
+ value: 0.38
240
  name: Cosine Accuracy@5
241
  - type: cosine_accuracy@10
242
+ value: 0.58
243
  name: Cosine Accuracy@10
244
  - type: cosine_precision@1
245
+ value: 0.26
246
  name: Cosine Precision@1
247
  - type: cosine_precision@3
248
  value: 0.2
 
251
  value: 0.14
252
  name: Cosine Precision@5
253
  - type: cosine_precision@10
254
+ value: 0.10599999999999998
255
  name: Cosine Precision@10
256
  - type: cosine_recall@1
257
+ value: 0.08253846153846153
258
  name: Cosine Recall@1
259
  - type: cosine_recall@3
260
+ value: 0.183986013986014
261
  name: Cosine Recall@3
262
  - type: cosine_recall@5
263
+ value: 0.21322843822843823
264
  name: Cosine Recall@5
265
  - type: cosine_recall@10
266
+ value: 0.30445687645687647
267
  name: Cosine Recall@10
268
  - type: cosine_ndcg@10
269
+ value: 0.261956835808035
270
  name: Cosine Ndcg@10
271
  - type: cosine_mrr@10
272
+ value: 0.3361349206349206
273
  name: Cosine Mrr@10
274
  - type: cosine_map@100
275
+ value: 0.23084417119066455
276
  name: Cosine Map@100
277
  - task:
278
  type: information-retrieval
 
282
  type: mteb/AILA_statutes
283
  metrics:
284
  - type: cosine_accuracy@1
285
+ value: 0.26
286
  name: Cosine Accuracy@1
287
  - type: cosine_accuracy@3
288
+ value: 0.44
289
  name: Cosine Accuracy@3
290
  - type: cosine_accuracy@5
291
+ value: 0.54
292
  name: Cosine Accuracy@5
293
  - type: cosine_accuracy@10
294
+ value: 0.7
295
  name: Cosine Accuracy@10
296
  - type: cosine_precision@1
297
+ value: 0.26
298
  name: Cosine Precision@1
299
  - type: cosine_precision@3
300
+ value: 0.16666666666666669
301
  name: Cosine Precision@3
302
  - type: cosine_precision@5
303
+ value: 0.14400000000000002
304
  name: Cosine Precision@5
305
  - type: cosine_precision@10
306
+ value: 0.10999999999999999
307
  name: Cosine Precision@10
308
  - type: cosine_recall@1
309
+ value: 0.071
310
  name: Cosine Recall@1
311
  - type: cosine_recall@3
312
+ value: 0.129
313
  name: Cosine Recall@3
314
  - type: cosine_recall@5
315
+ value: 0.17700000000000002
316
  name: Cosine Recall@5
317
  - type: cosine_recall@10
318
+ value: 0.2643333333333333
319
  name: Cosine Recall@10
320
  - type: cosine_ndcg@10
321
+ value: 0.23332317287231785
322
  name: Cosine Ndcg@10
323
  - type: cosine_mrr@10
324
+ value: 0.37441269841269836
325
  name: Cosine Mrr@10
326
  - type: cosine_map@100
327
+ value: 0.2043241006581302
328
  name: Cosine Map@100
329
  - task:
330
  type: information-retrieval
 
334
  type: mteb/legalbench_consumer_contracts_qa
335
  metrics:
336
  - type: cosine_accuracy@1
337
+ value: 0.45202020202020204
338
  name: Cosine Accuracy@1
339
  - type: cosine_accuracy@3
340
+ value: 0.6868686868686869
341
  name: Cosine Accuracy@3
342
  - type: cosine_accuracy@5
343
+ value: 0.7878787878787878
344
  name: Cosine Accuracy@5
345
  - type: cosine_accuracy@10
346
+ value: 0.8737373737373737
347
  name: Cosine Accuracy@10
348
  - type: cosine_precision@1
349
+ value: 0.45202020202020204
350
  name: Cosine Precision@1
351
  - type: cosine_precision@3
352
+ value: 0.22895622895622894
353
  name: Cosine Precision@3
354
  - type: cosine_precision@5
355
+ value: 0.15757575757575756
356
  name: Cosine Precision@5
357
  - type: cosine_precision@10
358
+ value: 0.08737373737373735
359
  name: Cosine Precision@10
360
  - type: cosine_recall@1
361
+ value: 0.45202020202020204
362
  name: Cosine Recall@1
363
  - type: cosine_recall@3
364
+ value: 0.6868686868686869
365
  name: Cosine Recall@3
366
  - type: cosine_recall@5
367
+ value: 0.7878787878787878
368
  name: Cosine Recall@5
369
  - type: cosine_recall@10
370
+ value: 0.8737373737373737
371
  name: Cosine Recall@10
372
  - type: cosine_ndcg@10
373
+ value: 0.660855212722782
374
  name: Cosine Ndcg@10
375
  - type: cosine_mrr@10
376
+ value: 0.5928561407728073
377
  name: Cosine Mrr@10
378
  - type: cosine_map@100
379
+ value: 0.5987644318492056
380
  name: Cosine Map@100
381
  - task:
382
  type: information-retrieval
 
386
  type: mteb/legalbench_corporate_lobbying
387
  metrics:
388
  - type: cosine_accuracy@1
389
+ value: 0.7705882352941177
390
  name: Cosine Accuracy@1
391
  - type: cosine_accuracy@3
392
+ value: 0.9088235294117647
393
  name: Cosine Accuracy@3
394
  - type: cosine_accuracy@5
395
+ value: 0.9382352941176471
396
  name: Cosine Accuracy@5
397
  - type: cosine_accuracy@10
398
+ value: 0.9705882352941176
399
  name: Cosine Accuracy@10
400
  - type: cosine_precision@1
401
+ value: 0.7705882352941177
402
  name: Cosine Precision@1
403
  - type: cosine_precision@3
404
+ value: 0.3029411764705882
405
  name: Cosine Precision@3
406
  - type: cosine_precision@5
407
+ value: 0.18764705882352936
408
  name: Cosine Precision@5
409
  - type: cosine_precision@10
410
+ value: 0.09705882352941174
411
  name: Cosine Precision@10
412
  - type: cosine_recall@1
413
+ value: 0.7705882352941177
414
  name: Cosine Recall@1
415
  - type: cosine_recall@3
416
+ value: 0.9088235294117647
417
  name: Cosine Recall@3
418
  - type: cosine_recall@5
419
+ value: 0.9382352941176471
420
  name: Cosine Recall@5
421
  - type: cosine_recall@10
422
+ value: 0.9705882352941176
423
  name: Cosine Recall@10
424
  - type: cosine_ndcg@10
425
+ value: 0.877258980240739
426
  name: Cosine Ndcg@10
427
  - type: cosine_mrr@10
428
+ value: 0.8466806722689075
429
  name: Cosine Mrr@10
430
  - type: cosine_map@100
431
+ value: 0.8476651359451062
432
  name: Cosine Map@100
433
  - task:
434
  type: information-retrieval
 
438
  type: mteb/legal_summarization
439
  metrics:
440
  - type: cosine_accuracy@1
441
+ value: 0.4894366197183099
442
  name: Cosine Accuracy@1
443
  - type: cosine_accuracy@3
444
+ value: 0.6408450704225352
445
  name: Cosine Accuracy@3
446
  - type: cosine_accuracy@5
447
+ value: 0.7147887323943662
448
  name: Cosine Accuracy@5
449
  - type: cosine_accuracy@10
450
+ value: 0.7816901408450704
451
  name: Cosine Accuracy@10
452
  - type: cosine_precision@1
453
+ value: 0.4894366197183099
454
  name: Cosine Precision@1
455
  - type: cosine_precision@3
456
+ value: 0.23591549295774647
457
  name: Cosine Precision@3
458
  - type: cosine_precision@5
459
+ value: 0.16619718309859152
460
  name: Cosine Precision@5
461
  - type: cosine_precision@10
462
+ value: 0.09753521126760564
463
  name: Cosine Precision@10
464
  - type: cosine_recall@1
465
+ value: 0.4368868514114993
466
  name: Cosine Recall@1
467
  - type: cosine_recall@3
468
+ value: 0.5753959362234009
469
  name: Cosine Recall@3
470
  - type: cosine_recall@5
471
+ value: 0.6440091305408207
472
  name: Cosine Recall@5
473
  - type: cosine_recall@10
474
+ value: 0.7159090909090909
475
  name: Cosine Recall@10
476
  - type: cosine_ndcg@10
477
+ value: 0.596027060399293
478
  name: Cosine Ndcg@10
479
  - type: cosine_mrr@10
480
+ value: 0.5833137715179968
481
  name: Cosine Mrr@10
482
  - type: cosine_map@100
483
+ value: 0.5567992166327345
484
  name: Cosine Map@100
485
  ---
486
 
 
591
 
592
  | Metric | mteb/AILA_casedocs | mteb/AILA_statutes | mteb/legalbench_consumer_contracts_qa | mteb/legalbench_corporate_lobbying | mteb/legal_summarization |
593
  |:--------------------|:-------------------|:-------------------|:--------------------------------------|:-----------------------------------|:-------------------------|
594
+ | cosine_accuracy@1 | 0.26 | 0.26 | 0.452 | 0.7706 | 0.4894 |
595
+ | cosine_accuracy@3 | 0.36 | 0.44 | 0.6869 | 0.9088 | 0.6408 |
596
+ | cosine_accuracy@5 | 0.38 | 0.54 | 0.7879 | 0.9382 | 0.7148 |
597
+ | cosine_accuracy@10 | 0.58 | 0.7 | 0.8737 | 0.9706 | 0.7817 |
598
+ | cosine_precision@1 | 0.26 | 0.26 | 0.452 | 0.7706 | 0.4894 |
599
+ | cosine_precision@3 | 0.2 | 0.1667 | 0.229 | 0.3029 | 0.2359 |
600
+ | cosine_precision@5 | 0.14 | 0.144 | 0.1576 | 0.1876 | 0.1662 |
601
+ | cosine_precision@10 | 0.106 | 0.11 | 0.0874 | 0.0971 | 0.0975 |
602
+ | cosine_recall@1 | 0.0825 | 0.071 | 0.452 | 0.7706 | 0.4369 |
603
+ | cosine_recall@3 | 0.184 | 0.129 | 0.6869 | 0.9088 | 0.5754 |
604
+ | cosine_recall@5 | 0.2132 | 0.177 | 0.7879 | 0.9382 | 0.644 |
605
+ | cosine_recall@10 | 0.3045 | 0.2643 | 0.8737 | 0.9706 | 0.7159 |
606
+ | **cosine_ndcg@10** | **0.262** | **0.2333** | **0.6609** | **0.8773** | **0.596** |
607
+ | cosine_mrr@10 | 0.3361 | 0.3744 | 0.5929 | 0.8467 | 0.5833 |
608
+ | cosine_map@100 | 0.2308 | 0.2043 | 0.5988 | 0.8477 | 0.5568 |
609
 
610
  <!--
611
  ## Bias, Risks and Limitations
 
819
  #### Non-Default Hyperparameters
820
 
821
  - `eval_strategy`: steps
822
+ - `per_device_train_batch_size`: 256
823
+ - `learning_rate`: 5e-06
824
  - `num_train_epochs`: 2
825
  - `warmup_ratio`: 0.1
826
  - `fp16`: True
 
833
  - `do_predict`: False
834
  - `eval_strategy`: steps
835
  - `prediction_loss_only`: True
836
+ - `per_device_train_batch_size`: 256
837
  - `per_device_eval_batch_size`: 8
838
  - `per_gpu_train_batch_size`: None
839
  - `per_gpu_eval_batch_size`: None
840
  - `gradient_accumulation_steps`: 1
841
  - `eval_accumulation_steps`: None
842
  - `torch_empty_cache_steps`: None
843
+ - `learning_rate`: 5e-06
844
  - `weight_decay`: 0.0
845
  - `adam_beta1`: 0.9
846
  - `adam_beta2`: 0.999
 
946
  </details>
947
 
948
  ### Training Logs
949
+ | Epoch | Step | mteb/AILA_casedocs_cosine_ndcg@10 | mteb/AILA_statutes_cosine_ndcg@10 | mteb/legalbench_consumer_contracts_qa_cosine_ndcg@10 | mteb/legalbench_corporate_lobbying_cosine_ndcg@10 | mteb/legal_summarization_cosine_ndcg@10 |
950
+ |:------:|:----:|:---------------------------------:|:---------------------------------:|:----------------------------------------------------:|:-------------------------------------------------:|:---------------------------------------:|
951
+ | 0 | 0 | 0.1972 | 0.2052 | 0.6560 | 0.8641 | 0.5900 |
952
+ | 0.4717 | 100 | 0.2409 | 0.2173 | 0.6624 | 0.8766 | 0.6055 |
953
+ | 0.9434 | 200 | 0.2489 | 0.2207 | 0.6553 | 0.8725 | 0.5998 |
954
+ | 1.4151 | 300 | 0.2619 | 0.2355 | 0.6641 | 0.8790 | 0.5992 |
955
+ | 1.8868 | 400 | 0.2620 | 0.2333 | 0.6609 | 0.8773 | 0.5960 |
 
 
 
 
 
 
 
 
 
 
 
 
956
 
957
 
958
  ### Framework Versions
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:9ff310237f7ad7bd604ddf873f325f93cfbe0e0b03a82a82b29cc8f3ac5c075e
3
  size 90864192
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:377c6c0f2f16b01e67bae8bf3d7ec4fa58444f4f55ca3465753fcaf6722e4bd4
3
  size 90864192