minor updates in publishing and logging results

main_backend.py

@@ -1,6 +1,7 @@
 import argparse
 import logging
 import pprint
+import os
 
 from huggingface_hub import snapshot_download
 
@@ -9,6 +10,8 @@ import src.backend.manage_requests as manage_requests
 import src.backend.sort_queue as sort_queue
 import src.envs as envs
 
+os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'expandable_segments:True'
+
 logging.basicConfig(level=logging.ERROR)
 pp = pprint.PrettyPrinter(width=80)
 
@@ -46,22 +49,30 @@ def run_auto_eval(args):
         logging.info("Sorted eval requests")
 
         print(f"Found {len(eval_requests)} {','.join(current_pending_status)} eval requests")
-
+        print(eval_requests)
         if len(eval_requests) == 0:
             print("No eval requests found. Exiting.")
             return
 
-        eval_request = eval_requests[0]
-        pp.pprint(eval_request)
-
-        manage_requests.set_eval_request(
-            api=envs.API,
-            eval_request=eval_request,
-            new_status=RUNNING_STATUS,
-            hf_repo=envs.QUEUE_REPO,
-            local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
-        )
-        logging.info("Set eval request to running, now running eval")
+        if args.model is not None:
+            eval_request = manage_requests.EvalRequest(
+                model=args.model,
+                status=PENDING_STATUS,
+                precision=args.precision
+            )
+            pp.pprint(eval_request)
+        else:
+            eval_request = eval_requests[0]
+            pp.pprint(eval_request)
+
+        # manage_requests.set_eval_request(
+        #     api=envs.API,
+        #     eval_request=eval_request,
+        #     new_status=RUNNING_STATUS,
+        #     hf_repo=envs.QUEUE_REPO,
+        #     local_dir=envs.EVAL_REQUESTS_PATH_BACKEND
+        # )
+        # logging.info("Set eval request to running, now running eval")
 
         run_eval_suite.run_evaluation(
             eval_request=eval_request,
@@ -70,6 +81,8 @@ def run_auto_eval(args):
             batch_size=1,
             device=envs.DEVICE,
             no_cache=True,
+            need_check=not args.publish,
+            write_results=args.update
         )
         logging.info("Eval finished, now setting status to finished")
     else:
@@ -87,6 +100,8 @@ def run_auto_eval(args):
             results_repo=envs.RESULTS_REPO,
             batch_size=1,
             device=envs.DEVICE,
+            need_check=not args.publish,
+            write_results=args.update
         )
         logging.info("Reproducibility eval finished")
 
@@ -98,6 +113,8 @@ def main():
     parser.add_argument("--reproduce", type=bool, default=False, help="Reproduce the evaluation results")
     parser.add_argument("--model", type=str, default=None, help="Your Model ID")
     parser.add_argument("--precision", type=str, default="float16", help="Precision of your model")
+    parser.add_argument("--publish", type=bool, default=False, help="whether directly publish the evaluation results on HF")
+    parser.add_argument("--update", type=bool, default=False, help="whether to update google drive files")
 
     args = parser.parse_args()
 

src/backend/run_eval_suite.py

@@ -15,7 +15,7 @@ logging.getLogger("openai").setLevel(logging.WARNING)
 
 def run_evaluation(eval_request: EvalRequest, batch_size, device,
                 local_dir: str, results_repo: str, no_cache=True, limit=None, 
-                need_check=True, write_results=True):
+                need_check=True, write_results=False):
     """
     Run the evaluation for a given model and upload the results.