Add authentication

app.py

@@ -4,6 +4,20 @@ import time
 import traceback
 from validation import validate_json, validate_croissant, validate_records, generate_validation_report
 import requests
+import requests, os
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+AUTH_HEADERS = {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}
+
+_original_get = requests.get
+def authenticated_get(url, *args, **kwargs):
+    if "headers" not in kwargs:
+        kwargs["headers"] = AUTH_HEADERS
+    else:
+        kwargs["headers"].update(AUTH_HEADERS)
+    return _original_get(url, *args, **kwargs)
+
+requests.get = authenticated_get
 
 def process_file(file):
     results = []

validation.py

@@ -2,6 +2,22 @@ import json
 import traceback
 import mlcroissant as mlc
 import func_timeout
+import os
+import requests
+
+HF_TOKEN = os.environ.get("HF_TOKEN")
+AUTH_HEADERS = {"Authorization": f"Bearer {HF_TOKEN}"} if HF_TOKEN else {}
+
+_original_get = requests.get
+
+def authenticated_get(url, *args, **kwargs):
+    if "headers" not in kwargs:
+        kwargs["headers"] = AUTH_HEADERS
+    else:
+        kwargs["headers"].update(AUTH_HEADERS)
+    return _original_get(url, *args, **kwargs)
+
+requests.get = authenticated_get
 
 WAIT_TIME = 10 * 60  # seconds