Spaces:
Running
Running
replace rs name w uuid, increase timeout
Browse files- validation.py +6 -6
validation.py
CHANGED
|
@@ -3,7 +3,7 @@ import traceback
|
|
| 3 |
import mlcroissant as mlc
|
| 4 |
import func_timeout
|
| 5 |
|
| 6 |
-
|
| 7 |
|
| 8 |
def validate_json(file_path):
|
| 9 |
"""Validate that the file is proper JSON."""
|
|
@@ -45,15 +45,15 @@ def validate_records(json_data):
|
|
| 45 |
|
| 46 |
for record_set in record_sets:
|
| 47 |
try:
|
| 48 |
-
records = dataset.records(record_set=record_set.
|
| 49 |
-
_ = func_timeout.func_timeout(
|
| 50 |
-
results.append(f"β
Record set '{record_set.
|
| 51 |
except func_timeout.exceptions.FunctionTimedOut:
|
| 52 |
-
error_message = f"β Record set '{record_set.
|
| 53 |
return False, error_message
|
| 54 |
except Exception as e:
|
| 55 |
error_details = traceback.format_exc()
|
| 56 |
-
error_message = f"β Record set '{record_set.
|
| 57 |
return False, error_message
|
| 58 |
|
| 59 |
return True, "\n".join(results)
|
|
|
|
| 3 |
import mlcroissant as mlc
|
| 4 |
import func_timeout
|
| 5 |
|
| 6 |
+
WAIT_TIME = 5 * 60 # seconds
|
| 7 |
|
| 8 |
def validate_json(file_path):
|
| 9 |
"""Validate that the file is proper JSON."""
|
|
|
|
| 45 |
|
| 46 |
for record_set in record_sets:
|
| 47 |
try:
|
| 48 |
+
records = dataset.records(record_set=record_set.uuid)
|
| 49 |
+
_ = func_timeout.func_timeout(WAIT_TIME, lambda: next(iter(records)))
|
| 50 |
+
results.append(f"β
Record set '{record_set.uuid}' passed validation.")
|
| 51 |
except func_timeout.exceptions.FunctionTimedOut:
|
| 52 |
+
error_message = f"β Record set '{record_set.uuid}' generation took too long (>60s)"
|
| 53 |
return False, error_message
|
| 54 |
except Exception as e:
|
| 55 |
error_details = traceback.format_exc()
|
| 56 |
+
error_message = f"β Record set '{record_set.uuid}' failed: {str(e)}\n\n{error_details}"
|
| 57 |
return False, error_message
|
| 58 |
|
| 59 |
return True, "\n".join(results)
|