Upload hf_utils.py with huggingface_hub
Browse files- hf_utils.py +0 -25
hf_utils.py
CHANGED
|
@@ -1,34 +1,9 @@
|
|
| 1 |
from pathlib import Path
|
| 2 |
|
| 3 |
-
from datasets import disable_caching, enable_caching, is_caching_enabled
|
| 4 |
from datasets.utils.py_utils import get_imports
|
| 5 |
|
| 6 |
from .file_utils import get_all_files_in_dir
|
| 7 |
|
| 8 |
-
HF_CACHING_ENABLED = False
|
| 9 |
-
|
| 10 |
-
|
| 11 |
-
class HFCachingContextManager:
|
| 12 |
-
def __init__(self, should_cache):
|
| 13 |
-
self.should_cache = should_cache
|
| 14 |
-
self.was_caching_enabled = None
|
| 15 |
-
|
| 16 |
-
def __enter__(self):
|
| 17 |
-
self.was_caching_enabled = is_caching_enabled()
|
| 18 |
-
if self.should_cache:
|
| 19 |
-
if not self.was_caching_enabled:
|
| 20 |
-
enable_caching()
|
| 21 |
-
else:
|
| 22 |
-
if self.was_caching_enabled:
|
| 23 |
-
disable_caching()
|
| 24 |
-
|
| 25 |
-
def __exit__(self, exc_type, exc_val, exc_tb):
|
| 26 |
-
if self.was_caching_enabled != is_caching_enabled():
|
| 27 |
-
if self.was_caching_enabled:
|
| 28 |
-
enable_caching()
|
| 29 |
-
else:
|
| 30 |
-
disable_caching()
|
| 31 |
-
|
| 32 |
|
| 33 |
def get_missing_imports(file, exclude=None):
|
| 34 |
if exclude is None:
|
|
|
|
| 1 |
from pathlib import Path
|
| 2 |
|
|
|
|
| 3 |
from datasets.utils.py_utils import get_imports
|
| 4 |
|
| 5 |
from .file_utils import get_all_files_in_dir
|
| 6 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 7 |
|
| 8 |
def get_missing_imports(file, exclude=None):
|
| 9 |
if exclude is None:
|