change auth token setting back
Browse files
src/axolotl/utils/data.py
CHANGED
@@ -60,12 +60,11 @@ def load_tokenized_prepared_datasets(
|
|
60 |
else Path(default_dataset_prepared_path) / ds_hash
|
61 |
)
|
62 |
dataset = None
|
63 |
-
use_auth_token = False
|
64 |
try:
|
65 |
if cfg.push_dataset_to_hub:
|
66 |
use_auth_token = True
|
67 |
dataset = load_dataset(
|
68 |
-
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=
|
69 |
)
|
70 |
dataset = dataset["train"]
|
71 |
except:
|
@@ -85,7 +84,7 @@ def load_tokenized_prepared_datasets(
|
|
85 |
ds = None
|
86 |
ds_from_hub = False
|
87 |
try:
|
88 |
-
load_dataset(d.path, streaming=True, use_auth_token=
|
89 |
ds_from_hub = True
|
90 |
except FileNotFoundError:
|
91 |
pass
|
@@ -104,7 +103,7 @@ def load_tokenized_prepared_datasets(
|
|
104 |
use_auth_token=use_auth_token,
|
105 |
)
|
106 |
else:
|
107 |
-
ds = load_dataset(d.path, streaming=False, use_auth_token=
|
108 |
else:
|
109 |
fp = hf_hub_download(
|
110 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
|
|
60 |
else Path(default_dataset_prepared_path) / ds_hash
|
61 |
)
|
62 |
dataset = None
|
|
|
63 |
try:
|
64 |
if cfg.push_dataset_to_hub:
|
65 |
use_auth_token = True
|
66 |
dataset = load_dataset(
|
67 |
+
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
|
68 |
)
|
69 |
dataset = dataset["train"]
|
70 |
except:
|
|
|
84 |
ds = None
|
85 |
ds_from_hub = False
|
86 |
try:
|
87 |
+
load_dataset(d.path, streaming=True, use_auth_token=True)
|
88 |
ds_from_hub = True
|
89 |
except FileNotFoundError:
|
90 |
pass
|
|
|
103 |
use_auth_token=use_auth_token,
|
104 |
)
|
105 |
else:
|
106 |
+
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
|
107 |
else:
|
108 |
fp = hf_hub_download(
|
109 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|