winglian commited on
Commit
d2a6f79
·
1 Parent(s): 98b1bce

change auth token setting back

Browse files
Files changed (1) hide show
  1. src/axolotl/utils/data.py +3 -4
src/axolotl/utils/data.py CHANGED
@@ -60,12 +60,11 @@ def load_tokenized_prepared_datasets(
60
  else Path(default_dataset_prepared_path) / ds_hash
61
  )
62
  dataset = None
63
- use_auth_token = False
64
  try:
65
  if cfg.push_dataset_to_hub:
66
  use_auth_token = True
67
  dataset = load_dataset(
68
- f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=use_auth_token
69
  )
70
  dataset = dataset["train"]
71
  except:
@@ -85,7 +84,7 @@ def load_tokenized_prepared_datasets(
85
  ds = None
86
  ds_from_hub = False
87
  try:
88
- load_dataset(d.path, streaming=True, use_auth_token=use_auth_token)
89
  ds_from_hub = True
90
  except FileNotFoundError:
91
  pass
@@ -104,7 +103,7 @@ def load_tokenized_prepared_datasets(
104
  use_auth_token=use_auth_token,
105
  )
106
  else:
107
- ds = load_dataset(d.path, streaming=False, use_auth_token=use_auth_token)
108
  else:
109
  fp = hf_hub_download(
110
  repo_id=d.path, repo_type="dataset", filename=d.data_files
 
60
  else Path(default_dataset_prepared_path) / ds_hash
61
  )
62
  dataset = None
 
63
  try:
64
  if cfg.push_dataset_to_hub:
65
  use_auth_token = True
66
  dataset = load_dataset(
67
+ f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
68
  )
69
  dataset = dataset["train"]
70
  except:
 
84
  ds = None
85
  ds_from_hub = False
86
  try:
87
+ load_dataset(d.path, streaming=True, use_auth_token=True)
88
  ds_from_hub = True
89
  except FileNotFoundError:
90
  pass
 
103
  use_auth_token=use_auth_token,
104
  )
105
  else:
106
+ ds = load_dataset(d.path, streaming=False, use_auth_token=True)
107
  else:
108
  fp = hf_hub_download(
109
  repo_id=d.path, repo_type="dataset", filename=d.data_files