File size: 1,184 Bytes
620e67e 9021f65 620e67e 81ddd1c 620e67e 9021f65 620e67e 81ddd1c 9021f65 95c3d5f 9021f65 95c3d5f 620e67e 9021f65 620e67e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 |
#!/usr/bin/env python3
from huggingface_hub import snapshot_download
import nltk
import os
import urllib.request
urls = [
"http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb",
]
repos = [
"InfiniFlow/text_concat_xgb_v1.0",
"InfiniFlow/deepdoc",
"BAAI/bge-large-zh-v1.5",
"BAAI/bge-reranker-v2-m3",
"maidalun1020/bce-embedding-base_v1",
"maidalun1020/bce-reranker-base_v1",
]
def download_model(repo_id):
local_dir = os.path.abspath(os.path.join("huggingface.co", repo_id))
os.makedirs(local_dir, exist_ok=True)
snapshot_download(repo_id=repo_id, local_dir=local_dir)
if __name__ == "__main__":
for url in urls:
filename = url.split("/")[-1]
print(f"Downloading {url}...")
if not os.path.exists(filename):
urllib.request.urlretrieve(url, filename)
local_dir = os.path.abspath('nltk_data')
for data in ['wordnet', 'punkt', 'punkt_tab']:
print(f"Downloading nltk {data}...")
nltk.download(data, download_dir=local_dir)
for repo_id in repos:
print(f"Downloading huggingface repo {repo_id}...")
download_model(repo_id)
|