Commit
·
10fd16b
1
Parent(s):
faa9e8b
Add code to create 'models' directory for caching fasttext model
Browse files
main.py
CHANGED
|
@@ -11,6 +11,7 @@ from huggingface_hub.utils import logging
|
|
| 11 |
from toolz import concat, groupby, valmap
|
| 12 |
from fastapi import FastAPI
|
| 13 |
from httpx import AsyncClient
|
|
|
|
| 14 |
|
| 15 |
app = FastAPI()
|
| 16 |
logger = logging.get_logger(__name__)
|
|
@@ -141,13 +142,13 @@ def yield_clean_rows(rows: Union[list[str], str], min_length: int = 3) -> Iterat
|
|
| 141 |
FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
|
| 142 |
|
| 143 |
# model = load_model(DEFAULT_FAST_TEXT_MODEL)
|
| 144 |
-
|
| 145 |
model = fasttext.load_model(
|
| 146 |
hf_hub_download(
|
| 147 |
"facebook/fasttext-language-identification",
|
| 148 |
"model.bin",
|
| 149 |
-
cache_dir="
|
| 150 |
-
local_dir="
|
| 151 |
local_dir_use_symlinks=False,
|
| 152 |
)
|
| 153 |
)
|
|
|
|
| 11 |
from toolz import concat, groupby, valmap
|
| 12 |
from fastapi import FastAPI
|
| 13 |
from httpx import AsyncClient
|
| 14 |
+
from pathlib import Path
|
| 15 |
|
| 16 |
app = FastAPI()
|
| 17 |
logger = logging.get_logger(__name__)
|
|
|
|
| 142 |
FASTTEXT_PREFIX_LENGTH = 9 # fasttext labels are formatted like "__label__eng_Latn"
|
| 143 |
|
| 144 |
# model = load_model(DEFAULT_FAST_TEXT_MODEL)
|
| 145 |
+
Path("models").mkdir(parents=True, exist_ok=True)
|
| 146 |
model = fasttext.load_model(
|
| 147 |
hf_hub_download(
|
| 148 |
"facebook/fasttext-language-identification",
|
| 149 |
"model.bin",
|
| 150 |
+
cache_dir="models",
|
| 151 |
+
local_dir="models",
|
| 152 |
local_dir_use_symlinks=False,
|
| 153 |
)
|
| 154 |
)
|