Spaces:
Sleeping
Sleeping
Add the num_workers parameter to the load_data function to enable multiprocessing when loading documents.
#5
by
Restodecoca
- opened
app.py
CHANGED
@@ -383,13 +383,14 @@ else:
|
|
383 |
# Caso contrário, apenas reutilizamos o que já existe.
|
384 |
if "docstore" not in st.session_state:
|
385 |
# Carregar documentos do diretório local
|
|
|
386 |
file_extractor = {".csv": CustomPandasCSVReader()}
|
387 |
documents = SimpleDirectoryReader(
|
388 |
input_dir=documents_path,
|
389 |
file_extractor=file_extractor,
|
390 |
filename_as_id=True,
|
391 |
recursive=True
|
392 |
-
).load_data()
|
393 |
|
394 |
documents = clean_documents(documents)
|
395 |
|
|
|
383 |
# Caso contrário, apenas reutilizamos o que já existe.
|
384 |
if "docstore" not in st.session_state:
|
385 |
# Carregar documentos do diretório local
|
386 |
+
count_cores = os.cpu_count()
|
387 |
file_extractor = {".csv": CustomPandasCSVReader()}
|
388 |
documents = SimpleDirectoryReader(
|
389 |
input_dir=documents_path,
|
390 |
file_extractor=file_extractor,
|
391 |
filename_as_id=True,
|
392 |
recursive=True
|
393 |
+
).load_data(num_workers=count_cores)
|
394 |
|
395 |
documents = clean_documents(documents)
|
396 |
|