Update utils.py

utils.py

@@ -87,6 +87,7 @@ german_stopwords = set(stopwords.words('german'))
 ANZAHL_DOCS = 5
 # Konstanten für Datei-Upload
 REPO_ID = "alexkueck/kkg_suche"
+STORAGE_REPO_ID = "alexkueck/kkg_files"
 REPO_TYPE = "space"
 
 ###############################
@@ -330,15 +331,57 @@ def split_documents_with_id(docs, text_splitter):
             splits.append(split_doc)
     return splits
 
+#######################################
+# Dokumente aus anderem Space laden
+#######################################
+#ein File aus dem Space mit der REPO_ID laden
+def download_file_from_hf(file_name, save_path):
+    url = f"https://huggingface.co/{STORAGE_REPO_ID}/resolve/main/{file_name}"
+    response = requests.get(url)
+    response.raise_for_status()  # Raise an error for bad status codes
+    with open(save_path, 'wb') as file:
+        file.write(response.content)
+    return save_path
+
+#Liste aller Files in dem Space mit der Repo_id
+def list_files_in_hf_repo(repo_id):
+    repo_info = api.list_repo_files(repo_id=repo_id)
+    return repo_info
+
+    
+
 ########################################
 #finally die Splits erzeugen und laden.....
 def document_loading_splitting():
     docs = []
     print("Directory Loader neu............................")
+
+
+
+    # Verzeichnis für heruntergeladene Dateien
+    download_dir = "downloaded_files"
+    os.makedirs(download_dir, exist_ok=True)
+
+    # Dateien im Hugging Face Space auflisten
+    files_in_repo = list_files_in_hf_repo(STORAGE_REPO_ID)
+
+    # Dateien aus dem Hugging Face Space herunterladen
+    for file_name in files_in_repo:
+        if file_name.endswith('.pdf') or file_name.endswith('.docx'):
+            local_file_path = os.path.join(download_dir, os.path.basename(file_name))
+            download_file_from_hf(file_name, local_file_path)
+
+    # Erstellen von DirectoryLoader für jeden Dateityp
+    pdf_loader = create_directory_loader('.pdf', download_dir)
+    word_loader = create_directory_loader('.word', download_dir)
+
+
+
+    """
     # kreiere einen DirectoryLoader für jeden file type
     pdf_loader = create_directory_loader('.pdf', CHROMA_PDF)
     word_loader = create_directory_loader('.word', CHROMA_WORD)
-    
+    """
     # Load the files
     pdf_documents = pdf_loader.load()
     word_documents = word_loader.load()
@@ -477,7 +520,7 @@ def upload_file_to_huggingface(file_path, upload_path):
     api.upload_file(
         path_or_fileobj=file_path,
         path_in_repo=upload_path,
-        repo_id=REPO_ID,
+        repo_id=STORAGE_REPO_ID,
         repo_type=REPO_TYPE,
         token=HF_WRITE
     )