Update app.py

app.py

@@ -65,11 +65,14 @@ YOUTUBE_DIR = "/youtube"
 
 ###############################################
 #URLs zu Dokumenten oder andere Inhalte, die einbezogen werden sollen
-PDF_URL       = "https://arxiv.org/pdf/2303.08774.pdf"
+#PDF_URL       = "https://arxiv.org/pdf/2303.08774.pdf"
 WEB_URL       = "https://openai.com/research/gpt-4"
 YOUTUBE_URL_1 = "https://www.youtube.com/watch?v=--khbXchTeE"
 YOUTUBE_URL_2 = "https://www.youtube.com/watch?v=hdhZwyf24mE"
-YOUTUBE_URL_3 = "https://www.youtube.com/watch?v=vw-KWfKwvTQ"
+#YOUTUBE_URL_3 = "https://www.youtube.com/watch?v=vw-KWfKwvTQ"
+
+#URL zu TIS dokumenten
+PDF_URL = https://huggingface.co/spaces/alexkueck/LIRAG/blob/main/chroma/1.%20Programmplanung%20in%20LIF15%20_%20MeisterNote.pdf
 
 
 ################################################
@@ -103,28 +106,57 @@ def add_file(history, file):
     history = history + [((file.name,), None)]
     return history
 
+# Funktion, um für einen best. File-typ ein directory-loader zu definieren
+def create_directory_loader(file_type, directory_path):
+    return DirectoryLoader(
+        path=directory_path,
+        glob=f"**/*{file_type}",
+        loader_cls=loaders[file_type],
+    )
     
 #die Inhalte splitten, um in Vektordatenbank entsprechend zu laden als Splits
 def document_loading_splitting():
     global splittet
+    
+    ##############################
     # Document loading
     docs = []
+    #verscheidene Dokument loaders:
+    loaders = {
+        '.pdf': PyMuPDFLoader,
+        '.word': UnstructuredWordDocumentLoader,
+    }
+
+
+    # kreiere einen DirectoryLoader für jeden file type
+    pdf_loader = create_directory_loader('.pdf', './chroma/pdf')
+    word_loader = create_directory_loader('.word', './chroma/word')
+    
+    
+    # Load the files
+    pdf_documents = pdf_loader.load()
+    word_documents = word_loader.load()
+
+    #alle zusammen in docs...
+    docs.extend(pdf_documents)
+    docs.extend(word_documents)
+
+    #andere loader...
     # Load PDF
-    loader = PyPDFLoader(PDF_URL)
-    docs.extend(loader.load())
+    #loader = PyPDFLoader(PDF_URL1)
+    #docs.extend(loader.load())
     # Load Web
-    loader = WebBaseLoader(WEB_URL)
-    docs.extend(loader.load())
+    #loader = WebBaseLoader(WEB_URL)
+    #docs.extend(loader.load())
     # Load YouTube
-    loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,
-                                               YOUTUBE_URL_2,
-                                               YOUTUBE_URL_3], PATH_WORK + YOUTUBE_DIR), 
-                           OpenAIWhisperParser())
-    docs.extend(loader.load())
+    #loader = GenericLoader(YoutubeAudioLoader([YOUTUBE_URL_1,YOUTUBE_URL_2], PATH_WORK + YOUTUBE_DIR), OpenAIWhisperParser())
+    #docs.extend(loader.load())
+
+    ################################
     # Document splitting
-    text_splitter = RecursiveCharacterTextSplitter(chunk_overlap = 150,
-                                                   chunk_size = 1500)
+    text_splitter = RecursiveCharacterTextSplitter(chunk_overlap = 150, chunk_size = 1500)
     splits = text_splitter.split_documents(docs)
+    
     #nur bei erster Anfrage mit "choma" wird gesplittet...
     splittet = True
     return splits