Update utils.py

utils.py

@@ -54,6 +54,10 @@ from langchain_core.pydantic_v1 import BaseModel, Field
 from langchain_core.runnables import RunnablePassthrough
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from chromadb.errors import InvalidDimensionException
+import fitz  # PyMuPDF
+import docx
+from langchain.document_loaders import DirectoryLoader
+from langchain.document_loaders.pydantic import Document
 #import io
 #from PIL import Image, ImageDraw, ImageOps, ImageFont
 #import base64
@@ -201,7 +205,7 @@ def clean_text(text):
 ##################################################
 ##################################################
 # Funktion, um für einen best. File-typ ein directory-loader zu definieren
-def create_directory_loader(file_type, directory_path):
+def create_directory_loaderBack(file_type, directory_path):
     #verscheidene Dokument loaders:
     loaders = {
         '.pdf': PyPDFLoader,
@@ -212,6 +216,64 @@ def create_directory_loader(file_type, directory_path):
         glob=f"**/*{file_type}",
         loader_cls=loaders[file_type],
     )
+
+#besseren directory Loader als CustomLoader definieren, der den inhalt des dokuemnts, die seitenzahlen, die überschriften und die pfadezu den dokumenten extrahieren
+def create_directory_loader(file_type, directory_path):
+    loaders = {
+        '.pdf': load_pdf_with_metadata,
+        '.word': load_word_with_metadata,
+    }
+
+    class CustomLoader:
+        def __init__(self, directory_path, file_type, loader_func):
+            self.directory_path = directory_path
+            self.file_type = file_type
+            self.loader_func = loader_func
+
+        def load(self):
+            documents = []
+            for root, _, files in os.walk(self.directory_path):
+                for file in files:
+                    if file.endswith(self.file_type):
+                        file_path = os.path.join(root, file)
+                        documents.extend(self.loader_func(file_path))
+            return documents
+
+    return CustomLoader(directory_path, file_type, loaders[file_type])
+    
+
+################################################
+# Custom Loader-Funktionen zu dem DirektoryLoader
+# Custom loader functions
+def load_pdf_with_metadata(file_path):
+    document = fitz.open(file_path)
+    documents = []
+    for page_num in range(len(document)):
+        page = document.load_page(page_num)
+        content = page.get_text("text")
+        metadata = {
+            "title": document.metadata.get("title", "Unbekannt"),
+            "page": page_num + 1,
+            "path": file_path
+        }
+        documents.append(Document(content=content, metadata=metadata))
+    return documents
+
+def load_word_with_metadata(file_path):
+    document = docx.Document(file_path)
+    metadata = {
+        "title": "Dokument",
+        "path": file_path
+    }
+    contents = []
+    for para in document.paragraphs:
+        content = para.text
+        # Hier wird keine Seitenzahl verwendet, aber Sie können zusätzliche Logik hinzufügen
+        contents.append(Document(content=content, metadata={**metadata, "page": 1}))
+    return contents
+
+    
+    
 ################################################    
 #die Inhalte splitten, um in Vektordatenbank entsprechend zu laden als Splits
 def document_loading_splitting():
@@ -252,9 +314,15 @@ def document_loading_splitting():
 ###########################################
 #Chroma DB die splits ablegen - vektorisiert...
 def document_storage_chroma(splits):
-    #HF embeddings--------------------------------------
-    vectorstore = Chroma.from_documents(documents = splits, embedding = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={"device": "cpu"}, encode_kwargs={'normalize_embeddings': False}),  persist_directory = PATH_WORK + CHROMA_DIR)  
+    # Embedding-Funktion definieren
+    embedding_fn = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2", model_kwargs={"device": "cpu"}, encode_kwargs={'normalize_embeddings': False})
+
+    # Vectorstore initialisieren und Dokumente hinzufügen
+    vectorstore = Chroma.from_documents(documents=splits, embedding=embedding_fn, persist_directory=CHROMA_DIR)
     retriever = vectorstore.as_retriever(search_kwargs = {"k": ANZAHL_DOCS})
+    # Persist the vectorstore to disk
+    vectorstore.persist()
+    
     return vectorstore, retriever
 
 ############################################
@@ -377,16 +445,16 @@ def extract_document_info(documents):
     extracted_info = []
     for doc in documents:
         info = {
-            'content': doc.page_content,
-            'page': doc.metadata['page'],
-            'path': doc.metadata['source']
+            'content' : doc["content"]
+            'metadaten' : doc["metadata"]
+            'titel' : metadaten.get("title", "Keine Überschrift")
+            'seite' : metadaten.get("page", "Unbekannte Seite")
+            'pfad' : metadaten.get("path", "Kein Pfad verfügbar")
         }
         extracted_info.append(info)
     return extracted_info
 
-
-
-
+ 
 
 
 ###################################################