Update utils.py
Browse files
utils.py
CHANGED
|
@@ -32,33 +32,27 @@ from langchain.chains import LLMChain, RetrievalQA
|
|
| 32 |
from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader, DirectoryLoader
|
| 33 |
#from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
|
| 34 |
#from langchain.document_loaders import GenericLoader
|
| 35 |
-
from langchain.schema import AIMessage, HumanMessage
|
| 36 |
-
from langchain_community.llms import HuggingFaceHub
|
| 37 |
-
from langchain_community.llms import HuggingFaceTextGenInference
|
| 38 |
#from langchain_community.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
|
| 39 |
from langchain_huggingface import HuggingFaceEmbeddings
|
| 40 |
-
from langchain_community.tools import DuckDuckGoSearchRun
|
| 41 |
from typing import Dict, TypedDict
|
| 42 |
from langchain_core.messages import BaseMessage
|
| 43 |
from langchain.prompts import PromptTemplate
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
#from langchain import hub
|
| 47 |
-
from langchain.prompts import PromptTemplate
|
| 48 |
-
from langchain.schema import Document
|
| 49 |
from langchain_community.vectorstores import Chroma
|
| 50 |
from langchain_core.messages import BaseMessage, FunctionMessage
|
| 51 |
from langchain_core.output_parsers import StrOutputParser
|
| 52 |
from langchain_core.pydantic_v1 import BaseModel, Field
|
| 53 |
from langchain_core.runnables import RunnablePassthrough
|
| 54 |
-
from langchain_core.utils.function_calling import convert_to_openai_tool
|
| 55 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 56 |
-
from langchain_community.vectorstores import Chroma
|
| 57 |
from chromadb.errors import InvalidDimensionException
|
| 58 |
import io
|
| 59 |
-
from PIL import Image, ImageDraw, ImageOps, ImageFont
|
| 60 |
-
import base64
|
| 61 |
-
from tempfile import NamedTemporaryFile
|
| 62 |
|
| 63 |
import nltk
|
| 64 |
from nltk.corpus import stopwords
|
|
@@ -134,6 +128,7 @@ urls = [
|
|
| 134 |
##################################################
|
| 135 |
#Normalisierung eines Prompts
|
| 136 |
##################################################
|
|
|
|
| 137 |
def normalise_prompt (prompt):
|
| 138 |
#alles Kleinbuchstaben
|
| 139 |
prompt_klein =prompt.lower()
|
|
@@ -288,7 +283,7 @@ def rag_chainback(prompt, db, k=3):
|
|
| 288 |
|
| 289 |
|
| 290 |
###############################################
|
| 291 |
-
#Langchain anlegen
|
| 292 |
###############################################
|
| 293 |
#langchain nutzen, um prompt an LLM zu leiten - llm und prompt sind austauschbar
|
| 294 |
def llm_chain(llm, prompt):
|
|
@@ -360,7 +355,7 @@ def generate_prompt_with_history_hf(prompt, history):
|
|
| 360 |
|
| 361 |
|
| 362 |
##########################################
|
| 363 |
-
#Hashing....
|
| 364 |
# Funktion zum Hashen des Eingabewerts
|
| 365 |
def hash_input(input_string):
|
| 366 |
return hashlib.sha256(input_string.encode()).hexdigest()
|
|
|
|
| 32 |
from langchain_community.document_loaders import PyPDFLoader, UnstructuredWordDocumentLoader, DirectoryLoader
|
| 33 |
#from langchain.document_loaders.blob_loaders.youtube_audio import YoutubeAudioLoader
|
| 34 |
#from langchain.document_loaders import GenericLoader
|
| 35 |
+
from langchain.schema import AIMessage, HumanMessage, Document
|
| 36 |
+
#from langchain_community.llms import HuggingFaceHub
|
| 37 |
+
#from langchain_community.llms import HuggingFaceTextGenInference
|
| 38 |
#from langchain_community.embeddings import HuggingFaceInstructEmbeddings, HuggingFaceEmbeddings, HuggingFaceBgeEmbeddings, HuggingFaceInferenceAPIEmbeddings
|
| 39 |
from langchain_huggingface import HuggingFaceEmbeddings
|
| 40 |
+
#from langchain_community.tools import DuckDuckGoSearchRun
|
| 41 |
from typing import Dict, TypedDict
|
| 42 |
from langchain_core.messages import BaseMessage
|
| 43 |
from langchain.prompts import PromptTemplate
|
| 44 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
from langchain_community.vectorstores import Chroma
|
| 46 |
from langchain_core.messages import BaseMessage, FunctionMessage
|
| 47 |
from langchain_core.output_parsers import StrOutputParser
|
| 48 |
from langchain_core.pydantic_v1 import BaseModel, Field
|
| 49 |
from langchain_core.runnables import RunnablePassthrough
|
|
|
|
| 50 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
|
|
| 51 |
from chromadb.errors import InvalidDimensionException
|
| 52 |
import io
|
| 53 |
+
#from PIL import Image, ImageDraw, ImageOps, ImageFont
|
| 54 |
+
#import base64
|
| 55 |
+
#from tempfile import NamedTemporaryFile
|
| 56 |
|
| 57 |
import nltk
|
| 58 |
from nltk.corpus import stopwords
|
|
|
|
| 128 |
##################################################
|
| 129 |
#Normalisierung eines Prompts
|
| 130 |
##################################################
|
| 131 |
+
#Zur zeit nicht im Gebrauch.............................
|
| 132 |
def normalise_prompt (prompt):
|
| 133 |
#alles Kleinbuchstaben
|
| 134 |
prompt_klein =prompt.lower()
|
|
|
|
| 283 |
|
| 284 |
|
| 285 |
###############################################
|
| 286 |
+
#Langchain anlegen für RAG Chaining
|
| 287 |
###############################################
|
| 288 |
#langchain nutzen, um prompt an LLM zu leiten - llm und prompt sind austauschbar
|
| 289 |
def llm_chain(llm, prompt):
|
|
|
|
| 355 |
|
| 356 |
|
| 357 |
##########################################
|
| 358 |
+
#Hashing.... Für die Validierung........
|
| 359 |
# Funktion zum Hashen des Eingabewerts
|
| 360 |
def hash_input(input_string):
|
| 361 |
return hashlib.sha256(input_string.encode()).hexdigest()
|