second

.env

@@ -0,0 +1,7 @@
+PERSIST_DIRECTORY=db
+MODEL_TYPE=GPT4All
+MODEL_PATH=models/ggml-gpt4all-j-v1.3-groovy.bin
+EMBEDDINGS_MODEL_NAME=all-MiniLM-L6-v2
+MODEL_N_CTX=1000
+MODEL_N_BATCH=8
+TARGET_SOURCE_CHUNKS=5

.gitattributes

@@ -0,0 +1,2 @@
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.sqlite3 filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+__pycache__
+source_documents
+models

README.md

@@ -0,0 +1,11 @@
+---
+title: Tsetlin-Chat
+emoji: ⚕
+colorFrom: gray
+colorTo: purple
+sdk: streamlit
+sdk_version: 1.2.0
+app_file: app.py
+models: models/gpt4all
+pinned: true
+---

app.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+from dotenv import load_dotenv
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.vectorstores import Chroma
+from langchain.llms import GPT4All, LlamaCpp
+import chromadb
+import os
+import argparse
+import time
+import streamlit as st
+from htmlTemplates import css, bot_template, user_template
+from langchain.memory import ConversationBufferMemory
+from langchain.chains import ConversationalRetrievalChain
+import langchain
+langchain.verbose = False
+if not load_dotenv():
+    print("Could not load .env file or it is empty. Please check if it exists and is readable.")
+    exit(1)
+
+embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME")
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+
+model_type = os.environ.get('MODEL_TYPE')
+model_path = os.environ.get('MODEL_PATH')
+model_n_ctx = os.environ.get('MODEL_N_CTX')
+model_n_batch = int(os.environ.get('MODEL_N_BATCH',8))
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+from constants import CHROMA_SETTINGS
+
+def handle_userinput(user_question):
+    response = st.session_state.conversation({'question': user_question})
+    st.session_state.chat_history = response['chat_history']
+
+    for i, message in enumerate(st.session_state.chat_history):
+        if i % 2 == 0:
+            st.write(user_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+        else:
+            st.write(bot_template.replace(
+                "{{MSG}}", message.content), unsafe_allow_html=True)
+
+
+def get_conversation_chain(llm, retriever):
+    #llm = ChatOpenAI()
+    #llm= GPT4All(model=model_path, max_tokens=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=False)
+    memory = ConversationBufferMemory(
+        memory_key='chat_history', return_messages=True)
+    conversation_chain = ConversationalRetrievalChain.from_llm(
+        llm=llm,
+        retriever=retriever,
+        memory=memory
+    )
+    return conversation_chain
+
+def main():
+    # Parse the command line arguments
+    args = parse_arguments()
+    st.set_page_config(page_title="Chat with multiple PDFs",
+                       page_icon=":books:")
+    st.write(css, unsafe_allow_html=True)
+    
+    if "conversation" not in st.session_state:
+        st.session_state.conversation = None
+    if "chat_history" not in st.session_state:
+        st.session_state.chat_history = None
+
+    st.header("Tsetlin LLM Powered Chatbot")
+    user_question = st.text_input("Ask a question about Tsetlin Machine:")
+
+    if user_question:
+        handle_userinput(user_question)
+    
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+    chroma_client = chromadb.PersistentClient(settings=CHROMA_SETTINGS , path=persist_directory)
+    db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS, client=chroma_client)
+    retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+    # activate/deactivate the streaming StdOut callback for LLMs
+    callbacks = [] if args.mute_stream else [StreamingStdOutCallbackHandler()]
+    # Prepare the LLM
+    #what is match equivalent in python 3.9?
+
+    llm = GPT4All(model=model_path, max_tokens=model_n_ctx, backend='gptj', n_batch=model_n_batch, callbacks=callbacks, verbose=False)
+
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents= not args.hide_source)
+    # Interactive questions and answers
+    
+    st.session_state.conversation = get_conversation_chain(llm, retriever)
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='privateGPT: Ask questions to your documents without an internet connection, '
+                                                 'using the power of LLMs.')
+    parser.add_argument("--hide-source", "-S", action='store_true',
+                        help='Use this flag to disable printing of source documents used for answers.')
+
+    parser.add_argument("--mute-stream", "-M",
+                        action='store_true',
+                        help='Use this flag to disable the streaming StdOut callback for LLMs.')
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    main()

constants.py

@@ -0,0 +1,16 @@
+import os
+from dotenv import load_dotenv
+from chromadb.config import Settings
+
+load_dotenv()
+
+# Define the folder for storing database
+PERSIST_DIRECTORY = os.environ.get('PERSIST_DIRECTORY')
+if PERSIST_DIRECTORY is None:
+    raise Exception("Please set the PERSIST_DIRECTORY environment variable")
+
+# Define the Chroma settings
+CHROMA_SETTINGS = Settings(
+        persist_directory=PERSIST_DIRECTORY,
+        anonymized_telemetry=False
+)

db/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faed94a1be48de21f54651ece93f3e27e36b8d70ece9812f105713ef8104fb35
+size 34115584

db/f3bf2f98-9353-4d29-83ca-7b0e011daddf/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a817b4dada67e3f64ef3ce4102e176fca480e057c2dac39212158444698da2d2
+size 8380000

db/f3bf2f98-9353-4d29-83ca-7b0e011daddf/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f9ef91d2871d89044cd5874a1689a09b8e45127f88b6f1e757908874e684a6c
+size 100

db/f3bf2f98-9353-4d29-83ca-7b0e011daddf/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa6a78fccf41054a748238c8168ecf7e2ddab452feeacadb00c10d6b093c404a
+size 20000

db/f3bf2f98-9353-4d29-83ca-7b0e011daddf/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ff4af9781581a47abe381aa548704f949a76960b1bca7a99253c81e43fb21fdc
+size 44752

htmlTemplates.py

@@ -0,0 +1,51 @@
+css = '''
+<style>
+.chat-message {
+    padding: 1.5rem; border-radius: 0.5rem; margin-bottom: 1rem; display: flex; box-shadow: 0 0 10px rgba(0,0,0,0.1);
+}
+.chat-message:hover {
+    box-shadow: 0 0 20px rgba(0,0,0,0.2);
+}
+.chat-message.user {
+    background-color: #2b313e
+}
+.chat-message.bot {
+    background-color: #475063
+}
+.chat-message .avatar {
+  width: 20%;
+  margin-right: 1rem;
+}
+.chat-message .avatar img {
+  max-width: 78px;
+  max-height: 78px;
+  border-radius: 50%;
+  object-fit: cover;
+}
+.chat-message .message {
+  width: 80%;
+  padding: 1rem;
+  border-radius: 0.5rem;
+  color: #fff;
+  font-family: Arial, sans-serif;
+}
+</style>
+'''
+
+bot_template = '''
+<div class="chat-message bot">
+    <div class="avatar">
+        <img src="https://i.ibb.co/cN0nmSj/Screenshot-2023-05-28-at-02-37-21.png" style="max-height: 78px; max-width: 78px; border-radius: 50%; object-fit: cover;">
+    </div>
+    <div class="message">{{MSG}}</div>
+</div>
+'''
+
+user_template = '''
+<div class="chat-message user">
+    <div class="avatar">
+        <img src="https://freeimage.host/i/JIFVVIf">
+    </div>    
+    <div class="message">{{MSG}}</div>
+</div>
+'''

ingest.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+import os
+import glob
+from typing import List
+from dotenv import load_dotenv
+from multiprocessing import Pool
+from tqdm import tqdm
+
+from langchain.document_loaders import (
+    CSVLoader,
+    EverNoteLoader,
+    PyMuPDFLoader,
+    TextLoader,
+    UnstructuredEmailLoader,
+    UnstructuredEPubLoader,
+    UnstructuredHTMLLoader,
+    UnstructuredMarkdownLoader,
+    UnstructuredODTLoader,
+    UnstructuredPowerPointLoader,
+    UnstructuredWordDocumentLoader,
+    PyPDFLoader
+)
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.vectorstores import Chroma
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.docstore.document import Document
+
+if not load_dotenv():
+    print("Could not load .env file or it is empty. Please check if it exists and is readable.")
+    exit(1)
+
+from constants import CHROMA_SETTINGS
+import chromadb
+
+# Load environment variables
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+source_directory = os.environ.get('SOURCE_DIRECTORY', 'source_documents')
+embeddings_model_name = os.environ.get('EMBEDDINGS_MODEL_NAME')
+chunk_size = 500
+chunk_overlap = 50
+
+
+# Custom document loaders
+# class MyElmLoader(UnstructuredEmailLoader):
+#     """Wrapper to fallback to text/plain when default does not work"""
+
+#     def load(self) -> List[Document]:
+#         """Wrapper adding fallback for elm without html"""
+#         try:
+#             try:
+#                 doc = UnstructuredEmailLoader.load(self)
+#             except ValueError as e:
+#                 if 'text/html content not found in email' in str(e):
+#                     # Try plain text
+#                     self.unstructured_kwargs["content_source"]="text/plain"
+#                     doc = UnstructuredEmailLoader.load(self)
+#                 else:
+#                     raise
+#         except Exception as e:
+#             # Add file_path to exception message
+#             raise type(e)(f"{self.file_path}: {e}") from e
+
+#         return doc
+
+
+# Map file extensions to document loaders and their arguments
+LOADER_MAPPING = {
+    ".csv": (CSVLoader, {}),
+    # ".docx": (Docx2txtLoader, {}),
+    ".doc": (UnstructuredWordDocumentLoader, {}),
+    ".docx": (UnstructuredWordDocumentLoader, {}),
+    ".enex": (EverNoteLoader, {}),
+    # ".eml": (MyElmLoader, {}),
+    ".epub": (UnstructuredEPubLoader, {}),
+    ".html": (UnstructuredHTMLLoader, {}),
+    ".md": (UnstructuredMarkdownLoader, {}),
+    ".odt": (UnstructuredODTLoader, {}),
+    # ".pdf": (PyMuPDFLoader, {}),
+    ".pdf": (PyPDFLoader, {}),
+    ".ppt": (UnstructuredPowerPointLoader, {}),
+    ".pptx": (UnstructuredPowerPointLoader, {}),
+    ".txt": (TextLoader, {"encoding": "utf8"}),
+    # Add more mappings for other file extensions and loaders as needed
+}
+
+
+def load_single_document(file_path: str) -> List[Document]:
+    ext = "." + file_path.rsplit(".", 1)[-1].lower()
+    if ext in LOADER_MAPPING:
+        loader_class, loader_args = LOADER_MAPPING[ext]
+        loader = loader_class(file_path, **loader_args)
+        return loader.load()
+
+    raise ValueError(f"Unsupported file extension '{ext}'")
+
+def load_documents(source_dir: str, ignored_files: List[str] = []) -> List[Document]:
+    """
+    Loads all documents from the source documents directory, ignoring specified files
+    """
+    all_files = []
+    for ext in LOADER_MAPPING:
+        all_files.extend(
+            glob.glob(os.path.join(source_dir, f"**/*{ext.lower()}"), recursive=True)
+        )
+        all_files.extend(
+            glob.glob(os.path.join(source_dir, f"**/*{ext.upper()}"), recursive=True)
+        )
+    filtered_files = [file_path for file_path in all_files if file_path not in ignored_files]
+
+    with Pool(processes=os.cpu_count()) as pool:
+        results = []
+        with tqdm(total=len(filtered_files), desc='Loading new documents', ncols=80) as pbar:
+            for i, docs in enumerate(pool.imap_unordered(load_single_document, filtered_files)):
+                results.extend(docs)
+                pbar.update()
+
+    return results
+
+def process_documents(ignored_files: List[str] = []) -> List[Document]:
+    """
+    Load documents and split in chunks
+    """
+    print(f"Loading documents from {source_directory}")
+    documents = load_documents(source_directory, ignored_files)
+    if not documents:
+        print("No new documents to load")
+        exit(0)
+    print(f"Loaded {len(documents)} new documents from {source_directory}")
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    texts = text_splitter.split_documents(documents)
+    print(f"Split into {len(texts)} chunks of text (max. {chunk_size} tokens each)")
+    return texts
+
+def does_vectorstore_exist(persist_directory: str, embeddings: HuggingFaceEmbeddings) -> bool:
+    """
+    Checks if vectorstore exists
+    """
+    db = Chroma(persist_directory=persist_directory, embedding_function=embeddings)
+    if not db.get()['documents']:
+        return False
+    return True
+
+def main():
+    # Create embeddings
+    embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+    # Chroma client
+    chroma_client = chromadb.PersistentClient(settings=CHROMA_SETTINGS , path=persist_directory)
+
+    if does_vectorstore_exist(persist_directory, embeddings):
+        # Update and store locally vectorstore
+        print(f"Appending to existing vectorstore at {persist_directory}")
+        db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS, client=chroma_client)
+        collection = db.get()
+        texts = process_documents([metadata['source'] for metadata in collection['metadatas']])
+        print(f"Creating embeddings. May take some minutes...")
+        db.add_documents(texts)
+    else:
+        # Create and store locally vectorstore
+        print("Creating new vectorstore")
+        texts = process_documents()
+        print(f"Creating embeddings. May take some minutes...")
+        db = Chroma.from_documents(texts, embeddings, persist_directory=persist_directory, client_settings=CHROMA_SETTINGS, client=chroma_client)
+    db.persist()
+    db = None
+
+    print(f"Ingestion complete! You can now run app.py to query your documents")
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -0,0 +1,29 @@
+[tool.poetry]
+name = "privategpt"
+version = "0.1.0"
+description = ""
+authors = ["Ivan Martinez <[email protected]>"]
+license = "Apache Version 2.0"
+readme = "README.md"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+langchain = "0.0.274"
+gpt4all = "1.0.8"
+chromadb = "0.4.7"
+llama-cpp-python = "0.1.81"
+urllib3 = "2.0.4"
+PyMuPDF = "1.23.1"
+python-dotenv = "^1.0.0"
+unstructured = "0.10.8"
+extract-msg = "0.45.0"
+tabulate = "^0.9.0"
+pandoc = "^2.3"
+pypandoc = "^1.11"
+tqdm = "4.66.1"
+sentence-transformers = "2.2.2"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

requirements.txt

@@ -0,0 +1,16 @@
+langchain==0.0.274
+streamlit==1.18.1
+altair==4
+gpt4all==1.0.8
+chromadb==0.4.7
+urllib3==2.0.4
+PyMuPDF==1.23.1
+python-dotenv==1.0.0
+unstructured==0.10.8
+extract-msg==0.45.0
+tabulate==0.9.0
+pandoc==2.3
+pypandoc==1.11
+tqdm==4.66.1
+sentence_transformers==2.2.2
+pypdf