using ctransformers instead of llama.cpp

app.py

@@ -5,9 +5,13 @@ from PyPDF2 import PdfReader
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
-from langchain.llms import LlamaCpp
+# from langchain.llms import LlamaCpp
 from langchain.vectorstores import Qdrant
 from qdrant_client.http import models
+from langchain.llms import CTransformers
+# from ctransformers import AutoModelForCausalLM
+
+
 
 
 
@@ -24,13 +28,23 @@ callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
 
 print("loading the LLM......................................")
 
-llm = LlamaCpp(
-    model_path="TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q8_0.gguf",
-    n_ctx=2048,
-    f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls
-    callback_manager=callback_manager,
-    verbose=True,
+# llm = LlamaCpp(
+#     model_path="TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q8_0.gguf",
+#     n_ctx=2048,
+#     f16_kv=True,  # MUST set to True, otherwise you will run into problem after a couple of calls
+#     callback_manager=callback_manager,
+#     verbose=True,
+# )
+
+llm = CTransformers(
+    model="TheBloke/Llama-2-7B-Chat-GGUF/llama-2-7b-chat.Q8_0.gguf", 
+    callbacks=[StreamingStdOutCallbackHandler()],
+    temperature = 0.2,
+    max_new_tokens = 1000,
+    
 )
+
+
 print("LLM loaded........................................")
 print("################################################################")