test

app.py

@@ -2,20 +2,28 @@ import os
 import gradio as gr
 #import time
 from langchain.llms import CTransformers
+from langchain.cache import InMemoryCache
+import langchain
 
 model_repo = os.getenv('HF_MODEL_REPO')
 model_bin = os.getenv('HF_MODEL_BIN')
 llm_config = {
-          'max_new_tokens': 256,
+          'max_new_tokens': 2048,
           'temperature': 0.8,
           'top_p': 0.5,
           'repetition_penalty': 1.1,
           }
 
-llm = CTransformers(model=model_repo, model_file=model_bin, config=llm_config, lib="avx2")
+llm = CTransformers(
+        model=model_repo,
+        model_file=model_bin,
+        config=llm_config,
+        lib="avx2"
+        )
+langchain.llm_cache = InMemoryCache()
 
 def response(prompt):
-    txt = llm(prompt)
+    txt = llm(prompt, reset=True, threads=2)
     return txt
 
 if __name__ == '__main__':