Create app.py

app.py

@@ -0,0 +1,76 @@
+import gradio as gr
+
+from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline
+from transformers import BitsAndBytesConfig
+import re
+from deep_translator import (GoogleTranslator,
+                             PonsTranslator,
+                             LingueeTranslator,
+                             MyMemoryTranslator,
+                             YandexTranslator,
+                             DeeplTranslator,
+                             QcriTranslator,
+                             single_detection,
+                             batch_detection)
+from pyaspeller import YandexSpeller
+
+def error_correct_pyspeller(sample_text):
+    """ grammer correction of input text"""
+    speller = YandexSpeller()
+    fixed = speller.spelled(sample_text)
+    return fixed
+
+def postprocerssing(inp_text: str):
+    """Post preocessing of the llm response"""
+    inp_text = re.sub('<[^>]+>', '', inp_text)
+    inp_text = inp_text.split('##', 1)[0]
+    inp_text = error_correct_pyspeller(inp_text)
+    return inp_text
+
+
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype="float16",
+    bnb_4bit_use_double_quant=True,
+)
+
+llm = HuggingFacePipeline.from_model_id(
+    model_id="Danielrahmai1991/nvlm_adapt_basic_model_16bit",
+    task="text-generation",
+    pipeline_kwargs=dict(
+        max_new_tokens=512,
+        do_sample=True,
+        repetition_penalty=1.15,
+        trust_remote_code= True,
+        temperature= 0.70
+
+    ),
+  model_kwargs={"quantization_config": quantization_config,
+                },
+    )
+
+chat_model = ChatHuggingFace(llm=llm)
+
+# history of the messages
+messages = []
+
+def run_llm(prompt):
+    global messages
+    print(f"question is {prompt}")
+    lang = single_detection(prompt, api_key='4ab77f25578d450f0902fb42c66d5e11')
+    if lang == 'en':
+        prompt = error_correct_pyspeller(prompt)
+    en_translated = GoogleTranslator(source='auto', target='en').translate(prompt)
+    messages.append({"role": "user", "content": en_translated})
+    ai_msg = chat_model.invoke(messages, skip_prompt = True)
+    response_of_llm = postprocerssing(ai_msg.content)
+    messages.append({"role": "assistant", "content": response_of_llm})
+    response_of_llm = GoogleTranslator(source='auto', target=lang).translate(response_of_llm)
+    print(f"out is: {response_of_llm}")
+    return response_of_llm
+
+demo = gr.Interface(fn=run_llm, inputs=["text", ], outputs="text")
+demo.launch(debug=True, share=True)
+
+