Upload 3 files

Voice_Assistant.py

@@ -0,0 +1,99 @@
+import speech_recognition as sr
+from transformers import pipeline
+from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
+
+from langchain.llms import HuggingFacePipeline
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.document_loaders import UnstructuredFileLoader
+
+import os
+import torch
+import pyttsx3
+
+import soundfile as sf
+from playsound import playsound
+from TTS.api import TTS
+from langchain.llms import Ollama
+
+
+# Loading RAG data
+loader = UnstructuredFileLoader("Foduu_KnowledgeBase.pdf")
+documents = loader.load()
+
+# Open-source embedding model
+embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-MiniLM-L6-v2")
+vectorstore = FAISS.from_documents(documents, embeddings)
+
+# Ollama Model
+ollama = Ollama(base_url='http://localhost:11434',model="llama3")
+qa = RetrievalQA.from_chain_type(llm=ollama, chain_type="stuff", retriever=vectorstore.as_retriever())
+
+# Speech recognition setup
+r = sr.Recognizer()
+
+# Initialize TTS with a model
+tts = TTS(model_name="tts_models/en/ljspeech/glow-tts")
+
+## Using Mozilla TTS (TTS)
+def speak(text):
+    """
+    Converts text to speech using Mozilla TTS, plays the audio, and then deletes the file.
+    """
+    try:
+        # Generate speech
+        output_file = "output.wav"
+        tts.tts_to_file(text=text, file_path=output_file)
+
+        # Play the speech
+        playsound(output_file)
+        os.remove(output_file)
+        print(f"Speech played and file {output_file} removed.")
+
+    except Exception as e:
+        print(f"Error: {e}")
+
+
+def listen():
+    """
+    Records audio and converts it to text using speech recognition.
+    """
+    with sr.Microphone() as source:
+        print("Listening...")
+        audio = r.listen(source)
+    try:
+        text = r.recognize_google(audio)
+        print(f"You said: {text}")
+        return text
+    except sr.UnknownValueError:
+        print("Could not understand audio")
+        speak('could not understand audio')
+        return None
+    except sr.RequestError as e:
+        print(f"Could not request results from Google Speech Recognition service; {e}")
+        return None
+
+
+def process_audio(text):
+    if text is not None:
+        try:
+            response = qa.run(text)
+            print(response)
+            speak(response)
+        except Exception as e:
+            print(f"An error occurred: {e}")
+            speak("Sorry, I'm having trouble processing that right now.")
+
+
+def main():
+    """
+    Main loop for the voice assistant.
+    """
+    while True:
+        text = listen()
+        process_audio(text)
+
+
+if __name__ == "__main__":
+    main()

readme.md

@@ -0,0 +1,77 @@
+# Voice Assistant with RAG and Speech Recognition
+
+This project implements a voice assistant that uses Retrieval-Augmented Generation (RAG) and speech recognition to provide responses to user queries. The assistant can listen to voice input, process it, and respond with synthesized speech based on the knowledge base you passed.
+
+## Features
+
+- Speech recognition using Google Speech Recognition
+- Text-to-Speech (TTS) using Mozilla TTS
+- RAG-based question answering using Langchain and FAISS
+- Integration with Ollama for language model processing
+
+## Prerequisites
+
+Before running this project, make sure you have the following dependencies installed:
+
+- Python 3.7+
+- PyTorch
+- Transformers
+- SpeechRecognition
+- pyttsx3
+- soundfile
+- playsound
+- TTS
+- Langchain
+- FAISS
+- Ollama
+
+Create an Conda environment
+```
+conda create -n VoiceAI python==3.10
+conda activate VoiceAI
+```
+You can install most of these dependencies using pip:
+```
+pip install torch transformers speechrecognition pyttsx3 soundfile playsound TTS langchain faiss-cpu
+```
+
+For Ollama, follow the installation instructions on their official website https://ollama.com/library/llama3.
+
+## Setup
+
+1. Clone this repository to your local machine.
+2. Install the required dependencies as mentioned above.
+3. Make sure you have the `KnowledgeBase.pdf` file in the same directory as the script. This file will be used to create the knowledge base for the RAG system.
+4. Ensure that Ollama is running on `http://localhost:11434` with the `llama3` model loaded.
+
+## Usage
+
+To run the voice assistant, execute the following command in your terminal:
+
+```
+python voice_assistant.py
+```
+
+The assistant will start listening for your voice input. Speak clearly into your microphone to ask questions or give commands. The assistant will process your input and respond with synthesized speech.
+
+## How It Works
+
+1. The script loads the knowledge base from `KnowledgeBase.pdf` and creates a FAISS vector store using sentence embeddings.
+2. It sets up a Retrieval QA chain using Ollama as the language model and the FAISS vector store as the retriever.
+3. The main loop continuously listens for voice input using the computer's microphone.
+4. When speech is detected, it's converted to text using Google's Speech Recognition service.
+5. The text query is then processed by the RAG system to generate a response.
+6. The response is converted to speech using Mozilla TTS and played back to the user.
+
+## Customization
+
+- To use a different knowledge base, replace `KnowledgeBase.pdf` with your own PDF file and update the filename in the script.
+- You can experiment with different embedding models by changing the `model_name` in the `HuggingFaceEmbeddings` initialization.
+- To use a different Ollama model, update the `model` parameter in the `Ollama` initialization.
+- Try to use other TTS frameworks - MeloTTS, coquiTTS, Mars5TTS.
+
+## Troubleshooting
+
+- If you encounter issues with speech recognition, ensure your microphone is properly connected and configured.
+- For TTS issues, make sure you have the necessary audio drivers installed on your system.
+- If the RAG system is not working as expected, check that your knowledge base PDF is properly formatted and contains relevant information.

requirements.txt

@@ -0,0 +1,12 @@
+SpeechRecognition==3.8.1
+transformers==4.29.2
+torch==1.13.1
+pyttsx3==2.90
+soundfile==0.10.3.post1
+playsound==1.2.2
+TTS==0.8.0
+langchain==0.0.184
+faiss-cpu==1.7.3
+Unstructured==0.6.6
+sentence-transformers==2.2.2
+pydantic==1.10.8