Fix async issues and improve error handling

rag_engine.py

@@ -40,7 +40,7 @@ class RAGEngine:
             logger.error(f"Error processing content: {str(e)}")
             raise
 
-    async def search_and_process(self, query: str, max_results: int = 5, similarity_k: int = 3) -> Dict:
+    def search_and_process(self, query: str, max_results: int = 5, similarity_k: int = 3) -> Dict:
         """Search the web and process results with RAG"""
         try:
             # Get web search results
@@ -49,31 +49,40 @@ class RAGEngine:
             if 'error' in web_results:
                 return {'error': web_results['error']}
             
+            if not web_results.get('results'):
+                return {'error': 'No results found'}
+            
             # Process and store new content
-            if 'results' in web_results and web_results['results']:
-                for result in web_results['results']:
-                    if 'content' in result:
+            for result in web_results['results']:
+                if result and isinstance(result, dict) and 'content' in result:
+                    try:
                         self.process_and_store_content(
                             result['content'],
                             metadata={'url': result.get('url'), 'title': result.get('title')}
                         )
+                    except Exception as e:
+                        logger.error(f"Error processing result: {str(e)}")
+                        continue
             
             # Perform similarity search if we have stored vectors
             if self.vector_store:
-                similar_docs = self.vector_store.similarity_search_with_score(
-                    query,
-                    k=similarity_k
-                )
-                
-                # Add similarity results to web results
-                web_results['similar_chunks'] = [
-                    {
-                        'content': doc[0].page_content,
-                        'metadata': doc[0].metadata,
-                        'similarity_score': doc[1]
-                    }
-                    for doc in similar_docs
-                ]
+                try:
+                    similar_docs = self.vector_store.similarity_search_with_score(
+                        query,
+                        k=similarity_k
+                    )
+                    
+                    # Add similarity results to web results
+                    web_results['similar_chunks'] = [
+                        {
+                            'content': doc[0].page_content,
+                            'metadata': doc[0].metadata,
+                            'similarity_score': float(doc[1])
+                        }
+                        for doc in similar_docs
+                    ]
+                except Exception as e:
+                    logger.error(f"Error in similarity search: {str(e)}")
             
             return web_results
             
@@ -91,7 +100,7 @@ class RAGEngine:
             {
                 'content': doc[0].page_content,
                 'metadata': doc[0].metadata,
-                'similarity_score': doc[1]
+                'similarity_score': float(doc[1])
             }
             for doc in similar_docs
         ]

search_engine.py

@@ -135,7 +135,9 @@ class WebSearchEngine:
         """Perform search and process results"""
         try:
             # Search using DuckDuckGo
-            search_results = list(self.ddgs.text(query, max_results=max_results))
+            search_results = []
+            for result in self.ddgs.text(query, max_results=max_results):
+                search_results.append(result)
             
             results = []
             for result in search_results: