Update app.py

app.py

@@ -222,7 +222,33 @@ class EnhancedGazaKnowledgeBase:
             self._load_vector_store()
         else:
             self._create_vector_store()
-            
+
+
+
+    def _batch_embed_with_retry(self, texts, batch_size=16, max_retries=3, delay=2):
+    """Embed texts in batches with retry fallback and logging"""
+    embeddings = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i+batch_size]
+        for attempt in range(max_retries):
+            try:
+                batch_embeddings = self.embedding_model.get_text_embedding_batch(batch)
+                embeddings.extend(batch_embeddings)
+                break  # Success
+            except Exception as e:
+                if attempt < max_retries - 1:
+                    logger.warning(f"Batch embedding failed (attempt {attempt+1}): {e}. Retrying...")
+                    time.sleep(delay * (attempt + 1))
+                else:
+                    logger.error(f"❌ Final failure embedding batch {i}-{i+len(batch)}: {e}")
+                    # Optionally fall back to single embeddings
+                    for text in batch:
+                        try:
+                            embeddings.append(self.embedding_model.get_text_embedding(text))
+                        except Exception as sub_e:
+                            logger.error(f"Failed to embed individual text: {sub_e}")
+    return embeddings
+
     def _load_vector_store(self):
         """Load existing vector store with error handling"""
         try:
@@ -314,10 +340,7 @@ class EnhancedGazaKnowledgeBase:
             logger.info("Training IVF index...")
             # Get some embeddings for training
             sample_texts = [doc.text[:500] for doc in documents[:100]]  # Sample for training
-            sample_embeddings = np.array([
-                self.embedding_model.get_text_embedding(text) 
-                for text in sample_texts
-            ]).astype('float32')
+            sample_embeddings = np.array(self._batch_embed_with_retry(sample_texts, batch_size=16)).astype('float32')
             faiss_index.train(sample_embeddings)
             logger.info("IVF index training completed")