Upload 20 files

.chromadb/20cb0d5e-499b-4d31-8a46-9a4bf83219b0/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a13e72541800c513c73dccea69f79e39cf4baef4fa23f7e117c0d6b0f5f99670
+size 3212000

.chromadb/20cb0d5e-499b-4d31-8a46-9a4bf83219b0/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ec6df10978b056a10062ed99efeef2702fa4a1301fad702b53dd2517103c746
+size 100

.chromadb/20cb0d5e-499b-4d31-8a46-9a4bf83219b0/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:939f9b658043836d45463f2f0a74346c2d2431220230ae6d8c2ad709814800bb
+size 4000

.chromadb/20cb0d5e-499b-4d31-8a46-9a4bf83219b0/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+size 0

.chromadb/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3484a9a12bf3911aa0a9d714f2a7b1c5a6b683276c2146cd4342065c375d29d5
+size 1622016

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+.chromadb/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text

SmartSearch/README.md

@@ -0,0 +1,19 @@
+# Smart Search 
+### Enhancing search experience with natural language
+
+
+The search should be able to support multiple llms providers
+
+* create embeddings
+* can do hybrid search
+* semantic search
+* keyword search
+
+
+
+
+
+
+
+
+

SmartSearch/database/annoydb.py

@@ -0,0 +1,89 @@
+from typing import List, Dict, Union, Any
+
+import numpy as np
+from annoy import AnnoyIndex
+
+from .vector_store import VectorStore
+
+class AnnoyDB(VectorStore):
+    def __init__(
+            self,
+            embedding_dim: int,
+            metric: str = 'angular'
+        ) -> None:
+        self.documents = []
+        self.metadata = []
+        self.embedding_dim = embedding_dim
+        
+        self.index = AnnoyIndex(embedding_dim, metric)
+        self.index_built = False
+        
+    def add_document(self, text: str, metadata: Dict[str, Any] = None):
+        """
+        Add a document to the search index.
+        
+        Args:
+            text: The document text
+            metadata: Optional metadata about the document
+        """
+        self.documents.append(text)
+        self.metadata.append(metadata or {})
+        
+        # Generate embedding using Sentence Transformers
+        embedding = self.model.encode(text, show_progress_bar=False)
+        
+        # Add to Annoy index
+        index_id = len(self.documents) - 1
+        self.index.add_item(index_id, embedding)
+        self.index_built = False
+
+    def add_documents(self, texts: List[str], embeddings: np.array, metadata_list: List[Dict[str, Any]] = None):
+        """
+        Batch add documents to the search index.
+        
+        Args:
+            texts: List of document texts
+            metadata_list: Optional list of metadata dictionaries
+        """
+        if metadata_list is None:
+            metadata_list = [{} for _ in texts]
+        
+        # Add documents and embeddings
+        print("Adding to index...")
+        for i, (text, metadata, embedding) in enumerate(zip(texts, metadata_list, embeddings)):
+            self.documents.append(text)
+            self.metadata.append(metadata)
+            self.index.add_item(len(self.documents) - 1, embedding)
+        
+        self.index_built = False
+        print("Done")
+        
+    def add_data(self, embedding: np.ndarray, document: str):
+        item_id = len(self.documents)
+        self.index.add_item(item_id, embedding)
+        self.documents.append(document)
+        
+    def build(self, num_trees:int = 10):
+        self.index.build(num_trees)
+        
+    def save(self, filepath: str):
+        self.index.save(filepath)
+        
+    def load(self, filepath: str):
+        self.index.load(filepath)
+        
+    def search(self, query_embedding: np.ndarray, top_k: int = 5) -> List[Dict[str, Union[str, float]]]:
+        indices, distances = self.index.get_nns_by_vector(
+            query_embedding, top_k, include_distances=True
+        )
+        
+        results = [
+            {
+                "document": self.documents[idx], 
+                "score": 1 / (1 + distance)  # Convert distance to similarity
+            } for idx, distance in zip(indices, distances)
+        ]
+        
+        return results
+        
+    

SmartSearch/database/chromadb.py

@@ -0,0 +1,213 @@
+from typing import Dict, Any, Optional, List
+
+import chromadb
+from chromadb.config import Settings
+from chromadb.api.types import (
+    Where,
+    GetResult,
+    QueryResult,
+)
+
+from ..embedding_provider import EmbeddingProvider
+from .vector_store import VectorStore
+
+
+class ChromaDB(VectorStore):
+    """
+    ChromaDB is an example of a vector-store class implementation.
+
+    See more:
+    https://github.com/chroma-core/chroma
+    """
+
+    def __init__(
+            self, 
+            configs: Dict[str, Any] = {},
+            db_path: str = ".chromadb", 
+            embedding_function: Optional[EmbeddingProvider] = None,
+            collection_name: Optional[str] = None,
+    ) -> None:
+        self.client = chromadb.PersistentClient(
+            path=db_path
+        )
+        self.configs = configs
+
+        self.embedding_function = embedding_function
+        self._collection_name = collection_name
+
+        self.collection = self.client.get_or_create_collection(
+            name = self.collection_name or "default_collection"
+        )
+
+        # self.logger = get_logger(self.__class__.__name__)
+
+    @property
+    def db_path(self) -> str:
+        return self.client.get_settings().persist_directory
+    
+    @db_path.setter
+    def db_path(self, value: str) -> None:
+        self.client = chromadb.PersistentClient(path=value)
+
+        self.collection = self.client.get_or_create_collection(
+            name = self.collection_name or "default_collection"
+        )
+
+    @property
+    def collection_name(self):
+        return self._collection_name
+    
+    @collection_name.setter
+    def collection_name(self, value):
+        self._collection_name = value
+        self.collection.modify(name=value)
+
+    def add_data(
+            self, 
+            documents: List[str],
+            ids: List[str],
+            metadatas: Optional[List[Dict[str, Any]]] = None, 
+            **optional_kwargs
+    ) -> None:
+        """
+        Add data to the collection by creating embeddings for them.
+
+        Args:
+            documents (List[str]): List of documents to add.
+            ids (List[str]): List of ids for the documents.
+            metadatas (Optional[List[Dict[str, Any]]]): List of metadata for the documents.
+            **optional_kwargs: Additional keyword arguments (see collection.add for more).
+        """
+
+        try:
+            params = {
+                "documents": documents,
+                "ids": ids,
+                **optional_kwargs
+            }
+
+            params["metadatas"] = metadatas or None
+            
+            # If an embedding function is provided, create embeddings for the documents
+            if self.embedding_function:
+                embeddings = self.embedding_function.embed_documents(documents)
+                params["embeddings"] = embeddings
+            
+            self.collection.add(**params)
+        except Exception as e:
+            # self.logger.error(f"Error adding data to collection: {e}")
+            print(f"Error adding data to collection: {e}")
+            raise e
+
+    def search(
+            self, 
+            query_text: Optional[List[str]] = None,  
+            query_embedding: Optional[List[List[float]]] = None,
+            n_results: int = 10,
+            **optional_kwargs
+    ) -> QueryResult:
+        """
+        Query the collection for similar documents.
+
+        Args:
+            query_text (Optional[List[str]]): List of query texts.
+            query_embedding (Optional[List[List[float]]]): List of query embeddings.
+            n_results (int): Number of results to return.
+            **optional_kwargs: Additional keyword arguments (see collection.query for more).
+
+        Returns:
+            QueryResult: The result of the query.
+        """
+
+        try:
+            if query_text is None and query_embedding is None:
+                raise ValueError("Either query_text or query_embedding must be provided.")
+            
+            params = {
+                "n_results": n_results,
+                **optional_kwargs
+            }
+
+            if query_text and query_embedding is None:
+                if self.embedding_function:
+                    query_embedding = self.embedding_function.embed_query(query_text)
+                    params["query_embeddings"] = query_embedding
+                else:
+                    params["query_text"] = query_text
+
+            elif query_embedding and query_text is None:
+                params["query_embeddings"] = query_embedding
+
+            elif query_embedding and query_text:
+                params["query_embeddings"] = query_embedding
+                
+                if self.embedding_function:
+                    embeddings = self.embedding_function.embed_query(query_text)
+                    params["query_embeddings"] = query_embedding.extend(embeddings)
+                else:
+                    params["query_text"] = query_text
+
+            return self.collection.query(**params)
+        except Exception as e:
+            # self.logger.error(f"Error querying data from collection: {e}")
+            print(f"Error querying data from collection: {e}")
+            raise e
+        
+    def query_by_id_or_metadata(
+            self, 
+            ids: Optional[List[str]] = None,
+	        where: Optional[Where] = None,
+            n_results: int = 10,
+            **optional_kwargs
+    ) -> GetResult:
+        """
+        Query the collection for similar documents.
+
+        Args:
+            ids (Optional[List[str]]): List of ids to query.
+            where (Optional[Where]): Where clause to query.
+            n_results (int): Number of results to return.
+            **optional_kwargs: Additional keyword arguments (see collection.get for more).
+
+        Returns:
+            GetResult: The result of the query.
+        """
+
+        try:
+            if ids is None and where is None:
+                raise ValueError("Either ids or where must be provided.")
+            
+            params = {
+                "n_results": n_results,
+                **optional_kwargs
+            }
+
+            if ids:
+                params["ids"] = ids
+            if where:
+                params["where"] = where
+
+            return self.collection.get(**params)
+        except Exception as e:
+            # self.logger.error(f"Error querying data from collection: {e}")
+            print(f"Error querying data from collection: {e}")
+            raise e
+        
+    def delete_collection(self, collection_name: Optional[str] = None) -> None:
+        """
+        Delete a specific collection from the ChromaDB.
+
+        Args:
+            collection_name (Optional[str]): Name of collection to delete. 
+                                             Uses class's collection_name if not provided.
+        """
+        try:
+            target_collection = collection_name or self.collection_name
+            if not target_collection:
+                raise ValueError("No collection name provided")
+            
+            self.client.delete_collection(name=target_collection)
+            print(f"Collection '{target_collection}' deleted successfully.")
+        except Exception as e:
+            print(f"Error deleting collection: {e}")
+        

SmartSearch/database/vector_store.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+
+from typing import Any
+
+class VectorStore(ABC):
+    @abstractmethod
+    def add_data(self, *args, **kwargs) -> Any:
+        """
+        Add data to the vector store
+        """
+        pass
+    
+    @abstractmethod
+    def search(self, *args, **kwargs) -> Any:
+        """
+        Search data from vector store
+        """
+        pass

SmartSearch/embedding_provider.py

@@ -0,0 +1,18 @@
+from abc import ABC, abstractmethod
+from typing import List
+import numpy as np
+
+class EmbeddingProvider(ABC):
+    """
+    Abstract class for the llm providers
+    """
+    @abstractmethod
+    def embed_documents(self, documents: List[str]) -> np.ndarray:
+        """Embed a list of documents"""
+        pass
+    
+    @abstractmethod
+    def embed_query(self, query: str) -> np.ndarray:
+        """Embed a query"""
+        pass
+    

SmartSearch/hybrid_search.py

@@ -0,0 +1,102 @@
+import os
+from typing import List, Dict, Union, Optional, Any
+import numpy as np
+
+from .embedding_provider import EmbeddingProvider
+from .database.annoydb import AnnoyDB
+from .keyword_search_provider import KeywordSearchProvider
+
+class HybridSearch:
+    def __init__(
+        self,
+        embedding_provider: EmbeddingProvider,
+        documents: List[str] = None,
+        ann_filepath: Optional[str] = None,
+        semantic_weight: float = 0.7, 
+        keyword_weight: float = 0.3
+    ) -> None:
+        self.embedding_provider = embedding_provider
+        self.documents = documents
+        
+        if ann_filepath and os.path.exists(ann_filepath):
+            self.index = AnnoyDB
+        self.embeddings = self.embedding_provider.embed_documents(documents)
+        
+        self.vector_db = AnnoyDB(
+            embedding_dim=self.embeddings.shape[1]
+        )
+        
+        for emb, doc in zip(self.embeddings, documents):
+            self.vector_db.add_data(emb, doc)
+        self.vector_db.build()
+        
+        # Keyword Search Setup
+        self.keyword_search = KeywordSearchProvider(documents)
+        
+        # Weights for hybrid search
+        self.semantic_weight = semantic_weight
+        self.keyword_weight = keyword_weight
+        
+        self.documents = documents
+        
+    def hybrid_search(self, query: str, top_k: int = 5) -> List[Dict[str, Union[str, float]]]:
+        # Embed query
+        query_embedding = self.embedding_provider.embed_query(query)
+        
+        # Perform semantic search
+        semantic_results = self.vector_db.search(query_embedding, top_k)
+        
+        # Perform keyword search
+        keyword_results = self.keyword_search.search(query, top_k)
+        
+        # Combine results with weighted scoring
+        combined_results = {}
+        
+        for result in semantic_results:
+            doc = result['document']
+            combined_results[doc] = {
+                'semantic_score': result['score'] * self.semantic_weight,
+                'keyword_score': 0,
+                'hybrid_score': result['score'] * self.semantic_weight
+            }
+        
+        for result in keyword_results:
+            doc = result['document']
+            if doc in combined_results:
+                combined_results[doc]['keyword_score'] = result['score'] * self.keyword_weight
+                combined_results[doc]['hybrid_score'] += result['score'] * self.keyword_weight
+            else:
+                combined_results[doc] = {
+                    'semantic_score': 0,
+                    'keyword_score': result['score'] * self.keyword_weight,
+                    'hybrid_score': result['score'] * self.keyword_weight
+                }
+        
+        # Sort and return top results
+        sorted_results = sorted(
+            [
+                {**{'document': doc}, **scores} 
+                for doc, scores in combined_results.items()
+            ], 
+            key=lambda x: x['hybrid_score'], 
+            reverse=True
+        )
+        
+        return sorted_results[:top_k]
+    
+    def set_weights(self, semantic_weight: float, keyword_weight: float):
+        """
+        Dynamically update search weights.
+        
+        Args:
+            semantic_weight: New weight for semantic search
+            keyword_weight: New weight for keyword search
+        """
+        if not (0 <= semantic_weight <= 1 and 0 <= keyword_weight <= 1):
+            raise ValueError("Weights must be between 0 and 1")
+        
+        if not np.isclose(semantic_weight + keyword_weight, 1.0):
+            raise ValueError("Semantic and keyword weights must sum to 1.0")
+        
+        self.semantic_weight = semantic_weight
+        self.keyword_weight = keyword_weight

SmartSearch/keyword_search_provider.py

@@ -0,0 +1,24 @@
+from typing import List, Dict, Union
+
+class KeywordSearchProvider:
+    def __init__(self, documents: List[str]):
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        self.vectorizer = TfidfVectorizer()
+        self.tfidf_matrix = self.vectorizer.fit_transform(documents)
+        self.documents = documents
+
+    def search(self, query: str, top_k: int = 5) -> List[Dict[str, Union[str, float]]]:
+        from sklearn.metrics.pairwise import cosine_similarity
+        query_vector = self.vectorizer.transform([query])
+        similarities = cosine_similarity(query_vector, self.tfidf_matrix)[0]
+        
+        # Get top-k results
+        top_indices = similarities.argsort()[-top_k:][::-1]
+        results = [
+            {
+                "document": self.documents[idx], 
+                "score": similarities[idx]
+            } for idx in top_indices
+        ]
+        
+        return results

SmartSearch/providers/OpenAIEmbedding.py

@@ -0,0 +1,125 @@
+from typing import List, Dict, Union, Optional
+import tiktoken
+from ..embedding_provider import EmbeddingProvider
+import numpy as np
+
+class OpenAIEmbedding(EmbeddingProvider):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        model: str = "text-embedding-3-small",
+        max_tokens: int = 8191
+    ) -> None:
+        """Initialize OpenAI embedding provider
+        
+        Args:
+            model_name (str, optional): Name of the embedding model. Default to "text-embedding-3-small"
+                more info: https://platform.openai.com/docs/models#embeddings
+            api_key: api_key for OpenAI
+        """
+        from openai import OpenAI
+        
+        self.client = OpenAI(api_key=api_key)
+        self.model = model
+        self.max_tokens = max_tokens
+        self.tokenizer = tiktoken.encoding_for_model(model)
+        
+    def _trancated_text(self, text: str) -> str:
+        """Truncate text into maximum token length
+
+        Args:
+            text (str): Input text
+
+        Returns:
+            str: Truncated text
+        """
+        tokens = self.tokenizer.encode(text)
+        truncated_tokens = tokens[:self.max_tokens]
+        return self.tokenizer.decode(truncated_tokens)
+        
+    def embed_documents(
+        self,
+        documents: List[str],
+        batch_size: int = 100
+    ) -> np.array:
+        """Embed a list of documents
+
+        Args:
+            documents (List[str]): List of documents to embed
+
+        Returns:
+            np.array: embeddings of documents
+        """
+        truncated_docs = [self._trancated_text(doc) for doc in documents]
+        
+        embeddings = []
+        for i in range(0, len(truncated_docs), batch_size):
+            batch = truncated_docs[i: i+batch_size]
+            
+            response = self.client.embeddings.create(
+                input=batch,
+                model=self.model
+            )
+            batch_embeddings = [
+                embed.embedding for embed in response.data
+            ]
+            embeddings.extend(batch_embeddings)
+            
+        return np.array(embeddings)
+        
+    def embed_query(self, query):
+        truncated_query = self._trancated_text(query)
+        
+        response = self.client.embeddings.create(
+            input=[truncated_query],
+            model=self.model
+        )
+        return np.array(response.data[0].embedding)
+    
+    def get_embedding_info(self) -> Dict[str, Union[str, int]]:
+        """
+        Get information about the current embedding configuration
+        
+        Returns:
+            Dict: Embedding configuration details
+        """
+        return {
+            "model": self.model,
+            "max_tokens": self.max_tokens,
+            "batch_size": 100,  # Default batch size
+        }
+    
+    def list_available_models(self) -> List[str]:
+        """
+        List available OpenAI embedding models
+        
+        Returns:
+            List[str]: Available embedding model names
+        """
+        return [
+            "text-embedding-ada-002",  # Most common
+            "text-embedding-3-small",  # Newer, more efficient
+            "text-embedding-3-large"   # Highest quality
+        ]
+    
+    def estimate_cost(self, num_documents: int) -> float:
+        """
+        Estimate embedding cost
+        
+        Args:
+            num_documents (int): Number of documents to embed
+        
+        Returns:
+            float: Estimated cost in USD
+        """
+        # Pricing as of 2024 (subject to change)
+        pricing = {
+            "text-embedding-ada-002": 0.0001 / 1000,  # $0.0001 per 1000 tokens
+            "text-embedding-3-small": 0.00006 / 1000,
+            "text-embedding-3-large": 0.00013 / 1000
+        }
+        
+        # Estimate tokens (assuming ~100 tokens per document)
+        total_tokens = num_documents * 100
+        
+        return total_tokens * pricing.get(self.model, pricing["text-embedding-ada-002"])

SmartSearch/providers/SentenceTransformerEmbedding.py

@@ -0,0 +1,85 @@
+from typing import List, Dict, Union
+from ..embedding_provider import EmbeddingProvider
+import numpy as np
+
+class SentenceTransformerEmbedding(EmbeddingProvider):
+    def __init__(
+        self,
+        model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+        device: str = None,
+        batch_size: int = 32,
+        normalize_embeddings: bool = True
+    ) -> None:
+        """Initialize sentence transformer embedding provider
+
+        Args:
+            model_name (str, optional): Name of the sentence tranformer model. Defaults to "sentence-transformers/all-MiniLM-L6-v2".
+        """
+        from sentence_transformers import SentenceTransformer
+        
+        if device is None:
+            import torch
+            self.device = "cuda" if torch.cuda.is_available() else "cpu"
+        
+        self.model = SentenceTransformer(model_name, device=device)
+        self.model_name = model_name
+        self.batch_size = batch_size
+        self.normalize_embeddings = normalize_embeddings
+        
+    def embed_documents(self, documents: List[str]) -> np.ndarray:
+        """Embed a list of documents
+
+        Args:
+            documents (List[str]): List of documents to embed
+        """
+        return self.model.encode(
+            documents,
+            batch_size=self.batch_size,
+            normalize_embeddings=self.normalize_embeddings
+        )
+        
+    def embed_query(self, query: str) -> np.ndarray:
+        """Embed a single query
+
+        Args:
+            query (str): Query to embed
+
+        Returns:
+            np.ndarray: Embedding vector
+        """
+        return self.model.encode(
+            query, 
+            normalize_embeddings=self.normalize_embeddings
+        )
+        
+    def get_model_info(self) -> Dict[str, Union[str, int]]:
+        """
+        Retrieve information about the current embedding model
+        
+        Returns:
+            Dict: Model information
+        """
+        return {
+            "model_name": self.model_name,
+            "device": self.device,
+            "batch_size": self.batch_size,
+            "normalize_embeddings": self.normalize_embeddings,
+            "embedding_dim": self.model.get_sentence_embedding_dimension()
+        }
+
+    def list_available_models(self) -> List[str]:
+        """
+        List some popular Sentence Transformer models
+        
+        Returns:
+            List[str]: Available model names
+        """
+        popular_models = [
+            "sentence-transformers/all-MiniLM-L6-v2",  # Small and fast
+            "sentence-transformers/all-mpnet-base-v2",  # High performance
+            "sentence-transformers/all-distilroberta-v1",  # Lightweight
+            "sentence-transformers/multi-qa-MiniLM-L6-cos-v1",  # Question Answering
+            "sentence-transformers/multi-qa-mpnet-base-cos-v1",  # Multilingual QA
+            "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"  # Multilingual
+        ]
+        return popular_models

SmartSearch/search_manager.py

@@ -0,0 +1,40 @@
+from typing import List
+
+from embedding_provider import EmbeddingProvider
+from database.annoydb import AnnoyDB
+
+class SearchManager:
+    def __init__(
+        self,
+        embedding_provider: EmbeddingProvider, 
+        documents: List[str], 
+        semantic_weight: float = 0.7, 
+        keyword_weight: float = 0.3
+    ) -> None:
+        """Smart Search Manager
+
+        Args:
+            embedding_provider (EmbeddingProvider): embedding provider
+            documents (List[str]): list of documents
+            semantic_weight (float, optional): _description_. Defaults to 0.7.
+            keyword_weight (float, optional): _description_. Defaults to 0.3.
+        """
+        self.embedding_provider = embedding_provider
+        self.semantic_embeddings = embedding_provider.embed_documents(documents)
+        
+        # Vector Database Setup
+        self.vector_db = AnnoyDB(
+            embedding_dim=self.semantic_embeddings.shape[1]
+        )
+        for emb, doc in zip(self.semantic_embeddings, documents):
+            self.vector_db.add_item(emb, doc)
+        self.vector_db.build()
+        
+        # Keyword Search Setup
+        self.keyword_search = KeywordSearchProvider(documents)
+        
+        # Weights for hybrid search
+        self.semantic_weight = semantic_weight
+        self.keyword_weight = keyword_weight
+        
+        self.documents = documents

app.py

@@ -0,0 +1,34 @@
+from typing import List
+from dataclasses import asdict
+import pandas as pd
+import gradio as gr
+
+from SmartSearch.database.chromadb import ChromaDB
+from SmartSearch.providers.SentenceTransformerEmbedding import SentenceTransformerEmbedding
+from utils import combine_metadata_with_distance
+st_chroma = ChromaDB(
+    embedding_function=SentenceTransformerEmbedding(model_name='all-mpnet-base-v2'),
+    collection_name="novel_mockup_collection"
+)
+
+# Function to search for products
+def search_novels(query, k):
+    result = st_chroma.search(query_text=query, n_results=k)
+    
+    result = combine_metadata_with_distance(result['metadatas'], result['distances'])
+    result = pd.DataFrame(result)
+    return result
+
+with gr.Blocks() as demo:
+    with gr.Row():
+        query = gr.Textbox(label="Search Query", placeholder="write a query to find the courses")
+    with gr.Row():
+        # search_type = gr.Dropdown(label="Search Type", choices=['semantic', 'keyword', 'hybrid'], value='hybrid')
+        k = gr.Number(label="Items Count", value=10)
+        # rerank = gr.Checkbox(value=True, label="Rerank")
+    results = gr.Dataframe(label="Search Results")
+    
+    search_button = gr.Button("Search", variant='primary')
+    search_button.click(fn=search_novels, inputs=[query, k], outputs=results)
+
+demo.launch()

requirements.txt

@@ -0,0 +1,116 @@
+-i https://pypi.org/simple
+aiofiles==23.2.1; python_version >= '3.7'
+annotated-types==0.7.0; python_version >= '3.8'
+anyio==4.8.0; python_version >= '3.9'
+asgiref==3.8.1; python_version >= '3.8'
+backoff==2.2.1; python_version >= '3.7' and python_version < '4.0'
+bcrypt==4.2.1; python_version >= '3.7'
+build==1.2.2.post1; python_version >= '3.8'
+cachetools==5.5.0; python_version >= '3.7'
+certifi==2024.12.14; python_version >= '3.6'
+charset-normalizer==3.4.1; python_version >= '3.7'
+chroma-hnswlib==0.7.6
+chromadb==0.6.2; python_version >= '3.9'
+click==8.1.8; python_version >= '3.7'
+coloredlogs==15.0.1; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+deprecated==1.2.15; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'
+durationpy==0.9
+exceptiongroup==1.2.2; python_version >= '3.7'
+fastapi==0.115.6; python_version >= '3.8'
+ffmpy==0.5.0; python_version >= '3.8' and python_version < '4.0'
+filelock==3.16.1; python_version >= '3.8'
+flatbuffers==24.12.23
+fsspec==2024.12.0; python_version >= '3.8'
+google-auth==2.37.0; python_version >= '3.7'
+googleapis-common-protos==1.66.0; python_version >= '3.7'
+gradio==5.11.0; python_version >= '3.10'
+gradio-client==1.5.3; python_version >= '3.10'
+grpcio==1.69.0; python_version >= '3.8'
+h11==0.14.0; python_version >= '3.7'
+httpcore==1.0.7; python_version >= '3.8'
+httptools==0.6.4; python_full_version >= '3.8.0'
+httpx==0.28.1; python_version >= '3.8'
+huggingface-hub==0.27.1; python_full_version >= '3.8.0'
+humanfriendly==10.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'
+idna==3.10; python_version >= '3.6'
+importlib-metadata==8.5.0; python_version >= '3.8'
+importlib-resources==6.5.2; python_version >= '3.9'
+jinja2==3.1.5; python_version >= '3.7'
+joblib==1.4.2; python_version >= '3.8'
+kubernetes==31.0.0; python_version >= '3.6'
+markdown-it-py==3.0.0; python_version >= '3.8'
+markupsafe==2.1.5; python_version >= '3.7'
+mdurl==0.1.2; python_version >= '3.7'
+mmh3==5.0.1; python_version >= '3.8'
+monotonic==1.6
+mpmath==1.3.0
+networkx==3.4.2; python_version >= '3.10'
+numpy==2.2.1; python_version >= '3.10'
+oauthlib==3.2.2; python_version >= '3.6'
+onnxruntime==1.20.1
+opentelemetry-api==1.29.0; python_version >= '3.8'
+opentelemetry-exporter-otlp-proto-common==1.29.0; python_version >= '3.8'
+opentelemetry-exporter-otlp-proto-grpc==1.29.0; python_version >= '3.8'
+opentelemetry-instrumentation==0.50b0; python_version >= '3.8'
+opentelemetry-instrumentation-asgi==0.50b0; python_version >= '3.8'
+opentelemetry-instrumentation-fastapi==0.50b0; python_version >= '3.8'
+opentelemetry-proto==1.29.0; python_version >= '3.8'
+opentelemetry-sdk==1.29.0; python_version >= '3.8'
+opentelemetry-semantic-conventions==0.50b0; python_version >= '3.8'
+opentelemetry-util-http==0.50b0; python_version >= '3.8'
+orjson==3.10.14; python_version >= '3.8'
+overrides==7.7.0; python_version >= '3.6'
+packaging==24.2; python_version >= '3.8'
+pandas==2.2.3; python_version >= '3.9'
+pillow==11.1.0; python_version >= '3.9'
+posthog==3.7.5
+protobuf==5.29.3; python_version >= '3.8'
+pyasn1==0.6.1; python_version >= '3.8'
+pyasn1-modules==0.4.1; python_version >= '3.8'
+pydantic==2.10.4; python_version >= '3.8'
+pydantic-core==2.27.2; python_version >= '3.8'
+pydub==0.25.1
+pygments==2.19.1; python_version >= '3.8'
+pypika==0.48.9
+pyproject-hooks==1.2.0; python_version >= '3.7'
+python-dateutil==2.9.0.post0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+python-dotenv==1.0.1; python_version >= '3.8'
+python-multipart==0.0.20; python_version >= '3.8'
+pytz==2024.2
+pyyaml==6.0.2; python_version >= '3.8'
+regex==2024.11.6; python_version >= '3.8'
+requests==2.32.3; python_version >= '3.8'
+requests-oauthlib==2.0.0; python_version >= '3.4'
+rich==13.9.4; python_full_version >= '3.8.0'
+rsa==4.9; python_version >= '3.6' and python_version < '4'
+ruff==0.8.6; python_version >= '3.7'
+safehttpx==0.1.6; python_version >= '3.10'
+safetensors==0.5.2; python_version >= '3.7'
+scikit-learn==1.6.0; python_version >= '3.9'
+scipy==1.15.0; python_version >= '3.10'
+semantic-version==2.10.0; python_version >= '2.7'
+sentence-transformers==3.3.1; python_version >= '3.9'
+shellingham==1.5.4; python_version >= '3.7'
+six==1.17.0; python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'
+sniffio==1.3.1; python_version >= '3.7'
+starlette==0.41.3; python_version >= '3.8'
+sympy==1.13.1; python_version >= '3.8'
+tenacity==9.0.0; python_version >= '3.8'
+threadpoolctl==3.5.0; python_version >= '3.8'
+tokenizers==0.21.0; python_version >= '3.7'
+tomli==2.2.1; python_version >= '3.8'
+tomlkit==0.13.2; python_version >= '3.8'
+torch==2.5.1; python_full_version >= '3.8.0'
+tqdm==4.67.1; python_version >= '3.7'
+transformers==4.47.1; python_full_version >= '3.9.0'
+typer==0.15.1; python_version >= '3.7'
+typing-extensions==4.12.2; python_version >= '3.8'
+tzdata==2024.2; python_version >= '2'
+urllib3==2.3.0; python_version >= '3.9'
+uvicorn[standard]==0.34.0; python_version >= '3.9'
+uvloop==0.21.0; python_full_version >= '3.8.0'
+watchfiles==1.0.3; python_version >= '3.9'
+websocket-client==1.8.0; python_version >= '3.8'
+websockets==14.1; python_version >= '3.9'
+wrapt==1.17.0; python_version >= '3.8'
+zipp==3.21.0; python_version >= '3.9'

utils.py

@@ -0,0 +1,24 @@
+def combine_metadata_with_distance(metadatas, distances):
+    # Flatten the nested lists if they are nested
+    metadatas = metadatas[0] if isinstance(metadatas[0], list) else metadatas
+    distances = distances[0] if isinstance(distances[0], list) else distances
+    print(metadatas)
+    if len(metadatas) != len(distances):
+        raise ValueError("Number of metadata entries must match the number of distances")
+    
+    combined_result = []
+    for metadata, distance in zip(metadatas, distances):
+        new_metadata = {
+            'title': metadata.get('title', ''),
+            'description': metadata.get('description', ''),
+            'price': metadata.get('price', ''),
+            'totalRatings': metadata.get('totalRatings', 0),
+            'reviewSummary': metadata.get('reviewSummary', ''),
+            'triggerWarning': metadata.get('triggerWarning', ''),
+            'distance': distance
+        }
+        combined_result.append(new_metadata)
+        
+    combined_result.sort(key=lambda x: x['distance'])
+
+    return combined_result