Spaces:
Running
Running
minor change
Browse files- app.py +1 -3
- doc_searcher_v2.py +30 -2
app.py
CHANGED
|
@@ -2,7 +2,6 @@ import nh3
|
|
| 2 |
from fastapi import FastAPI, Request
|
| 3 |
from doc_searcher import DocSearcher
|
| 4 |
from doc_searcher_v2 import DocSearcherV2
|
| 5 |
-
from suggestion_searcher import SuggestionSearcher
|
| 6 |
from huggingface_hub import login
|
| 7 |
from config import HUGGING_FACE_API_KEY, COLLECTION_NAME, API_KEY, COLLECTION_NAME_SUGGESTION
|
| 8 |
from fastapi.responses import StreamingResponse
|
|
@@ -14,7 +13,6 @@ app = FastAPI()
|
|
| 14 |
|
| 15 |
doc_searcher = DocSearcher(collection_name=COLLECTION_NAME)
|
| 16 |
doc_searcher_v2 = DocSearcherV2(collection_name=COLLECTION_NAME)
|
| 17 |
-
suggestion_searcher = SuggestionSearcher(collection_name=COLLECTION_NAME_SUGGESTION)
|
| 18 |
|
| 19 |
ALLOWED_API_KEY = str(API_KEY)
|
| 20 |
|
|
@@ -29,5 +27,5 @@ async def search(q: str, type: int, lt: str | None = None, offset: int = 0):
|
|
| 29 |
async def v2_search(q: str):
|
| 30 |
query = q.lower()
|
| 31 |
xss = nh3.clean(query)
|
| 32 |
-
data = await doc_searcher_v2.
|
| 33 |
return data
|
|
|
|
| 2 |
from fastapi import FastAPI, Request
|
| 3 |
from doc_searcher import DocSearcher
|
| 4 |
from doc_searcher_v2 import DocSearcherV2
|
|
|
|
| 5 |
from huggingface_hub import login
|
| 6 |
from config import HUGGING_FACE_API_KEY, COLLECTION_NAME, API_KEY, COLLECTION_NAME_SUGGESTION
|
| 7 |
from fastapi.responses import StreamingResponse
|
|
|
|
| 13 |
|
| 14 |
doc_searcher = DocSearcher(collection_name=COLLECTION_NAME)
|
| 15 |
doc_searcher_v2 = DocSearcherV2(collection_name=COLLECTION_NAME)
|
|
|
|
| 16 |
|
| 17 |
ALLOWED_API_KEY = str(API_KEY)
|
| 18 |
|
|
|
|
| 27 |
async def v2_search(q: str):
|
| 28 |
query = q.lower()
|
| 29 |
xss = nh3.clean(query)
|
| 30 |
+
data = await doc_searcher_v2.search_semantic(text=xss)
|
| 31 |
return data
|
doc_searcher_v2.py
CHANGED
|
@@ -14,7 +14,7 @@ class DocSearcherV2:
|
|
| 14 |
self.sparse_model = SparseTextEmbedding(SPARSE_MODEL)
|
| 15 |
self.qdrant_client = QdrantClient(QDRANT_URL,api_key=QDRANT_API_KEY,timeout=30)
|
| 16 |
|
| 17 |
-
async def
|
| 18 |
|
| 19 |
queries = [text]
|
| 20 |
dense_query = self.model.encode(text).tolist()
|
|
@@ -45,4 +45,32 @@ class DocSearcherV2:
|
|
| 45 |
|
| 46 |
scores = self.reranker.compute_logits(queries,data)
|
| 47 |
|
| 48 |
-
return scores
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 14 |
self.sparse_model = SparseTextEmbedding(SPARSE_MODEL)
|
| 15 |
self.qdrant_client = QdrantClient(QDRANT_URL,api_key=QDRANT_API_KEY,timeout=30)
|
| 16 |
|
| 17 |
+
async def search_semantic(self, text: str):
|
| 18 |
|
| 19 |
queries = [text]
|
| 20 |
dense_query = self.model.encode(text).tolist()
|
|
|
|
| 45 |
|
| 46 |
scores = self.reranker.compute_logits(queries,data)
|
| 47 |
|
| 48 |
+
return scores
|
| 49 |
+
|
| 50 |
+
async def search_keyword(self, text: str):
|
| 51 |
+
sparse_query = next(self.sparse_model.query_embed(text))
|
| 52 |
+
|
| 53 |
+
prefetch = [
|
| 54 |
+
models.Prefetch(
|
| 55 |
+
query=models.SparseVector(**sparse_query.as_object()),
|
| 56 |
+
using=SPARSE_MODEL,
|
| 57 |
+
limit=100
|
| 58 |
+
)
|
| 59 |
+
]
|
| 60 |
+
|
| 61 |
+
search_result = self.qdrant_client.query_points(
|
| 62 |
+
collection_name= "sl-list",
|
| 63 |
+
prefetch=prefetch,
|
| 64 |
+
query=models.FusionQuery(
|
| 65 |
+
fusion=models.Fusion.RRF,
|
| 66 |
+
),
|
| 67 |
+
with_payload=True,
|
| 68 |
+
limit = 100,
|
| 69 |
+
).points
|
| 70 |
+
|
| 71 |
+
data = []
|
| 72 |
+
|
| 73 |
+
for hit in search_result:
|
| 74 |
+
data.append(hit.payload["tekst"])
|
| 75 |
+
|
| 76 |
+
return data
|