quantization, reduce chunksize

client.py

@@ -1,7 +1,7 @@
 import os
 
 from dotenv import load_dotenv
-from qdrant_client import QdrantClient
+from qdrant_client import QdrantClient, models
 
 load_dotenv()
 
@@ -24,6 +24,13 @@ class HybridClient:
                 collection_name=collection,
                 vectors_config=self.qdrant_client.get_fastembed_vector_params(),
                 sparse_vectors_config=self.qdrant_client.get_fastembed_sparse_vector_params(),
+                quantization_config=models.ScalarQuantization(
+                    scalar=models.ScalarQuantizationConfig(
+                        type=models.ScalarType.INT8,
+                        quantile=0.99,
+                        always_ram=False,
+                    ),
+                ),
             )
             print(f"--- {collection} collection created")
             return collection
@@ -33,6 +40,8 @@ class HybridClient:
         documents = []
         for chunk in chunks:
             documents.append(chunk.pop("text"))
+            chunk.pop("color")
+            chunk.pop("size")
 
         self.qdrant_client.add(
             collection_name=collection,
@@ -52,3 +61,7 @@ class HybridClient:
         # Select and return metadata
         # metadata = [hit.metadata for hit in search_result]
         return search_result
+
+    def get_chapter_name(self, collection: str):
+        points = self.qdrant_client.retrieve(collection_name=collection, ids=[0])
+        return points[0]

preprocessing.py

@@ -10,14 +10,14 @@ def sort_text(chunks):
     right_column = []
 
     for chunk in chunks:
-        if chunk["x"] < x_threshold:
+        if chunk["coordinates"][0] < x_threshold:
             left_column.append(chunk)
         else:
             right_column.append(chunk)
 
     # Sort the chunks within each column based on the y-coordinate
-    left_column = sorted(left_column, key=lambda item: item["y"])
-    right_column = sorted(right_column, key=lambda item: item["y"])
+    left_column = sorted(left_column, key=lambda item: item["coordinates"][1])
+    right_column = sorted(right_column, key=lambda item: item["coordinates"][1])
 
     sorted_text = left_column + right_column
     return sorted_text
@@ -75,8 +75,7 @@ def get_chunks(doc):
                     {
                         "text": clean_text(text.strip()),
                         "page": page_num,
-                        "x": block["bbox"][0],
-                        "y": block["bbox"][1],
+                        "coordinates": [round(block["bbox"][0], 1), round(block["bbox"][1], 1)],
                         "color": majority_element(spans, "color"),
                         "size": majority_element(spans, "size"),
                     }

scraper.py

@@ -70,8 +70,7 @@ async def download(session: aiohttp.ClientSession, url: str, max_retries: int =
 async def upload_book(grade, subject, chapters=None):
     hclient = HybridClient()
 
-    book = await get_book(grade, subject)
-    print(type(book))
+    book = await get_book(grade, subject, chapters)
     for collection, pdf in book.items():
         print(collection)
         chunks = index_pdf(pdf, buffer=True)