File size: 1,945 Bytes
f315fdc e51e296 f315fdc b9afd09 e51e296 b9afd09 f315fdc b9afd09 f315fdc b9afd09 f315fdc b9afd09 f315fdc b9afd09 e51e296 b9afd09 e51e296 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 |
# modules/indexer.py
from typing import Dict, List
from openai import OpenAI
def index_videos(videos: List[Dict], collection, channel_url: str, batch_size: int = 50):
client = OpenAI()
total = len(videos)
print(f"[INDEX] Starting indexing for {total} videos (channel={channel_url})")
# Split into batches
for start in range(0, total, batch_size):
batch = videos[start:start + batch_size]
end = start + len(batch)
percent = round((end / total) * 100, 1)
print(f"[INDEX] Processing batch {start+1} → {end} of {total} — {percent}%")
# Prepare text inputs
texts = [f"{vid.get('title', '')} - {vid.get('description', '')}" for vid in batch]
# Call embeddings API in batch
response = client.embeddings.create(
input=texts,
model="text-embedding-3-small"
)
embeddings = [item.embedding for item in response.data]
# Build metadata + ids
metadatas, ids = [], []
for vid in batch:
metadata = {
"video_id": vid.get("video_id"),
"video_title": vid.get("title", ""),
"description": vid.get("description", ""),
"channel_url": channel_url,
}
if "channel_id" in vid:
metadata["channel_id"] = vid["channel_id"]
if "channel_title" in vid:
metadata["channel_title"] = vid["channel_title"]
metadatas.append(metadata)
ids.append(vid.get("video_id"))
# Insert in bulk
collection.add(
documents=texts,
embeddings=embeddings,
metadatas=metadatas,
ids=ids,
)
print(f"[INDEX] ✅ Indexed {len(batch)} videos (total so far: {end}/{total} — {percent}%)")
print(f"[INDEX] 🎉 Finished indexing {total} videos for channel={channel_url}")
return total
|