Upload folder using huggingface_hub

app.py

@@ -1,5 +1,7 @@
+import asyncio
 import os
 import re
+import threading
 import gradio as gr
 from gradio_modal import Modal
 import chromadb
@@ -13,6 +15,7 @@ from modules.indexer import index_videos
 from modules.answerer import answer_query, LLMAnswer, VideoItem, build_video_html
 from dotenv import load_dotenv
 
+from youtube_poller import start_poll
 from youtube_sync import sync_channels_from_youtube
 
 load_dotenv()
@@ -545,4 +548,7 @@ with gr.Blocks() as demo:
 if __name__ == "__main__":
     for msg in init():
         print(msg)
+    # Start polling in a background thread
+    poll_thread = threading.Thread(target=start_poll, daemon=True)
+    poll_thread.start()
     demo.launch()

modules/db.py

@@ -28,7 +28,7 @@ def get_collection():
 
 
 # modules/db.py
-def get_indexed_channels(collection):
+def get_indexed_channels(collection = get_collection()):
     results = collection.get(include=["metadatas"])
     channels = {}
 

pyproject.toml

@@ -7,6 +7,7 @@ requires-python = ">=3.13"
 dependencies = [
     "chromadb>=1.0.20",
     "dotenv>=0.9.9",
+    "feedparser>=6.0.11",
     "google-api-python-client>=2.179.0",
     "gradio>=5.44.0",
     "gradio-modal>=0.0.4",

requirements.txt

@@ -59,6 +59,8 @@ durationpy==0.10
     # via kubernetes
 fastapi==0.116.1
     # via gradio
+feedparser==6.0.11
+    # via youtube-surfer-ai-agent (pyproject.toml)
 ffmpy==0.6.1
     # via gradio
 filelock==3.19.1
@@ -295,6 +297,8 @@ safehttpx==0.1.6
     # via gradio
 semantic-version==2.10.0
     # via gradio
+sgmllib3k==1.0.0
+    # via feedparser
 shellingham==1.5.4
     # via typer
 six==1.17.0

uv.lock

@@ -358,6 +358,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/e5/47/d63c60f59a59467fda0f93f46335c9d18526d7071f025cb5b89d5353ea42/fastapi-0.116.1-py3-none-any.whl", hash = "sha256:c46ac7c312df840f0c9e220f7964bada936781bc4e2e6eb71f1c4d7553786565", size = 95631, upload-time = "2025-07-11T16:22:30.485Z" },
 ]
 
+[[package]]
+name = "feedparser"
+version = "6.0.11"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "sgmllib3k" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ff/aa/7af346ebeb42a76bf108027fe7f3328bb4e57a3a96e53e21fd9ef9dd6dd0/feedparser-6.0.11.tar.gz", hash = "sha256:c9d0407b64c6f2a065d0ebb292c2b35c01050cc0dc33757461aaabdc4c4184d5", size = 286197, upload-time = "2023-12-10T16:03:20.854Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/7c/d4/8c31aad9cc18f451c49f7f9cfb5799dadffc88177f7917bc90a66459b1d7/feedparser-6.0.11-py3-none-any.whl", hash = "sha256:0be7ee7b395572b19ebeb1d6aafb0028dee11169f1c934e0ed67d54992f4ad45", size = 81343, upload-time = "2023-12-10T16:03:19.484Z" },
+]
+
 [[package]]
 name = "ffmpy"
 version = "0.6.1"
@@ -1722,6 +1734,12 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/6a/23/8146aad7d88f4fcb3a6218f41a60f6c2d4e3a72de72da1825dc7c8f7877c/semantic_version-2.10.0-py2.py3-none-any.whl", hash = "sha256:de78a3b8e0feda74cabc54aab2da702113e33ac9d9eb9d2389bcf1f58b7d9177", size = 15552, upload-time = "2022-05-26T13:35:21.206Z" },
 ]
 
+[[package]]
+name = "sgmllib3k"
+version = "1.0.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9e/bd/3704a8c3e0942d711c1299ebf7b9091930adae6675d7c8f476a7ce48653c/sgmllib3k-1.0.0.tar.gz", hash = "sha256:7868fb1c8bfa764c1ac563d3cf369c381d1325d36124933a726f29fcdaa812e9", size = 5750, upload-time = "2010-08-24T14:33:52.445Z" }
+
 [[package]]
 name = "shellingham"
 version = "1.5.4"
@@ -2019,6 +2037,7 @@ source = { virtual = "." }
 dependencies = [
     { name = "chromadb" },
     { name = "dotenv" },
+    { name = "feedparser" },
     { name = "google-api-python-client" },
     { name = "gradio" },
     { name = "gradio-modal" },
@@ -2029,6 +2048,7 @@ dependencies = [
 requires-dist = [
     { name = "chromadb", specifier = ">=1.0.20" },
     { name = "dotenv", specifier = ">=0.9.9" },
+    { name = "feedparser", specifier = ">=6.0.11" },
     { name = "google-api-python-client", specifier = ">=2.179.0" },
     { name = "gradio", specifier = ">=5.44.0" },
     { name = "gradio-modal", specifier = ">=0.0.4" },

youtube_poller.py

@@ -0,0 +1,74 @@
+import feedparser
+from modules.db import get_collection, get_indexed_channels
+
+
+def fetch_channel_videos_rss(channel_id, max_results=50):
+    feed_url = f"https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
+    feed = feedparser.parse(feed_url)
+    videos = []
+    for entry in feed.entries[:max_results]:
+        videos.append(
+            {
+                "video_id": entry.yt_videoid,
+                "title": entry.title,
+                "published": entry.published,
+                "link": entry.link,
+                "channel_id": channel_id,
+            }
+        )
+    return videos
+
+
+def get_existing_video_ids(collection, channel_id):
+    # n_results: how many results to fetch; use a high number to get all entries
+    results = collection.get(where={"channel_id": channel_id})
+
+    existing_ids = set()
+    for metadata in results.get("metadatas", []):
+        if metadata and "video_id" in metadata:
+            existing_ids.add(metadata["video_id"])
+    return existing_ids
+
+
+def filter_new_videos(videos, existing_ids):
+    return [v for v in videos if v["video_id"] not in existing_ids]
+
+
+def add_to_chroma(collection, new_videos):
+    if not new_videos:
+        return
+    collection.add(
+        documents=[v["title"] for v in new_videos],
+        metadatas=[
+            {
+                "video_id": v["video_id"],
+                "channel_id": v["channel_id"],
+                "link": v["link"],
+            }
+            for v in new_videos
+        ],
+        ids=[v["video_id"] for v in new_videos],
+    )
+
+
+def incremental_update(collection, channel_id):
+    existing_ids = get_existing_video_ids(collection, channel_id)
+    latest_videos = fetch_channel_videos_rss(channel_id)
+    new_videos = filter_new_videos(latest_videos, existing_ids)
+
+    if new_videos:
+        add_to_chroma(collection, new_videos)
+        print(f"Added {len(new_videos)} new videos from {channel_id}")
+    else:
+        print(f"No new videos for {channel_id}")
+
+
+def start_poll():
+    import time
+
+    configured_channels = get_indexed_channels().keys()
+
+    while True:
+        for channel_id in configured_channels:
+            incremental_update(get_collection(), channel_id)
+        time.sleep(600)  # 10 minutes