Update app.py

app.py

@@ -18,12 +18,12 @@ HF_TOKEN = os.environ.get("RedditSemanticSearch")
 # Function to stream JSONL Reddit files from HF Hub
 from datasets import load_dataset
 
-# Load full Reddit dataset (assumes it's pre-split by subreddit or has a field)
-dataset = load_dataset("HuggingFaceGECLM/REDDIT_comments", split="askscience", "gaming", "technology", "todayilearned", "programming")
-
 # Define target subreddits
 target_subreddits = ["askscience", "gaming", "technology", "todayilearned", "programming"]
 
+# Load full Reddit dataset (assumes it's pre-split by subreddit or has a field)
+dataset_splits = [load_dataset("HuggingFaceGECLM/REDDIT_comments", split=sub, streaming=True) for sub in target_subreddits]
+
 # Filter only relevant subreddits
 dataset = dataset.filter(lambda x: x["subreddit"] in target_subreddits)