Update update_embeddings.py

update_embeddings.py

@@ -174,8 +174,17 @@ previous_embeddings = pd.read_parquet(previous_embed)
 # Find papers that are not in the previous embeddings
 new_papers = arxiv_metadata_split[~arxiv_metadata_split['id'].isin(previous_embeddings['id'])]
 
+# Number of new papers
+num_new_papers = len(new_papers)
+
+# What if there are no new papers?
+if num_new_papers == 0:
+    print(f"No new papers found for year: {year}")
+    print("Exiting")
+    sys.exit()
+
 # Create a column for embeddings
-print(f"Creating new embeddings for: {len(new_papers)} entries")
+print(f"Creating new embeddings for: {num_new_papers} entries")
 new_papers["vector"] = new_papers["abstract"].progress_apply(embed)
 
 # Rename columns