updates

app.py

@@ -58,8 +58,8 @@ def app(share=False, use_cluster_feats=False):
     instances, instance_ids = get_instances(cfg['instances_to_explain_path'])
 
     interp      = load_interp_space(cfg)
-    clustered_authors_df = interp['clustered_authors_df']
-    clustered_authors_df['fullText'] = clustered_authors_df['fullText'].map(lambda l: l[:5]) # Take at most 3 texts per author
+    clustered_authors_df = interp['clustered_authors_df'][:1000]
+    clustered_authors_df['fullText'] = clustered_authors_df['fullText']
 
     with gr.Blocks(title="Author Attribution Explainability Tool") as demo:
         # ── Big Centered Title ──────────────────────────────────────────

config/config.yaml

@@ -1,8 +1,8 @@
 # config.yaml
 instances_to_explain_path: "./datasets/hrs_explanations.json"
 instances_to_explain_url: "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/hrs_explanations_luar_clusters_18_balanced.json?/download=true"
-interp_space_path:    "./datasets/sentence_luar_interp_clusters_25/"
-interp_space_url:    "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/sentence_luar_interp_clusters_25.zip?download=true"
+interp_space_path:    "./datasets/sentence_luar_interp_space_2_35/"
+interp_space_url:    "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/sentence_luar_interp_space_2_35.zip?download=true"
 gram2vec_feats_path:      "./datasets/gram2vec_feats.csv"
 gram2vec_feats_url:      "https://huggingface.co/datasets/miladalsh/explanation_tool_files/resolve/main/gram2vec_feats.csv?download=true"
 

utils/interp_space_utils.py

@@ -78,7 +78,7 @@ def compute_g2v_features(clustered_authors_df: pd.DataFrame, task_authors_df: pd
             clustered_authors_df = pickle.load(f)
     
     else: # Else compute and cache
-        g2v_feats_df = vectorizer.from_documents(author_texts, batch_size=16)
+        g2v_feats_df = vectorizer.from_documents(author_texts, batch_size=8)
 
         print(f"Number of g2v features: {len(g2v_feats_df)}")
         print(f"Number of clustered_authors_df.authorID.tolist(): {len(clustered_authors_df.authorID.tolist())}")

utils/visualizations.py

@@ -14,7 +14,7 @@ from utils.interp_space_utils import compute_clusters_style_representation_3, co
 from utils.llm_feat_utils import split_features
 from utils.gram2vec_feat_utils import get_shorthand, get_fullform
 from gram2vec.feature_locator import find_feature_spans
-
+import umap
 import plotly.io as pio
 
 def clean_text(text: str) -> str:
@@ -132,8 +132,10 @@ def compute_tsne_with_cache(embeddings: np.ndarray, cache_path: str = 'datasets/
         return cache[hash_key]
     else:
         print("Computing t-SNE")
-        tsne_result = TSNE(n_components=2, learning_rate='auto',
-                           init='random', perplexity=3).fit_transform(embeddings)
+        # tsne_result = TSNE(n_components=2, learning_rate='auto',
+        #                    init='random', perplexity=3).fit_transform(embeddings)
+        tsne_result = umap.UMAP(n_components=2, n_neighbors=15, min_dist=0.0, metric='cosine').fit_transform(embeddings)
+        
         cache[hash_key] = tsne_result
         with open(cache_path, 'wb') as f:
             pkl.dump(cache, f)
@@ -147,7 +149,7 @@ def load_interp_space(cfg):
 
     # Load authors embeddings and their cluster labels
     clustered_authors_df = pd.read_pickle(clustered_authors_path)
-    clustered_authors_df = clustered_authors_df[clustered_authors_df.cluster_label != -1]
+    #clustered_authors_df = clustered_authors_df[clustered_authors_df.cluster_label != -1]
     author_embedding = clustered_authors_df.author_embedding.tolist()
     author_labels    = clustered_authors_df.cluster_label.tolist()
     author_ids      = clustered_authors_df.authorID.tolist()
@@ -276,12 +278,12 @@ def handle_zoom(event_json, bg_proj, bg_lbls, clustered_authors_df, task_authors
     print(f"Task authors: {len(task_authors_df)}, Clustered authors: {len(clustered_authors_df)}")
     merged_authors_df = pd.concat([task_authors_df, clustered_authors_df])
     print(f"Merged authors DataFrame:\n{len(merged_authors_df)}")
-    #style_analysis_response = {'features': [], 'spans': []}
-    style_analysis_response = compute_clusters_style_representation_3(
-        background_corpus_df=merged_authors_df,
-        cluster_ids=visible_authors,
-        cluster_label_clm_name='authorID',
-    )
+    style_analysis_response = {'features': [], 'spans': []}
+    # style_analysis_response = compute_clusters_style_representation_3(
+    #     background_corpus_df=merged_authors_df,
+    #     cluster_ids=visible_authors,
+    #     cluster_label_clm_name='authorID',
+    # )
 
     llm_feats = ['None'] + style_analysis_response['features']