Upload synonyms_preprocess.py

src/synonyms_preprocess.py

@@ -0,0 +1,43 @@
+import spacy
+import pickle
+from nltk.corpus import wordnet
+
+
+def load_spacy_values(filepath_model_spacy='model_spacy_synonyms', filepath_docs_spacy = 'dict_spacy_object.pkl'):
+
+    nlp = spacy.load(filepath_model_spacy)
+
+    
+    with open(filepath_docs_spacy, 'rb') as file:
+        dict_docs_spacy_bytes = pickle.load(file)
+    
+    dict_docs_spacy = {key: spacy.tokens.Doc(nlp.vocab).from_bytes(doc_bytes) for key, doc_bytes in dict_docs_spacy_bytes.items()}
+    
+    return nlp, dict_docs_spacy
+
+def find_antonyms(word):
+    antonyms = set()
+    syn_set = wordnet.synsets(word)
+    for syn in syn_set:
+        for lemma in syn.lemmas():
+            if lemma.antonyms():
+                antonyms.add(lemma.antonyms()[0].name())
+    return antonyms
+
+def find_synonyms(word, model, dict_embedding, dict_2000_tokens): #cluster_to_words, dbscan_model):
+    """
+    This function finds the most similar word in the same cluster, and excludes antonyms
+    """
+    antonyms = find_antonyms(word)
+    dict_2000_tokens_less_antonyms = [token for token in dict_2000_tokens if token not in antonyms]
+
+    word_embedding = model(word)
+
+    similarities=[]
+    
+    for token in dict_2000_tokens_less_antonyms:
+        similarities.append((token, dict_embedding.get(token).similarity(word_embedding)))
+    
+    most_similar_token = sorted(similarities, key=lambda item: -item[1])[0][0]
+
+    return most_similar_token