feat: auto-label-embed-cluster

notebooks/TFIDF.ipynb

@@ -18,7 +18,7 @@
     "import numpy as np\n",
     "import pandas as pd\n",
     "import matplotlib.pyplot as plt\n",
-    "import seaborn as sns\n",
+    "# import seaborn as sns\n",
     "import os\n",
     "\n",
     "import re\n",

requirements.txt

@@ -0,0 +1,8 @@
+numpy
+pandas
+matplotlib
+seaborn
+scikit-learn
+umap-learn
+nltk
+tensorflow-hub

src/main.py

@@ -1,9 +1,10 @@
 ######################################## IMPORTING REQUIRED LIBRARIES ####################################
 import os
 import sys
+import pandas as pd
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 data_folder = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'data')
-from utilities import get_data, input_filter, clean_data
+from utilities import get_data, input_filter, clean_data, autogenerate_labels
 
 
 ################################################## INPUTS ################################################
@@ -26,8 +27,15 @@ def data_clean_for_training(df):
 
 if __name__ == '__main__':
 
-    df = data_sourcing() ## testing the data sourcing endpoint
-    if df:
-        print("Data loaded successfully !!")
+    # df = data_sourcing() ## testing the data sourcing endpoint
+    # if df:
+    #     print("Data loaded successfully !!")
 
-    clean_df = data_clean_for_training(df)
+    # clean_df = data_clean_for_training(df)
+
+    df = pd.read_csv(f'{data_folder}/MMR_DATA.csv')
+    df = clean_data(df)
+    labelled_df, embeddings_df = autogenerate_labels(df)
+
+    labelled_df.to_csv(f'{data_folder}/MMR_DATA_CLEAN_LABELLED.csv', index=False)
+    embeddings_df.to_csv(f'{data_folder}/MMR_CLEAN_EMBEDDINGS.csv', index=False)

utilities/__init__.py

@@ -1,2 +1,3 @@
 from .data_loader import get_data, input_filter
-from .data_cleaner import clean_data
+from .data_cleaner import clean_data
+from .cluster_label import autogenerate_labels

utilities/cluster_label.py

@@ -0,0 +1,35 @@
+import numpy as numpy
+import pandas as pd
+import numpy as np
+import tensorflow_hub as hub
+from sklearn.cluster import KMeans
+from sklearn.preprocessing import StandardScaler
+
+
+def embed(input):
+    module_url = "https://tfhub.dev/google/universal-sentence-encoder/4"
+    model = hub.load(module_url)
+    return model(input)
+
+def generate_use_embeddings(data):
+    embeddings = embed(data)
+    embeddings = np.array(embeddings).tolist()
+    return embeddings
+
+def autogenerate_labels(df):
+    map_data = df['Map Data'].to_numpy()
+
+    embeddings_list = generate_use_embeddings(map_data)
+    np_embeddings = np.array(embeddings_list)
+    df_embeddings = pd.DataFrame(np_embeddings)
+    scaler = StandardScaler()
+    scaled_embeddings = scaler.fit_transform(np_embeddings)
+
+    n_clusters = 4
+    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
+    kmeans.fit(scaled_embeddings)
+
+    y_kmeans = kmeans.labels_
+
+    df['label'] = y_kmeans + 1
+    return df, df_embeddings

utilities/data_cleaner.py

@@ -22,6 +22,6 @@ def clean_text(text):
 def clean_data(df):
     df['Map Data'] = df['Map Data'].fillna('')
     df = df[df['Map Data'].str.len() > 0]
-    df = df[df['Map Data'].str.len() < 5000]
+    df = df[df['Map Data'].str.len() < 10000]
     # df['Map Data'] = df['Map Data'].apply(clean_text)
     return df