<reponame>mayank05942/Hate-Speech-Detection-in-Hindi-English-Codemixed-Data


""" Source https://towardsdatascience.com/multi-class-text-classification-with-doc2vec-logistic-regression-9da9947b43f4 """





import re
import sklearn
from gensim.models import Doc2Vec
import numpy as np
import pandas as pd
from sklearn import utils
from gensim.models.doc2vec import TaggedDocument
import multiprocessing
from sklearn.model_selection import train_test_split
from sklearn import model_selection, naive_bayes, svm
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier as RFC
from nltk.corpus import stopwords
import nltk
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier as RFC
from tqdm import tqdm

df = pd.read_csv("/home/mayank/Downloads/pmeans/IIITH_Codemixed_new.txt",sep = "\t",names=["sentence","label"]).dropna()

#Data Cleaning
#Removing all the punctuation marks and converting to lowercase


def clean_str(string):
    """Tokenization/string cleaning for all datasets except for SST.

    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    string = re.sub(r"[.,#!$%&;:{}=_`~()/\\]", "", string)
    return string.strip().lower()

def preprocessing(df):

    df['sentence'] = df['sentence'].astype(str).map(clean_str,na_action=None)
    df['label'] = df['label'].astype(str)
    
    
    """Removing most frequent words,rare words and stop words"""
    
    from collections import Counter
    cnt = Counter()
    for text in df["sentence"].values:
        for word in text.split():
            cnt[word] += 1
            
    cnt.most_common(10)
    
    FREQWORDS = set([w for (w, wc) in cnt.most_common(10)])
    
    def remove_freqwords(text):
        """custom function to remove the frequent words"""
        return " ".join([word for word in str(text).split() if word not in FREQWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_freqwords(text))
    
    n_rare_words = 10
    RAREWORDS = set([w for (w, wc) in cnt.most_common()[:-n_rare_words-1:-1]])
    
    def remove_rarewords(text):
        """custom function to remove the rare words"""
        return " ".join([word for word in str(text).split() if word not in RAREWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_rarewords(text))
    
    def remove_shorter_words(text):
      return " ".join([word for word in str(text).split() if len(word) >= 2])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_shorter_words(text))
    
    def remove_longer_words(text):
      """ remove words longer than 12 char """
      return " ".join([word for word in str(text).split() if len(word) <= 12])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_longer_words(text))
    
    def remove_words_digits(text):
      """ remove words with digits """
      return " ".join([word for word in str(text).split() if not any(c.isdigit() for c in word) ])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_words_digits(text))
    
    STOPWORDS = set(stopwords.words('english'))
    def remove_stopwords(text):
        """custom function to remove the stopwords"""
        return " ".join([word for word in str(text).split() if word not in STOPWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_stopwords(text))
    return df


df = preprocessing(df)

# import Word2Vec loading capabilities
from gensim.models import KeyedVectors

# Creating the model
print("Loading the pre-trained embeddings")
embed_lookup = KeyedVectors.load_word2vec_format('/home/mayank/Downloads/GoogleNews-vectors-negative300.bin', 
                                                 binary=True)
print("DONE")

#store pretrained vocab
pretrained_words = []
for word in embed_lookup.vocab:
    pretrained_words.append(word)

row_idx = 1

# get word/embedding in that row
word = pretrained_words[row_idx] # get words by index
embedding = embed_lookup[word] # embeddings by word

# vocab and embedding info
print("Size of Vocab: {}\n".format(len(pretrained_words)))
print('Word in vocab: {}\n'.format(word))
print('Length of embedding: {}\n'.format(len(embedding)))
#print('Associated embedding: \n', embedding)

# print a few common words
for i in range(100):
    print(pretrained_words[i])

train, test = train_test_split(df, test_size=0.3, random_state=42)



def tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text):
        for word in nltk.word_tokenize(sent):
            if len(word) < 2:
                continue
            tokens.append(word.lower())
    return tokens

train_tagged = train.apply(lambda r: TaggedDocument(words=tokenize_text(r['sentence']), tags=[r.label]), axis=1)
test_tagged = test.apply(lambda r: TaggedDocument(words=tokenize_text(r['sentence']), tags=[r.label]), axis=1)

print(train_tagged.values[1])

model= Doc2Vec(dm=1, vector_size=300,min_count=1,window = 5,workers = 4,neagtive = 6,epochs = 2,sample=1e-4)
model.build_vocab([x for x in tqdm(train_tagged.values)])

for epoch in range(30):
    model.train(utils.shuffle([x for x in tqdm(train_tagged.values)]), total_examples=len(train_tagged.values), epochs=1)
    model.alpha -= 0.002
    model.min_alpha = model.alpha

def vec_for_learning(model, tagged_docs):
    sents = tagged_docs.values
    targets, regressors = zip(*[(doc.tags[0], model.infer_vector(doc.words, steps=20)) for doc in sents])
    return targets, regressors



y_train, X_train = vec_for_learning(model, train_tagged)
y_test, X_test = vec_for_learning(model, test_tagged)


X_train = np.array(X_train)
X_test = np.array(X_test)
Y_test = np.array(y_test)
Y_train = np.array(y_train)


from keras.utils import to_categorical

    

# Y_train = to_categorical(Y_train,num_classes=3)
# Y_test = to_categorical(Y_test,num_classes=3)
# X_train=X_train.reshape(X_train.shape[0],X_train.shape[1], 1)
# X_test=X_test.reshape(X_test.shape[0],X_test.shape[1], 1)




 
# def train_classifier(X,y):
#     """ To perform grid search"""
#     param_grid = {'C': [0.1, 1, 10,100,1000],  
#               'gamma': [1, 0.1, 0.01], 
#               'kernel': ['rbf']} 
  
#     clf = GridSearchCV(svm.SVC(), param_grid, refit = True, verbose = 3)
#     clf.fit(X,y)
#     return clf 


# classifier = train_classifier(X_train,y_train)
# print (classifier.best_score_, "----------------Best Accuracy score on Cross Validation Sets")
# print (classifier.score(X_test,y_test))
# print(classifier.best_params_)


SVM = svm.SVC(kernel='rbf',C=1000, gamma=0.1)
SVM.fit(X_train,y_train)

pred_y = SVM.predict(X_test)

print(sklearn.metrics.classification_report(y_test, pred_y))







""" Source https://towardsdatascience.com/multi-class-text-classification-with-doc2vec-logistic-regression-9da9947b43f4 """





import re
import sklearn
from gensim.models import Doc2Vec
import numpy as np
import pandas as pd
from sklearn import utils
from gensim.models.doc2vec import TaggedDocument
import multiprocessing
from sklearn.model_selection import train_test_split
from sklearn import model_selection, naive_bayes, svm
from sklearn.metrics import accuracy_score
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier as RFC
from nltk.corpus import stopwords
import nltk
from sklearn.model_selection import cross_validate
from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier as RFC
from tqdm import tqdm

df = pd.read_csv("/home/mayank/Downloads/pmeans/IIITH_Codemixed_new.txt",sep = "\t",names=["sentence","label"]).dropna()

#Data Cleaning
#Removing all the punctuation marks and converting to lowercase


def clean_str(string):
    """Tokenization/string cleaning for all datasets except for SST.

    Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py
    """
    string = re.sub(r"[^A-Za-z0-9(),!?\'\`]", " ", string)
    string = re.sub(r"\'s", " \'s", string)
    string = re.sub(r"\'ve", " \'ve", string)
    string = re.sub(r"n\'t", " n\'t", string)
    string = re.sub(r"\'re", " \'re", string)
    string = re.sub(r"\'d", " \'d", string)
    string = re.sub(r"\'ll", " \'ll", string)
    string = re.sub(r",", " , ", string)
    string = re.sub(r"!", " ! ", string)
    string = re.sub(r"\(", " \( ", string)
    string = re.sub(r"\)", " \) ", string)
    string = re.sub(r"\?", " \? ", string)
    string = re.sub(r"\s{2,}", " ", string)
    string = re.sub(r"[.,#!$%&;:{}=_`~()/\\]", "", string)
    return string.strip().lower()

def preprocessing(df):

    df['sentence'] = df['sentence'].astype(str).map(clean_str,na_action=None)
    df['label'] = df['label'].astype(str)
    
    
    """Removing most frequent words,rare words and stop words"""
    
    from collections import Counter
    cnt = Counter()
    for text in df["sentence"].values:
        for word in text.split():
            cnt[word] += 1
            
    cnt.most_common(10)
    
    FREQWORDS = set([w for (w, wc) in cnt.most_common(10)])
    
    def remove_freqwords(text):
        """custom function to remove the frequent words"""
        return " ".join([word for word in str(text).split() if word not in FREQWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_freqwords(text))
    
    n_rare_words = 10
    RAREWORDS = set([w for (w, wc) in cnt.most_common()[:-n_rare_words-1:-1]])
    
    def remove_rarewords(text):
        """custom function to remove the rare words"""
        return " ".join([word for word in str(text).split() if word not in RAREWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_rarewords(text))
    
    def remove_shorter_words(text):
      return " ".join([word for word in str(text).split() if len(word) >= 2])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_shorter_words(text))
    
    def remove_longer_words(text):
      """ remove words longer than 12 char """
      return " ".join([word for word in str(text).split() if len(word) <= 12])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_longer_words(text))
    
    def remove_words_digits(text):
      """ remove words with digits """
      return " ".join([word for word in str(text).split() if not any(c.isdigit() for c in word) ])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_words_digits(text))
    
    STOPWORDS = set(stopwords.words('english'))
    def remove_stopwords(text):
        """custom function to remove the stopwords"""
        return " ".join([word for word in str(text).split() if word not in STOPWORDS])
    
    df["sentence"] = df["sentence"].apply(lambda text: remove_stopwords(text))
    return df


df = preprocessing(df)

# import Word2Vec loading capabilities
from gensim.models import KeyedVectors

# Creating the model
print("Loading the pre-trained embeddings")
embed_lookup = KeyedVectors.load_word2vec_format('/home/mayank/Downloads/GoogleNews-vectors-negative300.bin', 
                                                 binary=True)
print("DONE")

#store pretrained vocab
pretrained_words = []
for word in embed_lookup.vocab:
    pretrained_words.append(word)

row_idx = 1

# get word/embedding in that row
word = pretrained_words[row_idx] # get words by index
embedding = embed_lookup[word] # embeddings by word

# vocab and embedding info
print("Size of Vocab: {}\n".format(len(pretrained_words)))
print('Word in vocab: {}\n'.format(word))
print('Length of embedding: {}\n'.format(len(embedding)))
#print('Associated embedding: \n', embedding)

# print a few common words
for i in range(100):
    print(pretrained_words[i])

train, test = train_test_split(df, test_size=0.3, random_state=42)



def tokenize_text(text):
    tokens = []
    for sent in nltk.sent_tokenize(text):
        for word in nltk.word_tokenize(sent):
            if len(word) < 2:
                continue
            tokens.append(word.lower())
    return tokens

train_tagged = train.apply(lambda r: TaggedDocument(words=tokenize_text(r['sentence']), tags=[r.label]), axis=1)
test_tagged = test.apply(lambda r: TaggedDocument(words=tokenize_text(r['sentence']), tags=[r.label]), axis=1)

print(train_tagged.values[1])

model= Doc2Vec(dm=1, vector_size=300,min_count=1,window = 5,workers = 4,neagtive = 6,epochs = 2,sample=1e-4)
model.build_vocab([x for x in tqdm(train_tagged.values)])

for epoch in range(30):
    model.train(utils.shuffle([x for x in tqdm(train_tagged.values)]), total_examples=len(train_tagged.values), epochs=1)
    model.alpha -= 0.002
    model.min_alpha = model.alpha

def vec_for_learning(model, tagged_docs):
    sents = tagged_docs.values
    targets, regressors = zip(*[(doc.tags[0], model.infer_vector(doc.words, steps=20)) for doc in sents])
    return targets, regressors



y_train, X_train = vec_for_learning(model, train_tagged)
y_test, X_test = vec_for_learning(model, test_tagged)


X_train = np.array(X_train)
X_test = np.array(X_test)
Y_test = np.array(y_test)
Y_train = np.array(y_train)


from keras.utils import to_categorical

    

# Y_train = to_categorical(Y_train,num_classes=3)
# Y_test = to_categorical(Y_test,num_classes=3)
# X_train=X_train.reshape(X_train.shape[0],X_train.shape[1], 1)
# X_test=X_test.reshape(X_test.shape[0],X_test.shape[1], 1)




 
# def train_classifier(X,y):
#     """ To perform grid search"""
#     param_grid = {'C': [0.1, 1, 10,100,1000],  
#               'gamma': [1, 0.1, 0.01], 
#               'kernel': ['rbf']} 
  
#     clf = GridSearchCV(svm.SVC(), param_grid, refit = True, verbose = 3)
#     clf.fit(X,y)
#     return clf 


# classifier = train_classifier(X_train,y_train)
# print (classifier.best_score_, "----------------Best Accuracy score on Cross Validation Sets")
# print (classifier.score(X_test,y_test))
# print(classifier.best_params_)


SVM = svm.SVC(kernel='rbf',C=1000, gamma=0.1)
SVM.fit(X_train,y_train)

pred_y = SVM.predict(X_test)

print(sklearn.metrics.classification_report(y_test, pred_y))
Source https://towardsdatascience.com/multi-class-text-classification-with-doc2vec-logistic-regression-9da9947b43f4 #Data Cleaning #Removing all the punctuation marks and converting to lowercase Tokenization/string cleaning for all datasets except for SST.      Original taken from https://github.com/yoonkim/CNN_sentence/blob/master/process_data.py #!$%&;:{}=_`~()/\\]", "", string) Removing most frequent words,rare words and stop words custom function to remove the frequent words custom function to remove the rare words remove words longer than 12 char remove words with digits custom function to remove the stopwords # import Word2Vec loading capabilities # Creating the model #store pretrained vocab # get word/embedding in that row # get words by index # embeddings by word # vocab and embedding info #print('Associated embedding: \n', embedding) # print a few common words # Y_train = to_categorical(Y_train,num_classes=3) # Y_test = to_categorical(Y_test,num_classes=3) # X_train=X_train.reshape(X_train.shape[0],X_train.shape[1], 1) # X_test=X_test.reshape(X_test.shape[0],X_test.shape[1], 1) # def train_classifier(X,y): #     """ To perform grid search""" #     param_grid = {'C': [0.1, 1, 10,100,1000], #               'gamma': [1, 0.1, 0.01], #               'kernel': ['rbf']} #     clf = GridSearchCV(svm.SVC(), param_grid, refit = True, verbose = 3) #     clf.fit(X,y) #     return clf # classifier = train_classifier(X_train,y_train) # print (classifier.best_score_, "----------------Best Accuracy score on Cross Validation Sets") # print (classifier.score(X_test,y_test)) # print(classifier.best_params_)