
# coding: utf-8

# In[1]:

#News article classifier using pre-trained word2vec model trained on text8 corpus
#Features were extracted usig two techniques -  Avg. feature vectors and bag of centroids method
#The multi class labeled news corpus comprises of five categories and has been generated by scraping bbc and the guardian news websites 


# In[1]:

import pandas as pd
import nltk
import numpy as np
from os import listdir
import re


# In[2]:

loc='C:/Users/Ruchita/Desktop/document-classification/news/final'
categories=listdir(loc)
print(categories)
train_data=pd.DataFrame(columns=['article','label'])
test_data=pd.DataFrame(columns=['article','label'])
train_docs=[]
train_labels=[]
test_docs=[]
test_labels=[]
ct_dic={'business':1,'entertainment':2,'politics':3,'sports':4,'tech':5}
for ct in categories:
    floc=loc+"/"+ct
    j=listdir(floc)
    for i in j[:400]:
        file1=open(floc+"/"+i)
        txt=file1.read()
        test_docs.append(txt)
        test_labels.append(ct_dic[ct])
    for k in j[400:]:
        file2=open(floc+"/"+k)
        txt=file2.read()
        train_docs.append(txt)
        train_labels.append(ct_dic[ct])
train_data['article']=train_docs
train_data['label']=train_labels
test_data['article']=test_docs
test_data['label']=test_labels


# In[3]:

train_data = train_data.iloc[np.random.permutation(len(train_data))]
print(train_data.describe)
print(train_data.head())


# In[4]:

test_data = test_data.iloc[np.random.permutation(len(test_data))]
print(test_data.describe)
print(test_data.head())


# In[5]:

from nltk.stem import WordNetLemmatizer
from bs4 import BeautifulSoup
import re
from nltk.corpus import stopwords

def article_to_wordlist( article, remove_stopwords=True ):
    # Function to convert a document to a sequence of words,
    # optionally removing stop words.  Returns a list of words.
    #
    # 1. Remove HTML
    text = BeautifulSoup(article).get_text()
    #  
    # 2. Remove non-letters
    text = re.sub("[^a-zA-Z]"," ", text)
    #
    wnl=WordNetLemmatizer()
    # 3. Convert words to lower case and split them
    words = text.lower().split()
    #
    # 4. Optionally remove stop words (false by default)
    if remove_stopwords:
        stops = set(stopwords.words("english"))
        words = [w for w in words if not w in stops]
    #5. lemmatize
    words = [wnl.lemmatize(w) for w in words]
    
    # 6. Return a list of words
    return(words)


# In[6]:

import nltk.data
tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')


# In[7]:

def article_to_sentences( article, tokenizer, remove_stopwords=True ):
    # Function to split a review into parsed sentences. Returns a 
    # list of sentences, where each sentence is a list of words
    #
    # 1. Use the NLTK tokenizer to split the paragraph into sentences
    raw_sentences = tokenizer.tokenize(article.decode('utf-8').strip())
    #
    # 2. Loop over each sentence
    sentences = []
    for raw_sentence in raw_sentences:
        # If a sentence is empty, skip it
        if len(raw_sentence) > 0:
            # Otherwise, call review_to_wordlist to get a list of words
            sentences.append( article_to_wordlist( raw_sentence,               remove_stopwords ))
    #
    # Return the list of sentences (each sentence is a list of words,
    # so this returns a list of lists
    return sentences


# In[8]:

# Import the built-in logging module and configure it so that Word2Vec 
# creates nice output messages
import logging
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',    level=logging.INFO)


# In[9]:

#load pre-trained model
from gensim.models import Word2Vec
model = Word2Vec.load_word2vec_format('C:/Users/Ruchita/Desktop/document-classification/text8.bin', binary=True)


# In[10]:

print(type(model.syn0))
print(model.syn0.shape)


# In[11]:

model['mobile']


# In[12]:

import numpy as np  # Make sure that numpy is imported

def makeFeatureVec(words, model, num_features):
    # Function to average all of the word vectors in a given
    # paragraph
    #
    # Pre-initialize an empty numpy array (for speed)
    featureVec = np.zeros((num_features,),dtype="float32")
    #
    nwords = 0.
    # 
    # Index2word is a list that contains the names of the words in 
    # the model's vocabulary. Convert it to a set, for speed 
    index2word_set = set(model.index2word)
    #
    # Loop over each word in the review and, if it is in the model's
    # vocaublary, add its feature vector to the total
    for word in words:
        if word in index2word_set: 
            nwords = nwords + 1.
            featureVec = np.add(featureVec,model[word])
    # 
    # Divide the result by the number of words to get the average
    featureVec = np.divide(featureVec,nwords)
    return featureVec


def getAvgFeatureVecs(articles, model, num_features):
    # Given a set of reviews (each one a list of words), calculate 
    # the average feature vector for each one and return a 2D numpy array 
    # 
    # Initialize a counter
    counter = 0.
    # 
    # Preallocate a 2D numpy array, for speed
    articleFeatureVecs = np.zeros((len(articles),num_features),dtype="float32")
    # 
    # Loop through the reviews
    for article in articles:
        # Print a status message every 1000th review
        if counter%1000. == 0.:
            print "Article %d of %d" % (counter, len(articles))
        # Call the function (defined above) that makes average feature vectors
        articleFeatureVecs[counter] = makeFeatureVec(article, model,            num_features)
       
        # Increment the counter
        counter = counter + 1.
    return articleFeatureVecs


# In[18]:

clean_train_articles = []
print "Creating average feature vecs for train articles"
for article in train_data["article"]:
    clean_train_articles.append( article_to_wordlist( article,         remove_stopwords=True ))

x_train = getAvgFeatureVecs( clean_train_articles, model, 100 )
y_train = train_data["label"]

print "Creating average feature vecs for test articles"
clean_test_articles = []
for article in test_data["article"]:
    clean_test_articles.append( article_to_wordlist( article,         remove_stopwords=True ))

x_test = getAvgFeatureVecs( clean_test_articles, model, 100 )


# In[19]:

y_test = test_data["label"]


# In[20]:

print(len(x_train))
print(len(y_train))
print(len(x_test))
print(len(y_test))


# In[20]:

from sklearn.ensemble import RandomForestClassifier
forest = RandomForestClassifier( n_estimators = 100 )

print "Fitting a random forest to labeled training data..."
forest = forest.fit( x_train, y_train )
from sklearn.metrics import accuracy_score
result = forest.predict( x_test )
print(accuracy_score(y_test, result))


# In[21]:

from sklearn.linear_model import SGDClassifier
model2 = SGDClassifier(loss='modified_huber', n_iter=10, random_state=0, shuffle=True)
model2 = model2.fit(x_train,y_train)
result2=model2.predict(x_test)
print(accuracy_score(y_test,result2))


# In[48]:

from sklearn.linear_model import LogisticRegression
lr1 = LogisticRegression(C=50)
lr1 = lr1.fit(x_train, y_train)
result3 = lr1.predict(x_test)
print(accuracy_score(y_test,result3))


# In[13]:

from sklearn.cluster import KMeans
import time

start = time.time() # Start time

# Set "k" (num_clusters) to be 1/5th of the vocabulary size, or an
# average of 5 words per cluster
word_vectors = model.syn0
num_clusters = word_vectors.shape[0] / 50

# Initalize a k-means object and use it to extract centroids
kmeans_clustering = KMeans( n_clusters = num_clusters, max_iter=7, n_jobs=-1)
idx = kmeans_clustering.fit_predict( word_vectors )

# Get the end time and print how long the process took
end = time.time()
elapsed = end - start
print "Time taken for K Means clustering: ", elapsed, "seconds."


# In[14]:

word_centroid_map = dict(zip( model.index2word, idx ))
y_train = train_data['label']
y_test = test_data['label']


# In[15]:

for cluster in xrange(0,10):
    #
    # Print the cluster number  
    print "\nCluster %d" % cluster
    #
    # Find all of the words for that cluster number, and print them out
    words = []
    for i in xrange(0,len(word_centroid_map.values())):
        if( word_centroid_map.values()[i] == cluster ):
            words.append(word_centroid_map.keys()[i])
    print words


# In[16]:

def create_bag_of_centroids( wordlist, word_centroid_map ):
    #
    # The number of clusters is equal to the highest cluster index
    # in the word / centroid map
    num_centroids = max( word_centroid_map.values() ) + 1
    #
    # Pre-allocate the bag of centroids vector (for speed)
    bag_of_centroids = np.zeros( num_centroids, dtype="float32" )
    #
    # Loop over the words in the review. If the word is in the vocabulary,
    # find which cluster it belongs to, and increment that cluster count 
    # by one
    for word in wordlist:
        if word in word_centroid_map:
            index = word_centroid_map[word]
            bag_of_centroids[index] += 1
    #
    # Return the "bag of centroids"
    return bag_of_centroids


# In[21]:

train_centroids = np.zeros( (train_data["article"].size, num_clusters),     dtype="float32" )

# Transform the training set reviews into bags of centroids
print "training centroids"
counter = 0
for article in clean_train_articles:
    train_centroids[counter] = create_bag_of_centroids( article,         word_centroid_map )
    if counter%1000==0:
        print "Article %d of %d" % (counter, len(clean_train_articles))
    counter += 1
    

# Repeat for test reviews 
test_centroids = np.zeros(( test_data["article"].size, num_clusters),     dtype="float32" )

print('\n')
print "test centroids"

counter = 0
for article in clean_test_articles:
    test_centroids[counter] = create_bag_of_centroids( article,         word_centroid_map )
    if counter%1000==0:
        print "Article %d of %d" % (counter, len(clean_test_articles))
    counter += 1


# In[22]:

from sklearn.metrics import accuracy_score
from sklearn.linear_model import SGDClassifier


# In[23]:

from sklearn.ensemble import RandomForestClassifier
forest2 = RandomForestClassifier(n_estimators = 100)
forest2 = forest2.fit(train_centroids, y_train )
rc1 = forest2.predict( test_centroids )
print(accuracy_score(y_test, rc1))


# In[24]:

sgd2 = SGDClassifier(loss='modified_huber', n_iter=10, random_state=0, shuffle=True)
sgd2 = sgd2.fit(train_centroids, y_train)
rc2=sgd2.predict(test_centroids)
print(accuracy_score(y_test,rc2))


# In[38]:

from sklearn.linear_model import LogisticRegression
lr = LogisticRegression(C=0.008)
lr = lr.fit(train_centroids, y_train)
rc3 = lr.predict(test_centroids)
print(accuracy_score(y_test,rc3))

