<filename>Tfidf/SVM/test.py
# -*- coding: utf-8 -*-
"""crtest.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/19z0CxLn52gS1kQjdA-JhmSkp1-uWSjOg
"""

from google.colab import drive
drive.mount('/content/drive')

# Download the StanfordCoreNLP package
# We will use this package to extract verb phrases from sentences
!wget http://nlp.stanford.edu/software/stanford-corenlp-full-2018-10-05.zip

import zipfile

fh = open('stanford-corenlp-full-2018-10-05.zip', 'rb')
z = zipfile.ZipFile(fh)
for name in z.namelist():
    outpath = "/content/drive/My Drive/Causal_Relation_Extraction/Production/stanfordcorenlp"
    z.extract(name, outpath)
fh.close()

pip install stanfordcorenlp

import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, svm
from sklearn.metrics import accuracy_score
import pickle 
import re
import json

from stanfordcorenlp import StanfordCoreNLP
from nltk.tree import Tree
nlp = StanfordCoreNLP('/content/drive/My Drive/cicd/Production/stanfordcorenlp/stanford-corenlp-full-2018-10-05')

import nltk
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')

# form a parse tree of the sentence
# Extract the phrases
def extract_phrase(tree_str, label):
    phrases = []
    trees = Tree.fromstring(tree_str)
    for tree in trees:
        for subtree in tree.subtrees():
            if subtree.label() == label:
                t = subtree
                t = ' '.join(t.leaves())
                phrases.append(t)

    return phrases

# Check if the sentence contains a noun phrase and a verb phrase
def is_npvp(sentence):
  tree_str = nlp.parse(sentence)
  nps = extract_phrase(tree_str, 'NP')
  vps = extract_phrase(tree_str, 'VP')
  if len(nps) == 0 or len(vps) == 0:
    return False
  else:
    vpl = vps[0].split(" ")
    if len(vpl) > 1:
      return True
    else: 
      return False

def get_sent_token(Sent):
  # Step - a : Remove blank rows if any.
  rawSent = Sent.copy()
  rawSent['sentences'].dropna(inplace=True)
  # Step - b : Change all the sentences to lower case. This is required as python interprets 'dog' and 'DOG' differently
  rawSent['sentences'] = [entry.lower() for entry in rawSent['sentences']]

  # Step - c : Remove all special symbols 
  s_list = []
  for sentence in rawSent['sentences']:
      for k in sentence.split("\n"):
          s_list.append(re.sub(r"[^a-zA-Z0-9]+", ' ', k))

  # Find the name of the column by index
  n = rawSent.columns[0]
  # Drop that column
  rawSent.drop(n, axis = 1, inplace = True)

  # Put whatever series you want in its place
  rawSent[n] = s_list

  rel = [' consequently', 
         ' as a result ', 
         ' therefore', 
         ' as a consequence', 
         ' for this reason', 
         ' for all these reasons', 
         ' thus',
         ' because', 
         ' since ',
         ' cause of', 
         ' because of',
         ' after', 
         ' as '
         ]

  vsent = []    
  loc = []   
  for sent in rawSent['sentences']:
    if any(x in sent for x in rel):
      if is_npvp(sent):
        vsent.append(sent)
        i = rawSent.sentences[rawSent.sentences == sent].index.tolist()
        loc.append(i[0])

  if len(vsent) == 0:
    emp = pd.DataFrame()
    return emp
  else:
    polSent = pd.DataFrame(list(zip(vsent, loc)), 
                columns =['sentences', 'loc'])
    
    # Step - d : Tokenization : In this each entry in the rawSent will be broken into set of words
    polSent['sentences']= [word_tokenize(entry) for entry in polSent['sentences']]

    # Step - e : Remove Numeric symbols and perfom Word Stemming/Lemmenting.
    # WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun
    tag_map = defaultdict(lambda : wn.NOUN)
    tag_map['J'] = wn.ADJ
    tag_map['V'] = wn.VERB
    tag_map['R'] = wn.ADV
    for index,entry in enumerate(polSent['sentences']):
      # Declaring Empty List to store the words that follow the rules for this step
      Final_words = []
      # Initializing WordNetLemmatizer()
      word_Lemmatized = WordNetLemmatizer()
      # pos_tag function below will provide the 'tag' i.e if the word is Noun(N) or Verb(V) or something else.
      for word, tag in pos_tag(entry):
          # Below condition is to check for Stop words and consider only alphabets
          if word.isalpha():
              word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])
              Final_words.append(word_Final)
      # The final processed set of words for each iteration will be stored in 'sentences_final'
      polSent.loc[index,'sentences_final'] = str(Final_words)
    return polSent

def test():
  from sklearn.externals import joblib 

  # Load the model from the file 
  cfcd = joblib.load('/content/drive/My Drive/Causal_Relation_Extraction/Production/cr_model.pkl')
  text = input("Enter text: ") 

  if not text:
    print("No text entered")
  else:
    from nltk import tokenize

    sentences = tokenize.sent_tokenize(text)
    label = []
    for i in range(len(sentences)):
      label.append(0)

    rawSentences = pd.DataFrame(list(zip(sentences, label)), 
                columns =['sentences', 'label'])
    sent_tokens = get_sent_token(rawSentences)

    if sent_tokens.empty:
      print("No causes found")
      text_and_causes = get_json_from_pd(text, sent_tokens)
      print(text_and_causes)
    else:
      ind = sent_tokens['loc'].tolist()
      rawSentences = rawSentences.loc[ ind , : ]
      rawSentences.reset_index(drop = True, inplace = True)

      # Testing phase
      tf_features = pickle.load(open("/content/drive/My Drive/Causal_Relation_Extraction/Production/cr_features.pkl", 'rb'))

      sent_Tfidf = tf_features.transform(sent_tokens['sentences_final'])

      label_predictions = cfcd.predict(sent_Tfidf)
      n = rawSentences.columns[1]
      # Drop that column
      rawSentences.drop(n, axis = 1, inplace = True)

      rawSentences[n] = label_predictions.tolist()
      print(rawSentences)

      #convert to json format
      text_and_causes = get_json_from_pd(text, rawSentences)
      print(text_and_causes)

def get_json_from_pd(text, rawSentences):
  if rawSentences.empty:
    thisdict = {
      "text": text,
      "causes": []
    }
  else:
    causes_df = rawSentences[rawSentences.label == 1]
    causes = causes_df['sentences'].tolist()
    thisdict = {
      "text": text,
      "causes": causes
    }
  text_and_causes = json.dumps(thisdict)
  return text_and_causes

test()


<filename>Tfidf/SVM/test.py
# -*- coding: utf-8 -*-
"""crtest.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/19z0CxLn52gS1kQjdA-JhmSkp1-uWSjOg
"""

from google.colab import drive
drive.mount('/content/drive')

# Download the StanfordCoreNLP package
# We will use this package to extract verb phrases from sentences
!wget http://nlp.stanford.edu/software/stanford-corenlp-full-2018-10-05.zip

import zipfile

fh = open('stanford-corenlp-full-2018-10-05.zip', 'rb')
z = zipfile.ZipFile(fh)
for name in z.namelist():
    outpath = "/content/drive/My Drive/Causal_Relation_Extraction/Production/stanfordcorenlp"
    z.extract(name, outpath)
fh.close()

pip install stanfordcorenlp

import pandas as pd
import numpy as np
from nltk.tokenize import word_tokenize
from nltk import pos_tag
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer
from sklearn.preprocessing import LabelEncoder
from collections import defaultdict
from nltk.corpus import wordnet as wn
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn import model_selection, svm
from sklearn.metrics import accuracy_score
import pickle 
import re
import json

from stanfordcorenlp import StanfordCoreNLP
from nltk.tree import Tree
nlp = StanfordCoreNLP('/content/drive/My Drive/cicd/Production/stanfordcorenlp/stanford-corenlp-full-2018-10-05')

import nltk
nltk.download('punkt')
nltk.download('wordnet')
nltk.download('averaged_perceptron_tagger')
nltk.download('stopwords')

# form a parse tree of the sentence
# Extract the phrases
def extract_phrase(tree_str, label):
    phrases = []
    trees = Tree.fromstring(tree_str)
    for tree in trees:
        for subtree in tree.subtrees():
            if subtree.label() == label:
                t = subtree
                t = ' '.join(t.leaves())
                phrases.append(t)

    return phrases

# Check if the sentence contains a noun phrase and a verb phrase
def is_npvp(sentence):
  tree_str = nlp.parse(sentence)
  nps = extract_phrase(tree_str, 'NP')
  vps = extract_phrase(tree_str, 'VP')
  if len(nps) == 0 or len(vps) == 0:
    return False
  else:
    vpl = vps[0].split(" ")
    if len(vpl) > 1:
      return True
    else: 
      return False

def get_sent_token(Sent):
  # Step - a : Remove blank rows if any.
  rawSent = Sent.copy()
  rawSent['sentences'].dropna(inplace=True)
  # Step - b : Change all the sentences to lower case. This is required as python interprets 'dog' and 'DOG' differently
  rawSent['sentences'] = [entry.lower() for entry in rawSent['sentences']]

  # Step - c : Remove all special symbols 
  s_list = []
  for sentence in rawSent['sentences']:
      for k in sentence.split("\n"):
          s_list.append(re.sub(r"[^a-zA-Z0-9]+", ' ', k))

  # Find the name of the column by index
  n = rawSent.columns[0]
  # Drop that column
  rawSent.drop(n, axis = 1, inplace = True)

  # Put whatever series you want in its place
  rawSent[n] = s_list

  rel = [' consequently', 
         ' as a result ', 
         ' therefore', 
         ' as a consequence', 
         ' for this reason', 
         ' for all these reasons', 
         ' thus',
         ' because', 
         ' since ',
         ' cause of', 
         ' because of',
         ' after', 
         ' as '
         ]

  vsent = []    
  loc = []   
  for sent in rawSent['sentences']:
    if any(x in sent for x in rel):
      if is_npvp(sent):
        vsent.append(sent)
        i = rawSent.sentences[rawSent.sentences == sent].index.tolist()
        loc.append(i[0])

  if len(vsent) == 0:
    emp = pd.DataFrame()
    return emp
  else:
    polSent = pd.DataFrame(list(zip(vsent, loc)), 
                columns =['sentences', 'loc'])
    
    # Step - d : Tokenization : In this each entry in the rawSent will be broken into set of words
    polSent['sentences']= [word_tokenize(entry) for entry in polSent['sentences']]

    # Step - e : Remove Numeric symbols and perfom Word Stemming/Lemmenting.
    # WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun
    tag_map = defaultdict(lambda : wn.NOUN)
    tag_map['J'] = wn.ADJ
    tag_map['V'] = wn.VERB
    tag_map['R'] = wn.ADV
    for index,entry in enumerate(polSent['sentences']):
      # Declaring Empty List to store the words that follow the rules for this step
      Final_words = []
      # Initializing WordNetLemmatizer()
      word_Lemmatized = WordNetLemmatizer()
      # pos_tag function below will provide the 'tag' i.e if the word is Noun(N) or Verb(V) or something else.
      for word, tag in pos_tag(entry):
          # Below condition is to check for Stop words and consider only alphabets
          if word.isalpha():
              word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])
              Final_words.append(word_Final)
      # The final processed set of words for each iteration will be stored in 'sentences_final'
      polSent.loc[index,'sentences_final'] = str(Final_words)
    return polSent

def test():
  from sklearn.externals import joblib 

  # Load the model from the file 
  cfcd = joblib.load('/content/drive/My Drive/Causal_Relation_Extraction/Production/cr_model.pkl')
  text = input("Enter text: ") 

  if not text:
    print("No text entered")
  else:
    from nltk import tokenize

    sentences = tokenize.sent_tokenize(text)
    label = []
    for i in range(len(sentences)):
      label.append(0)

    rawSentences = pd.DataFrame(list(zip(sentences, label)), 
                columns =['sentences', 'label'])
    sent_tokens = get_sent_token(rawSentences)

    if sent_tokens.empty:
      print("No causes found")
      text_and_causes = get_json_from_pd(text, sent_tokens)
      print(text_and_causes)
    else:
      ind = sent_tokens['loc'].tolist()
      rawSentences = rawSentences.loc[ ind , : ]
      rawSentences.reset_index(drop = True, inplace = True)

      # Testing phase
      tf_features = pickle.load(open("/content/drive/My Drive/Causal_Relation_Extraction/Production/cr_features.pkl", 'rb'))

      sent_Tfidf = tf_features.transform(sent_tokens['sentences_final'])

      label_predictions = cfcd.predict(sent_Tfidf)
      n = rawSentences.columns[1]
      # Drop that column
      rawSentences.drop(n, axis = 1, inplace = True)

      rawSentences[n] = label_predictions.tolist()
      print(rawSentences)

      #convert to json format
      text_and_causes = get_json_from_pd(text, rawSentences)
      print(text_and_causes)

def get_json_from_pd(text, rawSentences):
  if rawSentences.empty:
    thisdict = {
      "text": text,
      "causes": []
    }
  else:
    causes_df = rawSentences[rawSentences.label == 1]
    causes = causes_df['sentences'].tolist()
    thisdict = {
      "text": text,
      "causes": causes
    }
  text_and_causes = json.dumps(thisdict)
  return text_and_causes

test()


# -*- coding: utf-8 -*- crtest.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/19z0CxLn52gS1kQjdA-JhmSkp1-uWSjOg # Download the StanfordCoreNLP package # We will use this package to extract verb phrases from sentences # form a parse tree of the sentence # Extract the phrases # Check if the sentence contains a noun phrase and a verb phrase # Step - a : Remove blank rows if any. # Step - b : Change all the sentences to lower case. This is required as python interprets 'dog' and 'DOG' differently # Step - c : Remove all special symbols # Find the name of the column by index # Drop that column # Put whatever series you want in its place # Step - d : Tokenization : In this each entry in the rawSent will be broken into set of words # Step - e : Remove Numeric symbols and perfom Word Stemming/Lemmenting. # WordNetLemmatizer requires Pos tags to understand if the word is noun or verb or adjective etc. By default it is set to Noun # Declaring Empty List to store the words that follow the rules for this step # Initializing WordNetLemmatizer() # pos_tag function below will provide the 'tag' i.e if the word is Noun(N) or Verb(V) or something else. # Below condition is to check for Stop words and consider only alphabets # The final processed set of words for each iteration will be stored in 'sentences_final' # Load the model from the file # Testing phase # Drop that column #convert to json format