<gh_stars>0
# -*- coding: utf-8 -*-
"""bm25_bert.ipynb

Automatically generated by Colaboratory.

Original file is located at
		https://colab.research.google.com/drive/1gGMMyFBKSJvZNuYsNRGcewX76rpctCOI

Importing required modules
"""

import nltk
import itertools
import os
from gensim.summarization.bm25 import BM25
from transformers import AutoTokenizer, AutoModelForQuestionAnswering, QuestionAnsweringPipeline
import json
import spacy
import math
from nltk.corpus import stopwords  
from nltk.tokenize import word_tokenize
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer, util
from sentence_transformers.cross_encoder import CrossEncoder
from qa_helper import KnowledgeGraph
import matplotlib.pyplot as plt
import seaborn as sns
from helper import preprocess_paragraph
import pandas as pd
from time import time

class bm25:

	def __init__(self, nlp):
		self.tokenize = lambda text: [token.lemma_ for token in nlp(text)]
		self.bm25 = None
		self.passages = None

	def preprocess(self, doc):
		passages = [p for p in doc.split('\n') if p and not p.startswith('=')]
		return passages

	def fit(self, docs):
			# passages = list(itertools.chain(*map(self.preprocess, docs)))
		corpus = [self.tokenize(p) for p in docs]
		self.bm25 = BM25(corpus)
		self.passages = docs

	def most_similar(self, question, topn=10):
		tokens = self.tokenize(question)
		# average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys())
		scores = self.bm25.get_scores(tokens)
		pairs = [(s, i) for i, s in enumerate(scores)]
		pairs.sort(reverse=True)
		passages = [self.passages[i] for _, i in pairs[:topn]]
		return passages
	
	def rankDocuments(self, query):
		tokens = self.tokenize(query)
		# average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys())
		scores = self.bm25.get_scores(tokens)
		return np.array([s for i,s in enumerate(scores)]).reshape((1,164))

class tfidf:

	def __init__(self, nlp):
		self.nlp = nlp
		self.tfidf_model = None
		self.documents = None

	def processSentence(self, sentence):
		sentence = sentence.lower() #Lowering sentences

		#Removing punctuations
		symbols = "!\"#$%&()*+-./:;<=>?@[\]^_`{|}~\n"
		sentence = sentence.translate(str.maketrans(symbols,' '*len(symbols)))

		#removing stopwords
		stop_words = set(stopwords.words('english')) 
		word_tokens = word_tokenize(sentence)  
		filtered_sentence = " ".join([w for w in word_tokens if w not in stop_words and len(w) > 1])

		#lemmatization
		doc = self.nlp(filtered_sentence)
		lemmatized_sentence = [t.lemma_ for t in doc]
		
		return lemmatized_sentence

	def preprocessDocument(self, passages):
		vocabulary = set()
		processed_passages = list()

		for p in range(len(passages)):
			lemmatized_sentence = self.processSentence(passages[p])
			processed_passages.append(" ".join(lemmatized_sentence))
			vocabulary.update(set(lemmatized_sentence))
		
		self.tfidf_model = TfidfVectorizer(vocabulary = list(vocabulary))
		self.documents = self.tfidf_model.fit_transform(processed_passages)
	
	def rankDocuments(self, query):
		query = " ".join(self.processSentence(query))
		queryVector = self.tfidf_model.transform([query])
		return np.transpose(cosine_similarity(self.documents, queryVector))

class sbert:

	def __init__(self):
		self.model = SentenceTransformer('msmarco-distilroberta-base-v2')
		self.document = None

	def fit(self, docs):
		self.document = self.model.encode(docs)
	
	def rankDocuments(self, query):
		query_encoded = self.model.encode(query)
		return util.pytorch_cos_sim(query_encoded, self.document).numpy()

class rerankPassages:

	def __init__(self, nlp):
		self.bm25_ranking = bm25(nlp)
		self.tfidf_ranking = tfidf(nlp)
		self.sbert_ranking = sbert()
		self.cross_encoder = CrossEncoder("cross-encoder/ms-marco-TinyBERT-L-6")
		self.kg = KnowledgeGraph('chatbot', 'password')
		self.document = None
	
	def fit(self, document):
		self.document = document
		self.bm25_ranking.fit(document)
		self.tfidf_ranking.preprocessDocument(document)
		self.sbert_ranking.fit(document)
	
	def matchParaSent(self, s, p):
		
		sList = s.split()
		if len(sList) < 1:
			return False
		count = 0
		for i in sList:
			if i in p:count += 1
	
		if count/len(sList) > 0.9: return True
		else: 
			return False

	def getSentences(self, query, n):
		return self.kg.retrieveSentences(query, n)
	def withKg(self, query, paras, t):
			sentences = self.kg.retrieveSentences(query, 10)

			for i in paras:
				avgScore = 0
				sentencesMatched = 0
				for s in sentences:
					sentence = s['sentence']
					score = s['score']
					if self.matchParaSent(sentence, i[0]):
						if sentence not in i[0]: print(sentence, i[0])
						# print(sentence, i[0])
						sentencesMatched += 1
						avgScore += score
				# if sentencesMatched == 0: sentencesMatched = 1
				i[1] = 1/(t + i[1]) + 1/(t + sentencesMatched)

			paras.sort(key = lambda x : x[1])
			return [i[0] for i in paras]

	def withCrossEncoder(self, query, paras):
		para_combination = [[query, p] for p in paras]

		score = self.cross_encoder.predict(para_combination)
		sim_scores_argsort = reversed(np.argsort(score))
		
		reranked_passages = list()
		for idx in sim_scores_argsort:
			reranked_passages.append(paras[idx])
		return reranked_passages

	def rankDocuments(self, query, mu, k):
		bm25_scores = self.bm25_ranking.rankDocuments(query)
		tfidf_scores = self.tfidf_ranking.rankDocuments(query)
		sbert_scores = self.sbert_ranking.rankDocuments(query)
		#Combined scoring
		# mu = 0.7
		# k = 10
		rrf = mu*sbert_scores + (1-mu)*tfidf_scores
		# rrf = 1/(k+c) + 1/(k + bm25_scores)
		# print(rrf)
		# print(np.shape(rrf))
		#retrive top k passages
		scores = rrf.tolist()
		score_passage = [(s,i) for i, s in enumerate(scores[0])]
		score_passage.sort(reverse = True)
		# return self.withKg(query, [[self.document[i[1]], i[0]] for i in score_passage[:4]], k)
		return self.withCrossEncoder(query, [self.document[i[1]] for i in score_passage[:5]])

		# return [self.document[i[1]] for i in score_passage[:4]]

class bert:

	def __init__(self, model):
		self.tokenizer = AutoTokenizer.from_pretrained(model)
		self.model = AutoModelForQuestionAnswering.from_pretrained(model)
		self.bert = QuestionAnsweringPipeline(model = self.model, tokenizer = self.tokenizer)
	
	def evaluateAnswer(self, question, sentence):
		answer = self.bert(question = question, context = sentence)
		return answer

def mergeAnswers(pr_paras, kg_sentences):
	s = set()
	for i in kg_sentences:
		# print(i)
		for p in pr_paras:
			# print(p)
			if i['sentence'] in p:
				s.add(p)
	return s

def completeAnswer(answer, para):
	for p in para:
		if answer in p:
			return p
	return None

def getTopics(filename):
	with open(filename) as file:
		data = json.load(file)
	topics = ['' for i in range(len(data["vertices"]["paragraphs"]))]
	edges = data["edges"]["main"]
	count = 0
	# print(edges)
	for e in edges:
		if e[1] == "about_concept":
			# print(e)
			topic_id = int(str(e[2])[2:])
			para_id = int(str(e[0])[2:])
			# print(topic_id, para_id)
			para_data = preprocess_paragraph(data["vertices"]["paragraphs"][para_id]["text"])
			if topics[para_id] == '' : topics[para_id] += data['vertices']['topics'][topic_id]['text'] + '|' + para_data
			else: topics[para_id] = data['vertices']['topics'][topic_id]['text'] + " | " + topics[para_id]
			count += 1
	return topics

def getPassages(filename):
		with open(filename) as file:
				data = json.load(file)
		passages = list()
		for i in data["vertices"]["paragraphs"]:
			passages.append(i["text"])
		return passages

def getQuestions(questionFile, answerFile):
	questions = list()
	with open(questionFile) as file:
		for line in file:
			questions.append([line[:-1]])
	questions[-1][0] = questions[-1][0] + "?"
	
	ind = 0
	with open(answerFile) as file:
		data = json.load(file)
		for q in data:
			for a in data[q]:
				if a['isCorrect'] and q == questions[ind][0]:
					questions[ind].append(a['sentence'].lstrip())
			if len(questions[ind]) == 1:
				questions[ind].append("")
			ind += 1

	return questions


def init(username, password):
	global retreivePassage

<<<<<<< HEAD
	passages = getTopics("/Users/osheensachdev/btp/iiitd_policy_chatbot/neo4j/iiit_website_graph.json")
=======
	passages = getTopics("C:/Users/mohni/Desktop/github_chatbot/iiitd_policy_chatbot/data/handbook_graph.json")
>>>>>>> 8dbc289bc06a21824413bf13e135e3b47c5b4f05
	print("Document loaded")

	SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
	nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])

	print("Fitting Corpus")
	retreivePassage = rerankPassages(nlp)
	retreivePassage.fit(passages)
	print("Done!")

def find_answer(q):
	global retreivePassage
	
	topAnswer = retreivePassage.rankDocuments(q, 0.4, 0)
	topSentences = retreivePassage.getSentences(q, 20)	

	each_para_answer = []
	sentencePicked = [False]*len(topSentences)

	for paras in topAnswer:
		maxScore = 0
		sentence = None
		ind, indChosen = 0,-1
		for sentences in topSentences:
			s = sentences['sentence']
			if retreivePassage.matchParaSent(s, paras) and maxScore < sentences['score']:
				indChosen = ind
				maxScore = sentences['score']
				sentence = sentences
			ind += 1
		if sentence != None and (not sentencePicked[indChosen]):
			sentencePicked[indChosen] = True
			each_para_answer.append(sentence)
			
	return each_para_answer


# questions = [
#              "How do I calculate cgpa",
#              "What is the normal load for UG students",
#              "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript",
#             " what is the process of registration?",
#             "how many seats are there in cse for admission?",
#              " what is the admission criteria for btech"
#              "I am in 1st year. Can I take overload?",
#              "I am in 2nd year. Can I take overload?",
#              "what happens if I miss the endsem because of a medical reason?",
#              "what happens if I fail a course?",
#              " what happens if I get an F grade in a course?",
#              "How can I calculate sgpa",
#              "What if I pass all my semesters",
#              "What about canteen",
#              "Will I get hostel",
#              "I dont know anything about IIIT",
#              "Who was <NAME>",
#              "Can i take 8 credits of online courses in a semester",
#              "how many credits do i need to graduate",
#              "how is my semester graded",
#              "what if I do more than 156 credits in my btech course",
#              "can I take up internships during a semester?",
#               "what is the i grade",
#               "can I replace a core course on getting an F grade?",
#               "how can I get the grade given to me in a course changed?",
#               "how will my cgpa be computed if I do more than 156 credits?",
#               "is there any rule for attendance?",
#               "how can I apply for a semester leave?",
#               "how can I apply for branch transfer from ece to cse",
#               "what is the minimum credit requirement for graduation?",
#               "what are the requirements to get an honors degree?",
#               "when is the convocation held?"
# ]

# for q in questions:
#   sentences = kg.retrieveSentences(q, 10)
#   passages = retreivePassage.rankDocuments(q, 0.7,50)
#   print(q)
#   paras = mergeAnswers(passages[:5], sentences[:5])
#   for i in range(2):
#     if len(paras) < 2: paras.update(mergeAnswers(passages[5*i:5*(i+1)], sentences))
#   print(paras)
	


# SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
# nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])
# retreivePassage = PassageRetrieval(nlp)
# retreivePassage.fit(passages)
# bertModel = bert("deepset/bert-base-cased-squad2")

# questions = [
						 # "How do I calculate cgpa",
						 # "what happens if I get an F grade in a course?"
						#  "What is the normal load for UG students",
						#  "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript",
						# " what is the process of registration?",
						# "how many seats are there in cse for admission?",
						#  " what is the admission criteria for btech",
						#  "I am in 1st year. Can I take overload?",
						#  "I am in 2nd year. Can I take overload?",
						#  "what happens if I miss the endsem because of a medical reason?",
						#  "what happens if I fail a course?",
						#  " what happens if I get an F grade in a course?",
						#  "How can I calculate sgpa",
						#  "What if I pass all my semesters",
						#  "What about canteen",
						#  "Will I get hostel",
						#  "I dont know anything about IIIT",
						#  "Who was <NAME>",
						#  "Can i take 8 credits of online courses in a semester",
						#  "how many credits do i need to graduate",
						#  "how is my semester graded",
						#  "what if I do more than 156 credits in my btech course",
						#  "can I take up internships during a semester?",
						#   "what is the i grade",
						#   "can I replace a core course on getting an F grade?",
						#   "how can I get the grade given to me in a course changed?",
						#   "how will my cgpa be computed if I do more than 156 credits?",
						#   "is there any rule for attendance?",
						#   "how can I apply for a semester leave?",
						#   "how can I apply for branch transfer from ece to cse",
						#   "what is the minimum credit requirement for graduation?",
						#   "what are the requirements to get an honors degree?",
						#   "when is the convocation held?"
# ]

if __name__ == "__main__":
	init()
	print(find_answer("When is the convocation held"))
	# passages = getTopics("../data/handbook_graph.json")
	# questionAnswerPair = getQuestions("../data/kg/questions.txt", "../data/kg/answers_2.json")
	# print("Document loaded")
	
	# SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
	# nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])

	# print("Fitting Corpus")
	# retreivePassage = rerankPassages(nlp)
	# retreivePassage.fit(passages)
	# print("Done!")

	# model_score = 0
	# para_score = 0
	# # mrr = 0
	# # indices = [0]*15
	# for q in questionAnswerPair[0:1]:
	# 	start = time()
	# 	topAnswer = retreivePassage.rankDocuments(q[0], 0.4, 0)
	# 	topSentences = retreivePassage.getSentences(q[0], 20)
	# 	for i in topSentences: print(i)

	# 	para_chosen = ''
	# 	topSentence = ''
	# 	for p in topAnswer:
	# 		maxScore = 0
	# 		for s in topSentences:
	# 			sentence = s['sentence']
	# 			score = s['score']
	# 			if retreivePassage.matchParaSent(sentence, p):
	# 				if maxScore < score:
	# 					maxScore = score
	# 					topSentences = sentence
	# 		if maxScore > 0:
	# 			para_chosen = p
	# 			break
	# 	print("Time for execution: ", time() - start)
	# 	print("-"*30)

	# 	flag = False
	# 	para_flag = False
	# 	for answer in q[1:]:
	# 		if retreivePassage.matchParaSent(answer, topSentences):
	# 			model_score += 1
	# 			para_score += 1
	# 			para_flag = True
	# 			flag = True
	# 			break
	# 		if (not para_flag) and retreivePassage.matchParaSent(answer, para_chosen):
	# 			para_score += 1
	# 			para_flag = True
	# 			break

	# 	if not para_flag:
	# 		print(q,"\n",para_chosen)
		# if not flag:
		#   print("Does not match any answer:", q[1:])
		# minIndexFound = float('inf')
		# flag = False
		# for ans in q[1:]:
		#   indexFound = 1
		#   for t in topAnswer:
		#     if retreivePassage.matchParaSent(ans, t):
		#       # print(mrr)
		#       print("condition satisfied")
		#       minIndexFound = min(minIndexFound, indexFound)
		#       flag = True
		#       break
		#     # else:
		#     #   maxSimilarity = max(score, maxSimilarity)
		#     indexFound += 1
		#   # print("still in")
		
		# mrr += 1/minIndexFound
		# if flag:
		#   indices[minIndexFound-1] += 1
		# if not flag:
		#   print(q[0],"\n",q[1:],"\n", topAnswer)

		# for i in range(2):
		#   if len(paras) < 2: paras.update(mergeAnswers(topAnswer[5*i:5*(i+1)], sentences))
		#   else: break

	#   # print(topAnswer)
	#   topAnswer = paras
	#   sentence = ""
	#   for i in topAnswer:
	#     # print(i)
	#     sentence += i + " "

	#   if len(topAnswer) == 0:
	#     print("No answer for question:",q[0])
	#     score += 1
	#     continue

	#   ans = bertModel.evaluateAnswer(q[0], sentence)
	#   print(len(sentence.split()))
	#   # print("Q:",q)
	#   # print("Ans:",ans,completeAnswer(ans["answer"], topAnswer))
	#   paras = completeAnswer(ans["answer"], topAnswer)
	#   exist = False
	#   for i in q[1:]:
	#     if i in paras: 
	#       exist = True
	#       score += 1
	#   if not exist:
	#     print("Q:",q)
	#     print("Ans:",ans,completeAnswer(ans["answer"], topAnswer))
	#     # print(len(sentence.split()))
	#   print("---------------------")

	# print((model_score/len(questionAnswerPair))*100)
	# print((para_score/len(questionAnswerPair))*100)

	# print(*indices)
	# print(score/len(questionAnswerPair))
	# df = pd.DataFrame([[indices[i],i+1] for i in range(len(indices))], columns = ['Count', 'Number of paragraphs'])
	# print(df)
	# sns.barplot(data = df, y = 'Count', x = 'Number of paragraphs')
	# plt.show()

# -*- coding: utf-8 -*-
"""bm25_bert.ipynb

Automatically generated by Colaboratory.

Original file is located at
		https://colab.research.google.com/drive/1gGMMyFBKSJvZNuYsNRGcewX76rpctCOI

Importing required modules
"""

import nltk
import itertools
import os
from gensim.summarization.bm25 import BM25
from transformers import AutoTokenizer, AutoModelForQuestionAnswering, QuestionAnsweringPipeline
import json
import spacy
import math
from nltk.corpus import stopwords  
from nltk.tokenize import word_tokenize
from collections import defaultdict
from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer, util
from sentence_transformers.cross_encoder import CrossEncoder
from qa_helper import KnowledgeGraph
import matplotlib.pyplot as plt
import seaborn as sns
from helper import preprocess_paragraph
import pandas as pd
from time import time

class bm25:

	def __init__(self, nlp):
		self.tokenize = lambda text: [token.lemma_ for token in nlp(text)]
		self.bm25 = None
		self.passages = None

	def preprocess(self, doc):
		passages = [p for p in doc.split('\n') if p and not p.startswith('=')]
		return passages

	def fit(self, docs):
			# passages = list(itertools.chain(*map(self.preprocess, docs)))
		corpus = [self.tokenize(p) for p in docs]
		self.bm25 = BM25(corpus)
		self.passages = docs

	def most_similar(self, question, topn=10):
		tokens = self.tokenize(question)
		# average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys())
		scores = self.bm25.get_scores(tokens)
		pairs = [(s, i) for i, s in enumerate(scores)]
		pairs.sort(reverse=True)
		passages = [self.passages[i] for _, i in pairs[:topn]]
		return passages
	
	def rankDocuments(self, query):
		tokens = self.tokenize(query)
		# average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys())
		scores = self.bm25.get_scores(tokens)
		return np.array([s for i,s in enumerate(scores)]).reshape((1,164))

class tfidf:

	def __init__(self, nlp):
		self.nlp = nlp
		self.tfidf_model = None
		self.documents = None

	def processSentence(self, sentence):
		sentence = sentence.lower() #Lowering sentences

		#Removing punctuations
		symbols = "!\"#$%&()*+-./:;<=>?@[\]^_`{|}~\n"
		sentence = sentence.translate(str.maketrans(symbols,' '*len(symbols)))

		#removing stopwords
		stop_words = set(stopwords.words('english')) 
		word_tokens = word_tokenize(sentence)  
		filtered_sentence = " ".join([w for w in word_tokens if w not in stop_words and len(w) > 1])

		#lemmatization
		doc = self.nlp(filtered_sentence)
		lemmatized_sentence = [t.lemma_ for t in doc]
		
		return lemmatized_sentence

	def preprocessDocument(self, passages):
		vocabulary = set()
		processed_passages = list()

		for p in range(len(passages)):
			lemmatized_sentence = self.processSentence(passages[p])
			processed_passages.append(" ".join(lemmatized_sentence))
			vocabulary.update(set(lemmatized_sentence))
		
		self.tfidf_model = TfidfVectorizer(vocabulary = list(vocabulary))
		self.documents = self.tfidf_model.fit_transform(processed_passages)
	
	def rankDocuments(self, query):
		query = " ".join(self.processSentence(query))
		queryVector = self.tfidf_model.transform([query])
		return np.transpose(cosine_similarity(self.documents, queryVector))

class sbert:

	def __init__(self):
		self.model = SentenceTransformer('msmarco-distilroberta-base-v2')
		self.document = None

	def fit(self, docs):
		self.document = self.model.encode(docs)
	
	def rankDocuments(self, query):
		query_encoded = self.model.encode(query)
		return util.pytorch_cos_sim(query_encoded, self.document).numpy()

class rerankPassages:

	def __init__(self, nlp):
		self.bm25_ranking = bm25(nlp)
		self.tfidf_ranking = tfidf(nlp)
		self.sbert_ranking = sbert()
		self.cross_encoder = CrossEncoder("cross-encoder/ms-marco-TinyBERT-L-6")
		self.kg = KnowledgeGraph('chatbot', 'password')
		self.document = None
	
	def fit(self, document):
		self.document = document
		self.bm25_ranking.fit(document)
		self.tfidf_ranking.preprocessDocument(document)
		self.sbert_ranking.fit(document)
	
	def matchParaSent(self, s, p):
		
		sList = s.split()
		if len(sList) < 1:
			return False
		count = 0
		for i in sList:
			if i in p:count += 1
	
		if count/len(sList) > 0.9: return True
		else: 
			return False

	def getSentences(self, query, n):
		return self.kg.retrieveSentences(query, n)
	def withKg(self, query, paras, t):
			sentences = self.kg.retrieveSentences(query, 10)

			for i in paras:
				avgScore = 0
				sentencesMatched = 0
				for s in sentences:
					sentence = s['sentence']
					score = s['score']
					if self.matchParaSent(sentence, i[0]):
						if sentence not in i[0]: print(sentence, i[0])
						# print(sentence, i[0])
						sentencesMatched += 1
						avgScore += score
				# if sentencesMatched == 0: sentencesMatched = 1
				i[1] = 1/(t + i[1]) + 1/(t + sentencesMatched)

			paras.sort(key = lambda x : x[1])
			return [i[0] for i in paras]

	def withCrossEncoder(self, query, paras):
		para_combination = [[query, p] for p in paras]

		score = self.cross_encoder.predict(para_combination)
		sim_scores_argsort = reversed(np.argsort(score))
		
		reranked_passages = list()
		for idx in sim_scores_argsort:
			reranked_passages.append(paras[idx])
		return reranked_passages

	def rankDocuments(self, query, mu, k):
		bm25_scores = self.bm25_ranking.rankDocuments(query)
		tfidf_scores = self.tfidf_ranking.rankDocuments(query)
		sbert_scores = self.sbert_ranking.rankDocuments(query)
		#Combined scoring
		# mu = 0.7
		# k = 10
		rrf = mu*sbert_scores + (1-mu)*tfidf_scores
		# rrf = 1/(k+c) + 1/(k + bm25_scores)
		# print(rrf)
		# print(np.shape(rrf))
		#retrive top k passages
		scores = rrf.tolist()
		score_passage = [(s,i) for i, s in enumerate(scores[0])]
		score_passage.sort(reverse = True)
		# return self.withKg(query, [[self.document[i[1]], i[0]] for i in score_passage[:4]], k)
		return self.withCrossEncoder(query, [self.document[i[1]] for i in score_passage[:5]])

		# return [self.document[i[1]] for i in score_passage[:4]]

class bert:

	def __init__(self, model):
		self.tokenizer = AutoTokenizer.from_pretrained(model)
		self.model = AutoModelForQuestionAnswering.from_pretrained(model)
		self.bert = QuestionAnsweringPipeline(model = self.model, tokenizer = self.tokenizer)
	
	def evaluateAnswer(self, question, sentence):
		answer = self.bert(question = question, context = sentence)
		return answer

def mergeAnswers(pr_paras, kg_sentences):
	s = set()
	for i in kg_sentences:
		# print(i)
		for p in pr_paras:
			# print(p)
			if i['sentence'] in p:
				s.add(p)
	return s

def completeAnswer(answer, para):
	for p in para:
		if answer in p:
			return p
	return None

def getTopics(filename):
	with open(filename) as file:
		data = json.load(file)
	topics = ['' for i in range(len(data["vertices"]["paragraphs"]))]
	edges = data["edges"]["main"]
	count = 0
	# print(edges)
	for e in edges:
		if e[1] == "about_concept":
			# print(e)
			topic_id = int(str(e[2])[2:])
			para_id = int(str(e[0])[2:])
			# print(topic_id, para_id)
			para_data = preprocess_paragraph(data["vertices"]["paragraphs"][para_id]["text"])
			if topics[para_id] == '' : topics[para_id] += data['vertices']['topics'][topic_id]['text'] + '|' + para_data
			else: topics[para_id] = data['vertices']['topics'][topic_id]['text'] + " | " + topics[para_id]
			count += 1
	return topics

def getPassages(filename):
		with open(filename) as file:
				data = json.load(file)
		passages = list()
		for i in data["vertices"]["paragraphs"]:
			passages.append(i["text"])
		return passages

def getQuestions(questionFile, answerFile):
	questions = list()
	with open(questionFile) as file:
		for line in file:
			questions.append([line[:-1]])
	questions[-1][0] = questions[-1][0] + "?"
	
	ind = 0
	with open(answerFile) as file:
		data = json.load(file)
		for q in data:
			for a in data[q]:
				if a['isCorrect'] and q == questions[ind][0]:
					questions[ind].append(a['sentence'].lstrip())
			if len(questions[ind]) == 1:
				questions[ind].append("")
			ind += 1

	return questions


def init(username, password):
	global retreivePassage

<<<<<<< HEAD
	passages = getTopics("/Users/osheensachdev/btp/iiitd_policy_chatbot/neo4j/iiit_website_graph.json")
=======
	passages = getTopics("C:/Users/mohni/Desktop/github_chatbot/iiitd_policy_chatbot/data/handbook_graph.json")
>>>>>>> 8dbc289bc06a21824413bf13e135e3b47c5b4f05
	print("Document loaded")

	SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
	nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])

	print("Fitting Corpus")
	retreivePassage = rerankPassages(nlp)
	retreivePassage.fit(passages)
	print("Done!")

def find_answer(q):
	global retreivePassage
	
	topAnswer = retreivePassage.rankDocuments(q, 0.4, 0)
	topSentences = retreivePassage.getSentences(q, 20)	

	each_para_answer = []
	sentencePicked = [False]*len(topSentences)

	for paras in topAnswer:
		maxScore = 0
		sentence = None
		ind, indChosen = 0,-1
		for sentences in topSentences:
			s = sentences['sentence']
			if retreivePassage.matchParaSent(s, paras) and maxScore < sentences['score']:
				indChosen = ind
				maxScore = sentences['score']
				sentence = sentences
			ind += 1
		if sentence != None and (not sentencePicked[indChosen]):
			sentencePicked[indChosen] = True
			each_para_answer.append(sentence)
			
	return each_para_answer


# questions = [
#              "How do I calculate cgpa",
#              "What is the normal load for UG students",
#              "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript",
#             " what is the process of registration?",
#             "how many seats are there in cse for admission?",
#              " what is the admission criteria for btech"
#              "I am in 1st year. Can I take overload?",
#              "I am in 2nd year. Can I take overload?",
#              "what happens if I miss the endsem because of a medical reason?",
#              "what happens if I fail a course?",
#              " what happens if I get an F grade in a course?",
#              "How can I calculate sgpa",
#              "What if I pass all my semesters",
#              "What about canteen",
#              "Will I get hostel",
#              "I dont know anything about IIIT",
#              "Who was <NAME>",
#              "Can i take 8 credits of online courses in a semester",
#              "how many credits do i need to graduate",
#              "how is my semester graded",
#              "what if I do more than 156 credits in my btech course",
#              "can I take up internships during a semester?",
#               "what is the i grade",
#               "can I replace a core course on getting an F grade?",
#               "how can I get the grade given to me in a course changed?",
#               "how will my cgpa be computed if I do more than 156 credits?",
#               "is there any rule for attendance?",
#               "how can I apply for a semester leave?",
#               "how can I apply for branch transfer from ece to cse",
#               "what is the minimum credit requirement for graduation?",
#               "what are the requirements to get an honors degree?",
#               "when is the convocation held?"
# ]

# for q in questions:
#   sentences = kg.retrieveSentences(q, 10)
#   passages = retreivePassage.rankDocuments(q, 0.7,50)
#   print(q)
#   paras = mergeAnswers(passages[:5], sentences[:5])
#   for i in range(2):
#     if len(paras) < 2: paras.update(mergeAnswers(passages[5*i:5*(i+1)], sentences))
#   print(paras)
	


# SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
# nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])
# retreivePassage = PassageRetrieval(nlp)
# retreivePassage.fit(passages)
# bertModel = bert("deepset/bert-base-cased-squad2")

# questions = [
						 # "How do I calculate cgpa",
						 # "what happens if I get an F grade in a course?"
						#  "What is the normal load for UG students",
						#  "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript",
						# " what is the process of registration?",
						# "how many seats are there in cse for admission?",
						#  " what is the admission criteria for btech",
						#  "I am in 1st year. Can I take overload?",
						#  "I am in 2nd year. Can I take overload?",
						#  "what happens if I miss the endsem because of a medical reason?",
						#  "what happens if I fail a course?",
						#  " what happens if I get an F grade in a course?",
						#  "How can I calculate sgpa",
						#  "What if I pass all my semesters",
						#  "What about canteen",
						#  "Will I get hostel",
						#  "I dont know anything about IIIT",
						#  "Who was <NAME>",
						#  "Can i take 8 credits of online courses in a semester",
						#  "how many credits do i need to graduate",
						#  "how is my semester graded",
						#  "what if I do more than 156 credits in my btech course",
						#  "can I take up internships during a semester?",
						#   "what is the i grade",
						#   "can I replace a core course on getting an F grade?",
						#   "how can I get the grade given to me in a course changed?",
						#   "how will my cgpa be computed if I do more than 156 credits?",
						#   "is there any rule for attendance?",
						#   "how can I apply for a semester leave?",
						#   "how can I apply for branch transfer from ece to cse",
						#   "what is the minimum credit requirement for graduation?",
						#   "what are the requirements to get an honors degree?",
						#   "when is the convocation held?"
# ]

if __name__ == "__main__":
	init()
	print(find_answer("When is the convocation held"))
	# passages = getTopics("../data/handbook_graph.json")
	# questionAnswerPair = getQuestions("../data/kg/questions.txt", "../data/kg/answers_2.json")
	# print("Document loaded")
	
	# SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm")
	# nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"])

	# print("Fitting Corpus")
	# retreivePassage = rerankPassages(nlp)
	# retreivePassage.fit(passages)
	# print("Done!")

	# model_score = 0
	# para_score = 0
	# # mrr = 0
	# # indices = [0]*15
	# for q in questionAnswerPair[0:1]:
	# 	start = time()
	# 	topAnswer = retreivePassage.rankDocuments(q[0], 0.4, 0)
	# 	topSentences = retreivePassage.getSentences(q[0], 20)
	# 	for i in topSentences: print(i)

	# 	para_chosen = ''
	# 	topSentence = ''
	# 	for p in topAnswer:
	# 		maxScore = 0
	# 		for s in topSentences:
	# 			sentence = s['sentence']
	# 			score = s['score']
	# 			if retreivePassage.matchParaSent(sentence, p):
	# 				if maxScore < score:
	# 					maxScore = score
	# 					topSentences = sentence
	# 		if maxScore > 0:
	# 			para_chosen = p
	# 			break
	# 	print("Time for execution: ", time() - start)
	# 	print("-"*30)

	# 	flag = False
	# 	para_flag = False
	# 	for answer in q[1:]:
	# 		if retreivePassage.matchParaSent(answer, topSentences):
	# 			model_score += 1
	# 			para_score += 1
	# 			para_flag = True
	# 			flag = True
	# 			break
	# 		if (not para_flag) and retreivePassage.matchParaSent(answer, para_chosen):
	# 			para_score += 1
	# 			para_flag = True
	# 			break

	# 	if not para_flag:
	# 		print(q,"\n",para_chosen)
		# if not flag:
		#   print("Does not match any answer:", q[1:])
		# minIndexFound = float('inf')
		# flag = False
		# for ans in q[1:]:
		#   indexFound = 1
		#   for t in topAnswer:
		#     if retreivePassage.matchParaSent(ans, t):
		#       # print(mrr)
		#       print("condition satisfied")
		#       minIndexFound = min(minIndexFound, indexFound)
		#       flag = True
		#       break
		#     # else:
		#     #   maxSimilarity = max(score, maxSimilarity)
		#     indexFound += 1
		#   # print("still in")
		
		# mrr += 1/minIndexFound
		# if flag:
		#   indices[minIndexFound-1] += 1
		# if not flag:
		#   print(q[0],"\n",q[1:],"\n", topAnswer)

		# for i in range(2):
		#   if len(paras) < 2: paras.update(mergeAnswers(topAnswer[5*i:5*(i+1)], sentences))
		#   else: break

	#   # print(topAnswer)
	#   topAnswer = paras
	#   sentence = ""
	#   for i in topAnswer:
	#     # print(i)
	#     sentence += i + " "

	#   if len(topAnswer) == 0:
	#     print("No answer for question:",q[0])
	#     score += 1
	#     continue

	#   ans = bertModel.evaluateAnswer(q[0], sentence)
	#   print(len(sentence.split()))
	#   # print("Q:",q)
	#   # print("Ans:",ans,completeAnswer(ans["answer"], topAnswer))
	#   paras = completeAnswer(ans["answer"], topAnswer)
	#   exist = False
	#   for i in q[1:]:
	#     if i in paras: 
	#       exist = True
	#       score += 1
	#   if not exist:
	#     print("Q:",q)
	#     print("Ans:",ans,completeAnswer(ans["answer"], topAnswer))
	#     # print(len(sentence.split()))
	#   print("---------------------")

	# print((model_score/len(questionAnswerPair))*100)
	# print((para_score/len(questionAnswerPair))*100)

	# print(*indices)
	# print(score/len(questionAnswerPair))
	# df = pd.DataFrame([[indices[i],i+1] for i in range(len(indices))], columns = ['Count', 'Number of paragraphs'])
	# print(df)
	# sns.barplot(data = df, y = 'Count', x = 'Number of paragraphs')
	# plt.show()
# -*- coding: utf-8 -*- bm25_bert.ipynb  Automatically generated by Colaboratory.  Original file is located at 		https://colab.research.google.com/drive/1gGMMyFBKSJvZNuYsNRGcewX76rpctCOI  Importing required modules # passages = list(itertools.chain(*map(self.preprocess, docs))) # average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys()) # average_idf = sum(map(lambda k: float(self.bm25.idf[k]), self.bm25.idf.keys())) / len(self.bm25.idf.keys()) #Lowering sentences #Removing punctuations #removing stopwords #lemmatization # print(sentence, i[0]) # if sentencesMatched == 0: sentencesMatched = 1 #Combined scoring # mu = 0.7 # k = 10 # rrf = 1/(k+c) + 1/(k + bm25_scores) # print(rrf) # print(np.shape(rrf)) #retrive top k passages # return self.withKg(query, [[self.document[i[1]], i[0]] for i in score_passage[:4]], k) # return [self.document[i[1]] for i in score_passage[:4]] # print(i) # print(p) # print(edges) # print(e) # print(topic_id, para_id) # questions = [ #              "How do I calculate cgpa", #              "What is the normal load for UG students", #              "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript", #             " what is the process of registration?", #             "how many seats are there in cse for admission?", #              " what is the admission criteria for btech" #              "I am in 1st year. Can I take overload?", #              "I am in 2nd year. Can I take overload?", #              "what happens if I miss the endsem because of a medical reason?", #              "what happens if I fail a course?", #              " what happens if I get an F grade in a course?", #              "How can I calculate sgpa", #              "What if I pass all my semesters", #              "What about canteen", #              "Will I get hostel", #              "I dont know anything about IIIT", #              "Who was <NAME>", #              "Can i take 8 credits of online courses in a semester", #              "how many credits do i need to graduate", #              "how is my semester graded", #              "what if I do more than 156 credits in my btech course", #              "can I take up internships during a semester?", #               "what is the i grade", #               "can I replace a core course on getting an F grade?", #               "how can I get the grade given to me in a course changed?", #               "how will my cgpa be computed if I do more than 156 credits?", #               "is there any rule for attendance?", #               "how can I apply for a semester leave?", #               "how can I apply for branch transfer from ece to cse", #               "what is the minimum credit requirement for graduation?", #               "what are the requirements to get an honors degree?", #               "when is the convocation held?" # ] # for q in questions: #   sentences = kg.retrieveSentences(q, 10) #   passages = retreivePassage.rankDocuments(q, 0.7,50) #   print(q) #   paras = mergeAnswers(passages[:5], sentences[:5]) #   for i in range(2): #     if len(paras) < 2: paras.update(mergeAnswers(passages[5*i:5*(i+1)], sentences)) #   print(paras) # SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm") # nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"]) # retreivePassage = PassageRetrieval(nlp) # retreivePassage.fit(passages) # bertModel = bert("deepset/bert-base-cased-squad2") # questions = [ # "How do I calculate cgpa", # "what happens if I get an F grade in a course?" #  "What is the normal load for UG students", #  "If I fail a course and take it again in the later semester, will my earlier course with F grade be removed from the transcript", # " what is the process of registration?", # "how many seats are there in cse for admission?", #  " what is the admission criteria for btech", #  "I am in 1st year. Can I take overload?", #  "I am in 2nd year. Can I take overload?", #  "what happens if I miss the endsem because of a medical reason?", #  "what happens if I fail a course?", #  " what happens if I get an F grade in a course?", #  "How can I calculate sgpa", #  "What if I pass all my semesters", #  "What about canteen", #  "Will I get hostel", #  "I dont know anything about IIIT", #  "Who was <NAME>", #  "Can i take 8 credits of online courses in a semester", #  "how many credits do i need to graduate", #  "how is my semester graded", #  "what if I do more than 156 credits in my btech course", #  "can I take up internships during a semester?", #   "what is the i grade", #   "can I replace a core course on getting an F grade?", #   "how can I get the grade given to me in a course changed?", #   "how will my cgpa be computed if I do more than 156 credits?", #   "is there any rule for attendance?", #   "how can I apply for a semester leave?", #   "how can I apply for branch transfer from ece to cse", #   "what is the minimum credit requirement for graduation?", #   "what are the requirements to get an honors degree?", #   "when is the convocation held?" # ] # passages = getTopics("../data/handbook_graph.json") # questionAnswerPair = getQuestions("../data/kg/questions.txt", "../data/kg/answers_2.json") # print("Document loaded") # SPACY_MODEL = os.environ.get("SPACY_MODEL", "en_core_web_sm") # nlp = spacy.load(SPACY_MODEL, disable = ["ner","parser","textcat"]) # print("Fitting Corpus") # retreivePassage = rerankPassages(nlp) # retreivePassage.fit(passages) # print("Done!") # model_score = 0 # para_score = 0 # # mrr = 0 # # indices = [0]*15 # for q in questionAnswerPair[0:1]: # 	start = time() # 	topAnswer = retreivePassage.rankDocuments(q[0], 0.4, 0) # 	topSentences = retreivePassage.getSentences(q[0], 20) # 	for i in topSentences: print(i) # 	para_chosen = '' # 	topSentence = '' # 	for p in topAnswer: # 		maxScore = 0 # 		for s in topSentences: # 			sentence = s['sentence'] # 			score = s['score'] # 			if retreivePassage.matchParaSent(sentence, p): # 				if maxScore < score: # 					maxScore = score # 					topSentences = sentence # 		if maxScore > 0: # 			para_chosen = p # 			break # 	print("Time for execution: ", time() - start) # 	print("-"*30) # 	flag = False # 	para_flag = False # 	for answer in q[1:]: # 		if retreivePassage.matchParaSent(answer, topSentences): # 			model_score += 1 # 			para_score += 1 # 			para_flag = True # 			flag = True # 			break # 		if (not para_flag) and retreivePassage.matchParaSent(answer, para_chosen): # 			para_score += 1 # 			para_flag = True # 			break # 	if not para_flag: # 		print(q,"\n",para_chosen) # if not flag: #   print("Does not match any answer:", q[1:]) # minIndexFound = float('inf') # flag = False # for ans in q[1:]: #   indexFound = 1 #   for t in topAnswer: #     if retreivePassage.matchParaSent(ans, t): #       # print(mrr) #       print("condition satisfied") #       minIndexFound = min(minIndexFound, indexFound) #       flag = True #       break #     # else: #     #   maxSimilarity = max(score, maxSimilarity) #     indexFound += 1 #   # print("still in") # mrr += 1/minIndexFound # if flag: #   indices[minIndexFound-1] += 1 # if not flag: #   print(q[0],"\n",q[1:],"\n", topAnswer) # for i in range(2): #   if len(paras) < 2: paras.update(mergeAnswers(topAnswer[5*i:5*(i+1)], sentences)) #   else: break #   # print(topAnswer) #   topAnswer = paras #   sentence = "" #   for i in topAnswer: #     # print(i) #     sentence += i + " " #   if len(topAnswer) == 0: #     print("No answer for question:",q[0]) #     score += 1 #     continue #   ans = bertModel.evaluateAnswer(q[0], sentence) #   print(len(sentence.split())) #   # print("Q:",q) #   # print("Ans:",ans,completeAnswer(ans["answer"], topAnswer)) #   paras = completeAnswer(ans["answer"], topAnswer) #   exist = False #   for i in q[1:]: #     if i in paras: #       exist = True #       score += 1 #   if not exist: #     print("Q:",q) #     print("Ans:",ans,completeAnswer(ans["answer"], topAnswer)) #     # print(len(sentence.split())) #   print("---------------------") # print((model_score/len(questionAnswerPair))*100) # print((para_score/len(questionAnswerPair))*100) # print(*indices) # print(score/len(questionAnswerPair)) # df = pd.DataFrame([[indices[i],i+1] for i in range(len(indices))], columns = ['Count', 'Number of paragraphs']) # print(df) # sns.barplot(data = df, y = 'Count', x = 'Number of paragraphs') # plt.show()