<filename>Part3-Article-NLP/question_answering/question_answering_inference_utility.py
"""
NB: The "question_answering" function has a different behavious from
the "main" from question_answering_main.py! The same goes for
"search_and_format_text" which now takes the raw text as an input
and not a filepath anymore!
"""

import warnings
warnings.filterwarnings("ignore")

import os
import csv
import json
import requests
import pandas as pd
import re
from question_answering_main import answer_prediction



def process_text(q):
    '''
    Function to clean up the text.
    '''
    stop_words = ['advertisement']
    query_tokens = [x for x in q.split() if x.lower() not in stop_words]
    processed_sentence = ' '.join(query_tokens)
    return processed_sentence

    
def search_and_format_text(processed_query, text):
    '''
    Function to format the paragraphs. Takes a string text as an input.
    '''
    final_paragraphs = text.split('\n\n')

    cleaned_paras = []
    for para in final_paragraphs:
        sentences = para.split('.')
        cleaned_para = '.'.join([process_text(x) for x in sentences])
        cleaned_paras.append(cleaned_para)

    return cleaned_paras


def question_answering(query, text, filter_numbers=False, threshold=0.5):
    ''' For a given question and a source text, returns the most probable answer.
    Inputs:
        - query (str)
        - text (str)
        - filter_numbers (bool): If set to True, only keeps answers that are numbers
        - threshold (float): Below this value, the model supposes that there is no
            correct answer to the given question.
    Output:
        - answer_ans (str)
        - para_ans (str)
        - prob_ans (float)
    '''
    
    paras_to_search = search_and_format_text(query, text)

    questions,answers,para,probs = answer_prediction(paras_to_search, query, 'bert-large-uncased-whole-word-masking-finetuned-squad-pytorch_model.bin','bert_config.json')
    questions_final, answers_final, paras_final, probs_final = zip(*sorted(zip(questions,answers,para,probs), reverse=True))
   
    result_str = []
    count = 0

    for i in range(int(len(answers_final))):
        if count < 3:
            if (answers_final[i] == 'empty'):
                pass
            if answers_final[i] != 'empty':
                count += 1
                result_str.append(answers_final[i])

            
    # print("3 most probable answers:")
    # print(list(set(result_str)))

    print('\n------------------------------------\n\n')

    for i, a in enumerate(answers_final):
        if a == 'empty':
            pass

        else:
            if not filter_numbers: # If we don't want to filter numbers...
                if probs_final[i] < threshold:
                    question_ans = questions_final[i]
                    answer_ans = 'No answer'
                    para_ans = ''
                    prob_ans = probs_final[i]

                    # print('No answer')
                    return answer_ans, para_ans, prob_ans
                
                else:
                    question_ans = questions_final[i]
                    answer_ans = answers_final[i]
                    para_ans = paras_final[i]
                    prob_ans = probs_final[i]
                    
                    # print(para_ans + '\n')
                    # print(question_ans+'\n')
                    # print(a+'\n')
                    # print(str(prob_ans)+'\n')
                    # print('------------------------------------\n')

                    return answer_ans, para_ans, prob_ans
            
            else: # If we want to filter numbers...
                if probs_final[i] < threshold:
                    question_ans = questions_final[i]
                    answer_ans = 'No answer'
                    para_ans = ''
                    prob_ans = probs_final[i]

                    # print('No answer')
                    return answer_ans, para_ans, prob_ans
                
                else:
                    question_ans = questions_final[i]
                    answer_ans = answers_final[i]
                    para_ans = paras_final[i]
                    prob_ans = probs_final[i]

                    # If no number is found in the answer...
                    if re.match(r"\d", answer_ans) is None:
                        continue # Look at the next probable answer
                    
                    # If at least one digit is in the answer...
                    else:
                        # print(para_ans + '\n')
                        # print(question_ans+'\n')
                        # print(a+'\n')
                        # print(str(prob_ans)+'\n')
                        # print('------------------------------------\n')

                        return answer_ans, para_ans, prob_ans




<filename>Part3-Article-NLP/question_answering/question_answering_inference_utility.py
"""
NB: The "question_answering" function has a different behavious from
the "main" from question_answering_main.py! The same goes for
"search_and_format_text" which now takes the raw text as an input
and not a filepath anymore!
"""

import warnings
warnings.filterwarnings("ignore")

import os
import csv
import json
import requests
import pandas as pd
import re
from question_answering_main import answer_prediction



def process_text(q):
    '''
    Function to clean up the text.
    '''
    stop_words = ['advertisement']
    query_tokens = [x for x in q.split() if x.lower() not in stop_words]
    processed_sentence = ' '.join(query_tokens)
    return processed_sentence

    
def search_and_format_text(processed_query, text):
    '''
    Function to format the paragraphs. Takes a string text as an input.
    '''
    final_paragraphs = text.split('\n\n')

    cleaned_paras = []
    for para in final_paragraphs:
        sentences = para.split('.')
        cleaned_para = '.'.join([process_text(x) for x in sentences])
        cleaned_paras.append(cleaned_para)

    return cleaned_paras


def question_answering(query, text, filter_numbers=False, threshold=0.5):
    ''' For a given question and a source text, returns the most probable answer.
    Inputs:
        - query (str)
        - text (str)
        - filter_numbers (bool): If set to True, only keeps answers that are numbers
        - threshold (float): Below this value, the model supposes that there is no
            correct answer to the given question.
    Output:
        - answer_ans (str)
        - para_ans (str)
        - prob_ans (float)
    '''
    
    paras_to_search = search_and_format_text(query, text)

    questions,answers,para,probs = answer_prediction(paras_to_search, query, 'bert-large-uncased-whole-word-masking-finetuned-squad-pytorch_model.bin','bert_config.json')
    questions_final, answers_final, paras_final, probs_final = zip(*sorted(zip(questions,answers,para,probs), reverse=True))
   
    result_str = []
    count = 0

    for i in range(int(len(answers_final))):
        if count < 3:
            if (answers_final[i] == 'empty'):
                pass
            if answers_final[i] != 'empty':
                count += 1
                result_str.append(answers_final[i])

            
    # print("3 most probable answers:")
    # print(list(set(result_str)))

    print('\n------------------------------------\n\n')

    for i, a in enumerate(answers_final):
        if a == 'empty':
            pass

        else:
            if not filter_numbers: # If we don't want to filter numbers...
                if probs_final[i] < threshold:
                    question_ans = questions_final[i]
                    answer_ans = 'No answer'
                    para_ans = ''
                    prob_ans = probs_final[i]

                    # print('No answer')
                    return answer_ans, para_ans, prob_ans
                
                else:
                    question_ans = questions_final[i]
                    answer_ans = answers_final[i]
                    para_ans = paras_final[i]
                    prob_ans = probs_final[i]
                    
                    # print(para_ans + '\n')
                    # print(question_ans+'\n')
                    # print(a+'\n')
                    # print(str(prob_ans)+'\n')
                    # print('------------------------------------\n')

                    return answer_ans, para_ans, prob_ans
            
            else: # If we want to filter numbers...
                if probs_final[i] < threshold:
                    question_ans = questions_final[i]
                    answer_ans = 'No answer'
                    para_ans = ''
                    prob_ans = probs_final[i]

                    # print('No answer')
                    return answer_ans, para_ans, prob_ans
                
                else:
                    question_ans = questions_final[i]
                    answer_ans = answers_final[i]
                    para_ans = paras_final[i]
                    prob_ans = probs_final[i]

                    # If no number is found in the answer...
                    if re.match(r"\d", answer_ans) is None:
                        continue # Look at the next probable answer
                    
                    # If at least one digit is in the answer...
                    else:
                        # print(para_ans + '\n')
                        # print(question_ans+'\n')
                        # print(a+'\n')
                        # print(str(prob_ans)+'\n')
                        # print('------------------------------------\n')

                        return answer_ans, para_ans, prob_ans




NB: The "question_answering" function has a different behavious from
 the "main" from question_answering_main.py! The same goes for
 "search_and_format_text" which now takes the raw text as an input
 and not a filepath anymore! Function to clean up the text. Function to format the paragraphs. Takes a string text as an input. For a given question and a source text, returns the most probable answer.
     Inputs:
         - query (str)
         - text (str)
         - filter_numbers (bool): If set to True, only keeps answers that are numbers
         - threshold (float): Below this value, the model supposes that there is no
             correct answer to the given question.
     Output:
         - answer_ans (str)
         - para_ans (str)
         - prob_ans (float) # print("3 most probable answers:") # print(list(set(result_str))) # If we don't want to filter numbers... # print('No answer') # print(para_ans + '\n') # print(question_ans+'\n') # print(a+'\n') # print(str(prob_ans)+'\n') # print('------------------------------------\n') # If we want to filter numbers... # print('No answer') # If no number is found in the answer... # Look at the next probable answer # If at least one digit is in the answer... # print(para_ans + '\n') # print(question_ans+'\n') # print(a+'\n') # print(str(prob_ans)+'\n') # print('------------------------------------\n')