Upload 9 files

app.py

@@ -0,0 +1,296 @@
+import torch
+import streamlit as st
+from datasets import Dataset
+from torch.utils.data import DataLoader
+from transformers import (
+    AutoTokenizer,
+    AutoModelForQuestionAnswering,
+    pipeline,
+)
+import spacy
+# import pandas as pd
+from lib.utils import ContextRetriever
+
+
+#### TO DO:######
+# build out functions for:
+#     * formatting input into document retrieval query (spaCy)
+#     * document retrieval based on query (wikipedia library)
+#     * document postprocessing into passages
+#     * ranking passage based on BM25 scores for query (rank_bm25)
+#     * feeding passages into RoBERTa an reporting answer(s) and passages as evidence
+# decide what to do with examples
+
+### CAN REMOVE:#####
+    # * context collection
+    # *
+
+########################
+### Helper functions ###
+########################
+
+# Build trainer using model and tokenizer from Hugging Face repo
+@st.cache_resource(show_spinner=False)
+def get_pipeline():
+    """
+    Load model and tokenizer from 🤗 repo
+    and build pipeline
+    Parameters: None
+    -----------
+    Returns:
+    --------
+    qa_pipeline : transformers.QuestionAnsweringPipeline
+        The question answering pipeline object
+    """
+    repo_id = 'etweedy/roberta-base-squad-v2'
+    qa_pipeline = pipeline(
+        task = 'question-answering',
+        model=repo_id,
+        tokenizer=repo_id,
+        handle_impossible_answer = True
+    )
+    return qa_pipeline
+
+@st.cache_resource(show_spinner=False)
+def get_spacy():
+    """
+    Load spaCy model for processing query
+    Parameters: None
+    -----------
+    Returns:
+    --------
+    nlp : spaCy.Pipe
+        Portion of 'en_core_web_sm' model pipeline
+        only containing tokenizer and part-of-speech
+        tagger
+    """
+    nlp = spacy.load(
+        'en_core_web_sm',
+        disable = ['ner','parser','textcat']
+    )
+    return nlp
+
+def generate_query(nlp,text):
+    """
+    Process text into a search query,
+    only retaining nouns, proper nouns,
+    numerals, verbs, and adjectives
+    Parameters:
+    -----------
+    nlp : spacy.Pipe
+        spaCy pipeline for processing search query
+    text : str
+        The input text to be processed
+    Returns:
+    --------
+    query : str
+        The condensed search query
+    """
+    tokens = nlp(text)
+    keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
+    query = ' '.join(token.text for token in tokens \
+                         if token.pos_ in keep)
+    return query
+
+def fill_in_example(i):
+    """
+    Function for context-question example button click
+    """
+    st.session_state['response'] = ''
+    st.session_state['question'] = ex_q[i]
+
+def clear_boxes():
+    """
+    Function for field clear button click
+    """
+    st.session_state['response'] = ''
+    st.session_state['question'] = ''
+
+# def get_examples():
+#     """
+#     Retrieve pre-made examples from a .csv file
+#     Parameters: None
+#     -----------
+#     Returns:
+#     --------
+#     questions, contexts : list, list
+#         Lists of examples of corresponding question-context pairs
+        
+#     """
+#     examples = pd.read_csv('examples.csv')
+#     questions = list(examples['question'])
+#     return questions
+
+#############
+### Setup ###
+#############
+    
+# Set mps or cuda device if available
+if torch.backends.mps.is_available():
+    device = "mps"
+elif torch.cuda.is_available():
+    device = "cuda"
+else:
+    device = "cpu"
+
+# Initialize session state variables
+if 'response' not in st.session_state:
+    st.session_state['response'] = ''
+if 'question' not in st.session_state:
+    st.session_state['question'] = ''
+
+# Retrieve trained RoBERTa pipeline for Q&A
+# and spaCy pipeline for processing search query
+with st.spinner('Loading the model...'):
+    qa_pipeline = get_pipeline()
+    nlp = get_spacy()
+
+# # Grab example question-context pairs from csv file
+# ex_q, ex_c = get_examples()
+
+###################
+### App content ###
+###################
+
+# Intro text
+st.header('RoBERTa answer retieval')
+st.markdown('''
+This app demonstrates the answer-retrieval capabilities of a fine-tuned RoBERTa (Robustly optimized Bidirectional Encoder Representations from Transformers) model.
+
+Please type in a question and click submit.  When you do, a few things will happen:
+1. A Wikipedia search will be performed based on your question
+2. Candidate passages will be ranked based on a similarity score as compared to your question
+3. RoBERTa will search the best candidate passages to find the answer to your question
+
+If the model cannot find the answer to your question, it will tell you so.
+''')
+with st.expander('Click to read more about the model...'):
+    st.markdown('''
+* [Click here](https://huggingface.co/etweedy/roberta-base-squad-v2) to visit the Hugging Face model card for this fine-tuned model.
+* To create this model, the [RoBERTa base model](https://huggingface.co/roberta-base) was fine-tuned on Version 2 of [SQuAD (Stanford Question Answering Dataset)](https://huggingface.co/datasets/squad_v2), a dataset of context-question-answer triples.
+* The objective of the model is "extractive question answering", the task of retrieving the answer to the question from a given context text corpus.
+* SQuAD Version 2 incorporates the 100,000 samples from Version 1.1, along with 50,000 'unanswerable' questions, i.e. samples in the question cannot be answered using the context given.
+* The original base RoBERTa model was introduced in [this paper](https://arxiv.org/abs/1907.11692) and [this repository](https://github.com/facebookresearch/fairseq/tree/main/examples/roberta).  Here's a citation for that base model:
+```bibtex
+@article{DBLP:journals/corr/abs-1907-11692,
+  author    = {Yinhan Liu and
+               Myle Ott and
+               Naman Goyal and
+               Jingfei Du and
+               Mandar Joshi and
+               Danqi Chen and
+               Omer Levy and
+               Mike Lewis and
+               Luke Zettlemoyer and
+               Veselin Stoyanov},
+  title     = {RoBERTa: {A} Robustly Optimized {BERT} Pretraining Approach},
+  journal   = {CoRR},
+  volume    = {abs/1907.11692},
+  year      = {2019},
+  url       = {http://arxiv.org/abs/1907.11692},
+  archivePrefix = {arXiv},
+  eprint    = {1907.11692},
+  timestamp = {Thu, 01 Aug 2019 08:59:33 +0200},
+  biburl    = {https://dblp.org/rec/journals/corr/abs-1907-11692.bib},
+  bibsource = {dblp computer science bibliography, https://dblp.org}
+}
+```
+''')
+# st.markdown('''
+# Please type or paste a context paragraph and question you'd like to ask about it.  The model will attempt to answer the question based on the context you provided.  If the model cannot find the answer in the context, it will tell you so - the model is also trained to recognize when the context doesn't provide the answer.
+
+# Your results will appear below the question field when the model is finished running.
+
+# Alternatively, you can try an example by clicking one of the buttons below:
+# ''')
+
+# Generate containers in order
+# example_container = st.container()
+input_container = st.container()
+button_container = st.container()
+response_container = st.container()
+
+###########################
+### Populate containers ###
+###########################
+
+# Populate example button container
+# with example_container:
+#     ex_cols = st.columns(len(ex_q)+1)
+#     for i in range(len(ex_q)):
+#         with ex_cols[i]:
+#             st.button(
+#                 label = f'Try example {i+1}',
+#                 key = f'ex_button_{i+1}',
+#                 on_click = fill_in_example,
+#                 args=(i,),
+#             )
+#     with ex_cols[-1]:
+#         st.button(
+#             label = "Clear all fields",
+#             key = "clear_button",
+#             on_click = clear_boxes,
+#         )
+
+# Populate user input container
+with input_container:
+    with st.form(key='input_form',clear_on_submit=False):
+        # Question input field
+        question = st.text_input(
+            label='Question',
+            value=st.session_state['question'],
+            key='question_field',
+            label_visibility='hidden',
+            placeholder='Enter your question here.',
+        )
+        # Form submit button
+        query_submitted = st.form_submit_button("Submit")
+        if query_submitted:
+            # update question, context in session state
+            st.session_state['question'] = question
+            with st.spinner('Retrieving documentation...'):
+                query = generate_query(nlp,question)
+                retriever = ContextRetriever()
+                retriever.get_pageids(query)
+                retriever.get_pages()
+                retriever.get_paragraphs()
+                retriever.rank_paragraphs(question)
+            with st.spinner('Generating response...'):
+                # Loop through best_paragraph contexts
+                # looking for answer in each
+                best_answer = ""
+                for context in retriever.best_paragraphs:
+                    input = {
+                        'context':context,
+                        'question':st.session_state['question'],
+                    }
+                    # Pass to QA pipeline
+                    response = qa_pipeline(**input)
+                    if response['answer']!='':
+                        best_answer = response['answer']
+                        best_context = context
+                        break
+            # Update response in session state
+            if best_answer == "":
+                st.session_state['response'] = "I cannot find the answer to your question."
+            else:
+                st.session_state['response'] = f"""
+                    My answer is: {best_answer}
+                    
+                    ...and here's where I found it:
+                    
+                    {best_context}
+                """
+
+# Button for clearing the form
+with button_container:
+    st.button(
+            label = "Clear all fields",
+            key = "clear_button",
+            on_click = clear_boxes,
+        )
+
+# Display response
+with response_container:
+    st.write(st.session_state['response'])
+            

lib/.ipynb_checkpoints/utils-checkpoint.py

@@ -0,0 +1,154 @@
+import requests, wikipedia, re, spacy
+from rank_bm25 import BM25Okapi
+import torch
+from datasets import Dataset
+from torch.utils.data import DataLoader
+from transformers import (
+    AutoTokenizer,
+    AutoModelForQuestionAnswering,
+    pipeline,
+)
+
+class QueryProcessor:
+    """
+    Processes text into queries using a spaCy model
+    """
+    def __init__(self):
+        self.keep = {'PROPN', 'NUM', 'VERB', 'NOUN', 'ADJ'}
+        self.nlp = spacy.load(
+            'en_core_web_sm',
+            disable = ['ner','parser','textcat']
+        )
+
+    def generate_query(self,text):
+        """
+        Process text into a search query,
+        only retaining nouns, proper nouns numerals, verbs, and adjectives
+        Parameters:
+        -----------
+        text : str
+            The input text to be processed
+        Returns:
+        --------
+        query : str
+            The condensed search query
+        """
+        tokens = self.nlp(text)
+        query = ' '.join(token.text for token in tokens \
+                         if token.pos_ in self.keep)
+        return query
+
+class ContextRetriever:
+    """
+    Retrieves documents from Wikipedia based on a query,
+    and prepared context paragraphs for a RoBERTa model
+    """
+    def __init__(self,url='https://en.wikipedia.org/w/api.php'):
+        self.url = url
+        self.pageids = None
+        self.pages = None
+        self.paragraphs = None
+        
+    def get_pageids(self,query):
+        """
+        Retrieve page ids corresponding to a search query
+        Parameters:
+        -----------
+        query : str
+            A query to use for Wikipedia page search
+        Returns: None, but stores:
+        --------
+        self.pageids : list(int)
+            A list of Wikipedia page ids corresponding to search results
+        """
+        params = {
+            'action':'query',
+            'list':'search',
+            'srsearch':query,
+            'format':'json',
+        }
+        results = requests.get(self.url, params=params).json()
+        pageids = [page['pageid'] for page in results['query']['search']]
+        self.pageids = pageids
+    
+    def get_pages(self):
+        """
+        Use MediaWiki API to retrieve page content corresponding to
+        entries of self.pageids
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.pages : list(str)
+            Entries are content of pages corresponding to entries of self.pageid
+        """
+        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
+        self.pages = []
+        for pageid in self.pageids:
+            try:
+                self.pages.append(wikipedia.page(pageid=pageid,auto_suggest=False).content)
+            except wikipedia.DisambiguationError as e:
+                continue
+
+    def get_paragraphs(self):
+        """
+        Process self.pages into list of paragraphs from pages
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.paragraphs : list(str)
+            List of paragraphs from all pages in self.pages, in order of self.pages
+        """
+        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
+        # Content from WikiMedia has these headings. We only grab content appearing
+        # before the first instance of any of these
+        pattern = '|'.join([
+            '== References ==',
+            '== Further reading ==',
+            '== External links',
+            '== See also ==',
+            '== Sources ==',
+            '== Notes ==',
+            '== Further references ==',
+            '== Footnotes ==',
+            '=== Notes ===',
+            '=== Sources ===',
+            '=== Citations ===',
+        ])
+        pattern = re.compile(pattern)
+        paragraphs = []
+        for page in self.pages:
+            # Truncate page to the first index of the start of a matching heading,
+            # or the end of the page if no matches exist
+            idx = min([match.start() for match in pattern.finditer(page)]+[len(page)])
+            page = page[:idx]
+            # Split into paragraphs, omitting lines with headings (start with '='),
+            # empty lines, or lines like '\t\t' or '\t\t\t' which sometimes appear
+            paragraphs += [
+                p for p in page.split('\n') if p \
+                and not p.startswith('=') \
+                and not p.startswith('\t\t')
+            ]
+        self.paragraphs = paragraphs
+
+    def rank_paragraphs(self,query,topn=10):
+        """
+        Ranks the elements of self.paragraphs in descending order
+        by relevance to query using BM25F, and returns top topn results
+        Parameters:
+        -----------
+        query : str
+            The query to use in ranking paragraphs by relevance
+        topn : int
+            The number of most relevant paragraphs to return
+        Returns: None, but stores
+        --------
+        self.best_paragraphs : list(str)
+            The topn most relevant paragraphs to the query
+        """
+        tokenized_paragraphs = [p.split(" ") for p in self.paragraphs]
+        bm25 = BM25Okapi(tokenized_paragraphs)
+        tokenized_query = query.split(" ")
+        self.best_paragraphs = bm25.get_top_n(tokenized_query,self.paragraphs,n=topn)
+

lib/utils.py

@@ -0,0 +1,125 @@
+import requests, wikipedia, re
+from rank_bm25 import BM25Okapi
+import torch
+from datasets import Dataset
+from torch.utils.data import DataLoader
+from transformers import (
+    AutoTokenizer,
+    AutoModelForQuestionAnswering,
+    pipeline,
+)
+
+class ContextRetriever:
+    """
+    Retrieves documents from Wikipedia based on a query,
+    and prepared context paragraphs for a RoBERTa model
+    """
+    def __init__(self,url='https://en.wikipedia.org/w/api.php'):
+        self.url = url
+        self.pageids = None
+        self.pages = None
+        self.paragraphs = None
+        
+    def get_pageids(self,query):
+        """
+        Retrieve page ids corresponding to a search query
+        Parameters:
+        -----------
+        query : str
+            A query to use for Wikipedia page search
+        Returns: None, but stores:
+        --------
+        self.pageids : list(int)
+            A list of Wikipedia page ids corresponding to search results
+        """
+        params = {
+            'action':'query',
+            'list':'search',
+            'srsearch':query,
+            'format':'json',
+        }
+        results = requests.get(self.url, params=params).json()
+        pageids = [page['pageid'] for page in results['query']['search']]
+        self.pageids = pageids
+    
+    def get_pages(self):
+        """
+        Use MediaWiki API to retrieve page content corresponding to
+        entries of self.pageids
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.pages : list(str)
+            Entries are content of pages corresponding to entries of self.pageid
+        """
+        assert self.pageids is not None, "No pageids exist. Get pageids first using self.get_pageids"
+        self.pages = []
+        for pageid in self.pageids:
+            try:
+                self.pages.append(wikipedia.page(pageid=pageid,auto_suggest=False).content)
+            except wikipedia.DisambiguationError as e:
+                continue
+
+    def get_paragraphs(self):
+        """
+        Process self.pages into list of paragraphs from pages
+        Parameters: None
+        -----------
+        Returns: None, but stores
+        --------
+        self.paragraphs : list(str)
+            List of paragraphs from all pages in self.pages, in order of self.pages
+        """
+        assert self.pages is not None, "No page content exists. Get pages first using self.get_pages"
+        # Content from WikiMedia has these headings. We only grab content appearing
+        # before the first instance of any of these
+        pattern = '|'.join([
+            '== References ==',
+            '== Further reading ==',
+            '== External links',
+            '== See also ==',
+            '== Sources ==',
+            '== Notes ==',
+            '== Further references ==',
+            '== Footnotes ==',
+            '=== Notes ===',
+            '=== Sources ===',
+            '=== Citations ===',
+        ])
+        pattern = re.compile(pattern)
+        paragraphs = []
+        for page in self.pages:
+            # Truncate page to the first index of the start of a matching heading,
+            # or the end of the page if no matches exist
+            idx = min([match.start() for match in pattern.finditer(page)]+[len(page)])
+            page = page[:idx]
+            # Split into paragraphs, omitting lines with headings (start with '='),
+            # empty lines, or lines like '\t\t' or '\t\t\t' which sometimes appear
+            paragraphs += [
+                p for p in page.split('\n') if p \
+                and not p.startswith('=') \
+                and not p.startswith('\t\t')
+            ]
+        self.paragraphs = paragraphs
+
+    def rank_paragraphs(self,query,topn=10):
+        """
+        Ranks the elements of self.paragraphs in descending order
+        by relevance to query using BM25F, and returns top topn results
+        Parameters:
+        -----------
+        query : str
+            The query to use in ranking paragraphs by relevance
+        topn : int
+            The number of most relevant paragraphs to return
+        Returns: None, but stores
+        --------
+        self.best_paragraphs : list(str)
+            The topn most relevant paragraphs to the query
+        """
+        tokenized_paragraphs = [p.split(" ") for p in self.paragraphs]
+        bm25 = BM25Okapi(tokenized_paragraphs)
+        tokenized_query = query.split(" ")
+        self.best_paragraphs = bm25.get_top_n(tokenized_query,self.paragraphs,n=topn)
+

requirements.txt

@@ -0,0 +1,92 @@
+aiohttp==3.8.4
+aiosignal==1.3.1
+altair==5.0.1
+async-timeout==4.0.2
+attrs==23.1.0
+beautifulsoup4==4.12.2
+blinker==1.6.2
+blis==0.7.9
+cachetools==5.3.1
+catalogue==2.0.8
+certifi==2023.5.7
+charset-normalizer==3.2.0
+click==8.1.4
+confection==0.1.0
+cymem==2.0.7
+datasets==2.13.1
+decorator==5.1.1
+dill==0.3.6
+filelock==3.12.2
+frozenlist==1.3.3
+fsspec==2023.6.0
+gitdb==4.0.10
+GitPython==3.1.31
+huggingface-hub==0.16.4
+idna==3.4
+importlib-metadata==6.8.0
+Jinja2==3.1.2
+jsonschema==4.18.0
+jsonschema-specifications==2023.6.1
+langcodes==3.3.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+mdurl==0.1.2
+mpmath==1.3.0
+multidict==6.0.4
+multiprocess==0.70.14
+murmurhash==1.0.9
+networkx==3.1
+numpy==1.25.1
+packaging==23.1
+pandas==2.0.3
+pathy==0.10.2
+Pillow==9.5.0
+preshed==3.0.8
+protobuf==4.23.4
+pyarrow==12.0.1
+pydantic==1.10.11
+pydeck==0.8.1b0
+Pygments==2.15.1
+Pympler==1.0.1
+python-dateutil==2.8.2
+pytz==2023.3
+pytz-deprecation-shim==0.1.0.post0
+PyYAML==6.0
+rank-bm25==0.2.2
+referencing==0.29.1
+regex==2023.6.3
+requests==2.31.0
+rich==13.4.2
+rpds-py==0.8.10
+safetensors==0.3.1
+six==1.16.0
+smart-open==6.3.0
+smmap==5.0.0
+soupsieve==2.4.1
+spacy==3.6.0
+en_core_web_sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.6.0/en_core_web_sm-3.6.0-py3-none-any.whl
+spacy-legacy==3.0.12
+spacy-loggers==1.0.4
+srsly==2.4.6
+streamlit==1.24.1
+sympy==1.12
+tenacity==8.2.2
+thinc==8.1.10
+tokenizers==0.13.3
+toml==0.10.2
+toolz==0.12.0
+torch==2.0.1
+tornado==6.3.2
+tqdm==4.65.0
+transformers==4.30.2
+typer==0.8.0
+typing_extensions==4.7.1
+tzdata==2023.3
+tzlocal==4.3.1
+urllib3==2.0.3
+validators==0.20.0
+wasabi==1.1.2
+wikipedia==1.4.0
+xxhash==3.2.0
+yarl==1.9.2
+zipp==3.16.0