spacy changes 5

code/quiz_gen_new.py

@@ -225,6 +225,7 @@ if raw_text != None and raw_text != '':
     summary_text = summarizer(raw_text,summary_model,summary_tokenizer)
     ans_list =  get_keywords(raw_text,summary_text)
     questions = []
+    
     for ans in ans_list:
         ques = get_question(summary_text,ans,question_model,question_tokenizer)
         if ques not in questions:    

code/quiz_gen_new2.py

@@ -0,0 +1,290 @@
+# !pip install --quiet transformers==4.5.0
+# !pip install --quiet sentencepiece==0.1.95
+# !pip install --quiet git+https://github.com/boudinfl/pke.git@dc4d5f21e0ffe64c4df93c46146d29d1c522476b
+# pip install git+https://github.com/boudinfl/pke.git
+# !pip install --quiet nltk==3.2.5
+
+
+# pip install git+https://github.com/boudinfl/pke.git@dc4d5f21e0ffe64c4df93c46146d29d1c522476b
+# pip install spacy==3.1.3
+# pip install textwrap3==0.9.2
+# pip install flashtext==2.7
+
+
+import streamlit as st
+from textwrap3 import wrap
+from flashtext import KeywordProcessor
+import torch, random, nltk, string, traceback, sys, os, requests, datetime
+import numpy as np
+import pandas as pd
+from transformers import T5ForConditionalGeneration,T5Tokenizer
+# st.write("Import pke")
+import pke
+
+def set_seed(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+set_seed(42)
+
+
+@st.cache(allow_output_mutation = True)
+def load_model():
+    nltk.download('punkt')
+    nltk.download('brown')
+    nltk.download('wordnet')
+    nltk.download('stopwords')
+    nltk.download('wordnet')
+    nltk.download('omw-1.4')    
+    summary_model = T5ForConditionalGeneration.from_pretrained('t5-base')
+    summary_tokenizer = T5Tokenizer.from_pretrained('t5-base')
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    summary_model = summary_model.to(device)
+    question_model = T5ForConditionalGeneration.from_pretrained('ramsrigouthamg/t5_squad_v1')
+    question_tokenizer = T5Tokenizer.from_pretrained('ramsrigouthamg/t5_squad_v1')
+    question_model = question_model.to(device)
+    return summary_model, summary_tokenizer, question_tokenizer, question_model 
+
+from nltk.corpus import wordnet as wn
+from nltk.tokenize import sent_tokenize
+from nltk.corpus import stopwords
+
+def postprocesstext (content):
+    final=""
+    for sent in sent_tokenize(content):
+        sent = sent.capitalize()
+        final = final +" "+sent
+    return final
+
+def summarizer(text,model,tokenizer):
+    text = text.strip().replace("\n"," ")
+    text = "summarize: "+text
+    # print (text)
+    max_len = 512
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    encoding = tokenizer.encode_plus(text,max_length=max_len, pad_to_max_length=False,\
+        truncation=True, return_tensors="pt").to(device)
+    input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+    outs = model.generate(input_ids=input_ids,
+                                  attention_mask=attention_mask,
+                                  early_stopping=True,
+                                  num_beams=3,
+                                  num_return_sequences=1,
+                                  no_repeat_ngram_size=2,
+                                  min_length = 75,
+                                  max_length=300)
+    dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+    summary = dec[0]
+    summary = postprocesstext(summary)
+    summary= summary.strip()
+    return summary
+
+
+def get_nouns_multipartite(content):
+    out=[]
+    try:
+        extractor = pke.unsupervised.MultipartiteRank()
+        extractor.load_document(input=content)
+        #    not contain punctuation marks or stopwords as candidates.
+        pos = {'PROPN','NOUN'}
+        #pos = {'PROPN','NOUN'}
+        stoplist = list(string.punctuation)
+        stoplist += ['-lrb-', '-rrb-', '-lcb-', '-rcb-', '-lsb-', '-rsb-']
+        stoplist += stopwords.words('english')
+        # extractor.candidate_selection(pos=pos, stoplist=stoplist)
+        extractor.candidate_selection(pos=pos)
+        # 4. build the Multipartite graph and rank candidates using random walk,
+        #    alpha controls the weight adjustment mechanism, see TopicRank for
+        #    threshold/method parameters.
+        extractor.candidate_weighting(alpha=1.1,
+                                      threshold=0.75,
+                                      method='average')
+        keyphrases = extractor.get_n_best(n=15)
+        for val in keyphrases:
+            out.append(val[0])
+    except Exception as e:
+        out = []
+        traceback.print_exc()
+        print("EXCEPTION: {}".format(e))
+    return out
+
+def get_keywords(originaltext,summarytext):
+    keywords = get_nouns_multipartite(originaltext)
+    print ("keywords unsummarized: ",keywords)
+    keyword_processor = KeywordProcessor()
+    for keyword in keywords:
+        keyword_processor.add_keyword(keyword)
+    keywords_found = keyword_processor.extract_keywords(summarytext)
+    keywords_found = list(set(keywords_found))
+    print("keywords_found in summarized: ",keywords_found)
+    important_keywords =[]
+    for keyword in keywords:
+        if keyword in keywords_found:
+            important_keywords.append(keyword)
+    return important_keywords[:5]
+
+def get_question(context,answer,model,tokenizer):
+    text = "context: {} answer: {}".format(context,answer)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    encoding = tokenizer.encode_plus(text,max_length=384, pad_to_max_length=False,\
+        truncation=True, return_tensors="pt").to(device)
+    input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+    outs = model.generate(input_ids=input_ids,
+                                  attention_mask=attention_mask,
+                                  early_stopping=True,
+                                  num_beams=5,
+                                  num_return_sequences=1,
+                                  no_repeat_ngram_size=2,
+                                  max_length=72)
+    dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+    Question = dec[0].replace("question:","")
+    Question= Question.strip()
+    return Question
+
+
+def csv_downloader(df):
+    res = df.to_csv(index=False,sep="\t").encode('utf-8')
+    st.download_button(
+    label="Download logs data as CSV separated by tab",
+    data=res,
+    file_name='df_quiz_log_file.csv',
+    mime='text/csv')
+
+def load_file():
+    """Load text from file"""
+    uploaded_file = st.file_uploader("Upload Files",type=['txt'])
+    if uploaded_file is not None:
+        if uploaded_file.type == "text/plain":
+            raw_text = str(uploaded_file.read(),"utf-8")
+        return raw_text
+
+
+def get_related_word(word):
+    url = "https://api.datamuse.com/words"
+    querystring = {"ml":word}
+    responses = requests.request("GET", url, params=querystring)
+    related_words = []
+    count = 0
+    responses = responses.json()
+    for res in responses:
+        if count >= 4:
+            break
+        if res["word"]!=word and res["word"]!="":
+            related_words.append(res["word"])
+            count += 1
+    return related_words
+
+
+def get_final_option_list(ans,other_options):
+    option1 = ans
+    option2,option3,option4 = "dummy","dummy","dummy"
+    try:
+        option2 = other_options[0]
+    except:
+        pass
+    try:
+        option3 = other_options[1]
+    except:
+        pass
+    try:
+        option4 = other_options[2]
+    except:
+        pass
+    final_options = [option1,option2,option3,option4]
+    random.shuffle(final_options)
+    final_options = tuple(final_options)
+    ans_index= 0
+    for i in range(4):
+        if final_options[i] == ans:
+            ans_index = i
+    return final_options, ans_index
+
+def load_raw_text():
+    return " Elon Musk has shown again he can influence the digital currency market with just his tweets. After saying that his electric vehicle-making company Tesla will not accept payments in Bitcoin because of environmental concerns, he tweeted that he was working with developers of Dogecoin to improve system transaction efficiency. Following the two distinct statements from him, the world's largest cryptocurrency hit a two-month low, while Dogecoin rallied by about 20 percent. The SpaceX CEO has in recent months often tweeted in support of Dogecoin, but rarely for Bitcoin. In a recent tweet, Musk put out a statement from Tesla that it was concerned about the rapidly increasing use of fossil fuels for Bitcoin (price in India) mining and transaction, and hence was suspending vehicle purchases using the cryptocurrency. A day later he again tweeted saying, To be clear, I strongly believe in crypto, but it can't drive a massive increase in fossil fuel use, especially coal. It triggered a downward spiral for Bitcoin value but the cryptocurrency has stabilised since. A number of Twitter users welcomed Musk's statement. One of them said it's time people started realising that Dogecoin is here to stay and another referred to Musk's previous assertion that crypto could become the world's future currency."
+
+# def get_final_option_list(ans,other_options):
+#     option1 = ans
+#     option2,option3,option4 = "dummy","dummy","dummy"
+#     try:
+#         option2 = other_options[0]
+#     except:
+#         pass
+#     try:
+#         option3 = other_options[1]
+#     except:
+#         pass
+#     try:
+#         option4 = other_options[2]
+#     except:
+#         pass
+#     final_options = [option1,option2,option3,option4]
+#     #st.write(final_options)
+#     random.shuffle(final_options)
+#     #st.write(final_options)
+#     final_options = [None]+final_options 
+#     final_options = tuple(final_options)
+#     #st.write(final_options)    
+#     return final_options
+
+st.markdown('![Visitor count](https://shields-io-visitor-counter.herokuapp.com/badge?page=https://share.streamlit.io/https://huggingface.co/spaces/aakashgoel12/getmcq&label=VisitorsCount&labelColor=000000&logo=GitHub&logoColor=FFFFFF&color=1D70B8&style=for-the-badge)')
+
+# Loading Model
+summary_model, summary_tokenizer, question_tokenizer, question_model =load_model()
+
+# App title and description
+st.title("Exam Assistant")
+st.write("Upload text, Get ready for answering autogenerated questions")
+
+# Load file
+st.text("Disclaimer: This app stores user's input for model improvement purposes !!")
+
+# Load file
+
+default_text = load_raw_text()
+raw_text = st.text_area("Enter text here", default_text, height=400, max_chars=1000000, )
+
+# raw_text = load_file()
+start_time = str(datetime.datetime.now())
+if raw_text != None and raw_text != '':
+
+    # Display text
+    # with st.expander("See text"):
+    #     st.write(raw_text)
+
+    summary_text = summarizer(raw_text,summary_model,summary_tokenizer)
+    ans_list =  get_keywords(raw_text,summary_text)
+    questions = []
+    
+    for idx,ans in enumerate(ans_list):
+        ques = get_question(summary_text,ans,question_model,question_tokenizer)
+        if ques not in questions:    
+            other_options = get_related_word(ans)
+            final_options = get_final_option_list(ans,other_options)
+            final_options, ans_index = get_final_option_list(ans,other_options)
+            # st.write(final_options)
+            html_str = f"""
+            <div>
+            <p>
+            {idx+1}: <b> {ques} </b>
+            </p>
+            </div>
+            """
+            html_str += f' <p style="color:Green;"><b> {final_options[0]} </b></p> ' if ans_index == 0 else f' <p><b> {final_options[0]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[1]} </b></p> ' if ans_index == 1 else f' <p><b> {final_options[1]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[2]} </b></p> ' if ans_index == 2 else f' <p><b> {final_options[2]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[3]} </b></p> ' if ans_index == 3 else f' <p><b> {final_options[3]} </b></p> '
+            html_str += f"""
+            """
+            st.markdown(html_str , unsafe_allow_html=True)
+            st.markdown("-----")
+            # st.write("="*50)
+        questions.append(ques)
+    output_path = "results/df_quiz_log_file.csv"
+    res_df = pd.DataFrame({"TimeStamp":[start_time]*len(questions),\
+        "Input":[str(raw_text)]*len(questions),\
+        "Question":questions,"Answer":ans_list})
+    # res_df.to_csv(output_path, mode='a', index=False, sep="\t", header= not os.path.exists(output_path))
+    # st.dataframe(pd.read_csv(output_path,sep="\t").tail(5))
+    # csv_downloader(pd.read_csv(output_path,sep="\t"))