change readme and improvements

README.md

@@ -5,7 +5,7 @@ colorFrom: yellow
 colorTo: green
 sdk: streamlit
 sdk_version: 1.10.0
-app_file: code/quiz_gen_new2.py
+app_file: code/quiz_gen_new3.py
 pinned: false
 ---
 

code/helper.py.py

@@ -0,0 +1,155 @@
+import nltk, pke, string, torch, requests, random
+from nltk.tokenize import sent_tokenize
+from nltk.corpus import stopwords
+from flashtext import KeywordProcessor
+
+def postprocesstext(content):
+    final=""
+    for sent in sent_tokenize(content):
+        sent = sent.capitalize()
+        final = final +" "+sent
+    return final
+
+def summarizer(text,model,tokenizer):
+    text = text.strip().replace("\n"," ")
+    text = "summarize: "+text
+    # print (text)
+    max_len = 512
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    encoding = tokenizer.encode_plus(text,max_length=max_len, pad_to_max_length=False,\
+        truncation=True, return_tensors="pt").to(device)
+    input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+    outs = model.generate(input_ids=input_ids,
+                                  attention_mask=attention_mask,
+                                  early_stopping=True,
+                                  num_beams=3,
+                                  num_return_sequences=1,
+                                  no_repeat_ngram_size=2,
+                                  min_length = 75,
+                                  max_length=300)
+    dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+    summary = dec[0]
+    summary = postprocesstext(summary)
+    summary= summary.strip()
+    return summary
+
+def get_nouns_multipartite(content):
+    out=[]
+    try:
+        extractor = pke.unsupervised.MultipartiteRank()
+        extractor.load_document(input=content)
+        #    not contain punctuation marks or stopwords as candidates.
+        pos = {'PROPN','NOUN'}
+        #pos = {'PROPN','NOUN'}
+        stoplist = list(string.punctuation)
+        stoplist += ['-lrb-', '-rrb-', '-lcb-', '-rcb-', '-lsb-', '-rsb-']
+        stoplist += stopwords.words('english')
+        # extractor.candidate_selection(pos=pos, stoplist=stoplist)
+        extractor.candidate_selection(pos=pos)
+        # 4. build the Multipartite graph and rank candidates using random walk,
+        #    alpha controls the weight adjustment mechanism, see TopicRank for
+        #    threshold/method parameters.
+        extractor.candidate_weighting(alpha=1.1,
+                                      threshold=0.75,
+                                      method='average')
+        keyphrases = extractor.get_n_best(n=15)
+        for val in keyphrases:
+            out.append(val[0])
+    except Exception as e:
+        out = []
+        #traceback.print_exc()
+        print("EXCEPTION: {}".format(e))
+    return out
+
+def filter_overlap_words(l):
+    nl = []
+    for i in range(len(l)):
+        temp_l = l[0:i]+l[i+1:]
+        inside=False
+        for j in temp_l:
+            if l[i] not in j:
+                if l[i] not in nl:
+                    nl.append(l[i])
+                    inside = True
+            else:
+                if inside:
+                    nl.remove(l[i])
+                break
+    return nl    
+
+def get_keywords(originaltext,summarytext):
+    keywords = get_nouns_multipartite(originaltext)
+    print ("keywords unsummarized: ",keywords)
+    keyword_processor = KeywordProcessor()
+    for keyword in keywords:
+        keyword_processor.add_keyword(keyword)
+    keywords_found = keyword_processor.extract_keywords(summarytext)
+    keywords_found = list(set(keywords_found))
+    print("keywords_found in summarized: ",keywords_found)
+    important_keywords =[]
+    for keyword in keywords:
+        if keyword in keywords_found:
+            important_keywords.append(keyword)
+    ## find keywords which don't have common word .. 
+    imp_words = filter_overlap_words(important_keywords)
+    return imp_words[:4]
+
+def get_question(context,answer,model,tokenizer):
+    text = "context: {} answer: {}".format(context,answer)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    encoding = tokenizer.encode_plus(text,max_length=384, pad_to_max_length=False,\
+        truncation=True, return_tensors="pt").to(device)
+    input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
+    outs = model.generate(input_ids=input_ids,
+                                  attention_mask=attention_mask,
+                                  early_stopping=True,
+                                  num_beams=5,
+                                  num_return_sequences=1,
+                                  no_repeat_ngram_size=2,
+                                  max_length=72)
+    dec = [tokenizer.decode(ids,skip_special_tokens=True) for ids in outs]
+    Question = dec[0].replace("question:","")
+    Question= Question.strip()
+    return Question
+
+def get_related_word(word):
+    url = "https://api.datamuse.com/words"
+    querystring = {"ml":word}
+    responses = requests.request("GET", url, params=querystring)
+    related_words = []
+    count = 0
+    responses = responses.json()
+    for res in responses:
+        if count >= 4:
+            break
+        if res["word"]!=word and res["word"]!="":
+            related_words.append(res["word"])
+            count += 1
+    return related_words
+
+def get_final_option_list(ans,other_options):
+    option1 = ans
+    option2,option3,option4 = "dummy","dummy","dummy"
+    try:
+        option2 = other_options[0]
+    except:
+        pass
+    try:
+        option3 = other_options[1]
+    except:
+        pass
+    try:
+        option4 = other_options[2]
+    except:
+        pass
+    final_options = [option1,option2,option3,option4]
+    random.shuffle(final_options)
+    final_options = tuple(final_options)
+    ans_index= 0
+    for i in range(4):
+        if final_options[i] == ans:
+            ans_index = i
+    return final_options, ans_index
+
+def load_raw_text():
+    return " Elon Musk has shown again he can influence the digital currency market with just his tweets. After saying that his electric vehicle-making company Tesla will not accept payments in Bitcoin because of environmental concerns, he tweeted that he was working with developers of Dogecoin to improve system transaction efficiency. Following the two distinct statements from him, the world's largest cryptocurrency hit a two-month low, while Dogecoin rallied by about 20 percent. The SpaceX CEO has in recent months often tweeted in support of Dogecoin, but rarely for Bitcoin. In a recent tweet, Musk put out a statement from Tesla that it was concerned about the rapidly increasing use of fossil fuels for Bitcoin (price in India) mining and transaction, and hence was suspending vehicle purchases using the cryptocurrency. A day later he again tweeted saying, To be clear, I strongly believe in crypto, but it can't drive a massive increase in fossil fuel use, especially coal. It triggered a downward spiral for Bitcoin value but the cryptocurrency has stabilised since. A number of Twitter users welcomed Musk's statement. One of them said it's time people started realising that Dogecoin is here to stay and another referred to Musk's previous assertion that crypto could become the world's future currency."

code/quiz_gen_new2.py

@@ -1,16 +1,3 @@
-# !pip install --quiet transformers==4.5.0
-# !pip install --quiet sentencepiece==0.1.95
-# !pip install --quiet git+https://github.com/boudinfl/pke.git@dc4d5f21e0ffe64c4df93c46146d29d1c522476b
-# pip install git+https://github.com/boudinfl/pke.git
-# !pip install --quiet nltk==3.2.5
-
-
-# pip install git+https://github.com/boudinfl/pke.git@dc4d5f21e0ffe64c4df93c46146d29d1c522476b
-# pip install spacy==3.1.3
-# pip install textwrap3==0.9.2
-# pip install flashtext==2.7
-
-
 import streamlit as st
 from textwrap3 import wrap
 from flashtext import KeywordProcessor
@@ -18,7 +5,6 @@ import torch, random, nltk, string, traceback, sys, os, requests, datetime
 import numpy as np
 import pandas as pd
 from transformers import T5ForConditionalGeneration,T5Tokenizer
-# st.write("Import pke")
 import pke
 
 def set_seed(seed: int):
@@ -29,7 +15,6 @@ def set_seed(seed: int):
 
 set_seed(42)
 
-
 @st.cache(allow_output_mutation = True)
 def load_model():
     nltk.download('punkt')
@@ -37,13 +22,15 @@ def load_model():
     nltk.download('wordnet')
     nltk.download('stopwords')
     nltk.download('wordnet')
-    nltk.download('omw-1.4')    
-    summary_model = T5ForConditionalGeneration.from_pretrained('t5-base')
-    summary_tokenizer = T5Tokenizer.from_pretrained('t5-base')
+    nltk.download('omw-1.4')
+    summary_mod_name = os.environ.getattribute("summary_mod_name")
+    question_mod_name = os.environ.getattribute("question_mod_name")
+    summary_model = T5ForConditionalGeneration.from_pretrained(summary_mod_name)
+    summary_tokenizer = T5Tokenizer.from_pretrained(summary_mod_name)
     device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
     summary_model = summary_model.to(device)
-    question_model = T5ForConditionalGeneration.from_pretrained('ramsrigouthamg/t5_squad_v1')
-    question_tokenizer = T5Tokenizer.from_pretrained('ramsrigouthamg/t5_squad_v1')
+    question_model = T5ForConditionalGeneration.from_pretrained(question_mod_name)
+    question_tokenizer = T5Tokenizer.from_pretrained(question_mod_name)
     question_model = question_model.to(device)
     return summary_model, summary_tokenizer, question_tokenizer, question_model 
 

code/quiz_gen_new3.py

@@ -0,0 +1,124 @@
+import streamlit as st
+from textwrap3 import wrap
+from flashtext import KeywordProcessor
+import torch, random, nltk, string, traceback, sys, os, requests, datetime
+import numpy as np
+import pandas as pd
+from transformers import T5ForConditionalGeneration,T5Tokenizer
+import pke
+from helper import postprocesstext, summarizer, get_nouns_multipartite, get_keywords,\
+    get_question, get_related_word, get_final_option_list, load_raw_text
+
+
+def set_seed(seed: int):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+set_seed(42)
+
+@st.cache(allow_output_mutation = True)
+def load_model():
+    nltk.download('punkt')
+    nltk.download('brown')
+    nltk.download('wordnet')
+    nltk.download('stopwords')
+    nltk.download('wordnet')
+    nltk.download('omw-1.4')
+    summary_mod_name = os.environ.getattribute("summary_mod_name")
+    question_mod_name = os.environ.getattribute("question_mod_name")
+    summary_model = T5ForConditionalGeneration.from_pretrained(summary_mod_name)
+    summary_tokenizer = T5Tokenizer.from_pretrained(summary_mod_name)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    summary_model = summary_model.to(device)
+    question_model = T5ForConditionalGeneration.from_pretrained(question_mod_name)
+    question_tokenizer = T5Tokenizer.from_pretrained(question_mod_name)
+    question_model = question_model.to(device)
+    return summary_model, summary_tokenizer, question_tokenizer, question_model 
+
+from nltk.corpus import wordnet as wn
+from nltk.tokenize import sent_tokenize
+from nltk.corpus import stopwords
+
+def csv_downloader(df):
+    res = df.to_csv(index=False,sep="\t").encode('utf-8')
+    st.download_button(
+    label="Download logs data as CSV separated by tab",
+    data=res,
+    file_name='df_quiz_log_file_v1.csv',
+    mime='text/csv')
+
+def load_file():
+    """Load text from file"""
+    uploaded_file = st.file_uploader("Upload Files",type=['txt'])
+    if uploaded_file is not None:
+        if uploaded_file.type == "text/plain":
+            raw_text = str(uploaded_file.read(),"utf-8")
+        return raw_text
+
+st.markdown('![Visitor count](https://shields-io-visitor-counter.herokuapp.com/badge?page=https://share.streamlit.io/https://huggingface.co/spaces/aakashgoel12/getmcq&label=VisitorsCount&labelColor=000000&logo=GitHub&logoColor=FFFFFF&color=1D70B8&style=for-the-badge)')
+
+# Loading Model
+summary_model, summary_tokenizer, question_tokenizer, question_model =load_model()
+
+# App title and description
+st.title("Exam Assistant")
+st.write("Upload text, Get ready for answering autogenerated questions")
+
+# Load file
+st.text("Disclaimer: This app stores user's input for model improvement purposes !!")
+
+# Load file
+
+default_text = load_raw_text()
+raw_text = st.text_area("Enter text here", default_text, height=400, max_chars=1000000, )
+
+# raw_text = load_file()
+start_time = str(datetime.datetime.now())
+if raw_text != None and raw_text != '':
+
+    # Display text
+    # with st.expander("See text"):
+    #     st.write(raw_text)
+
+    summary_text = summarizer(raw_text,summary_model,summary_tokenizer)
+    ans_list =  get_keywords(raw_text,summary_text)
+    questions = []
+    option1=[];option2=[];option3=[];option4=[]
+    for idx,ans in enumerate(ans_list):
+        ques = get_question(summary_text,ans,question_model,question_tokenizer)
+        other_options = get_related_word(ans)
+        final_options = get_final_option_list(ans,other_options)
+        final_options, ans_index = get_final_option_list(ans,other_options)
+        option1.append(final_options[0]);option2.append(final_options[1])
+        option3.append(final_options[2]);option4.append(final_options[3])                        
+        if ques not in questions:
+            # st.write(final_options)
+            html_str = f"""
+            <div>
+            <p>
+            {idx+1}: <b> {ques} </b>
+            </p>
+            </div>
+            """
+            html_str += f' <p style="color:Green;"><b> {final_options[0]} </b></p> ' if ans_index == 0 else f' <p><b> {final_options[0]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[1]} </b></p> ' if ans_index == 1 else f' <p><b> {final_options[1]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[2]} </b></p> ' if ans_index == 2 else f' <p><b> {final_options[2]} </b></p> '
+            html_str += f' <p style="color:Green;"><b> {final_options[3]} </b></p> ' if ans_index == 3 else f' <p><b> {final_options[3]} </b></p> '
+            html_str += f"""
+            """
+            st.markdown(html_str , unsafe_allow_html=True)
+            st.markdown("-----")
+        questions.append(ques)
+    output_path = "results/df_quiz_log_file_v1.csv"
+    res_df = pd.DataFrame({"TimeStamp":[start_time]*len(questions),\
+        "Input":[str(raw_text)]*len(questions),\
+        "Question":questions,"Option1":[option1],\
+        "Option2":[option2],\
+        "Option3":[option3],\
+        "Option4":[option4],\
+        "Correct Answer":ans_list})
+    res_df.to_csv(output_path, mode='a', index=False, sep="\t", header= not os.path.exists(output_path))
+    st.dataframe(pd.read_csv(output_path,sep="\t").tail(5))
+    csv_downloader(pd.read_csv(output_path,sep="\t"))