app.py

import pandas as pd
import requests
import isort
import black
import flair
import time
from bs4 import BeautifulSoup
import re
import numpy as np
import os

from flair.data import Sentence
from flair.models import SequenceTagger
from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline


import string
import textwrap
import tweepy
import gradio as gr





URL = "https://www.formula1.com/content/fom-website/en/latest/all.xml"

api_key = os.environ['api_key']
secret_api_key = os.environ['secret_api_key']
access_token = os.environ['access_token']
secret_access_token = os.environ['secret_access_token']
bearer_token = os.environ['bearer_token']

def get_xml(url):
    # xpath is only for formula1
    # use urllib.parse to check for formula1.com website or other news
    xml = pd.read_xml(url,xpath='channel/item')
    return xml

cols_list = ['title', 'description', 'link', 'creator', 'guid']

previous_xml = pd.DataFrame(columns=cols_list)


# care taken to only consider results where there are more words not a single word quotes
def extract_quote(string):
    # Use the re.findall function to extract the quoted text
    results = re.findall(r'[“\"](.*?)[”\"]', string)
    quotes = []
    for result in results:
        split_result = result.split()
        if len(split_result) >3:
            quotes.append(result)   
    
    return quotes



def get_names(text):
    # # load the NER tagger
    tagger = SequenceTagger.load('ner')
    
    sentence = Sentence(text)
    tagger.predict(sentence)
    
    names = []
    for label in sentence.get_labels('ner'):
        if label.value == "PER":       
            names.append(f"{label.data_point.text}")
            
     # convert to a set to remove some of the repetitions
    names = list(set(names))
        
    return names

def get_text(new_articles_df):
    """
    quotes outputs a list of quotes 
    """
    
    dfs_dict = {}
    
    for article in new_articles_df.iterrows():
         
        link = article[1]["guid"]
        request = requests.get(link)
        soup = BeautifulSoup(request.content, "html.parser")
        # class_ below will be different for different websites
        s = soup.find("div", class_="col-lg-8 col-xl-7 offset-xl-1 f1-article--content")
        lines = s.find_all("p")
        text_content = pd.DataFrame(data={"text": []})
        for i, line in enumerate(lines):
            df = pd.DataFrame(data={"text": [line.text]})
            text_content = pd.concat([text_content, df], ignore_index=True)

        strongs = s.find_all("strong")
        strong_content = pd.DataFrame(data={"text": []})
        for i, strong in enumerate(strongs):
            if i > 0:
                df = pd.DataFrame(data={"text": [strong.text]})
                strong_content = pd.concat([strong_content, df], ignore_index=True)
        # df has content
        df = text_content[~text_content["text"].isin(strong_content["text"])].reset_index(
                    drop=True
                )
#         df["quote"] = df["text"].apply(lambda row: extract_quote(row))
#         # combine all rows into context
        
        context = ""
        
        for i,row in df.iterrows():
            context += f" {row['text']}"
            
            
        quotes = extract_quote(context)
        # to save some time not computing unnecessary NER
        if len(quotes) != 0:
            speakers = get_names(context)
        else:
            speakers = ()
        
        dfs_dict[link] = {'context':context, 'quotes':quotes, 'speakers':speakers}   
        
    return dfs_dict   

def load_speaker_model():  

    model_name = f"deepset/xlm-roberta-large-squad2"
    
    tokenizer = AutoTokenizer.from_pretrained(model_name)
    
    model = AutoModelForQuestionAnswering.from_pretrained(model_name)
    
    question_answerer = pipeline("question-answering", model=model, tokenizer=tokenizer)

    return question_answerer

question_answerer = load_speaker_model()

def remove_punctuations(text):
    
    modified_text = "".join([character for character in text if character not in string.punctuation])
    modified_text = modified_text.lstrip(" ")
    modified_text = modified_text.rstrip(" ")
    
    return modified_text


def get_speaker_quotes(dfs_dict, question_answerer):
    
    speaker_quote = []
    
    
    for link in dfs_dict:
        context = dfs_dict[link]['context']
        quotes = dfs_dict[link]['quotes']
        potential_speakers = dfs_dict[link]['speakers']
        if len(quotes) != 0:
            #loop through the list of quotes
            for quote in quotes:  
                # max_seq_len == 384 : https://huggingface.co/deepset/roberta-base-squad2
                full_quote = quote
                if len(quote) >380:
                    quote = quote[:384]
                    
                
                speaker_dict = question_answerer(question=f"Who said '{quote}'?", context=context)
                
                speaker = speaker_dict['answer']
                if len(speaker) >0:
                    speaker = remove_punctuations(speaker_dict['answer'])
                
                
                
                if speaker not in potential_speakers:
                    speaker = ""
                    quote = ""
                else:
                    pair = {'speaker':speaker, 'quote': quote, 'source':link}
                    speaker_quote.append(pair)
                    
    return speaker_quote
                    



def post_to_twitter():
    twitter_api_key = api_key
    twitter_secret_api_key = secret_api_key
    twitter_access_token = access_token
    twitter_secret_access_token = secret_access_token
    twitter_bearer_token = bearer_token

    api = tweepy.Client(bearer_token=twitter_bearer_token, consumer_key=twitter_api_key,
                    consumer_secret=twitter_secret_api_key, access_token=twitter_access_token,
                    access_token_secret=twitter_secret_access_token,wait_on_rate_limit=True
                    )
    #tweet = api.create_tweet(text=post_title, in_reply_to_tweet_id=in_reply_to_tweet_id)
    
    return api



def split_near_space(string, max_length):
    # Split the string into lines based on the maximum line width, breaking only at spaces
    lines = textwrap.wrap(string, width=max_length,)
    return lines   

def send_tweets(speaker_quote):
    for i, pair in enumerate(speaker_quote):
        speaker = pair['speaker']
        quote = pair['quote']
        source = pair['source']
        
        total_tweet_length = len(speaker) + len(quote) + 10 # 10 is for emojis and #f1 hashtag

        tweet_text = f"🗣️ | {speaker}: '{quote}'"
        
        api = post_to_twitter()
        

        if total_tweet_length < 280:
            try:
                first_tweet = api.create_tweet(text=tweet_text, )
                first_tweet_id = first_tweet.data['id']
                second_tweet = api.create_tweet(text=f"Source: {source}", in_reply_to_tweet_id=first_tweet_id)
            except:
                continue

        else:
            quotes_list = split_near_space(quote, 280 - len(speaker) -10)
            thread_id = None
            try:
                for i, quote in enumerate(quotes_list):
                    tweet_text = f"'...{quote}...'"
                    if i == 0:                        
                        tweet_text = f"🗣️ | {speaker}: '{quote}...'"

                    if i ==len(quotes_list) -1:
                        tweet_text = f"'...{quote}'"

                    recent_tweet =  api.create_tweet(text=tweet_text, in_reply_to_tweet_id=thread_id)
                    thread_id = recent_tweet.data['id']


                last_tweet = api.create_tweet(text=f"Source: {source}", in_reply_to_tweet_id=thread_id)
            except:
                continue
    
def check_updates(every=300):
    while True:
        time.sleep(every) 
        latest_xml = get_xml(URL)
        if ~previous_xml.equals(latest_xml):
            print('New articles found')
            new_articles_df = latest_xml[~latest_xml["guid"].isin(previous_xml["guid"])]

            # loops through new articles and gets the necessary text, quotes and speakers
            dfs_dict = get_text(new_articles_df)
            speaker_quote = get_speaker_quotes(dfs_dict, question_answerer)
            send_tweets(speaker_quote)
                
        else:
            print('No New article is found')




demo = gr.Interface(fn=check_updates, inputs="number", outputs="text", analytics_enabled=True)
demo.launch(max_threads=1, show_api=False)