app.py

import tweepy as tw
import streamlit as st
import pandas as pd
import torch
import numpy as np
import regex as re
import pysentimiento
import geopy
import matplotlib.pyplot as plt


from pysentimiento.preprocessing import preprocess_tweet
from geopy.geocoders import Nominatim
from transformers import pipeline

from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from transformers import AutoTokenizer, AutoModelForSequenceClassification,AdamW
tokenizer = AutoTokenizer.from_pretrained('hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021')
model = AutoModelForSequenceClassification.from_pretrained("hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021")

model_checkpoint = "hackathon-pln-es/twitter_sexismo-finetuned-robertuito-exist2021" 
pipeline_nlp = pipeline("text-classification", model=model_checkpoint)

import torch
if torch.cuda.is_available():  
    device = torch.device(	"cuda")
    print('I will use the GPU:', torch.cuda.get_device_name(0))
    
else:
    print('No GPU available, using the CPU instead.')
    device = torch.device("cpu")

    
consumer_key = "BjipwQslVG4vBdy4qK318KnoA"
consumer_secret = "3fzL70v9faklrPgvTi3zbofw9rwk92fgGdtAslFkFYt8kGmqBJ"
access_token = "1217853705086799872-Y5zEChpTeKccuLY3XJRXDPPZhNrlba"
access_token_secret = "pqQ5aFSJxzJ2xnI6yhVtNjQO36FOu8DBOH6DtUrPAU54J"
auth = tw.OAuthHandler(consumer_key, consumer_secret)
auth.set_access_token(access_token, access_token_secret)
api = tw.API(auth, wait_on_rate_limit=True)

def preprocess(text):
    #text=text.lower()
    # remove hyperlinks
    text = re.sub(r'https?:\/\/.*[\r\n]*', '', text)
    text = re.sub(r'http?:\/\/.*[\r\n]*', '', text)
    #Replace &amp, &lt, &gt with &,<,> respectively
    text=text.replace(r'&amp;?',r'and')
    text=text.replace(r'&lt;',r'<')
    text=text.replace(r'&gt;',r'>')
    #remove hashtag sign
    #text=re.sub(r"#","",text)   
    #remove mentions
    text = re.sub(r"(?:\@)\w+", '', text)
    #text=re.sub(r"@","",text)
    #remove non ascii chars
    text=text.encode("ascii",errors="ignore").decode()
    #remove some puncts (except . ! ?)
    text=re.sub(r'[:"#$%&\*+,-/:;<=>@\\^_`{|}~]+','',text)
    text=re.sub(r'[!]+','!',text)
    text=re.sub(r'[?]+','?',text)
    text=re.sub(r'[.]+','.',text)
    text=re.sub(r"'","",text)
    text=re.sub(r"\(","",text)
    text=re.sub(r"\)","",text)
    text=" ".join(text.split())
    return text


def highlight_survived(s):
    return ['background-color: red']*len(s) if (s.Sexista == 1) else ['background-color: green']*len(s)

def color_survived(val):
    color = 'red' if val=='Sexista' else 'white'
    return f'background-color: {color}'


st.set_page_config(layout="wide")
st.markdown('<style>body{background-color: Blue;}</style>',unsafe_allow_html=True)

colT1,colT2 = st.columns([2,8])
with colT2:
   # st.title('Analisis de comentarios sexistas en Twitter') 
    st.markdown(""" <style> .font {
    font-size:40px ; font-family: 'Cooper Black'; color: #06bf69;} 
    </style> """, unsafe_allow_html=True)
    st.markdown('<p class="font">Análisis de comentarios sexistas en Twitter</p>', unsafe_allow_html=True)
    
    st.markdown(""" <style> .font1 {
    font-size:28px ; font-family: 'Times New Roman'; color: #8d33ff;} 
    </style> """, unsafe_allow_html=True)

    st.markdown(""" <style> .font2 {
    font-size:16px ; font-family: 'Times New Roman'; color: #3358ff;} 
    </style> """, unsafe_allow_html=True)





def analizar_tweets(search_words, number_of_tweets ):
  tweets = api.user_timeline(screen_name = search_words,tweet_mode="extended", count= number_of_tweets)
  tweet_list = [i.full_text for i in tweets]
  text= pd.DataFrame(tweet_list)
  text[0] = text[0].apply(preprocess_tweet)
  text_list = text[0].tolist()
  result = []
  for text in text_list:
      if (text.startswith('RT')):
          continue
      else:
          prediction = pipeline_nlp(text)
          for predic in prediction:
              etiqueta = {'Tweets': text,'Prediccion': predic['label'], 'Probabilidad': predic['score']}
              result.append(etiqueta)
              
  df = pd.DataFrame(result)
  df['Prediccion'] = np.where( df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
  tabla = st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
    
  return tabla

def analizar_frase(frase):
    
  #palabra = frase.split()
  #palabra = frase
  predictions = pipeline_nlp(frase)

  # convierte las predicciones en una lista de diccionarios
  data = [{'Texto': frase, 'Prediccion': prediction['label'], 'Probabilidad': prediction['score']} for prediction in predictions]
  
  # crea un DataFrame a partir de la lista de diccionarios
  df = pd.DataFrame(data)
  df['Prediccion'] = np.where( df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
  # muestra el DataFrame
 #st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
  tabla = st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
  
  return tabla

def tweets_localidad(buscar_localidad):
    try:
        geolocator = Nominatim(user_agent="nombre_del_usuario")
        location = geolocator.geocode(buscar_localidad)
        radius = "10km"
        tweets = api.search_tweets(q="",lang="es",geocode=f"{location.latitude},{location.longitude},{radius}", count = 50, tweet_mode="extended")
        
        tweet_list = [i.full_text for i in tweets]
        text= pd.DataFrame(tweet_list)
        text[0] = text[0].apply(preprocess)
        text_list = text[0].tolist()
        result = []
        for text in text_list:
            if (text.startswith('RT')):
                continue
            else:
                prediction = pipeline_nlp(text)
                for predic in prediction:
                    etiqueta = {'Tweets': text,'Prediccion': predic['label'], 'Probabilidad': predic['score']}
                    result.append(etiqueta) 
    except AttributeError:
        st.text("No existe ninguna localidad con ese nombre") 
                 
    df = pd.DataFrame(result)
    df['Prediccion'] = np.where( df['Prediccion'] == 'LABEL_1', 'Sexista', 'No Sexista')
    #tabla = st.table(df.reset_index(drop=True).head(30).style.applymap(color_survived, subset=['Prediccion']))
    #df['Tweets'] = df['Tweets'].str.replace('RT|@', '')
    df=df[df["Prediccion"] == 'Sexista']
    tabla = st.table(df.reset_index(drop=True).head(50).style.applymap(color_survived, subset=['Prediccion']))
        
    df_sexista = df[df['Prediccion']=="Sexista"]
    df_no_sexista = df[df['Probabilidad'] > 0]
    sexista = len(df_sexista)
    no_sexista = len(df_no_sexista)
    
    # Crear un gráfico de barras
    labels = ['Sexista  ', ' No sexista']
    counts = [sexista, no_sexista]
    plt.bar(labels, counts)
    plt.xlabel('Categoría')
    plt.ylabel('Cantidad de tweets')
    plt.title('Cantidad de tweets sexistas y no sexistas')
    plt.figure(figsize=(10,6))
    plt.show()
    st.pyplot()
      
    st.set_option('deprecation.showPyplotGlobalUse', False)
        
    return tabla    

def run():   
 with st.form("my_form"):
   col,buff1, buff2 = st.columns([2,2,1])
   st.write("Escoja una Opción")
   search_words = col.text_input("Introduzca el termino, usuario o localidad para analizar y pulse el check correspondiente")
   number_of_tweets = col.number_input('Introduzca número de tweets a analizar. Máximo 50', 0,50,0)
   termino=st.checkbox('Término')
   usuario=st.checkbox('Usuario')
   localidad=st.checkbox('Localidad')
   submit_button = col.form_submit_button(label='Analizar')
   error =False
     
   if submit_button:
            # Condición para el caso de que esten dos check seleccionados
            if ( termino == False and usuario == False and localidad == False):
                st.text('Error no se ha seleccionado ningun check')
                error=True
            elif ( termino == True and usuario == True and localidad == True):
                st.text('Error se han seleccionado varios check')
                error=True
                
            if (error == False):
                if (termino):
                  analizar_frase(search_words)
                    
                elif (usuario):
                    analizar_tweets(search_words,number_of_tweets)
                elif (localidad):
                    tweets_localidad(search_words)
                
run()