<s> from totalimpactwebapp import json_sqlalchemy <EOL> from util import commit <EOL> from util import cached_property <EOL> from util import dict_from_dir <EOL> from util import as_int_or_float_if_possible <EOL> from totalimpactwebapp import db <EOL> from totalimpactwebapp . tweeter import Tweeter <EOL> from birdy . twitter import AppClient , TwitterApiError , TwitterRateLimitError , TwitterClientError <EOL> from collections import defaultdict <EOL> from sqlalchemy import case <EOL> import os <EOL> import re <EOL> import datetime <EOL> import logging <EOL> logger = logging . getLogger ( '<STR_LIT>' ) <EOL> def tweets_from_tiids ( tiids ) : <EOL> if not tiids : <EOL> return [ ] <EOL> tweets = db . session . query ( Tweet ) . filter ( Tweet . tiid . in_ ( tiids ) ) . all ( ) <EOL> return tweets <EOL> def get_product_tweets_for_profile ( profile_id ) : <EOL> tweets = db . session . query ( Tweet ) . filter ( Tweet . profile_id == profile_id ) . all ( ) <EOL> response = defaultdict ( list ) <EOL> for tweet in tweets : <EOL> if tweet . tiid and tweet . tweet_text : <EOL> response [ tweet . tiid ] . append ( tweet ) <EOL> return response <EOL> def store_tweet_payload_and_tweeter_from_twitter ( payload_dicts_from_twitter , tweets ) : <EOL> tweets_by_tweet_id = defaultdict ( list ) <EOL> for tweet in tweets : <EOL> tweets_by_tweet_id [ tweet . tweet_id ] . append ( tweet ) <EOL> for payload_dict in payload_dicts_from_twitter : <EOL> tweet_id = payload_dict [ "<STR_LIT>" ] <EOL> logger . debug ( "<STR_LIT>" . format ( <EOL> tweet_id = tweet_id ) ) <EOL> for tweet in tweets_by_tweet_id [ tweet_id ] : <EOL> if not tweet . payload : <EOL> tweet . payload = payload_dict <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> tweet_id = tweet_id , tiid = tweet . tiid ) ) <EOL> if "<STR_LIT:user>" in payload_dict : <EOL> try : <EOL> tweet . tweeter . set_attributes_from_twitter_data ( payload_dict [ "<STR_LIT:user>" ] ) <EOL> except AttributeError : <EOL> tweeter = Tweeter . query . get ( tweet . screen_name ) <EOL> if not tweeter : <EOL> tweeter = Tweeter ( screen_name = tweet . screen_name ) <EOL> db . session . add ( tweeter ) <EOL> tweeter . set_attributes_from_twitter_data ( payload_dict [ "<STR_LIT:user>" ] ) <EOL> tweet . tweeter = tweeter <EOL> commit ( db ) <EOL> if tweet . tweeter : <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> screen_name = tweet . tweeter . screen_name ) ) <EOL> def flag_deleted_tweets ( tweet_ids ) : <EOL> if not tweet_ids : <EOL> return None <EOL> for tweet in Tweet . query . filter ( Tweet . tweet_id . in_ ( tweet_ids ) ) . all ( ) : <EOL> tweet . is_deleted = True <EOL> db . session . merge ( tweet ) <EOL> def handle_all_tweets ( data , tweets ) : <EOL> store_tweet_payload_and_tweeter_from_twitter ( data , tweets ) <EOL> tweet_ids = [ tweet . tweet_id for tweet in tweets ] <EOL> tweet_ids_with_response = [ tweet [ "<STR_LIT>" ] for tweet in data ] <EOL> tweet_ids_without_response = [ tweet for tweet in tweet_ids if tweet not in tweet_ids_with_response ] <EOL> flag_deleted_tweets ( tweet_ids_without_response ) <EOL> return True <EOL> class AppDictClient ( AppClient ) : <EOL> @ staticmethod <EOL> def get_json_object_hook ( data ) : <EOL> return data <EOL> def get_and_save_tweet_text_and_tweeter_followers ( tweets ) : <EOL> client = AppDictClient ( <EOL> os . getenv ( "<STR_LIT>" ) , <EOL> os . getenv ( "<STR_LIT>" ) , <EOL> access_token = os . getenv ( "<STR_LIT>" ) <EOL> ) <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> num = len ( tweets ) ) ) <EOL> group_size = <NUM_LIT:100> <EOL> list_of_groups = [ tweets [ i : i + group_size ] for i in range ( <NUM_LIT:0> , len ( tweets ) , group_size ) ] <EOL> for tweet_subset in list_of_groups : <EOL> tweet_id_string = "<STR_LIT:U+002C>" . join ( [ tweet . tweet_id for tweet in tweet_subset ] ) <EOL> try : <EOL> response = client . api . statuses . lookup . post ( id = tweet_id_string , trim_user = False ) <EOL> handle_all_tweets ( response . data , tweet_subset ) <EOL> except TwitterApiError , e : <EOL> logger . exception ( "<STR_LIT>" ) <EOL> except TwitterClientError , e : <EOL> logger . exception ( "<STR_LIT>" ) <EOL> except TwitterRateLimitError , e : <EOL> logger . exception ( "<STR_LIT>" ) <EOL> return <EOL> def hydrate_twitter_text_and_followers ( profile_id , altmetric_twitter_posts ) : <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> profile_id = profile_id ) ) <EOL> tweets_to_hydrate_from_twitter = [ ] <EOL> tweets = Tweet . query . filter ( Tweet . profile_id == profile_id ) <EOL> tweet_dict = dict ( [ ( ( tweet . tweet_id , tweet . tiid ) , tweet ) for tweet in tweets ] ) <EOL> for tiid , post_list in altmetric_twitter_posts . iteritems ( ) : <EOL> for post in post_list : <EOL> tweet_id = post [ "<STR_LIT>" ] <EOL> screen_name = post [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> if ( tweet_id , tiid ) in tweet_dict . keys ( ) : <EOL> tweet = tweet_dict [ ( tweet_id , tiid ) ] <EOL> if not tweet . tweet_text and not tweet . is_deleted : <EOL> tweets_to_hydrate_from_twitter . append ( tweet ) <EOL> else : <EOL> if not Tweet . query . get ( ( tweet_id , tiid ) ) : <EOL> tweet = Tweet ( tweet_id = tweet_id , tiid = tiid ) <EOL> tweet . set_attributes_from_altmetric_post ( post ) <EOL> tweet . profile_id = profile_id <EOL> tweets_to_hydrate_from_twitter . append ( tweet ) <EOL> db . session . add ( tweet ) <EOL> if not tweet . tweeter : <EOL> tweeter = Tweeter . query . get ( screen_name ) <EOL> if not tweeter : <EOL> tweeter = Tweeter ( screen_name = screen_name ) <EOL> db . session . add ( tweeter ) <EOL> tweeter . set_attributes_from_altmetric_post ( post ) <EOL> commit ( db ) <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> profile_id = profile_id ) ) <EOL> if tweets_to_hydrate_from_twitter : <EOL> commit ( db ) <EOL> tweet_ids = [ tweet . tweet_id for tweet in tweets_to_hydrate_from_twitter ] <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> profile_id = profile_id ) ) <EOL> get_and_save_tweet_text_and_tweeter_followers ( tweets_to_hydrate_from_twitter ) <EOL> commit ( db ) <EOL> else : <EOL> logger . info ( u"<STR_LIT>" . format ( <EOL> profile_id = profile_id ) ) <EOL> return <EOL> handle_workaround_join_string = "<STR_LIT>" <EOL> class Tweet ( db . Model ) : <EOL> tweet_id = db . Column ( db . Text , primary_key = True ) <EOL> tiid = db . Column ( db . Text , primary_key = True ) <EOL> profile_id = db . Column ( db . Integer , db . ForeignKey ( '<STR_LIT>' ) ) <EOL> screen_name = db . Column ( db . Text , db . ForeignKey ( '<STR_LIT>' ) ) <EOL> tweet_timestamp = db . Column ( db . DateTime ( ) ) <EOL> payload = db . Column ( json_sqlalchemy . JSONAlchemy ( db . Text ) ) <EOL> is_deleted = db . Column ( db . Boolean ) <EOL> tweet_url = db . Column ( db . Text ) <EOL> country = db . Column ( db . Text ) <EOL> followers_at_time_of_tweet = db . Column ( db . Integer ) <EOL> tweeter = db . relationship ( <EOL> '<STR_LIT>' , <EOL> lazy = '<STR_LIT>' , <EOL> cascade = '<STR_LIT:all>' , <EOL> backref = db . backref ( "<STR_LIT>" ) , <EOL> uselist = False , <EOL> primaryjoin = handle_workaround_join_string <EOL> ) <EOL> def __init__ ( self , ** kwargs ) : <EOL> if "<STR_LIT>" in kwargs : <EOL> payload_dict = kwargs [ "<STR_LIT>" ] <EOL> kwargs [ "<STR_LIT>" ] = payload_dict [ "<STR_LIT>" ] <EOL> kwargs [ "<STR_LIT>" ] = payload_dict [ "<STR_LIT:user>" ] [ "<STR_LIT>" ] <EOL> kwargs [ "<STR_LIT>" ] = payload_dict <EOL> kwargs [ "<STR_LIT>" ] = datetime . datetime . strptime ( payload_dict [ "<STR_LIT>" ] , r"<STR_LIT>" ) <EOL> if not "<STR_LIT>" in kwargs : <EOL> try : <EOL> kwargs [ "<STR_LIT>" ] = payload_dict [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> except ( AttributeError , TypeError ) : <EOL> pass <EOL> super ( Tweet , self ) . __init__ ( ** kwargs ) <EOL> @ classmethod <EOL> def most_recent_tweet_id ( cls , screen_name ) : <EOL> screen_name = screen_name . replace ( "<STR_LIT:@>" , "<STR_LIT>" ) <EOL> q = db . session . query ( Tweet ) . filter ( Tweet . screen_name == screen_name ) . order_by ( Tweet . tweet_timestamp . desc ( ) ) <EOL> tweet = q . first ( ) <EOL> try : <EOL> tweet_id = tweet . tweet_id <EOL> except AttributeError : <EOL> tweet_id = None <EOL> return tweet_id <EOL> @ cached_property <EOL> def tweet_text ( self ) : <EOL> try : <EOL> return self . payload [ "<STR_LIT:text>" ] <EOL> except TypeError : <EOL> return None <EOL> @ cached_property <EOL> def tweet_text_with_links ( self ) : <EOL> if self . tweet_text is None : <EOL> return None <EOL> ret = self . tweet_text <EOL> ret = re . sub ( r"<STR_LIT>" , r"<STR_LIT>" , ret ) <EOL> for url_info in self . urls : <EOL> my_link = u"<STR_LIT>" . format ( <EOL> url = url_info [ "<STR_LIT>" ] , <EOL> display_url = url_info [ "<STR_LIT>" ] <EOL> ) <EOL> ret = re . sub ( r"<STR_LIT>" , my_link , ret , <NUM_LIT:1> ) <EOL> ret = re . sub ( r"<STR_LIT>" , r"<STR_LIT>" , ret ) <EOL> ret = re . sub ( r"<STR_LIT>" , r"<STR_LIT>" , ret ) <EOL> return ret <EOL> @ cached_property <EOL> def urls ( self ) : <EOL> try : <EOL> return self . payload [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> except TypeError : <EOL> return None <EOL> except KeyError : <EOL> return [ ] <EOL> @ cached_property <EOL> def has_country ( self ) : <EOL> return self . country != None <EOL> def set_attributes_from_altmetric_post ( self , post ) : <EOL> self . tweet_id = post [ "<STR_LIT>" ] <EOL> self . screen_name = post [ "<STR_LIT>" ] [ "<STR_LIT>" ] <EOL> self . tweet_timestamp = post [ "<STR_LIT>" ] <EOL> if "<STR_LIT>" in post [ "<STR_LIT>" ] : <EOL> self . country = post [ "<STR_LIT>" ] [ "<STR_LIT>" ] . get ( "<STR_LIT>" , None ) <EOL> return self <EOL> def __repr__ ( self ) : <EOL> return u'<STR_LIT>' . format ( <EOL> tweet_id = self . tweet_id , <EOL> profile_id = self . profile_id , <EOL> screen_name = self . screen_name , <EOL> timestamp = self . tweet_timestamp ) <EOL> def to_dict ( self ) : <EOL> attributes_to_ignore = [ <EOL> "<STR_LIT>" <EOL> ] <EOL> ret = dict_from_dir ( self , attributes_to_ignore ) <EOL> return ret <EOL> twitter_example_contents = """<STR_LIT>""" </s>
