<s> import logging <EOL> import time <EOL> import urllib <EOL> import urlparse <EOL> from google . appengine . ext import db <EOL> from base . constants import CONSTANTS <EOL> import base . util <EOL> from datasources import googlereader <EOL> class FeedInfoData ( db . Model ) : <EOL> title = db . StringProperty ( indexed = False ) <EOL> item_ids = db . StringListProperty ( indexed = False ) <EOL> item_timestamps_usec = db . ListProperty ( long , indexed = False ) <EOL> @ classmethod <EOL> def kind ( cls ) : <EOL> return '<STR_LIT>' <EOL> class FeedInfo ( object ) : <EOL> def __init__ ( self , feed_url , title , item_ids , item_timestamps_usec ) : <EOL> self . feed_url = feed_url <EOL> self . title = title <EOL> self . item_ids = item_ids <EOL> self . item_timestamps_usec = item_timestamps_usec <EOL> def as_json_dict ( self ) : <EOL> item_count = len ( self . item_ids ) <EOL> oldest_item_timestamp_msec = - <NUM_LIT:1> <EOL> if item_count : <EOL> oldest_item_timestamp_msec = int ( self . item_timestamps_usec [ <NUM_LIT:0> ] / <NUM_LIT:1000> ) <EOL> return { <EOL> '<STR_LIT>' : self . feed_url , <EOL> '<STR_LIT>' : self . title , <EOL> '<STR_LIT>' : item_count , <EOL> '<STR_LIT>' : oldest_item_timestamp_msec , <EOL> } <EOL> def get_feed_info ( html_or_feed_url ) : <EOL> feed_url = googlereader . lookup_feed_url ( html_or_feed_url ) <EOL> if not feed_url : <EOL> return None <EOL> return get_feed_info_from_feed_url ( feed_url ) <EOL> def get_feed_info_from_feed_url ( feed_url ) : <EOL> feed_info = FeedInfoData . get_by_key_name ( feed_url ) <EOL> if not feed_info : <EOL> title = googlereader . lookup_feed_title ( feed_url ) or urlparse . urlparse ( feed_url ) . netloc <EOL> item_refs = googlereader . get_feed_item_refs ( feed_url ) <EOL> if not item_refs : <EOL> return { } <EOL> feed_info = FeedInfoData ( <EOL> key_name = feed_url , <EOL> title = title , <EOL> item_ids = [ i . id for i in item_refs ] , <EOL> item_timestamps_usec = [ i . timestamp_usec for i in item_refs ] , <EOL> ) <EOL> feed_info . put ( ) <EOL> return FeedInfo ( <EOL> feed_url = feed_url , <EOL> title = feed_info . title , <EOL> item_ids = feed_info . item_ids , <EOL> item_timestamps_usec = feed_info . item_timestamps_usec ) <EOL> class SubscriptionData ( db . Model ) : <EOL> reader_stream_id = db . StringProperty ( indexed = False ) <EOL> feed_url = db . StringProperty ( indexed = False ) <EOL> frequency = db . StringProperty ( ) <EOL> frequency_modulo = db . IntegerProperty ( ) <EOL> position = db . IntegerProperty ( indexed = False ) <EOL> @ classmethod <EOL> def kind ( cls ) : <EOL> return '<STR_LIT>' <EOL> class Subscription ( object ) : <EOL> def __init__ ( self , id , reader_stream_id , feed_url , frequency , frequency_modulo , position ) : <EOL> self . id = id <EOL> self . reader_stream_id = reader_stream_id <EOL> self . feed_url = feed_url <EOL> self . frequency = frequency <EOL> self . frequency_modulo = frequency_modulo <EOL> self . position = position <EOL> @ staticmethod <EOL> def from_datastore ( subscription ) : <EOL> return Subscription ( <EOL> subscription . key ( ) . name ( ) , <EOL> subscription . reader_stream_id , <EOL> subscription . feed_url , <EOL> subscription . frequency , <EOL> subscription . frequency_modulo , <EOL> subscription . position ) <EOL> def save ( self ) : <EOL> subscription = SubscriptionData ( <EOL> key_name = self . id , <EOL> reader_stream_id = self . reader_stream_id , <EOL> feed_url = self . feed_url , <EOL> frequency = self . frequency , <EOL> frequency_modulo = self . frequency_modulo , <EOL> position = self . position ) <EOL> subscription . put ( ) <EOL> def create_reader_stream ( self , intro_html_url , intro_title , intro_body ) : <EOL> googlereader . set_stream_public ( self . reader_stream_id , is_public = True ) <EOL> self . advance ( ) <EOL> googlereader . create_note ( <EOL> title = intro_title , <EOL> body = intro_body , <EOL> url = intro_html_url , <EOL> source_url = CONSTANTS . APP_URL , <EOL> source_title = CONSTANTS . APP_NAME , <EOL> share = False , <EOL> additional_stream_ids = [ self . reader_stream_id ] ) <EOL> def advance ( self ) : <EOL> feed_info = get_feed_info_from_feed_url ( self . feed_url ) <EOL> if self . position == len ( feed_info . item_ids ) : <EOL> return <EOL> item_id = feed_info . item_ids [ self . position ] <EOL> retry_count = <NUM_LIT:0> <EOL> while True : <EOL> try : <EOL> googlereader . edit_item_tags ( <EOL> item_id , <EOL> origin_stream_id = '<STR_LIT>' % self . feed_url , <EOL> add_tags = [ self . reader_stream_id ] ) <EOL> break <EOL> except : <EOL> retry_count += <NUM_LIT:1> <EOL> if retry_count == <NUM_LIT:5> : <EOL> logging . error ( '<STR_LIT>' <EOL> '<STR_LIT>' % self . id ) <EOL> return <EOL> self . position += <NUM_LIT:1> <EOL> self . save ( ) <EOL> def get_subscription_feed_url ( self ) : <EOL> return '<STR_LIT>' % urllib . quote ( self . reader_stream_id . encode ( '<STR_LIT:utf-8>' ) ) <EOL> def get_subscription_reader_url ( self ) : <EOL> return '<STR_LIT>' % urllib . quote ( self . reader_stream_id . encode ( '<STR_LIT:utf-8>' ) ) <EOL> def as_json_dict ( self ) : <EOL> return { <EOL> '<STR_LIT>' : self . get_subscription_feed_url ( ) , <EOL> '<STR_LIT>' : self . get_subscription_reader_url ( ) , <EOL> } <EOL> def get_modulo_for_frequency ( frequency ) : <EOL> if frequency == '<STR_LIT>' : <EOL> return <NUM_LIT:0> <EOL> else : <EOL> days_since_epoch = int ( time . time ( ) / ( <NUM_LIT> * <NUM_LIT> ) ) <EOL> if frequency == '<STR_LIT>' : <EOL> return days_since_epoch % <NUM_LIT:2> <EOL> else : <EOL> return days_since_epoch % <NUM_LIT:7> <EOL> def get_subscription_by_id ( id ) : <EOL> subscription = SubscriptionData . get_by_key_name ( id ) <EOL> if not subscription : <EOL> return None <EOL> return Subscription . from_datastore ( subscription ) <EOL> def get_subscriptions_with_frequency_and_modulo ( frequency , frequency_modulo ) : <EOL> query = SubscriptionData . all ( ) <EOL> query . filter ( '<STR_LIT>' , frequency ) <EOL> query . filter ( '<STR_LIT>' , frequency_modulo ) <EOL> subscriptions = [ ] <EOL> for result in query : <EOL> subscriptions . append ( Subscription . from_datastore ( result ) ) <EOL> return subscriptions <EOL> def _get_nearest_item_index ( feed_info , date ) : <EOL> timestamp_usec = time . mktime ( date . utctimetuple ( ) ) * <NUM_LIT> <EOL> index = <NUM_LIT:0> <EOL> for item_timestamp_usec in feed_info . item_timestamps_usec : <EOL> if item_timestamp_usec >= timestamp_usec : <EOL> break <EOL> index += <NUM_LIT:1> <EOL> return min ( index , len ( feed_info . item_timestamps_usec ) - <NUM_LIT:1> ) <EOL> def get_start_item_contents ( feed_url , start_date ) : <EOL> feed_info = get_feed_info_from_feed_url ( feed_url ) <EOL> item_index = _get_nearest_item_index ( feed_info , start_date ) <EOL> item_id = feed_info . item_ids [ item_index ] <EOL> return googlereader . get_item_contents ( item_id ) <EOL> def create_subscription ( feed_url , start_date , frequency ) : <EOL> feed_info = get_feed_info_from_feed_url ( feed_url ) <EOL> start_position = _get_nearest_item_index ( feed_info , start_date ) <EOL> feed_title = feed_info . title [ <NUM_LIT:0> : <NUM_LIT:100> ] <EOL> subscription_id = base . util . generate_id ( '<STR_LIT:s>' ) <EOL> reader_tag_name = googlereader . sanitize_tag_name ( <EOL> '<STR_LIT>' % ( feed_title , subscription_id ) ) <EOL> reader_stream_id = '<STR_LIT>' % ( <EOL> googlereader . FEED_PLAYBACK_USER_ID , reader_tag_name ) <EOL> subscription = Subscription ( <EOL> subscription_id , <EOL> reader_stream_id , <EOL> feed_url , <EOL> frequency , <EOL> get_modulo_for_frequency ( frequency ) , <EOL> start_position ) <EOL> subscription . save ( ) <EOL> return subscription </s>
