<s> import os <EOL> import time <EOL> import calendar <EOL> import uuid <EOL> import urlparse <EOL> import urllib2 <EOL> import filters <EOL> import util <EOL> import Queue <EOL> import logging <EOL> import safe_pickle <EOL> from settings import settings <EOL> def cmp_timestamp ( a , b ) : <EOL> return cmp ( a . timestamp , b . timestamp ) <EOL> def create_id ( entry ) : <EOL> keys = [ '<STR_LIT:id>' , '<STR_LIT>' , '<STR_LIT:title>' ] <EOL> values = tuple ( util . get ( entry , key , None ) for key in keys ) <EOL> return values if any ( values ) else uuid . uuid4 ( ) . hex <EOL> class Item ( object ) : <EOL> def __init__ ( self , feed , id ) : <EOL> self . feed = feed <EOL> self . id = id <EOL> self . timestamp = int ( time . time ( ) ) <EOL> self . received = int ( time . time ( ) ) <EOL> self . title = '<STR_LIT>' <EOL> self . description = '<STR_LIT>' <EOL> self . link = '<STR_LIT>' <EOL> self . author = '<STR_LIT>' <EOL> self . read = False <EOL> @ property <EOL> def time_since ( self ) : <EOL> return util . time_since ( self . timestamp ) <EOL> class Feed ( object ) : <EOL> def __init__ ( self , url ) : <EOL> self . uuid = uuid . uuid4 ( ) . hex <EOL> self . url = url <EOL> self . username = None <EOL> self . password = None <EOL> self . enabled = True <EOL> self . last_poll = <NUM_LIT:0> <EOL> self . interval = settings . DEFAULT_POLLING_INTERVAL <EOL> self . etag = None <EOL> self . modified = None <EOL> self . title = '<STR_LIT>' <EOL> self . link = '<STR_LIT>' <EOL> self . clicks = <NUM_LIT:0> <EOL> self . item_count = <NUM_LIT:0> <EOL> self . color = None <EOL> self . id_list = [ ] <EOL> self . id_set = set ( ) <EOL> def make_copy ( self ) : <EOL> feed = Feed ( self . url ) <EOL> for key in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:title>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> value = getattr ( self , key ) <EOL> setattr ( feed , key , value ) <EOL> return feed <EOL> def copy_from ( self , feed ) : <EOL> for key in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:title>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> value = getattr ( feed , key ) <EOL> setattr ( self , key , value ) <EOL> @ property <EOL> def favicon_url ( self ) : <EOL> components = urlparse . urlsplit ( self . link ) <EOL> scheme , domain = components [ : <NUM_LIT:2> ] <EOL> return '<STR_LIT>' % ( scheme , domain ) <EOL> @ property <EOL> def favicon_path ( self ) : <EOL> components = urlparse . urlsplit ( self . link ) <EOL> scheme , domain = components [ : <NUM_LIT:2> ] <EOL> path = '<STR_LIT>' % domain <EOL> return os . path . abspath ( path ) <EOL> @ property <EOL> def has_favicon ( self ) : <EOL> return os . path . exists ( self . favicon_path ) <EOL> def download_favicon ( self ) : <EOL> try : <EOL> dir , name = os . path . split ( self . favicon_path ) <EOL> os . makedirs ( dir ) <EOL> except Exception : <EOL> pass <EOL> try : <EOL> opener = urllib2 . build_opener ( util . get_proxy ( ) ) <EOL> f = opener . open ( self . favicon_url ) <EOL> data = f . read ( ) <EOL> f . close ( ) <EOL> f = open ( self . favicon_path , '<STR_LIT:wb>' ) <EOL> f . write ( data ) <EOL> f . close ( ) <EOL> except Exception : <EOL> pass <EOL> def clear_cache ( self ) : <EOL> self . id_list = [ ] <EOL> self . id_set = set ( ) <EOL> self . etag = None <EOL> self . modified = None <EOL> def clean_cache ( self , size ) : <EOL> for id in self . id_list [ : - size ] : <EOL> self . id_set . remove ( id ) <EOL> self . id_list = self . id_list [ - size : ] <EOL> def should_poll ( self ) : <EOL> if not self . enabled : <EOL> return False <EOL> now = int ( time . time ( ) ) <EOL> duration = now - self . last_poll <EOL> return duration >= self . interval <EOL> def poll ( self , timestamp , filters ) : <EOL> logging . info ( '<STR_LIT>' % self . url ) <EOL> result = [ ] <EOL> self . last_poll = timestamp <EOL> username = util . decode_password ( self . username ) <EOL> password = util . decode_password ( self . password ) <EOL> d = util . parse ( self . url , username , password , self . etag , self . modified ) <EOL> self . etag = util . get ( d , '<STR_LIT>' , None ) <EOL> self . modified = util . get ( d , '<STR_LIT>' , None ) <EOL> feed = util . get ( d , '<STR_LIT>' , None ) <EOL> if feed : <EOL> self . title = self . title or util . get ( feed , '<STR_LIT:title>' , '<STR_LIT>' ) <EOL> self . link = self . link or util . get ( feed , '<STR_LIT>' , self . url ) <EOL> entries = util . get ( d , '<STR_LIT>' , [ ] ) <EOL> for entry in reversed ( entries ) : <EOL> id = create_id ( entry ) <EOL> if id in self . id_set : <EOL> continue <EOL> self . item_count += <NUM_LIT:1> <EOL> self . id_list . append ( id ) <EOL> self . id_set . add ( id ) <EOL> item = Item ( self , id ) <EOL> item . timestamp = calendar . timegm ( util . get ( entry , '<STR_LIT>' , time . gmtime ( ) ) ) <EOL> item . title = util . format ( util . get ( entry , '<STR_LIT:title>' , '<STR_LIT>' ) , settings . POPUP_TITLE_LENGTH ) <EOL> item . description = util . format ( util . get ( entry , '<STR_LIT:description>' , '<STR_LIT>' ) , settings . POPUP_BODY_LENGTH ) <EOL> item . link = util . get ( entry , '<STR_LIT>' , '<STR_LIT>' ) <EOL> item . author = util . format ( util . get ( entry , '<STR_LIT>' , '<STR_LIT>' ) ) <EOL> if all ( filter . filter ( item ) for filter in filters ) : <EOL> result . append ( item ) <EOL> self . clean_cache ( settings . FEED_CACHE_SIZE ) <EOL> return result <EOL> class Filter ( object ) : <EOL> def __init__ ( self , code , ignore_case = True , whole_word = True , feeds = None ) : <EOL> self . uuid = uuid . uuid4 ( ) . hex <EOL> self . enabled = True <EOL> self . code = code <EOL> self . ignore_case = ignore_case <EOL> self . whole_word = whole_word <EOL> self . feeds = set ( feeds ) if feeds else set ( ) <EOL> self . inputs = <NUM_LIT:0> <EOL> self . outputs = <NUM_LIT:0> <EOL> def make_copy ( self ) : <EOL> filter = Filter ( self . code , self . ignore_case , self . whole_word , self . feeds ) <EOL> for key in [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> value = getattr ( self , key ) <EOL> setattr ( filter , key , value ) <EOL> return filter <EOL> def copy_from ( self , filter ) : <EOL> for key in [ '<STR_LIT>' , '<STR_LIT:code>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] : <EOL> value = getattr ( filter , key ) <EOL> setattr ( self , key , value ) <EOL> def filter ( self , item ) : <EOL> if not self . enabled : <EOL> return True <EOL> if self . feeds and item . feed not in self . feeds : <EOL> return True <EOL> self . inputs += <NUM_LIT:1> <EOL> rule = filters . parse ( self . code ) <EOL> if rule . evaluate ( item , self . ignore_case , self . whole_word ) : <EOL> self . outputs += <NUM_LIT:1> <EOL> return True <EOL> else : <EOL> return False <EOL> class FeedManager ( object ) : <EOL> def __init__ ( self ) : <EOL> self . feeds = [ ] <EOL> self . items = [ ] <EOL> self . filters = [ ] <EOL> def add_feed ( self , feed ) : <EOL> logging . info ( '<STR_LIT>' % feed . url ) <EOL> self . feeds . append ( feed ) <EOL> def remove_feed ( self , feed ) : <EOL> logging . info ( '<STR_LIT>' % feed . url ) <EOL> self . feeds . remove ( feed ) <EOL> for filter in self . filters : <EOL> filter . feeds . discard ( feed ) <EOL> def add_filter ( self , filter ) : <EOL> logging . info ( '<STR_LIT>' % filter . code ) <EOL> self . filters . append ( filter ) <EOL> def remove_filter ( self , filter ) : <EOL> logging . info ( '<STR_LIT>' % filter . code ) <EOL> self . filters . remove ( filter ) <EOL> def should_poll ( self ) : <EOL> return any ( feed . should_poll ( ) for feed in self . feeds ) <EOL> def poll ( self ) : <EOL> now = int ( time . time ( ) ) <EOL> jobs = Queue . Queue ( ) <EOL> results = Queue . Queue ( ) <EOL> feeds = [ feed for feed in self . feeds if feed . should_poll ( ) ] <EOL> for feed in feeds : <EOL> jobs . put ( feed ) <EOL> count = len ( feeds ) <EOL> logging . info ( '<STR_LIT>' ) <EOL> for i in range ( min ( count , settings . MAX_WORKER_THREADS ) ) : <EOL> util . start_thread ( self . worker , now , jobs , results ) <EOL> while count : <EOL> items = results . get ( ) <EOL> count -= <NUM_LIT:1> <EOL> if items : <EOL> yield items <EOL> logging . info ( '<STR_LIT>' ) <EOL> def worker ( self , now , jobs , results ) : <EOL> while True : <EOL> try : <EOL> feed = jobs . get ( False ) <EOL> except Queue . Empty : <EOL> break <EOL> try : <EOL> items = feed . poll ( now , self . filters ) <EOL> items . sort ( cmp = cmp_timestamp ) <EOL> if items and not feed . has_favicon : <EOL> feed . download_favicon ( ) <EOL> results . put ( items ) <EOL> jobs . task_done ( ) <EOL> except Exception : <EOL> results . put ( [ ] ) <EOL> jobs . task_done ( ) <EOL> def purge_items ( self , max_age ) : <EOL> now = int ( time . time ( ) ) <EOL> feeds = set ( self . feeds ) <EOL> for item in list ( self . items ) : <EOL> age = now - item . received <EOL> if age > max_age or item . feed not in feeds : <EOL> self . items . remove ( item ) <EOL> def load ( self , path = '<STR_LIT>' ) : <EOL> logging . info ( '<STR_LIT>' % path ) <EOL> try : <EOL> data = safe_pickle . load ( path ) <EOL> except Exception : <EOL> data = ( [ ] , [ ] , [ ] ) <EOL> if len ( data ) == <NUM_LIT:2> : <EOL> self . feeds , self . items = data <EOL> self . filters = [ ] <EOL> else : <EOL> self . feeds , self . items , self . filters = data <EOL> attributes = { <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT>' : <NUM_LIT:0> , <EOL> '<STR_LIT:username>' : None , <EOL> '<STR_LIT:password>' : None , <EOL> '<STR_LIT>' : None , <EOL> } <EOL> for feed in self . feeds : <EOL> for name , value in attributes . iteritems ( ) : <EOL> if not hasattr ( feed , name ) : <EOL> setattr ( feed , name , value ) <EOL> if not hasattr ( feed , '<STR_LIT>' ) : <EOL> feed . id_list = list ( feed . id_set ) <EOL> logging . info ( '<STR_LIT>' % ( len ( self . feeds ) , len ( self . items ) , len ( self . filters ) ) ) <EOL> def save ( self , path = '<STR_LIT>' ) : <EOL> logging . info ( '<STR_LIT>' % path ) <EOL> data = ( self . feeds , self . items , self . filters ) <EOL> safe_pickle . save ( path , data ) <EOL> def clear_item_history ( self ) : <EOL> logging . info ( '<STR_LIT>' ) <EOL> del self . items [ : ] <EOL> def clear_feed_cache ( self ) : <EOL> logging . info ( '<STR_LIT>' ) <EOL> for feed in self . feeds : <EOL> feed . clear_cache ( ) </s>
