<s> """<STR_LIT>""" <EOL> import feedparser <EOL> import pycurl <EOL> import time <EOL> import sys <EOL> import curses <EOL> from string import Template <EOL> from cStringIO import StringIO <EOL> from readability . readability import Document <EOL> from BeautifulSoup import BeautifulSoup <EOL> from textwrap import wrap <EOL> from sites import interests <EOL> UA = "<STR_LIT>" <EOL> feedparser . USER_AGENT = UA <EOL> FEED_MEMO_HASH = { } <EOL> FEED_MEMOIZE_INTERVAL = <NUM_LIT> <EOL> WINDOW_COLS = <NUM_LIT> <EOL> WINDOW_ROWS = <NUM_LIT> <EOL> def initscr ( screen ) : <EOL> """<STR_LIT>""" <EOL> global WINDOW_COLS , WINDOW_ROWS <EOL> screen . refresh ( ) <EOL> win = curses . newwin ( <NUM_LIT:0> , <NUM_LIT:0> ) <EOL> WINDOW_ROWS , WINDOW_COLS = win . getmaxyx ( ) <EOL> try : <EOL> curses . wrapper ( initscr ) <EOL> except curses . error : <EOL> pass <EOL> def purge_expired ( data_hash , interval ) : <EOL> """<STR_LIT>""" <EOL> expired = [ ] <EOL> for key , val in data_hash . items ( ) : <EOL> set_time = val [ <NUM_LIT:0> ] <EOL> if ( time . time ( ) - set_time ) > interval : <EOL> expired . append ( key ) <EOL> for ex_k in expired : <EOL> del data_hash [ ex_k ] <EOL> def parse_feed ( url , interval = FEED_MEMOIZE_INTERVAL ) : <EOL> """<STR_LIT>""" <EOL> purge_expired ( FEED_MEMO_HASH , interval ) <EOL> if FEED_MEMO_HASH . has_key ( url ) : <EOL> return FEED_MEMO_HASH [ url ] [ <NUM_LIT:1> ] <EOL> else : <EOL> feed = feedparser . parse ( url ) <EOL> if feed : <EOL> if feed . version and len ( feed . version ) > <NUM_LIT:0> : <EOL> FEED_MEMO_HASH [ url ] = ( time . time ( ) , feed ) <EOL> return feed <EOL> def _read_item_data ( feed_url , get_fn ) : <EOL> """<STR_LIT>""" <EOL> data = [ ] <EOL> feed = parse_feed ( feed_url ) <EOL> if feed is not None : <EOL> for entry in feed . entries : <EOL> try : <EOL> data . append ( get_fn ( entry ) ) <EOL> except AttributeError , detail : <EOL> print >> sys . stderr , detail <EOL> data . append ( None ) <EOL> return data <EOL> def item_titles ( feed_url ) : <EOL> """<STR_LIT>""" <EOL> return _read_item_data ( feed_url , lambda x : x . title ) <EOL> def strip_url_parameters ( url ) : <EOL> """<STR_LIT>""" <EOL> return url . split ( '<STR_LIT:?>' ) [ <NUM_LIT:0> ] <EOL> def item_links ( feed_url , strip_parameters ) : <EOL> """<STR_LIT>""" <EOL> links = _read_item_data ( feed_url , lambda x : x . link ) <EOL> if strip_parameters : <EOL> return map ( strip_url_parameters , links ) <EOL> else : <EOL> return links <EOL> def load_url ( url , referrer = None ) : <EOL> """<STR_LIT>""" <EOL> data = None <EOL> databuffer = StringIO ( ) <EOL> curl = pycurl . Curl ( ) <EOL> curl . setopt ( pycurl . URL , url ) <EOL> curl . setopt ( pycurl . FOLLOWLOCATION , <NUM_LIT:1> ) <EOL> curl . setopt ( pycurl . CONNECTTIMEOUT , <NUM_LIT:5> ) <EOL> curl . setopt ( pycurl . TIMEOUT , <NUM_LIT:8> ) <EOL> curl . setopt ( pycurl . WRITEFUNCTION , databuffer . write ) <EOL> curl . setopt ( pycurl . USERAGENT , UA ) <EOL> curl . setopt ( pycurl . COOKIEFILE , '<STR_LIT>' ) <EOL> if referrer is not None : <EOL> curl . setopt ( pycurl . REFERER , referrer ) <EOL> try : <EOL> curl . perform ( ) <EOL> data = databuffer . getvalue ( ) <EOL> except Exception : <EOL> pass <EOL> curl . close ( ) <EOL> return data <EOL> OUTPUT_FORMAT = Template ( """<STR_LIT>""" ) <EOL> def get_items ( feed_url , strip_parameters = True ) : <EOL> """<STR_LIT>""" <EOL> titles = filter ( None , item_titles ( feed_url ) ) <EOL> links = filter ( None , item_links ( feed_url , strip_parameters ) ) <EOL> if len ( titles ) == <NUM_LIT:0> or len ( links ) == <NUM_LIT:0> : <EOL> print "<STR_LIT>" , feed_url <EOL> output = [ ] <EOL> posts = { } <EOL> for i , title in enumerate ( titles ) : <EOL> try : <EOL> output . append ( OUTPUT_FORMAT . substitute ( num = ( i + <NUM_LIT:1> ) , <EOL> title = title , <EOL> link = links [ i ] ) ) <EOL> posts [ ( i + <NUM_LIT:1> ) ] = str ( links [ i ] . encode ( '<STR_LIT:utf-8>' ) ) <EOL> except KeyError : <EOL> pass <EOL> return u'<STR_LIT>' . join ( output ) , posts <EOL> def get_article ( url , referrer = None ) : <EOL> """<STR_LIT>""" <EOL> html = load_url ( url , referrer ) <EOL> if html is not None : <EOL> doc_html = Document ( html ) . summary ( html_partial = True ) <EOL> clean_html = doc_html . replace ( '<STR_LIT>' , u'<STR_LIT:&>' ) . replace ( u'<STR_LIT>' , u'<STR_LIT:\n>' ) <EOL> return BeautifulSoup ( clean_html ) . getText ( separator = u'<STR_LIT:U+0020>' ) . replace ( u'<STR_LIT:U+0020>' , u'<STR_LIT:U+0020>' ) <EOL> def prompt_user ( prompt ) : <EOL> """<STR_LIT>""" <EOL> print prompt , <EOL> reply = sys . stdin . readline ( ) . rstrip ( ) . lstrip ( ) <EOL> return reply <EOL> def scroll_output ( data , <EOL> rows = ( WINDOW_ROWS - <NUM_LIT:1> ) , <EOL> cols = WINDOW_COLS , <EOL> prompt = '<STR_LIT>' , <EOL> wrap_data = True , <EOL> choice_fn = None ) : <EOL> """<STR_LIT>""" <EOL> if wrap_data : <EOL> margin = u'<STR_LIT:U+0020>' * ( cols / <NUM_LIT:4> ) <EOL> lines = [ ] <EOL> for line in wrap ( data , cols / <NUM_LIT:2> ) : <EOL> lines . append ( margin + line ) <EOL> else : <EOL> lines = data . splitlines ( ) <EOL> for i , line in enumerate ( lines ) : <EOL> if i > <NUM_LIT:0> and i % rows == <NUM_LIT:0> : <EOL> user_choice = prompt_user ( prompt ) <EOL> if choice_fn is not None : <EOL> if choice_fn ( user_choice ) : <EOL> break <EOL> print line <EOL> def show_feed_menu ( ) : <EOL> """<STR_LIT>""" <EOL> if len ( interests ) == <NUM_LIT:0> : <EOL> print "<STR_LIT>" <EOL> else : <EOL> print '<STR_LIT>' . format ( '<STR_LIT>' ) , '<STR_LIT>' <EOL> print '<STR_LIT>' . format ( '<STR_LIT>' ) , '<STR_LIT>' <EOL> for code , feed_data in interests . items ( ) : <EOL> if feed_data . has_key ( '<STR_LIT:url>' ) : <EOL> feed_desc = feed_data [ '<STR_LIT:url>' ] <EOL> if feed_data . has_key ( '<STR_LIT>' ) : <EOL> feed_desc = feed_data [ '<STR_LIT>' ] <EOL> print '<STR_LIT>' . format ( code ) , feed_desc <EOL> def get_news ( ) : <EOL> """<STR_LIT>""" <EOL> option_prompt = '<STR_LIT>' <EOL> no_content = '<STR_LIT>' <EOL> while True : <EOL> feed = prompt_user ( "<STR_LIT>" ) <EOL> if len ( feed ) == <NUM_LIT:0> : <EOL> break <EOL> elif "<STR_LIT:!>" == feed : <EOL> show_feed_menu ( ) <EOL> else : <EOL> feed_referrer = None <EOL> try : <EOL> feed_data = interests [ feed . lower ( ) ] <EOL> feed_url = feed_data [ '<STR_LIT:url>' ] <EOL> strip_parameters = True <EOL> if feed_data . has_key ( '<STR_LIT>' ) : <EOL> strip_parameters = feed_data [ '<STR_LIT>' ] <EOL> if feed_data . has_key ( '<STR_LIT>' ) : <EOL> feed_referrer = feed_data [ '<STR_LIT>' ] <EOL> menu , links = get_items ( feed_url , strip_parameters ) <EOL> except KeyError : <EOL> menu , links = get_items ( feed ) <EOL> if len ( links ) == <NUM_LIT:0> : <EOL> print no_content , feed <EOL> break <EOL> else : <EOL> options = links . keys ( ) <EOL> bad_option = "<STR_LIT>" % ( min ( options ) , <EOL> max ( options ) ) <EOL> def _display_article ( user_choice ) : <EOL> """<STR_LIT>""" <EOL> break_menu = False <EOL> try : <EOL> choice = int ( user_choice ) <EOL> if choice in options : <EOL> article = get_article ( links [ choice ] , feed_referrer ) <EOL> if article is not None : <EOL> scroll_output ( article ) <EOL> break_menu = True <EOL> else : <EOL> print no_content , links [ choice ] <EOL> else : <EOL> print bad_option <EOL> except ValueError : <EOL> pass <EOL> return break_menu <EOL> scroll_output ( menu , <EOL> wrap_data = False , <EOL> prompt = option_prompt , <EOL> choice_fn = _display_article ) <EOL> while True : <EOL> choice = prompt_user ( <EOL> "<STR_LIT>" "<STR_LIT>" <EOL> % ( min ( options ) , max ( options ) ) ) <EOL> if <NUM_LIT:0> == len ( choice ) : <EOL> break <EOL> if _display_article ( choice ) : <EOL> break <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> get_news ( ) </s>
