<s> import codecs <EOL> import urllib2 <EOL> import cStringIO as StringIO <EOL> from lxml import etree <EOL> rss_urls = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> def parseHtml ( html ) : <EOL> parser = etree . HTMLParser ( encoding = '<STR_LIT:utf8>' ) <EOL> tree = etree . parse ( StringIO . StringIO ( html ) , parser ) <EOL> return tree <EOL> def parseXml ( xml ) : <EOL> parser = etree . XMLParser ( encoding = '<STR_LIT:utf8>' ) <EOL> tree = etree . parse ( StringIO . StringIO ( xml ) , parser ) <EOL> return tree <EOL> def getPage ( url ) : <EOL> file = urllib2 . urlopen ( url ) <EOL> content = file . read ( ) <EOL> file . close ( ) <EOL> return content <EOL> def getLinks ( rss_url ) : <EOL> content = getPage ( rss_url ) <EOL> tree = parseXml ( content ) <EOL> return tree . xpath ( '<STR_LIT>' ) <EOL> def getNewsText ( news_url ) : <EOL> content = getPage ( news_url ) <EOL> tree = parseHtml ( content ) <EOL> for paragraph in tree . xpath ( "<STR_LIT>" ) : <EOL> yield paragraph . strip ( ) <EOL> def crawelCategory ( rss_url ) : <EOL> links = getLinks ( rss_url ) <EOL> for link in links [ <NUM_LIT:1> : ] : <EOL> yield '<STR_LIT:U+0020>' . join ( getNewsText ( link ) ) <EOL> def main ( ) : <EOL> with codecs . open ( '<STR_LIT>' , '<STR_LIT>' , encoding = '<STR_LIT:utf8>' ) as file : <EOL> for url in rss_urls : <EOL> for text in crawelCategory ( url ) : <EOL> print '<STR_LIT>' % len ( text ) <EOL> print >> file , text <EOL> print '<STR_LIT>' <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
