<s> import sys <EOL> import urllib <EOL> import re <EOL> import json <EOL> import socket <EOL> socket . setdefaulttimeout ( <NUM_LIT:10> ) <EOL> from bs4 import BeautifulSoup <EOL> cache = { } <EOL> for line in open ( sys . argv [ <NUM_LIT:1> ] ) : <EOL> fields = line . rstrip ( '<STR_LIT:\n>' ) . split ( '<STR_LIT:\t>' ) <EOL> sid = fields [ <NUM_LIT:0> ] <EOL> uid = fields [ <NUM_LIT:1> ] <EOL> tweet = None <EOL> text = "<STR_LIT>" <EOL> if cache . has_key ( sid ) : <EOL> text = cache [ sid ] <EOL> else : <EOL> try : <EOL> f = urllib . urlopen ( "<STR_LIT>" % ( uid , sid ) ) <EOL> html = f . read ( ) . replace ( "<STR_LIT>" , "<STR_LIT>" ) + "<STR_LIT>" <EOL> soup = BeautifulSoup ( html ) <EOL> jstt = soup . find_all ( "<STR_LIT:p>" , "<STR_LIT>" ) <EOL> tweets = list ( set ( [ x . get_text ( ) for x in jstt ] ) ) <EOL> if ( len ( tweets ) ) > <NUM_LIT:1> : <EOL> continue <EOL> text = tweets [ <NUM_LIT:0> ] <EOL> cache [ sid ] = tweets [ <NUM_LIT:0> ] <EOL> for j in soup . find_all ( "<STR_LIT:input>" , "<STR_LIT>" , id = "<STR_LIT>" ) : <EOL> js = json . loads ( j [ '<STR_LIT:value>' ] ) <EOL> if ( js . has_key ( "<STR_LIT>" ) ) : <EOL> tweet = js [ "<STR_LIT>" ] [ "<STR_LIT:status>" ] <EOL> text = js [ "<STR_LIT>" ] [ "<STR_LIT:status>" ] [ "<STR_LIT:text>" ] <EOL> cache [ sid ] = text <EOL> break <EOL> except Exception : <EOL> continue <EOL> if ( tweet != None and tweet [ "<STR_LIT>" ] != sid ) : <EOL> text = "<STR_LIT>" <EOL> cache [ sid ] = "<STR_LIT>" <EOL> text = text . replace ( '<STR_LIT:\n>' , '<STR_LIT:U+0020>' , ) <EOL> text = re . sub ( r'<STR_LIT>' , '<STR_LIT:U+0020>' , text ) <EOL> print "<STR_LIT:\t>" . join ( fields + [ text ] ) . encode ( '<STR_LIT:utf-8>' ) </s>
