<s> import sys <EOL> import pycurl <EOL> try : <EOL> import signal <EOL> from signal import SIGPIPE , SIG_IGN <EOL> signal . signal ( signal . SIGPIPE , signal . SIG_IGN ) <EOL> except ImportError : <EOL> pass <EOL> num_conn = <NUM_LIT:10> <EOL> try : <EOL> if sys . argv [ <NUM_LIT:1> ] == "<STR_LIT:->" : <EOL> urls = sys . stdin . readlines ( ) <EOL> else : <EOL> urls = open ( sys . argv [ <NUM_LIT:1> ] ) . readlines ( ) <EOL> if len ( sys . argv ) >= <NUM_LIT:3> : <EOL> num_conn = int ( sys . argv [ <NUM_LIT:2> ] ) <EOL> except : <EOL> print ( "<STR_LIT>" % sys . argv [ <NUM_LIT:0> ] ) <EOL> raise SystemExit <EOL> queue = [ ] <EOL> for url in urls : <EOL> url = url . strip ( ) <EOL> if not url or url [ <NUM_LIT:0> ] == "<STR_LIT:#>" : <EOL> continue <EOL> filename = "<STR_LIT>" % ( len ( queue ) + <NUM_LIT:1> ) <EOL> queue . append ( ( url , filename ) ) <EOL> assert queue , "<STR_LIT>" <EOL> num_urls = len ( queue ) <EOL> num_conn = min ( num_conn , num_urls ) <EOL> assert <NUM_LIT:1> <= num_conn <= <NUM_LIT> , "<STR_LIT>" <EOL> print ( "<STR_LIT>" % ( pycurl . version , pycurl . COMPILE_LIBCURL_VERSION_NUM ) ) <EOL> print ( "<STR_LIT>" , num_urls , "<STR_LIT>" , num_conn , "<STR_LIT>" ) <EOL> m = pycurl . CurlMulti ( ) <EOL> m . handles = [ ] <EOL> for i in range ( num_conn ) : <EOL> c = pycurl . Curl ( ) <EOL> c . fp = None <EOL> c . setopt ( pycurl . FOLLOWLOCATION , <NUM_LIT:1> ) <EOL> c . setopt ( pycurl . MAXREDIRS , <NUM_LIT:5> ) <EOL> c . setopt ( pycurl . CONNECTTIMEOUT , <NUM_LIT:30> ) <EOL> c . setopt ( pycurl . TIMEOUT , <NUM_LIT> ) <EOL> c . setopt ( pycurl . NOSIGNAL , <NUM_LIT:1> ) <EOL> m . handles . append ( c ) <EOL> freelist = m . handles [ : ] <EOL> num_processed = <NUM_LIT:0> <EOL> while num_processed < num_urls : <EOL> while queue and freelist : <EOL> url , filename = queue . pop ( <NUM_LIT:0> ) <EOL> c = freelist . pop ( ) <EOL> c . fp = open ( filename , "<STR_LIT:wb>" ) <EOL> c . setopt ( pycurl . URL , url ) <EOL> c . setopt ( pycurl . WRITEDATA , c . fp ) <EOL> m . add_handle ( c ) <EOL> c . filename = filename <EOL> c . url = url <EOL> while <NUM_LIT:1> : <EOL> ret , num_handles = m . perform ( ) <EOL> if ret != pycurl . E_CALL_MULTI_PERFORM : <EOL> break <EOL> while <NUM_LIT:1> : <EOL> num_q , ok_list , err_list = m . info_read ( ) <EOL> for c in ok_list : <EOL> c . fp . close ( ) <EOL> c . fp = None <EOL> m . remove_handle ( c ) <EOL> print ( "<STR_LIT>" , c . filename , c . url , c . getinfo ( pycurl . EFFECTIVE_URL ) ) <EOL> freelist . append ( c ) <EOL> for c , errno , errmsg in err_list : <EOL> c . fp . close ( ) <EOL> c . fp = None <EOL> m . remove_handle ( c ) <EOL> print ( "<STR_LIT>" , c . filename , c . url , errno , errmsg ) <EOL> freelist . append ( c ) <EOL> num_processed = num_processed + len ( ok_list ) + len ( err_list ) <EOL> if num_q == <NUM_LIT:0> : <EOL> break <EOL> m . select ( <NUM_LIT:1.0> ) <EOL> for c in m . handles : <EOL> if c . fp is not None : <EOL> c . fp . close ( ) <EOL> c . fp = None <EOL> c . close ( ) <EOL> m . close ( ) </s>
