<s> import codecs <EOL> import json <EOL> import logging <EOL> import xmlrpclib <EOL> import sys <EOL> from gevent import monkey <EOL> from gevent . pool import Pool <EOL> import requests <EOL> monkey . patch_all ( ) <EOL> logging . basicConfig ( level = logging . INFO , format = '<STR_LIT>' ) <EOL> logger = logging . getLogger ( ) <EOL> download_count = <NUM_LIT:0> <EOL> not_found_count = <NUM_LIT:0> <EOL> session = requests . Session ( ) <EOL> def download_package_dict ( package_name ) : <EOL> """<STR_LIT>""" <EOL> global download_count , not_found_count <EOL> download_count += <NUM_LIT:1> <EOL> if download_count % <NUM_LIT:1000> == <NUM_LIT:0> : <EOL> logger . info ( '<STR_LIT>' , download_count , not_found_count ) <EOL> response = session . get ( '<STR_LIT>' % package_name ) <EOL> if response . status_code == requests . codes . ok : <EOL> return response . json ( ) <EOL> elif response . status_code == requests . codes . not_found : <EOL> not_found_count += <NUM_LIT:1> <EOL> else : <EOL> logger . warning ( '<STR_LIT>' , package_name , response . status_code ) <EOL> def download_all_package_dicts ( num_greenlets = <NUM_LIT:5> , limit = None ) : <EOL> """<STR_LIT>""" <EOL> package_names = xmlrpclib . ServerProxy ( '<STR_LIT>' ) . list_packages ( ) [ : limit ] <EOL> logger . info ( '<STR_LIT>' , len ( package_names ) ) <EOL> return Pool ( num_greenlets ) . imap ( download_package_dict , package_names ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> with codecs . open ( '<STR_LIT>' , '<STR_LIT:wb>' , encoding = '<STR_LIT:utf-8>' ) as out_file : <EOL> for package_dict in download_all_package_dicts ( limit = int ( sys . argv [ <NUM_LIT:1> ] ) if len ( sys . argv ) > <NUM_LIT:1> else None ) : <EOL> if package_dict : <EOL> out_file . write ( json . dumps ( package_dict ) ) <EOL> out_file . write ( '<STR_LIT:\n>' ) </s>
