<s> import urllib <EOL> from tornado import escape <EOL> from tornado import gen <EOL> from tornado . httpclient import AsyncHTTPClient <EOL> from tornado . options import options <EOL> import tornado . ioloop <EOL> from libs . client import GetPage , sync_loop_call , formula , update_file <EOL> from libs . geo import match_geoname , match_world_geoname <EOL> import time <EOL> github_china = [ ] <EOL> github_world = [ ] <EOL> temp_github_world = [ ] <EOL> temp_github_china = [ ] <EOL> current_china_page = <NUM_LIT:1> <EOL> current_world_page = <NUM_LIT:1> <EOL> china_location_map = { } <EOL> china_map = { } <EOL> for city in options . city_list : <EOL> china_map [ city ] = { "<STR_LIT>" : <NUM_LIT:0> , "<STR_LIT>" : <NUM_LIT:6> } <EOL> world_location_map = { } <EOL> world_map = { } <EOL> for country_code in options . country_code_list : <EOL> world_map [ country_code ] = { "<STR_LIT>" : <NUM_LIT:0> , "<STR_LIT>" : <NUM_LIT:6> } <EOL> AsyncHTTPClient . configure ( "<STR_LIT>" ) <EOL> def wash ( users ) : <EOL> user_names = [ ] <EOL> new_users = [ ] <EOL> for user in users : <EOL> if user [ '<STR_LIT>' ] not in user_names : <EOL> user_names . append ( user [ '<STR_LIT>' ] ) <EOL> new_users . append ( user ) <EOL> return new_users <EOL> @ gen . coroutine <EOL> def contribute ( login ) : <EOL> resp = yield GetPage ( options . contribution_url ( login ) ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> resp = escape . json_decode ( resp . body ) <EOL> all_contribute = sum ( [ day [ <NUM_LIT:1> ] for day in resp ] ) <EOL> else : <EOL> all_contribute = <NUM_LIT:0> <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( resp . code , resp . message ) ) <EOL> raise gen . Return ( all_contribute ) <EOL> @ sync_loop_call ( <NUM_LIT:32> * <NUM_LIT:1000> ) <EOL> @ gen . coroutine <EOL> def update_china_user ( ) : <EOL> global github_china <EOL> global current_china_page <EOL> global temp_github_china <EOL> options . logger . info ( "<STR_LIT>" % current_china_page ) <EOL> resp = yield search_china ( current_china_page ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> resp = escape . json_decode ( resp . body ) <EOL> users = resp [ "<STR_LIT>" ] <EOL> for user in users : <EOL> contributions = yield contribute ( user [ "<STR_LIT>" ] ) <EOL> temp_github_china . append ( { <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT:name>" : user [ "<STR_LIT:name>" ] or "<STR_LIT>" , <EOL> "<STR_LIT:location>" : user [ "<STR_LIT:location>" ] , <EOL> "<STR_LIT>" : "<STR_LIT>" + user [ "<STR_LIT>" ] <EOL> + urllib . urlencode ( { "<STR_LIT:s>" : <NUM_LIT> } ) , <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : contributions , <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : contributions + formula ( user [ "<STR_LIT>" ] ) <EOL> } ) <EOL> temp_github_china = wash ( temp_github_china ) <EOL> current_china_page += <NUM_LIT:1> <EOL> if len ( github_china ) < len ( temp_github_china ) : <EOL> github_china = temp_github_china [ : ] <EOL> github_china = sorted ( github_china , key = lambda d : d [ '<STR_LIT>' ] , reverse = True ) <EOL> elif resp . code == <NUM_LIT> : <EOL> github_china = temp_github_china [ : ] <EOL> github_china = sorted ( github_china , key = lambda d : d [ '<STR_LIT>' ] , reverse = True ) <EOL> temp_github_china = [ ] <EOL> current_china_page = <NUM_LIT:1> <EOL> options . logger . info ( "<STR_LIT>" ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( current_china_page , resp . code , resp . message ) ) <EOL> @ sync_loop_call ( <NUM_LIT:64> * <NUM_LIT:1000> ) <EOL> @ gen . coroutine <EOL> def update_world_user ( ) : <EOL> global github_world <EOL> global current_world_page <EOL> global temp_github_world <EOL> options . logger . info ( "<STR_LIT>" % current_world_page ) <EOL> resp = yield search_world ( current_world_page ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> resp = escape . json_decode ( resp . body ) <EOL> users = resp [ "<STR_LIT>" ] <EOL> for user in users : <EOL> contributions = yield contribute ( user [ "<STR_LIT>" ] ) <EOL> temp_github_world . append ( { <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT:name>" : user [ "<STR_LIT:name>" ] or "<STR_LIT>" , <EOL> "<STR_LIT:location>" : user [ "<STR_LIT:location>" ] , <EOL> "<STR_LIT>" : "<STR_LIT>" + user [ "<STR_LIT>" ] <EOL> + urllib . urlencode ( { "<STR_LIT:s>" : <NUM_LIT> } ) , <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : contributions , <EOL> "<STR_LIT>" : user [ "<STR_LIT>" ] , <EOL> "<STR_LIT>" : contributions + formula ( user [ "<STR_LIT>" ] ) <EOL> } ) <EOL> temp_github_world = wash ( temp_github_world ) <EOL> current_world_page += <NUM_LIT:1> <EOL> if len ( github_world ) < len ( temp_github_world ) : <EOL> github_world = temp_github_world [ : ] <EOL> github_world = sorted ( github_world , key = lambda d : d [ '<STR_LIT>' ] , reverse = True ) <EOL> elif resp . code == <NUM_LIT> : <EOL> github_world = temp_github_world [ : ] <EOL> github_world = sorted ( github_world , key = lambda d : d [ '<STR_LIT>' ] , reverse = True ) <EOL> temp_github_world = [ ] <EOL> current_world_page = <NUM_LIT:1> <EOL> options . logger . info ( "<STR_LIT>" ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( current_world_page , resp . code , resp . message ) ) <EOL> @ gen . coroutine <EOL> def search_china ( page ) : <EOL> url = options . api_url + "<STR_LIT>" + str ( page ) + "<STR_LIT>" <EOL> resp = yield GetPage ( url ) <EOL> options . logger . info ( "<STR_LIT>" % url ) <EOL> raise gen . Return ( resp ) <EOL> @ gen . coroutine <EOL> def search_world ( page ) : <EOL> url = options . api_url + "<STR_LIT>" + str ( page ) + "<STR_LIT>" <EOL> options . logger . info ( "<STR_LIT>" % url ) <EOL> resp = yield GetPage ( url ) <EOL> raise gen . Return ( resp ) <EOL> @ sync_loop_call ( <NUM_LIT> * <NUM_LIT:1000> ) <EOL> @ gen . coroutine <EOL> def update_china_location ( ) : <EOL> global china_location_map <EOL> global china_map <EOL> if not china_location_map : <EOL> resp = yield GetPage ( options . api_url + options . location_map_gist ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> resp = escape . json_decode ( resp . body ) <EOL> raw_url = resp [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> resp = yield GetPage ( raw_url ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> china_location_map = escape . json_decode ( resp . body ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( resp . code , resp . message ) ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % ( resp . code , resp . message ) ) <EOL> else : <EOL> if int ( time . time ( ) ) % <NUM_LIT> < <NUM_LIT> : <EOL> resp = yield update_file ( options . api_url + options . location_map_gist , <EOL> "<STR_LIT>" , <EOL> china_location_map ) <EOL> if resp . code != <NUM_LIT:200> : <EOL> options . logger . error ( "<STR_LIT>" % ( resp . code , resp . message ) ) <EOL> temp_china_map = { } <EOL> for city in options . city_list : <EOL> temp_china_map [ city ] = { "<STR_LIT>" : <NUM_LIT:0> , "<STR_LIT>" : <NUM_LIT:6> } <EOL> for user in github_china : <EOL> try : <EOL> location = user [ "<STR_LIT:location>" ] . lower ( ) <EOL> location = '<STR_LIT:U+002C>' . join ( location . strip ( ) . split ( ) ) <EOL> except Exception , e : <EOL> options . logger . error ( "<STR_LIT>" % e ) <EOL> continue <EOL> if location == "<STR_LIT>" : <EOL> continue <EOL> if location in china_location_map : <EOL> city = china_location_map [ location ] <EOL> else : <EOL> city = yield match_geoname ( location ) <EOL> if city : <EOL> temp_china_map [ city ] [ "<STR_LIT>" ] += <NUM_LIT:1> <EOL> china_location_map [ location ] = city <EOL> else : <EOL> options . logger . warning ( "<STR_LIT>" % location ) <EOL> china_map = temp_china_map . copy ( ) <EOL> @ sync_loop_call ( <NUM_LIT> * <NUM_LIT:1000> ) <EOL> @ gen . coroutine <EOL> def update_world_location ( ) : <EOL> options . logger . info ( "<STR_LIT>" ) <EOL> global world_location_map <EOL> global world_map <EOL> if not world_location_map : <EOL> resp = yield GetPage ( options . api_url + options . world_location_map_gist ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> resp = escape . json_decode ( resp . body ) <EOL> raw_url = resp [ "<STR_LIT>" ] [ "<STR_LIT>" ] [ '<STR_LIT>' ] <EOL> resp = yield GetPage ( raw_url ) <EOL> if resp . code == <NUM_LIT:200> : <EOL> world_location_map = escape . json_decode ( resp . body ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( resp . code , resp . message ) ) <EOL> else : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( resp . code , resp . message ) ) <EOL> else : <EOL> if int ( time . time ( ) ) % <NUM_LIT> < <NUM_LIT> : <EOL> resp = yield update_file ( options . api_url + options . world_location_map_gist , <EOL> "<STR_LIT>" , <EOL> world_location_map ) <EOL> if resp . code != <NUM_LIT:200> : <EOL> options . logger . error ( "<STR_LIT>" % <EOL> ( resp . code , resp . message ) ) <EOL> temp_world_map = { } <EOL> for country_code in options . country_code_list : <EOL> temp_world_map [ country_code ] = { "<STR_LIT>" : <NUM_LIT:0> , "<STR_LIT>" : <NUM_LIT:6> } <EOL> for user in github_world : <EOL> if not user [ "<STR_LIT:location>" ] : <EOL> continue <EOL> try : <EOL> location = user [ "<STR_LIT:location>" ] . strip ( ) <EOL> location = '<STR_LIT:U+002C>' . join ( filter ( lambda d : d , <EOL> '<STR_LIT:U+002C>' . join ( location . split ( ) ) . split ( '<STR_LIT:U+002C>' ) ) ) <EOL> except Exception , e : <EOL> options . logger . error ( "<STR_LIT>" % e ) <EOL> continue <EOL> if location in world_location_map : <EOL> country_code = world_location_map [ location ] <EOL> else : <EOL> country_code = yield match_world_geoname ( location ) <EOL> if country_code : <EOL> temp_world_map [ country_code ] [ "<STR_LIT>" ] += <NUM_LIT:1> <EOL> world_location_map [ location ] = country_code <EOL> else : <EOL> options . logger . warning ( "<STR_LIT>" % location ) <EOL> world_map = temp_world_map . copy ( ) <EOL> if __name__ == "<STR_LIT:__main__>" : <EOL> update_china_user ( ) <EOL> update_world_user ( ) <EOL> update_china_location ( ) <EOL> update_world_location ( ) <EOL> tornado . ioloop . IOLoop . instance ( ) . start ( ) </s>
