<s> import cProfile <EOL> import pstats <EOL> import multiprocessing as mp <EOL> from os import path , remove <EOL> from shutil import rmtree <EOL> from sys import exit <EOL> from time import time , sleep <EOL> import argparse <EOL> import plyvel <EOL> import ujson as json <EOL> from imposm . parser import OSMParser <EOL> from shapely . geometry import Polygon <EOL> import shapely . speedups <EOL> from settings import wantedTags <EOL> def prep_args ( ) : <EOL> parser = argparse . ArgumentParser ( ) <EOL> parser . add_argument ( <EOL> '<STR_LIT:source>' , <EOL> help = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> default = '<STR_LIT>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> type = str , <EOL> default = False ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> type = int , <EOL> default = <NUM_LIT:20> ) <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' , <EOL> action = '<STR_LIT:store_true>' ) , <EOL> parser . add_argument ( <EOL> '<STR_LIT>' , <EOL> help = '<STR_LIT>' <EOL> '<STR_LIT>' , <EOL> type = int , <EOL> default = <NUM_LIT> ) <EOL> return parser <EOL> args = vars ( prep_args ( ) . parse_args ( ) ) <EOL> args [ '<STR_LIT>' ] = args [ '<STR_LIT>' ] + '<STR_LIT>' <EOL> if args [ '<STR_LIT>' ] : <EOL> print args <EOL> def file_prep ( db_only = False ) : <EOL> if not db_only : <EOL> if path . isfile ( args [ '<STR_LIT>' ] ) : <EOL> if args [ '<STR_LIT>' ] : <EOL> remove ( args [ '<STR_LIT>' ] ) <EOL> else : <EOL> print '<STR_LIT>' + args [ '<STR_LIT>' ] <EOL> print ( '<STR_LIT>' + args [ '<STR_LIT>' ] + <EOL> '<STR_LIT>' ) <EOL> exit ( ) <EOL> if path . isdir ( '<STR_LIT>' ) : <EOL> rmtree ( '<STR_LIT>' ) <EOL> if path . isdir ( '<STR_LIT>' ) : <EOL> rmtree ( '<STR_LIT>' ) <EOL> class Ways ( ) : <EOL> groups = set ( ) <EOL> def __init__ ( self , db ) : <EOL> self . db = db <EOL> def way ( self , ways ) : <EOL> for id , tags , refs in ways : <EOL> if len ( tags ) and args [ '<STR_LIT>' ] > len ( refs ) > <NUM_LIT:1> and refs [ <NUM_LIT:0> ] == refs [ - <NUM_LIT:1> ] : <EOL> id = str ( id ) <EOL> tags [ '<STR_LIT>' ] = '<STR_LIT>' + id <EOL> self . db . put ( id , json . dumps ( [ refs , tags ] ) ) <EOL> if not args [ '<STR_LIT>' ] : <EOL> self . put_refs ( refs ) <EOL> def put_refs ( self , refs ) : <EOL> for ref in refs : <EOL> self . groups . add ( round_down ( ref , args [ '<STR_LIT>' ] ) ) <EOL> class Nodes ( ) : <EOL> batch = [ ] <EOL> first = True <EOL> def __init__ ( self , file ) : <EOL> self . file = file <EOL> def node ( self , nodes ) : <EOL> for id , tags , coords in nodes : <EOL> if len ( tags ) : <EOL> lat , lon = coords <EOL> lat = "<STR_LIT>" % lat <EOL> lon = "<STR_LIT>" % lon <EOL> tags [ '<STR_LIT>' ] = '<STR_LIT>' + str ( id ) <EOL> feature = { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : [ float ( lat ) , float ( lon ) ] } , <EOL> '<STR_LIT>' : tags <EOL> } <EOL> self . batch . append ( json . dumps ( feature ) ) <EOL> if len ( self . batch ) > <NUM_LIT> : <EOL> self . batch_write ( ) <EOL> def batch_write ( self ) : <EOL> if self . first : <EOL> self . file . write ( '<STR_LIT>' . join ( self . batch ) ) <EOL> self . first = False <EOL> else : <EOL> self . file . write ( '<STR_LIT>' + '<STR_LIT>' . join ( self . batch ) ) <EOL> self . batch = [ ] <EOL> class Coords ( ) : <EOL> def __init__ ( self , db , needed ) : <EOL> self . db = db <EOL> self . needed = needed <EOL> def coord ( self , coords ) : <EOL> for id , lat , lon in coords : <EOL> if round_down ( id , args [ '<STR_LIT>' ] ) in self . needed : <EOL> self . db . put ( str ( id ) , str ( lat ) + '<STR_LIT:U+002C>' + str ( lon ) ) <EOL> def coord_precache ( self , coords ) : <EOL> for id , lat , lon in coords : <EOL> self . db . put ( str ( id ) , str ( lat ) + '<STR_LIT:U+002C>' + str ( lon ) ) <EOL> def tag_filter ( tags ) : <EOL> if args [ '<STR_LIT>' ] and args [ '<STR_LIT>' ] not in tags : <EOL> for key in tags . keys ( ) : <EOL> del tags [ key ] <EOL> else : <EOL> for key in tags . keys ( ) : <EOL> if key not in wantedTags : <EOL> del tags [ key ] <EOL> else : <EOL> if wantedTags [ key ] == '<STR_LIT:*>' or tags [ key ] in wantedTags [ key ] : <EOL> a = <NUM_LIT:1> <EOL> else : <EOL> del tags [ key ] <EOL> def round_down ( num , divisor ) : <EOL> if divisor == <NUM_LIT:0> : <EOL> divisor = <NUM_LIT:1> <EOL> if divisor == <NUM_LIT:1> : <EOL> return num <EOL> return num - ( num % divisor ) <EOL> def include_queue ( queue ) : <EOL> build_POIs . queue = queue <EOL> def process ( output ) : <EOL> process . writeDone = False <EOL> queue = mp . Queue ( ) <EOL> pool = mp . Pool ( None , include_queue , [ queue ] , <NUM_LIT> ) <EOL> go = pool . map_async ( build_POIs , waysDB . iterator ( ) , callback = all_done ) <EOL> sleep ( <NUM_LIT:1> ) <EOL> while True : <EOL> if write ( output , queue ) : <EOL> break <EOL> go . wait ( ) <EOL> pool . terminate ( ) <EOL> pool . join ( ) <EOL> def all_done ( necessary_arg ) : <EOL> process . writeDone = True <EOL> def build_POIs ( ( id , string ) ) : <EOL> queue = build_POIs . queue <EOL> refs , tags = json . loads ( string ) <EOL> polygon = build_polygon ( refs ) <EOL> if polygon and polygon . is_valid : <EOL> tags [ '<STR_LIT>' ] = polygon . area * <NUM_LIT> <EOL> if tags [ '<STR_LIT>' ] > <NUM_LIT:0.0> : <EOL> centroid = polygon . centroid <EOL> feature = { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : [ <EOL> float ( "<STR_LIT>" % centroid . x ) , <EOL> float ( "<STR_LIT>" % centroid . y ) ] } , <EOL> '<STR_LIT>' : tags <EOL> } <EOL> queue . put ( json . dumps ( feature ) ) <EOL> def build_polygon ( refs ) : <EOL> coords = [ ] <EOL> for ref in refs : <EOL> coord = coordsDB . get ( str ( ref ) ) <EOL> if coord : <EOL> coord = map ( float , coord . split ( '<STR_LIT:U+002C>' ) ) <EOL> coords . append ( coord ) <EOL> else : <EOL> return False <EOL> if len ( coords ) > <NUM_LIT:2> : <EOL> polygon = Polygon ( coords ) <EOL> if polygon . is_valid : <EOL> return polygon <EOL> else : <EOL> return polygon . buffer ( <NUM_LIT:0.0> ) <EOL> else : <EOL> return False <EOL> def write ( file , queue ) : <EOL> toFile = [ ] <EOL> while not queue . empty ( ) : <EOL> toFile . append ( queue . get_nowait ( ) ) <EOL> if len ( toFile ) : <EOL> file . write ( '<STR_LIT>' + '<STR_LIT>' . join ( toFile ) ) <EOL> if process . writeDone : <EOL> if queue . empty ( ) : <EOL> return True <EOL> else : <EOL> sleep ( <NUM_LIT> ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> if args [ '<STR_LIT>' ] : <EOL> prW = cProfile . Profile ( ) <EOL> prW . enable ( ) <EOL> start = time ( ) <EOL> shapely . speedups . enable ( ) <EOL> file_prep ( ) <EOL> waysDB = plyvel . DB ( <EOL> '<STR_LIT>' , <EOL> create_if_missing = True , <EOL> error_if_exists = True , <EOL> write_buffer_size = <NUM_LIT> * <NUM_LIT> ) <EOL> coordsDB = plyvel . DB ( <EOL> '<STR_LIT>' , <EOL> create_if_missing = True , <EOL> error_if_exists = True , <EOL> write_buffer_size = <NUM_LIT> * <NUM_LIT> ) <EOL> output = open ( args [ '<STR_LIT>' ] , '<STR_LIT:a>' ) <EOL> output . write ( '<STR_LIT>' ) <EOL> ways = Ways ( waysDB ) <EOL> nodes = Nodes ( output ) <EOL> coords = Coords ( coordsDB , ways . groups ) <EOL> if args [ '<STR_LIT>' ] : <EOL> p = OSMParser ( coords_callback = coords . coord_precache ) <EOL> print '<STR_LIT>' <EOL> p . parse ( args [ '<STR_LIT:source>' ] ) <EOL> coords = None <EOL> del coords <EOL> p = OSMParser ( <EOL> ways_callback = ways . way , <EOL> ways_tag_filter = tag_filter , <EOL> nodes_callback = nodes . node , <EOL> nodes_tag_filter = tag_filter ) <EOL> print '<STR_LIT>' <EOL> p . parse ( args [ '<STR_LIT:source>' ] ) <EOL> nodes . batch_write ( ) <EOL> if not args [ '<STR_LIT>' ] : <EOL> p = OSMParser ( coords_callback = coords . coord ) <EOL> print '<STR_LIT>' <EOL> p . parse ( args [ '<STR_LIT:source>' ] ) <EOL> coords = None <EOL> del coords <EOL> del p , ways , nodes <EOL> print '<STR_LIT>' <EOL> process ( output ) <EOL> output . write ( '<STR_LIT>' ) <EOL> file_prep ( True ) <EOL> print '<STR_LIT>' + str ( args [ '<STR_LIT>' ] ) <EOL> print '<STR_LIT>' + str ( round ( time ( ) - start , <NUM_LIT:2> ) ) + '<STR_LIT>' <EOL> if args [ '<STR_LIT>' ] : <EOL> prW . disable ( ) <EOL> ps = pstats . Stats ( prW ) <EOL> ps . sort_stats ( '<STR_LIT:time>' ) <EOL> a = ps . print_stats ( <NUM_LIT:30> ) </s>
