<s> from __future__ import print_function <EOL> import bz2 <EOL> import collections <EOL> import json <EOL> import os <EOL> import shutil <EOL> import sqlite3 <EOL> import tempfile <EOL> import urllib2 <EOL> COMP_TYPES = [ <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT:30> , <EOL> <NUM_LIT> , <EOL> <NUM_LIT> , <EOL> ] <EOL> SQLITE_DUMP_URL = "<STR_LIT>" <EOL> def download_database ( destination_path ) : <EOL> decompressor = bz2 . BZ2Decompressor ( ) <EOL> response = urllib2 . urlopen ( SQLITE_DUMP_URL ) <EOL> total_bytes = int ( response . info ( ) . getheader ( '<STR_LIT>' ) . strip ( ) ) <EOL> chunk = <NUM_LIT> <EOL> bytes_read = <NUM_LIT:0> <EOL> with open ( destination_path , '<STR_LIT:wb>' ) as f : <EOL> while <NUM_LIT:1> : <EOL> data = response . read ( chunk ) <EOL> if not data : <EOL> break <EOL> bytes_read += len ( data ) <EOL> print ( "<STR_LIT>" % ( <EOL> bytes_read , <EOL> total_bytes , <EOL> float ( bytes_read ) / total_bytes * <NUM_LIT:100> , <EOL> ) ) <EOL> decompressed_data = decompressor . decompress ( data ) <EOL> if decompressed_data is not None : <EOL> f . write ( decompressed_data ) <EOL> def build_all_types ( cursor ) : <EOL> inv_type_materials = collections . defaultdict ( list ) <EOL> for row in cursor . execute ( '<STR_LIT>' ) : <EOL> inv_type_materials [ row [ '<STR_LIT>' ] ] . append ( { <EOL> '<STR_LIT>' : row [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : row [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : row [ '<STR_LIT>' ] , <EOL> } ) <EOL> for ( type_id , <EOL> group_id , <EOL> type_name , <EOL> volume , <EOL> market_group_id ) in cursor . execute ( '''<STR_LIT>''' ) : <EOL> print ( "<STR_LIT>" % type_name ) <EOL> try : <EOL> type_name = type_name . decode ( '<STR_LIT:utf-8>' ) <EOL> except UnicodeDecodeError : <EOL> continue <EOL> has_market = market_group_id is not None <EOL> d = { <EOL> '<STR_LIT>' : type_id , <EOL> '<STR_LIT>' : group_id , <EOL> '<STR_LIT>' : type_name , <EOL> '<STR_LIT>' : volume or <NUM_LIT:0.0> , <EOL> '<STR_LIT>' : has_market , <EOL> } <EOL> if ( <EOL> group_id in COMP_TYPES and <EOL> type_id in inv_type_materials <EOL> ) : <EOL> d [ '<STR_LIT>' ] = [ { '<STR_LIT>' : material [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : material [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : material [ '<STR_LIT>' ] } <EOL> for material <EOL> in inv_type_materials [ type_id ] ] <EOL> yield d <EOL> def main ( ) : <EOL> temp_dir = tempfile . mkdtemp ( ) <EOL> try : <EOL> db_path = os . path . join ( temp_dir , '<STR_LIT>' ) <EOL> print ( "<STR_LIT>" % db_path ) <EOL> download_database ( db_path ) <EOL> print ( "<STR_LIT>" ) <EOL> conn = sqlite3 . connect ( db_path ) <EOL> conn . row_factory = sqlite3 . Row <EOL> conn . text_factory = str <EOL> c = conn . cursor ( ) <EOL> print ( "<STR_LIT>" ) <EOL> all_types = list ( build_all_types ( c ) ) <EOL> types_output_file = '<STR_LIT>' <EOL> print ( "<STR_LIT>" % types_output_file ) <EOL> with open ( types_output_file , '<STR_LIT:w>' ) as f : <EOL> f . write ( json . dumps ( all_types , indent = <NUM_LIT:2> ) ) <EOL> finally : <EOL> shutil . rmtree ( temp_dir ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> main ( ) </s>
