<s> import os <EOL> import json <EOL> import logging <EOL> import unicodecsv <EOL> from xlsxwriter import Workbook <EOL> log = logging . getLogger ( __name__ ) <EOL> SOURCE_FIELDS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:source>' , '<STR_LIT>' ] <EOL> SORT_ENITTY = [ '<STR_LIT>' , '<STR_LIT:name>' , '<STR_LIT:type>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT:text>' ] <EOL> def get_headers ( rows ) : <EOL> headers = set ( ) <EOL> for row in rows : <EOL> headers . update ( row . keys ( ) ) <EOL> if '<STR_LIT:id>' in headers : <EOL> headers . remove ( '<STR_LIT:id>' ) <EOL> return headers <EOL> def csv_generate ( file_name , rows ) : <EOL> if not len ( rows ) : <EOL> return <EOL> headers = get_headers ( rows ) <EOL> if not len ( headers ) : <EOL> return <EOL> log . info ( '<STR_LIT>' , file_name , len ( rows ) ) <EOL> with open ( file_name , '<STR_LIT:w>' ) as fh : <EOL> writer = unicodecsv . writer ( fh ) <EOL> writer . writerow ( headers ) <EOL> for row in rows : <EOL> writer . writerow ( [ row . get ( h ) for h in headers ] ) <EOL> def json_generate ( file_name , jsons ) : <EOL> log . info ( '<STR_LIT>' , file_name ) <EOL> with open ( file_name , '<STR_LIT:w>' ) as fh : <EOL> json . dump ( { '<STR_LIT>' : jsons } , fh ) <EOL> def xlsx_generate ( file_name , sheets ) : <EOL> log . info ( '<STR_LIT>' , file_name ) <EOL> with open ( file_name , '<STR_LIT:w>' ) as fh : <EOL> workbook = Workbook ( fh ) <EOL> header_format = workbook . add_format ( { <EOL> '<STR_LIT>' : True , <EOL> '<STR_LIT>' : <NUM_LIT:1> <EOL> } ) <EOL> for title , rows in sheets : <EOL> if not len ( rows ) : <EOL> continue <EOL> headers = get_headers ( rows ) <EOL> if not len ( headers ) : <EOL> continue <EOL> worksheet = workbook . add_worksheet ( title ) <EOL> worksheet . strings_to_numbers = False <EOL> worksheet . strings_to_urls = False <EOL> worksheet . strings_to_formulas = False <EOL> row_idx = <NUM_LIT:0> <EOL> headers = None <EOL> for row in rows : <EOL> if headers is None : <EOL> headers = row . keys ( ) <EOL> for c , cell in enumerate ( headers ) : <EOL> worksheet . write ( row_idx , c , cell , header_format ) <EOL> row_idx += <NUM_LIT:1> <EOL> col_idx = <NUM_LIT:0> <EOL> for cell in headers : <EOL> worksheet . write ( row_idx , col_idx , row . get ( cell ) ) <EOL> col_idx += <NUM_LIT:1> <EOL> row_idx += <NUM_LIT:1> <EOL> worksheet . freeze_panes ( <NUM_LIT:1> , <NUM_LIT:0> ) <EOL> workbook . close ( ) <EOL> def dump_query ( manager , prefix , name , q ) : <EOL> uids = set ( ) <EOL> entities = [ ] <EOL> jsons = [ ] <EOL> file_base = os . path . join ( prefix , name ) <EOL> meta = { <EOL> '<STR_LIT>' : [ { <EOL> '<STR_LIT:label>' : '<STR_LIT>' , <EOL> '<STR_LIT:file>' : name + '<STR_LIT>' <EOL> } ] , <EOL> '<STR_LIT>' : name + '<STR_LIT>' , <EOL> '<STR_LIT>' : name + '<STR_LIT>' <EOL> } <EOL> for entity in manager . _entities . find ( ** q ) : <EOL> uids . add ( entity . get ( '<STR_LIT>' ) ) <EOL> json_repr = entity . pop ( '<STR_LIT>' , None ) <EOL> if json_repr is not None : <EOL> jsons . append ( json . loads ( json_repr ) ) <EOL> entities . append ( entity ) <EOL> entities = sorted ( entities , key = lambda e : e . get ( '<STR_LIT:name>' ) ) <EOL> csv_generate ( file_base + '<STR_LIT>' , entities ) <EOL> sheets = [ ( '<STR_LIT>' , entities ) ] <EOL> tables = { <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : manager . _other_names , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : manager . _identities , <EOL> ( '<STR_LIT>' , '<STR_LIT>' ) : manager . _addresses <EOL> } <EOL> for ( label , section ) , table in tables . items ( ) : <EOL> q = table . table . select ( ) . where ( table . table . c . uid . in_ ( uids ) ) <EOL> rows = list ( manager . engine . query ( q ) ) <EOL> rows = sorted ( rows , key = lambda r : r . get ( '<STR_LIT:name>' ) ) <EOL> csv_generate ( '<STR_LIT>' % ( file_base , section ) , rows ) <EOL> sheets . append ( ( label , rows ) ) <EOL> meta [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT:label>' : label , <EOL> '<STR_LIT:file>' : '<STR_LIT>' % ( name , section ) <EOL> } ) <EOL> xlsx_generate ( file_base + '<STR_LIT>' , sheets ) <EOL> json_generate ( file_base + '<STR_LIT>' , jsons ) <EOL> return meta <EOL> def dump_db ( manager , outdir ) : <EOL> try : <EOL> os . makedirs ( outdir ) <EOL> except : <EOL> pass <EOL> log . info ( '<STR_LIT>' ) <EOL> meta = { <EOL> '<STR_LIT:data>' : dump_query ( manager , outdir , "<STR_LIT>" , { } ) , <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> for source in manager . _entities . distinct ( * SOURCE_FIELDS ) : <EOL> source_id = source . get ( '<STR_LIT>' ) <EOL> if source_id is None : <EOL> continue <EOL> log . info ( '<STR_LIT>' , source . get ( '<STR_LIT:source>' ) ) <EOL> q = { '<STR_LIT>' : source_id } <EOL> source [ '<STR_LIT:data>' ] = dump_query ( manager , outdir , source_id . lower ( ) , q ) <EOL> meta [ '<STR_LIT>' ] . append ( source ) <EOL> with open ( os . path . join ( outdir , '<STR_LIT>' ) , '<STR_LIT:w>' ) as fh : <EOL> json . dump ( meta , fh , indent = <NUM_LIT:2> ) </s>
