<s> """<STR_LIT>""" <EOL> import json <EOL> import requests <EOL> PANDA_API = '<STR_LIT>' <EOL> PANDA_AUTH_PARAMS = { <EOL> '<STR_LIT:email>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : '<STR_LIT>' <EOL> } <EOL> PANDA_DATASET_SLUG = '<STR_LIT>' <EOL> PANDA_DATASET_URL = '<STR_LIT>' % ( PANDA_API , PANDA_DATASET_SLUG ) <EOL> PANDA_DATA_URL = '<STR_LIT>' % ( PANDA_API , PANDA_DATASET_SLUG ) <EOL> PANDA_BULK_UPDATE_SIZE = <NUM_LIT:1000> <EOL> COUCHDB_ROOT_URL = '<STR_LIT>' <EOL> COUCHDB_ROWS_URL = '<STR_LIT>' <EOL> COUCHDB_CHANGES_URL = '<STR_LIT>' <EOL> COLUMNS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> LAST_SEQ_FILENAME = '<STR_LIT>' <EOL> def panda_get ( url , params = { } ) : <EOL> params . update ( PANDA_AUTH_PARAMS ) <EOL> return requests . get ( url , params = params ) <EOL> def panda_put ( url , data , params = { } ) : <EOL> params . update ( PANDA_AUTH_PARAMS ) <EOL> return requests . put ( url , data , params = params , headers = { '<STR_LIT:Content-Type>' : '<STR_LIT:application/json>' } ) <EOL> def panda_delete ( url ) : <EOL> return requests . delete ( url , params = PANDA_AUTH_PARAMS , headers = { '<STR_LIT:Content-Type>' : '<STR_LIT:application/json>' } ) <EOL> def write_last_seq ( last_seq ) : <EOL> with open ( LAST_SEQ_FILENAME , '<STR_LIT:w>' ) as f : <EOL> f . write ( str ( last_seq ) ) <EOL> def read_last_seq ( ) : <EOL> with open ( LAST_SEQ_FILENAME ) as f : <EOL> return f . read ( ) . strip ( ) <EOL> def couchdb_row_to_panda_data ( row ) : <EOL> return { <EOL> '<STR_LIT:data>' : [ row [ '<STR_LIT>' ] , row [ '<STR_LIT>' ] , row [ '<STR_LIT>' ] ] , <EOL> '<STR_LIT>' : row [ '<STR_LIT>' ] <EOL> } <EOL> response = panda_get ( PANDA_DATASET_URL ) <EOL> if response . status_code == <NUM_LIT> : <EOL> dataset = { <EOL> '<STR_LIT:name>' : '<STR_LIT>' , <EOL> '<STR_LIT:description>' : '<STR_LIT>' <EOL> } <EOL> response = panda_put ( PANDA_DATASET_URL , json . dumps ( dataset ) , params = { '<STR_LIT>' : '<STR_LIT:U+002C>' . join ( COLUMNS ) } ) <EOL> response = requests . get ( COUCHDB_CHANGES_URL ) <EOL> data = json . loads ( response . content ) <EOL> write_last_seq ( data [ '<STR_LIT>' ] ) <EOL> response = requests . get ( COUCHDB_ROWS_URL ) <EOL> data = json . loads ( response . content ) <EOL> put_data = { <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> for i , row in enumerate ( data [ '<STR_LIT>' ] ) : <EOL> put_data [ '<STR_LIT>' ] . append ( couchdb_row_to_panda_data ( row [ '<STR_LIT:value>' ] ) ) <EOL> if i and i % PANDA_BULK_UPDATE_SIZE == <NUM_LIT:0> : <EOL> print '<STR_LIT>' % PANDA_BULK_UPDATE_SIZE <EOL> panda_put ( PANDA_DATA_URL , json . dumps ( put_data ) ) <EOL> put_data [ '<STR_LIT>' ] = [ ] <EOL> if put_data [ '<STR_LIT>' ] : <EOL> print '<STR_LIT>' % len ( put_data [ '<STR_LIT>' ] ) <EOL> panda_put ( PANDA_DATA_URL , json . dumps ( put_data ) ) <EOL> else : <EOL> last_seq = read_last_seq ( ) <EOL> response = requests . get ( COUCHDB_CHANGES_URL , params = { '<STR_LIT>' : last_seq } ) <EOL> data = json . loads ( response . content ) <EOL> delete_ids = [ ] <EOL> put_data = { <EOL> '<STR_LIT>' : [ ] <EOL> } <EOL> for i , row in enumerate ( data [ '<STR_LIT>' ] ) : <EOL> if row . get ( '<STR_LIT>' , False ) : <EOL> delete_ids . append ( row [ '<STR_LIT:id>' ] ) <EOL> continue <EOL> doc_id = row [ '<STR_LIT:id>' ] <EOL> detail_response = requests . get ( '<STR_LIT>' % ( COUCHDB_ROOT_URL , doc_id ) ) <EOL> detail_data = json . loads ( detail_response . content ) <EOL> put_data [ '<STR_LIT>' ] . append ( couchdb_row_to_panda_data ( detail_data ) ) <EOL> if i and i % PANDA_BULK_UPDATE_SIZE == <NUM_LIT:0> : <EOL> print '<STR_LIT>' % PANDA_BULK_UPDATE_SIZE <EOL> panda_put ( PANDA_DATA_URL , json . dumps ( put_data ) ) <EOL> put_data [ '<STR_LIT>' ] = [ ] <EOL> if put_data [ '<STR_LIT>' ] : <EOL> print '<STR_LIT>' % len ( put_data [ '<STR_LIT>' ] ) <EOL> panda_put ( PANDA_DATA_URL , json . dumps ( put_data ) ) <EOL> if delete_ids : <EOL> print '<STR_LIT>' % len ( delete_ids ) <EOL> for deleted in delete_ids : <EOL> response = panda_delete ( '<STR_LIT>' % ( PANDA_DATA_URL , deleted ) ) <EOL> write_last_seq ( data [ '<STR_LIT>' ] ) <EOL> print '<STR_LIT>' </s>
