def push_json_file(json_file,
                   url,
                   dry_run=False,
                   batch_size=100,
                   anonymize_fields=[],
                   remove_fields=[],
                   rename_fields=[]):
    """
    read the json file provided and POST in batches no bigger than the
    batch_size specified to the specified url.

    """
    batch = []
    json_data = json.loads(json_file.read())

    if isinstance(json_data, list):
        for item in json_data:

            # anonymize fields
            for field_name in anonymize_fields:
                if field_name in item:
                    item[field_name] = md5sum(item[field_name])

            # remove fields
            for field_name in remove_fields:
                if field_name in item:
                    del item[field_name]

            # rename fields
            for (field_name, new_field_name) in rename_fields:
                if field_name in item:
                    item[new_field_name] = item[field_name]
                    del item[field_name]

            batch.append(item)

            if len(batch) >= batch_size:
                post(batch,
                     url,
                     dry_run=dry_run)
                batch = []

        if len(batch) > 0:
            post(batch,
                 url,
                 dry_run=dry_run)

    else:
        post(json_data,
             url,
             dry_run=dry_run)
def push_json_file ( json_file , url , dry_run = False , batch_size = 100 , anonymize_fields = [ ] , remove_fields = [ ] , rename_fields = [ ] ) : batch = [ ] json_data = json . loads ( json_file . read ( ) ) if isinstance ( json_data , list ) : for item in json_data : for field_name in anonymize_fields : if field_name in item : item [ field_name ] = md5sum ( item [ field_name ] ) for field_name in remove_fields : if field_name in item : del item [ field_name ] for ( field_name , new_field_name ) in rename_fields : if field_name in item : item [ new_field_name ] = item [ field_name ] del item [ field_name ] batch . append ( item ) if len ( batch ) >= batch_size : post ( batch , url , dry_run = dry_run ) batch = [ ] if len ( batch ) > 0 : post ( batch , url , dry_run = dry_run ) else : post ( json_data , url , dry_run = dry_run )
read the json file provided and POST in batches no bigger than the
    batch_size specified to the specified url.