invenio_migrator/legacy/cli.py
def dump(thing, query, from_date, file_prefix, chunk_size, limit, thing_flags):
    """Dump data from Invenio legacy."""
    init_app_context()

    file_prefix = file_prefix if file_prefix else '{0}_dump'.format(thing)

    kwargs = dict((f.strip('-').replace('-', '_'), True) for f in thing_flags)

    try:
        thing_func = collect_things_entry_points()[thing]
    except KeyError:
        click.Abort(
            '{0} is not in the list of available things to migrate: '
            '{1}'.format(thing, collect_things_entry_points()))

    click.echo("Querying {0}...".format(thing))
    count, items = thing_func.get(query, from_date, limit=limit, **kwargs)

    progress_i = 0  # Progress bar counter
    click.echo("Dumping {0}...".format(thing))
    with click.progressbar(length=count) as bar:
        for i, chunk_ids in enumerate(grouper(items, chunk_size)):
            with open('{0}_{1}.json'.format(file_prefix, i), 'w') as fp:
                fp.write("[\n")
                for _id in chunk_ids:
                    try:
                        json.dump(
                            thing_func.dump(_id, from_date, **kwargs),
                            fp,
                            default=set_serializer
                        )
                        fp.write(",")
                    except Exception as e:
                        click.secho("Failed dump {0} {1} ({2})".format(
                            thing, _id, e.message), fg='red')
                    progress_i += 1
                    bar.update(progress_i)

                # Strip trailing comma.
                fp.seek(fp.tell()-1)
                fp.write("\n]")
def dump(thing, query, from_date, file_prefix, chunk_size, limit, thing_flags):
    init_app_context()

    file_prefix = file_prefix if file_prefix else '{0}_dump'.format(thing)

    kwargs = dict((f.strip('-').replace('-', '_'), True) for f in thing_flags)

    try:
        thing_func = collect_things_entry_points()[thing]
    except KeyError:
        click.Abort(
            '{0} is not in the list of available things to migrate: '
            '{1}'.format(thing, collect_things_entry_points()))

    click.echo("Querying {0}...".format(thing))
    count, items = thing_func.get(query, from_date, limit=limit, **kwargs)

    progress_i = 0  # Progress bar counter
    click.echo("Dumping {0}...".format(thing))
    with click.progressbar(length=count) as bar:
        for i, chunk_ids in enumerate(grouper(items, chunk_size)):
            with open('{0}_{1}.json'.format(file_prefix, i), 'w') as fp:
                fp.write("[\n")
                for _id in chunk_ids:
                    try:
                        json.dump(
                            thing_func.dump(_id, from_date, **kwargs),
                            fp,
                            default=set_serializer
                        )
                        fp.write(",")
                    except Exception as e:
                        click.secho("Failed dump {0} {1} ({2})".format(
                            thing, _id, e.message), fg='red')
                    progress_i += 1
                    bar.update(progress_i)

                # Strip trailing comma.
                fp.seek(fp.tell()-1)
                fp.write("\n]")
def dump ( thing , query , from_date , file_prefix , chunk_size , limit , thing_flags ) : init_app_context ( ) file_prefix = file_prefix if file_prefix else '{0}_dump' . format ( thing ) kwargs = dict ( ( f . strip ( '-' ) . replace ( '-' , '_' ) , True ) for f in thing_flags ) try : thing_func = collect_things_entry_points ( ) [ thing ] except KeyError : click . Abort ( '{0} is not in the list of available things to migrate: ' '{1}' . format ( thing , collect_things_entry_points ( ) ) ) click . echo ( "Querying {0}..." . format ( thing ) ) count , items = thing_func . get ( query , from_date , limit = limit , * * kwargs ) progress_i = 0 # Progress bar counter click . echo ( "Dumping {0}..." . format ( thing ) ) with click . progressbar ( length = count ) as bar : for i , chunk_ids in enumerate ( grouper ( items , chunk_size ) ) : with open ( '{0}_{1}.json' . format ( file_prefix , i ) , 'w' ) as fp : fp . write ( "[\n" ) for _id in chunk_ids : try : json . dump ( thing_func . dump ( _id , from_date , * * kwargs ) , fp , default = set_serializer ) fp . write ( "," ) except Exception as e : click . secho ( "Failed dump {0} {1} ({2})" . format ( thing , _id , e . message ) , fg = 'red' ) progress_i += 1 bar . update ( progress_i ) # Strip trailing comma. fp . seek ( fp . tell ( ) - 1 ) fp . write ( "\n]" )
Dump data from Invenio legacy.
Dump data from Invenio legacy .