def verify(full, dataset_uri):
    """Verify the integrity of a dataset.
    """
    dataset = dtoolcore.DataSet.from_uri(dataset_uri)
    all_okay = True

    generated_manifest = dataset.generate_manifest()
    generated_identifiers = set(generated_manifest["items"].keys())
    manifest_identifiers = set(dataset.identifiers)

    for i in generated_identifiers.difference(manifest_identifiers):
        message = "Unknown item: {} {}".format(
            i,
            generated_manifest["items"][i]["relpath"]
        )
        click.secho(message, fg="red")
        all_okay = False

    for i in manifest_identifiers.difference(generated_identifiers):
        message = "Missing item: {} {}".format(
            i,
            dataset.item_properties(i)["relpath"]
        )
        click.secho(message, fg="red")
        all_okay = False

    for i in manifest_identifiers.intersection(generated_identifiers):
        generated_hash = generated_manifest["items"][i]["size_in_bytes"]
        manifest_hash = dataset.item_properties(i)["size_in_bytes"]
        if generated_hash != manifest_hash:
            message = "Altered item size: {} {}".format(
                i,
                dataset.item_properties(i)["relpath"]
            )
            click.secho(message, fg="red")
            all_okay = False

    if full:
        for i in manifest_identifiers.intersection(generated_identifiers):
            generated_hash = generated_manifest["items"][i]["hash"]
            manifest_hash = dataset.item_properties(i)["hash"]
            if generated_hash != manifest_hash:
                message = "Altered item hash: {} {}".format(
                    i,
                    dataset.item_properties(i)["relpath"]
                )
                click.secho(message, fg="red")
                all_okay = False

    if not all_okay:
        sys.exit(1)
    else:
        click.secho("All good :)", fg="green")
def verify ( full , dataset_uri ) : dataset = dtoolcore . DataSet . from_uri ( dataset_uri ) all_okay = True generated_manifest = dataset . generate_manifest ( ) generated_identifiers = set ( generated_manifest [ "items" ] . keys ( ) ) manifest_identifiers = set ( dataset . identifiers ) for i in generated_identifiers . difference ( manifest_identifiers ) : message = "Unknown item: {} {}" . format ( i , generated_manifest [ "items" ] [ i ] [ "relpath" ] ) click . secho ( message , fg = "red" ) all_okay = False for i in manifest_identifiers . difference ( generated_identifiers ) : message = "Missing item: {} {}" . format ( i , dataset . item_properties ( i ) [ "relpath" ] ) click . secho ( message , fg = "red" ) all_okay = False for i in manifest_identifiers . intersection ( generated_identifiers ) : generated_hash = generated_manifest [ "items" ] [ i ] [ "size_in_bytes" ] manifest_hash = dataset . item_properties ( i ) [ "size_in_bytes" ] if generated_hash != manifest_hash : message = "Altered item size: {} {}" . format ( i , dataset . item_properties ( i ) [ "relpath" ] ) click . secho ( message , fg = "red" ) all_okay = False if full : for i in manifest_identifiers . intersection ( generated_identifiers ) : generated_hash = generated_manifest [ "items" ] [ i ] [ "hash" ] manifest_hash = dataset . item_properties ( i ) [ "hash" ] if generated_hash != manifest_hash : message = "Altered item hash: {} {}" . format ( i , dataset . item_properties ( i ) [ "relpath" ] ) click . secho ( message , fg = "red" ) all_okay = False if not all_okay : sys . exit ( 1 ) else : click . secho ( "All good :)" , fg = "green" )
Verify the integrity of a dataset.