def marvcli_run(ctx, datasets, deps, excluded_nodes, force, force_dependent,
                force_deps, keep, keep_going, list_nodes,
                list_dependent, selected_nodes, update_detail,
                update_listing, cachesize, collections):
    """Run nodes for selected datasets.

    Datasets are specified by a list of set ids, or --collection
    <name>, use --collection=* to run for all collections. --node in
    conjunction with --collection=* will pick those collections for
    which the selected nodes are configured.

    Set ids may be abbreviated to any uniquely identifying
    prefix. Suffix a prefix by '+' to match multiple.

    """
    if collections and datasets:
        ctx.fail('--collection and DATASETS are mutually exclusive')

    if list_dependent and not selected_nodes:
        ctx.fail('--list-dependent needs at least one selected --node')

    if not any([datasets, collections, list_nodes]):
        click.echo(ctx.get_help())
        ctx.exit(1)

    deps = 'force' if force_deps else deps
    force = force_deps or force

    site = create_app().site

    if '*' in collections:
        if selected_nodes:
            collections = [k for k, v in site.collections.items()
                           if set(v.nodes).issuperset(selected_nodes)]
            if not collections:
                ctx.fail('No collections have all selected nodes')
        else:
            collections = None
    else:
        for col in collections:
            if col not in site.collections:
                ctx.fail('Unknown collection: {}'.format(col))

    if list_nodes:
        for col in (collections or sorted(site.collections.keys())):
            click.echo('{}:'.format(col))
            for name in sorted(site.collections[col].nodes):
                if name == 'dataset':
                    continue
                click.echo('    {}'.format(name))
        return

    if list_dependent:
        for col in (collections or sorted(site.collections.keys())):
            click.echo('{}:'.format(col))
            dependent = {x for name in selected_nodes
                         for x in site.collections[col].nodes[name].dependent}
            for name in sorted(x.name for x in dependent):
                click.echo('    {}'.format(name))
        return

    errors = []

    setids = [SetID(x) for x in parse_setids(datasets)]
    if not setids:
        query = db.session.query(Dataset.setid)\
                           .filter(Dataset.discarded.isnot(True))\
                           .filter(Dataset.status.op('&')(STATUS_MISSING) == 0)
        if collections is not None:
            query = query.filter(Dataset.collection.in_(collections))
        setids = (SetID(x[0]) for x in query)

    for setid in setids:
        if IPDB:
            site.run(setid, selected_nodes, deps, force, keep,
                     force_dependent, update_detail, update_listing,
                     excluded_nodes, cachesize=cachesize)
        else:
            try:
                site.run(setid, selected_nodes, deps, force, keep,
                         force_dependent, update_detail, update_listing,
                         excluded_nodes, cachesize=cachesize)
            except UnknownNode as e:
                ctx.fail('Collection {} has no node {}'.format(*e.args))
            except NoResultFound:
                click.echo('ERROR: unknown {!r}'.format(setid), err=True)
                if not keep_going:
                    raise
            except BaseException as e:
                errors.append(setid)
                if isinstance(e, KeyboardInterrupt):
                    log.warn('KeyboardInterrupt: aborting')
                    raise
                elif isinstance(e, DirectoryAlreadyExists):
                    click.echo("""
ERROR: Directory for node run already exists:
{!r}
In case no other node run is in progress, this is a bug which you are kindly
asked to report, providing information regarding any previous, failed node runs.
""".format(e.args[0]), err=True)
                    if not keep_going:
                        ctx.abort()
                else:
                    log.error('Exception occured for dataset %s:', setid, exc_info=True)
                    log.error('Error occured for dataset %s: %s', setid, e)
                    if not keep_going:
                        ctx.exit(1)
    if errors:
        log.error('There were errors for %r', errors)
def marvcli_run ( ctx , datasets , deps , excluded_nodes , force , force_dependent , force_deps , keep , keep_going , list_nodes , list_dependent , selected_nodes , update_detail , update_listing , cachesize , collections ) : if collections and datasets : ctx . fail ( '--collection and DATASETS are mutually exclusive' ) if list_dependent and not selected_nodes : ctx . fail ( '--list-dependent needs at least one selected --node' ) if not any ( [ datasets , collections , list_nodes ] ) : click . echo ( ctx . get_help ( ) ) ctx . exit ( 1 ) deps = 'force' if force_deps else deps force = force_deps or force site = create_app ( ) . site if '*' in collections : if selected_nodes : collections = [ k for k , v in site . collections . items ( ) if set ( v . nodes ) . issuperset ( selected_nodes ) ] if not collections : ctx . fail ( 'No collections have all selected nodes' ) else : collections = None else : for col in collections : if col not in site . collections : ctx . fail ( 'Unknown collection: {}' . format ( col ) ) if list_nodes : for col in ( collections or sorted ( site . collections . keys ( ) ) ) : click . echo ( '{}:' . format ( col ) ) for name in sorted ( site . collections [ col ] . nodes ) : if name == 'dataset' : continue click . echo ( '    {}' . format ( name ) ) return if list_dependent : for col in ( collections or sorted ( site . collections . keys ( ) ) ) : click . echo ( '{}:' . format ( col ) ) dependent = { x for name in selected_nodes for x in site . collections [ col ] . nodes [ name ] . dependent } for name in sorted ( x . name for x in dependent ) : click . echo ( '    {}' . format ( name ) ) return errors = [ ] setids = [ SetID ( x ) for x in parse_setids ( datasets ) ] if not setids : query = db . session . query ( Dataset . setid ) . filter ( Dataset . discarded . isnot ( True ) ) . filter ( Dataset . status . op ( '&' ) ( STATUS_MISSING ) == 0 ) if collections is not None : query = query . filter ( Dataset . collection . in_ ( collections ) ) setids = ( SetID ( x [ 0 ] ) for x in query ) for setid in setids : if IPDB : site . run ( setid , selected_nodes , deps , force , keep , force_dependent , update_detail , update_listing , excluded_nodes , cachesize = cachesize ) else : try : site . run ( setid , selected_nodes , deps , force , keep , force_dependent , update_detail , update_listing , excluded_nodes , cachesize = cachesize ) except UnknownNode as e : ctx . fail ( 'Collection {} has no node {}' . format ( * e . args ) ) except NoResultFound : click . echo ( 'ERROR: unknown {!r}' . format ( setid ) , err = True ) if not keep_going : raise except BaseException as e : errors . append ( setid ) if isinstance ( e , KeyboardInterrupt ) : log . warn ( 'KeyboardInterrupt: aborting' ) raise elif isinstance ( e , DirectoryAlreadyExists ) : click . echo ( """
ERROR: Directory for node run already exists:
{!r}
In case no other node run is in progress, this is a bug which you are kindly
asked to report, providing information regarding any previous, failed node runs.
""" . format ( e . args [ 0 ] ) , err = True ) if not keep_going : ctx . abort ( ) else : log . error ( 'Exception occured for dataset %s:' , setid , exc_info = True ) log . error ( 'Error occured for dataset %s: %s' , setid , e ) if not keep_going : ctx . exit ( 1 ) if errors : log . error ( 'There were errors for %r' , errors )
Run nodes for selected datasets.

    Datasets are specified by a list of set ids, or --collection
    <name>, use --collection=* to run for all collections. --node in
    conjunction with --collection=* will pick those collections for
    which the selected nodes are configured.

    Set ids may be abbreviated to any uniquely identifying
    prefix. Suffix a prefix by '+' to match multiple.