def export_namespace(graph, namespace, directory=None, cacheable=False):
    """Exports all names and missing names from the given namespace to its own BEL Namespace files in the given
    directory.

    Could be useful during quick and dirty curation, where planned namespace building is not a priority.

    :param pybel.BELGraph graph: A BEL graph
    :param str namespace: The namespace to process
    :param str directory: The path to the directory where to output the namespace. Defaults to the current working
                      directory returned by :func:`os.getcwd`
    :param bool cacheable: Should the namespace be cacheable? Defaults to ``False`` because, in general, this operation
                        will probably be used for evil, and users won't want to reload their entire cache after each
                        iteration of curation.
    """
    directory = os.getcwd() if directory is None else directory
    path = os.path.join(directory, '{}.belns'.format(namespace))

    with open(path, 'w') as file:
        log.info('Outputting to %s', path)
        right_names = get_names_by_namespace(graph, namespace)
        log.info('Graph has %d correct names in %s', len(right_names), namespace)
        wrong_names = get_incorrect_names_by_namespace(graph, namespace)
        log.info('Graph has %d incorrect names in %s', len(right_names), namespace)
        undefined_ns_names = get_undefined_namespace_names(graph, namespace)
        log.info('Graph has %d names in missing namespace %s', len(right_names), namespace)

        names = (right_names | wrong_names | undefined_ns_names)

        if 0 == len(names):
            log.warning('%s is empty', namespace)

        write_namespace(
            namespace_name=namespace,
            namespace_keyword=namespace,
            namespace_domain='Other',
            author_name=graph.authors,
            author_contact=graph.contact,
            citation_name=graph.name,
            values=names,
            cacheable=cacheable,
            file=file
        )

Exports all names and missing names from the given namespace to its own BEL Namespace files in the given directory .


def lint_file(in_file, out_file=None):
    """Helps remove extraneous whitespace from the lines of a file

    :param file in_file: A readable file or file-like
    :param file out_file: A writable file or file-like
    """
    for line in in_file:
        print(line.strip(), file=out_file)

def lint_file ( in_file , out_file = None ) : for line in in_file : print ( line . strip ( ) , file = out_file )


def lint_directory(source, target):
    """Adds a linted version of each document in the source directory to the target directory

    :param str source: Path to directory to lint
    :param str target: Path to directory to output
    """
    for path in os.listdir(source):
        if not path.endswith('.bel'):
            continue

        log.info('linting: %s', path)
        with open(os.path.join(source, path)) as i, open(os.path.join(target, path), 'w') as o:
            lint_file(i, o)

def
lint_directory
(
source
,
target
)
:
for
path
in
os
.
listdir
(
source
)
:
if
not
path
.
endswith
(
'.bel'
)
:
continue
log
.
info
(
'linting: %s'
,
path
)
with
open
(
os
.
path
.
join
(
source
,
path
)
)
as
i
,
open
(
os
.
path
.
join
(
target
,
path
)
,
'w'
)
as
o
:
lint_file
(
i
,
o
)


def get_entrez_gene_data(entrez_ids: Iterable[Union[str, int]]):
    """Get gene info from Entrez."""
    url = PUBMED_GENE_QUERY_URL.format(','.join(str(x).strip() for x in entrez_ids))
    response = requests.get(url)
    tree = ElementTree.fromstring(response.content)

    return {
        element.attrib['uid']: {
            'summary': _sanitize(element.find('Summary').text),
            'description': element.find('Description').text
        }
        for element in tree.findall('./DocumentSummarySet/DocumentSummary')
    }

def get_entrez_gene_data ( entrez_ids : Iterable [ Union [ str , int ] ] ) : url = PUBMED_GENE_QUERY_URL . format ( ',' . join ( str ( x ) . strip ( ) for x in entrez_ids ) ) response = requests . get ( url ) tree = ElementTree . fromstring ( response . content ) return { element . attrib [ 'uid' ] : { 'summary' : _sanitize ( element . find ( 'Summary' ) . text ) , 'description' : element . find ( 'Description' ) . text } for element in tree . findall ( './DocumentSummarySet/DocumentSummary' ) }


def make_pubmed_gene_group(entrez_ids: Iterable[Union[str, int]]) -> Iterable[str]:
    """Builds a skeleton for gene summaries

    :param entrez_ids: A list of Entrez Gene identifiers to query the PubMed service
    :return: An iterator over statement lines for NCBI Entrez Gene summaries
    """
    url = PUBMED_GENE_QUERY_URL.format(','.join(str(x).strip() for x in entrez_ids))
    response = requests.get(url)
    tree = ElementTree.fromstring(response.content)

    for x in tree.findall('./DocumentSummarySet/DocumentSummary'):
        yield '\n# {}'.format(x.find('Description').text)
        yield 'SET Citation = {{"Other", "PubMed Gene", "{}"}}'.format(x.attrib['uid'])
        yield 'SET Evidence = "{}"'.format(x.find('Summary').text.strip().replace('\n', ''))
        yield '\nUNSET Evidence\nUNSET Citation'


def write_boilerplate(name: str,
                      version: Optional[str] = None,
                      description: Optional[str] = None,
                      authors: Optional[str] = None,
                      contact: Optional[str] = None,
                      copyright: Optional[str] = None,
                      licenses: Optional[str] = None,
                      disclaimer: Optional[str] = None,
                      namespace_url: Optional[Mapping[str, str]] = None,
                      namespace_patterns: Optional[Mapping[str, str]] = None,
                      annotation_url: Optional[Mapping[str, str]] = None,
                      annotation_patterns: Optional[Mapping[str, str]] = None,
                      annotation_list: Optional[Mapping[str, Set[str]]] = None,
                      pmids: Optional[Iterable[Union[str, int]]] = None,
                      entrez_ids: Optional[Iterable[Union[str, int]]] = None,
                      file: Optional[TextIO] = None,
                      ) -> None:
    """Write a boilerplate BEL document, with standard document metadata, definitions.

    :param name: The unique name for this BEL document
    :param contact: The email address of the maintainer
    :param description: A description of the contents of this document
    :param authors: The authors of this document
    :param version: The version. Defaults to current date in format ``YYYYMMDD``.
    :param copyright: Copyright information about this document
    :param licenses: The license applied to this document
    :param disclaimer: The disclaimer for this document
    :param namespace_url: an optional dictionary of {str name: str URL} of namespaces
    :param namespace_patterns: An optional dictionary of {str name: str regex} namespaces
    :param annotation_url: An optional dictionary of {str name: str URL} of annotations
    :param annotation_patterns: An optional dictionary of {str name: str regex} of regex annotations
    :param annotation_list: An optional dictionary of {str name: set of names} of list annotations
    :param pmids: A list of PubMed identifiers to auto-populate with citation and abstract
    :param entrez_ids: A list of Entrez identifiers to autopopulate the gene summary as evidence
    :param file: A writable file or file-like. If None, defaults to :data:`sys.stdout`
    """
    lines = make_knowledge_header(
        name=name,
        version=version or '1.0.0',
        description=description,
        authors=authors,
        contact=contact,
        copyright=copyright,
        licenses=licenses,
        disclaimer=disclaimer,
        namespace_url=namespace_url,
        namespace_patterns=namespace_patterns,
        annotation_url=annotation_url,
        annotation_patterns=annotation_patterns,
        annotation_list=annotation_list,
    )

    for line in lines:
        print(line, file=file)

    if pmids is not None:
        for line in make_pubmed_abstract_group(pmids):
            print(line, file=file)

    if entrez_ids is not None:
        for line in make_pubmed_gene_group(entrez_ids):
            print(line, file=file)

def write_boilerplate ( name : str , version : Optional [ str ] = None , description : Optional [ str ] = None , authors : Optional [ str ] = None , contact : Optional [ str ] = None , copyright : Optional [ str ] = None , licenses : Optional [ str ] = None , disclaimer : Optional [ str ] = None , namespace_url : Optional [ Mapping [ str , str ] ] = None , namespace_patterns : Optional [ Mapping [ str , str ] ] = None , annotation_url : Optional [ Mapping [ str , str ] ] = None , annotation_patterns : Optional [ Mapping [ str , str ] ] = None , annotation_list : Optional [ Mapping [ str , Set [ str ] ] ] = None , pmids : Optional [ Iterable [ Union [ str , int ] ] ] = None , entrez_ids : Optional [ Iterable [ Union [ str , int ] ] ] = None , file : Optional [ TextIO ] = None , ) -> None : lines = make_knowledge_header ( name = name , version = version or '1.0.0' , description = description , authors = authors , contact = contact , copyright = copyright , licenses = licenses , disclaimer = disclaimer , namespace_url = namespace_url , namespace_patterns = namespace_patterns , annotation_url = annotation_url , annotation_patterns = annotation_patterns , annotation_list = annotation_list , ) for line in lines : print ( line , file = file ) if pmids is not None : for line in make_pubmed_abstract_group ( pmids ) : print ( line , file = file ) if entrez_ids is not None : for line in make_pubmed_gene_group ( entrez_ids ) : print ( line , file = file )

Write a boilerplate BEL document with standard document metadata definitions .


def get_subgraph_by_node_search(graph: BELGraph, query: Strings) -> BELGraph:
    """Get a sub-graph induced over all nodes matching the query string.

    :param graph: A BEL Graph
    :param query: A query string or iterable of query strings for node names

    Thinly wraps :func:`search_node_names` and :func:`get_subgraph_by_induction`.
    """
    nodes = search_node_names(graph, query)
    return get_subgraph_by_induction(graph, nodes)

def get_subgraph_by_node_search ( graph : BELGraph , query : Strings ) -> BELGraph : nodes = search_node_names ( graph , query ) return get_subgraph_by_induction ( graph , nodes )

Get a sub - graph induced over all nodes matching the query string .


def get_largest_component(graph: BELGraph) -> BELGraph:
    """Get the giant component of a graph."""
    biggest_component_nodes = max(nx.weakly_connected_components(graph), key=len)
    return subgraph(graph, biggest_component_nodes)

def get_largest_component ( graph : BELGraph ) -> BELGraph : biggest_component_nodes = max ( nx . weakly_connected_components ( graph ) , key = len ) return subgraph ( graph , biggest_component_nodes )


def random_by_nodes(graph: BELGraph, percentage: Optional[float] = None) -> BELGraph:
    """Get a random graph by inducing over a percentage of the original nodes.

    :param graph: A BEL graph
    :param percentage: The percentage of edges to keep
    """
    percentage = percentage or 0.9

    assert 0 < percentage <= 1

    nodes = graph.nodes()
    n = int(len(nodes) * percentage)

    subnodes = random.sample(nodes, n)

    result = graph.subgraph(subnodes)

    update_node_helper(graph, result)

    return result

def random_by_nodes ( graph : BELGraph , percentage : Optional [ float ] = None ) -> BELGraph : percentage = percentage or 0.9 assert 0 < percentage <= 1 nodes = graph . nodes ( ) n = int ( len ( nodes ) * percentage ) subnodes = random . sample ( nodes , n ) result = graph . subgraph ( subnodes ) update_node_helper ( graph , result ) return result

Get a random graph by inducing over a percentage of the original nodes .


def random_by_edges(graph: BELGraph, percentage: Optional[float] = None) -> BELGraph:
    """Get a random graph by keeping a certain percentage of original edges.

    :param graph: A BEL graph
    :param percentage: What percentage of eges to take
    """
    percentage = percentage or 0.9
    assert 0 < percentage <= 1

    edges = graph.edges(keys=True)
    n = int(graph.number_of_edges() * percentage)

    subedges = random.sample(edges, n)

    rv = graph.fresh_copy()

    for u, v, k in subedges:
        safe_add_edge(rv, u, v, k, graph[u][v][k])

    update_node_helper(graph, rv)

    return rv

def random_by_edges ( graph : BELGraph , percentage : Optional [ float ] = None ) -> BELGraph : percentage = percentage or 0.9 assert 0 < percentage <= 1 edges = graph . edges ( keys = True ) n = int ( graph . number_of_edges ( ) * percentage ) subedges = random . sample ( edges , n ) rv = graph . fresh_copy ( ) for u , v , k in subedges : safe_add_edge ( rv , u , v , k , graph [ u ] [ v ] [ k ] ) update_node_helper ( graph , rv ) return rv

Get a random graph by keeping a certain percentage of original edges .


def shuffle_node_data(graph: BELGraph, key: str, percentage: Optional[float] = None) -> BELGraph:
    """Shuffle the node's data.

    Useful for permutation testing.

    :param graph: A BEL graph
    :param key: The node data dictionary key
    :param percentage: What percentage of possible swaps to make
    """
    percentage = percentage or 0.3
    assert 0 < percentage <= 1

    n = graph.number_of_nodes()
    swaps = int(percentage * n * (n - 1) / 2)

    result: BELGraph = graph.copy()

    for _ in range(swaps):
        s, t = random.sample(result.node, 2)
        result.nodes[s][key], result.nodes[t][key] = result.nodes[t][key], result.nodes[s][key]

    return result

def shuffle_node_data ( graph : BELGraph , key : str , percentage : Optional [ float ] = None ) -> BELGraph : percentage = percentage or 0.3 assert 0 < percentage <= 1 n = graph . number_of_nodes ( ) swaps = int ( percentage * n * ( n - 1 ) / 2 ) result : BELGraph = graph . copy ( ) for _ in range ( swaps ) : s , t = random . sample ( result . node , 2 ) result . nodes [ s ] [ key ] , result . nodes [ t ] [ key ] = result . nodes [ t ] [ key ] , result . nodes [ s ] [ key ] return result


def shuffle_relations(graph: BELGraph, percentage: Optional[str] = None) -> BELGraph:
    """Shuffle the relations.

    Useful for permutation testing.

    :param graph: A BEL graph
    :param percentage: What percentage of possible swaps to make
    """
    percentage = percentage or 0.3
    assert 0 < percentage <= 1

    n = graph.number_of_edges()
    swaps = int(percentage * n * (n - 1) / 2)

    result: BELGraph = graph.copy()

    edges = result.edges(keys=True)

    for _ in range(swaps):
        (s1, t1, k1), (s2, t2, k2) = random.sample(edges, 2)
        result[s1][t1][k1], result[s2][t2][k2] = result[s2][t2][k2], result[s1][t1][k1]

    return result

def shuffle_relations ( graph : BELGraph , percentage : Optional [ str ] = None ) -> BELGraph : percentage = percentage or 0.3 assert 0 < percentage <= 1 n = graph . number_of_edges ( ) swaps = int ( percentage * n * ( n - 1 ) / 2 ) result : BELGraph = graph . copy ( ) edges = result . edges ( keys = True ) for _ in range ( swaps ) : ( s1 , t1 , k1 ) , ( s2 , t2 , k2 ) = random . sample ( edges , 2 ) result [ s1 ] [ t1 ] [ k1 ] , result [ s2 ] [ t2 ] [ k2 ] = result [ s2 ] [ t2 ] [ k2 ] , result [ s1 ] [ t1 ] [ k1 ] return result


def is_edge_consistent(graph, u, v):
    """Check if all edges between two nodes have the same relation.

    :param pybel.BELGraph graph: A BEL Graph
    :param tuple u: The source BEL node
    :param tuple v: The target BEL node
    :return: If all edges from the source to target node have the same relation
    :rtype: bool
    """
    if not graph.has_edge(u, v):
        raise ValueError('{} does not contain an edge ({}, {})'.format(graph, u, v))

    return 0 == len(set(d[RELATION] for d in graph.edge[u][v].values()))

def
is_edge_consistent
(
graph
,
u
,
v
)
:
if
not
graph
.
has_edge
(
u
,
v
)
:
raise
ValueError
(
'{} does not contain an edge ({}, {})'
.
format
(
graph
,
u
,
v
)
)
return
0
==
len
(
set
(
d
[
RELATION
]
for
d
in
graph
.
edge
[
u
]
[
v
]
.
values
(
)
)
)

Check if all edges between two nodes have the same relation .


def rewire_targets(graph, rewiring_probability):
    """Rewire a graph's edges' target nodes.

    - For BEL graphs, assumes edge consistency (all edges between two given nodes are have the same relation)
    - Doesn't make self-edges

    :param pybel.BELGraph graph: A BEL graph
    :param float rewiring_probability: The probability of rewiring (between 0 and 1)
    :return: A rewired BEL graph
    """
    if not all_edges_consistent(graph):
        raise ValueError('{} is not consistent'.format(graph))

    result = graph.copy()
    nodes = result.nodes()

    for u, v in result.edges():
        if random.random() < rewiring_probability:
            continue

        w = random.choice(nodes)

        while w == u or result.has_edge(u, w):
            w = random.choice(nodes)

        result.add_edge(w, v)
        result.remove_edge(u, v)

    return result

def
rewire_targets
(
graph
,
rewiring_probability
)
:
if
not
all_edges_consistent
(
graph
)
:
raise
ValueError
(
'{} is not consistent'
.
format
(
graph
)
)
result
=
graph
.
copy
(
)
nodes
=
result
.
nodes
(
)
for
u
,
v
in
result
.
edges
(
)
:
if
random
.
random
(
)
<
rewiring_probability
:
continue
w
=
random
.
choice
(
nodes
)
while
w
==
u
or
result
.
has_edge
(
u
,
w
)
:
w
=
random
.
choice
(
nodes
)
result
.
add_edge
(
w
,
v
)
result
.
remove_edge
(
u
,
v
)
return
result


def self_edge_filter(_: BELGraph, source: BaseEntity, target: BaseEntity, __: str) -> bool:
    """Check if the source and target nodes are the same."""
    return source == target

def self_edge_filter ( _ : BELGraph , source : BaseEntity , target : BaseEntity , __ : str ) -> bool : return source == target

Check if the source and target nodes are the same .


def has_protein_modification_increases_activity(graph: BELGraph,
                                                source: BaseEntity,
                                                target: BaseEntity,
                                                key: str,
                                                ) -> bool:
    """Check if pmod of source causes activity of target."""
    edge_data = graph[source][target][key]
    return has_protein_modification(graph, source) and part_has_modifier(edge_data, OBJECT, ACTIVITY)

Check if pmod of source causes activity of target .


def has_degradation_increases_activity(data: Dict) -> bool:
    """Check if the degradation of source causes activity of target."""
    return part_has_modifier(data, SUBJECT, DEGRADATION) and part_has_modifier(data, OBJECT, ACTIVITY)

def has_degradation_increases_activity ( data : Dict ) -> bool : return part_has_modifier ( data , SUBJECT , DEGRADATION ) and part_has_modifier ( data , OBJECT , ACTIVITY )

Check if the degradation of source causes activity of target .


def has_translocation_increases_activity(data: Dict) -> bool:
    """Check if the translocation of source causes activity of target."""
    return part_has_modifier(data, SUBJECT, TRANSLOCATION) and part_has_modifier(data, OBJECT, ACTIVITY)

def has_translocation_increases_activity ( data : Dict ) -> bool : return part_has_modifier ( data , SUBJECT , TRANSLOCATION ) and part_has_modifier ( data , OBJECT , ACTIVITY )

Check if the translocation of source causes activity of target .


def complex_has_member(graph: BELGraph, complex_node: ComplexAbundance, member_node: BaseEntity) -> bool:
    """Does the given complex contain the member?"""
    return any(  # TODO can't you look in the members of the complex object (if it's enumerated)
        v == member_node
        for _, v, data in graph.out_edges(complex_node, data=True)
        if data[RELATION] == HAS_COMPONENT
    )


def complex_increases_activity(graph: BELGraph, u: BaseEntity, v: BaseEntity, key: str) -> bool:
    """Return if the formation of a complex with u increases the activity of v."""
    return (
        isinstance(u, (ComplexAbundance, NamedComplexAbundance)) and
        complex_has_member(graph, u, v) and
        part_has_modifier(graph[u][v][key], OBJECT, ACTIVITY)
    )

def complex_increases_activity ( graph : BELGraph , u : BaseEntity , v : BaseEntity , key : str ) -> bool : return ( isinstance ( u , ( ComplexAbundance , NamedComplexAbundance ) ) and complex_has_member ( graph , u , v ) and part_has_modifier ( graph [ u ] [ v ] [ key ] , OBJECT , ACTIVITY ) )

Return if the formation of a complex with u increases the activity of v .


def find_activations(graph: BELGraph):
    """Find edges that are A - A, meaning that some conditions in the edge best describe the interaction."""
    for u, v, key, data in graph.edges(keys=True, data=True):
        if u != v:
            continue

        bel = graph.edge_to_bel(u, v, data)

        line = data.get(LINE)

        if line is None:
            continue  # this was inferred, so need to investigate another way

        elif has_protein_modification_increases_activity(graph, u, v, key):
            print(line, '- pmod changes -', bel)
            find_related(graph, v, data)

        elif has_degradation_increases_activity(data):
            print(line, '- degradation changes -', bel)
            find_related(graph, v, data)

        elif has_translocation_increases_activity(data):
            print(line, '- translocation changes -', bel)
            find_related(graph, v, data)

        elif complex_increases_activity(graph, u, v, key):
            print(line, '- complex changes - ', bel)
            find_related(graph, v, data)

        elif has_same_subject_object(graph, u, v, key):
            print(line, '- same sub/obj -', bel)

        else:
            print(line, '- *** - ', bel)

Find edges that are A - A meaning that some conditions in the edge best describe the interaction .


def summarize_edge_filter(graph: BELGraph, edge_predicates: EdgePredicates) -> None:
    """Print a summary of the number of edges passing a given set of filters."""
    passed = count_passed_edge_filter(graph, edge_predicates)
    print('{}/{} edges passed {}'.format(
        passed, graph.number_of_edges(),
        (
            ', '.join(edge_filter.__name__ for edge_filter in edge_predicates)
            if isinstance(edge_predicates, Iterable) else
            edge_predicates.__name__
        )
    ))

def
summarize_edge_filter
(
graph
:
BELGraph
,
edge_predicates
:
EdgePredicates
)
->
:
passed
=
count_passed_edge_filter
(
graph
,
edge_predicates
)
print
(
'{}/{} edges passed {}'
.
format
(
passed
,
graph
.
number_of_edges
(
)
,
(
', '
.
join
(
edge_filter
.
__name__
for
edge_filter
in
edge_predicates
)
if
isinstance
(
edge_predicates
,
Iterable
)
else
edge_predicates
.
__name__
)
)
)

Print a summary of the number of edges passing a given set of filters .


def build_edge_data_filter(annotations: Mapping, partial_match: bool = True) -> EdgePredicate: # noqa: D202
    """Build a filter that keeps edges whose data dictionaries are super-dictionaries to the given dictionary.

    :param annotations: The annotation query dict to match
    :param partial_match: Should the query values be used as partial or exact matches? Defaults to :code:`True`.
    """

    @edge_predicate
    def annotation_dict_filter(data: EdgeData) -> bool:
        """A filter that matches edges with the given dictionary as a sub-dictionary."""
        return subdict_matches(data, annotations, partial_match=partial_match)

    return annotation_dict_filter

Build a filter that keeps edges whose data dictionaries are super - dictionaries to the given dictionary .


def build_pmid_exclusion_filter(pmids: Strings) -> EdgePredicate:
    """Fail for edges with citations whose references are one of the given PubMed identifiers.

    :param pmids: A PubMed identifier or list of PubMed identifiers to filter against
    """
    if isinstance(pmids, str):
        @edge_predicate
        def pmid_exclusion_filter(data: EdgeData) -> bool:
            """Fail for edges with PubMed citations matching the contained PubMed identifier.

            :return: If the edge has a PubMed citation with the contained PubMed identifier
            """
            return has_pubmed(data) and data[CITATION][CITATION_REFERENCE] != pmids

    elif isinstance(pmids, Iterable):
        pmids = set(pmids)

        @edge_predicate
        def pmid_exclusion_filter(data: EdgeData) -> bool:
            """Pass for edges with PubMed citations matching one of the contained PubMed identifiers.

            :return: If the edge has a PubMed citation with one of the contained PubMed identifiers
            """
            return has_pubmed(data) and data[CITATION][CITATION_REFERENCE] not in pmids

    else:
        raise TypeError

    return pmid_exclusion_filter

Fail for edges with citations whose references are one of the given PubMed identifiers .


def node_has_namespace(node: BaseEntity, namespace: str) -> bool:
    """Pass for nodes that have the given namespace."""
    ns = node.get(NAMESPACE)
    return ns is not None and ns == namespace

def node_has_namespace ( node : BaseEntity , namespace : str ) -> bool : ns = node . get ( NAMESPACE ) return ns is not None and ns == namespace

Pass for nodes that have the given namespace .


def node_has_namespaces(node: BaseEntity, namespaces: Set[str]) -> bool:
    """Pass for nodes that have one of the given namespaces."""
    ns = node.get(NAMESPACE)
    return ns is not None and ns in namespaces

def node_has_namespaces ( node : BaseEntity , namespaces : Set [ str ] ) -> bool : ns = node . get ( NAMESPACE ) return ns is not None and ns in namespaces

Pass for nodes that have one of the given namespaces .


def get_cutoff(value: float, cutoff: Optional[float] = None) -> int:
    """Assign if a value is greater than or less than a cutoff."""
    cutoff = cutoff if cutoff is not None else 0

    if value > cutoff:
        return 1

    if value < (-1 * cutoff):
        return - 1

    return 0

def get_cutoff ( value : float , cutoff : Optional [ float ] = None ) -> int : cutoff = cutoff if cutoff is not None else 0 if value > cutoff : return 1 if value < ( - 1 * cutoff ) : return - 1 return 0

Assign if a value is greater than or less than a cutoff .


def calculate_concordance_helper(graph: BELGraph,
                                 key: str,
                                 cutoff: Optional[float] = None,
                                 ) -> Tuple[int, int, int, int]:
    """Help calculate network-wide concordance

    Assumes data already annotated with given key

    :param graph: A BEL graph
    :param key: The node data dictionary key storing the logFC
    :param cutoff: The optional logFC cutoff for significance
    """
    scores = defaultdict(int)

    for u, v, k, d in graph.edges(keys=True, data=True):
        c = edge_concords(graph, u, v, k, d, key, cutoff=cutoff)
        scores[c] += 1

    return (
        scores[Concordance.correct],
        scores[Concordance.incorrect],
        scores[Concordance.ambiguous],
        scores[Concordance.unassigned],
    )

def calculate_concordance_helper ( graph : BELGraph , key : str , cutoff : Optional [ float ] = None , ) -> Tuple [ int , int , int , int ] : scores = defaultdict ( int ) for u , v , k , d in graph . edges ( keys = True , data = True ) : c = edge_concords ( graph , u , v , k , d , key , cutoff = cutoff ) scores [ c ] += 1 return ( scores [ Concordance . correct ] , scores [ Concordance . incorrect ] , scores [ Concordance . ambiguous ] , scores [ Concordance . unassigned ] , )

Help calculate network - wide concordance


def calculate_concordance(graph: BELGraph, key: str, cutoff: Optional[float] = None,
                          use_ambiguous: bool = False) -> float:
    """Calculates network-wide concordance.

    Assumes data already annotated with given key

    :param graph: A BEL graph
    :param key: The node data dictionary key storing the logFC
    :param cutoff: The optional logFC cutoff for significance
    :param use_ambiguous: Compare to ambiguous edges as well
    """
    correct, incorrect, ambiguous, _ = calculate_concordance_helper(graph, key, cutoff=cutoff)

    try:
        return correct / (correct + incorrect + (ambiguous if use_ambiguous else 0))
    except ZeroDivisionError:
        return -1.0

def calculate_concordance ( graph : BELGraph , key : str , cutoff : Optional [ float ] = None , use_ambiguous : bool = False ) -> float : correct , incorrect , ambiguous , _ = calculate_concordance_helper ( graph , key , cutoff = cutoff ) try : return correct / ( correct + incorrect + ( ambiguous if use_ambiguous else 0 ) ) except ZeroDivisionError : return - 1.0


def one_sided(value: float, distribution: List[float]) -> float:
    """Calculate the one-sided probability of getting a value more extreme than the distribution."""
    assert distribution
    return sum(value < element for element in distribution) / len(distribution)

def one_sided ( value : float , distribution : List [ float ] ) -> float : assert distribution return sum ( value < element for element in distribution ) / len ( distribution )

Calculate the one - sided probability of getting a value more extreme than the distribution .


def calculate_concordance_probability(graph: BELGraph,
                                      key: str,
                                      cutoff: Optional[float] = None,
                                      permutations: Optional[int] = None,
                                      percentage: Optional[float] = None,
                                      use_ambiguous: bool = False,
                                      permute_type: str = 'shuffle_node_data',
                                      ) -> Tuple[float, List[float], float]:
    """Calculates a graph's concordance as well as its statistical probability.

    :param graph: A BEL graph
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :param int permutations: The number of random permutations to test. Defaults to 500
    :param float percentage: The percentage of the graph's edges to maintain. Defaults to 0.9
    :param bool use_ambiguous: Compare to ambiguous edges as well
    :returns: A triple of the concordance score, the null distribution, and the p-value.
    """
    if permute_type == 'random_by_edges':
        permute_func = partial(random_by_edges, percentage=percentage)
    elif permute_type == 'shuffle_node_data':
        permute_func = partial(shuffle_node_data, key=key, percentage=percentage)
    elif permute_type == 'shuffle_relations':
        permute_func = partial(shuffle_relations, percentage=percentage)
    else:
        raise ValueError('Invalid permute_type: {}'.format(permute_type))

    graph: BELGraph = graph.copy()
    collapse_to_genes(graph)
    collapse_all_variants(graph)

    score = calculate_concordance(graph, key, cutoff=cutoff)

    distribution = []

    for _ in range(permutations or 500):
        permuted_graph = permute_func(graph)
        permuted_graph_scores = calculate_concordance(permuted_graph, key, cutoff=cutoff, use_ambiguous=use_ambiguous)
        distribution.append(permuted_graph_scores)

    return score, distribution, one_sided(score, distribution)

def
calculate_concordance_probability
(
graph
:
BELGraph
,
key
:
str
,
cutoff
:
Optional
[
float
]
=
,
permutations
:
Optional
[
int
]
=
,
percentage
:
Optional
[
float
]
=
,
use_ambiguous
:
bool
=
False
,
permute_type
:
str
=
'shuffle_node_data'
,
)
->
Tuple
[
float
,
List
[
float
]
,
float
]
:
if
permute_type
==
'random_by_edges'
:
permute_func
=
partial
(
random_by_edges
,
percentage
=
percentage
)
elif
permute_type
==
'shuffle_node_data'
:
permute_func
=
partial
(
shuffle_node_data
,
key
=
key
,
percentage
=
percentage
)
elif
permute_type
==
'shuffle_relations'
:
permute_func
=
partial
(
shuffle_relations
,
percentage
=
percentage
)
else
:
raise
ValueError
(
'Invalid permute_type: {}'
.
format
(
permute_type
)
)
graph
:
BELGraph
=
graph
.
copy
(
)
collapse_to_genes
(
graph
)
collapse_all_variants
(
graph
)
score
=
calculate_concordance
(
graph
,
key
,
cutoff
=
cutoff
)
distribution
=
[
]
for
_
in
range
(
permutations
or
500
)
:
permuted_graph
=
permute_func
(
graph
)
permuted_graph_scores
=
calculate_concordance
(
permuted_graph
,
key
,
cutoff
=
cutoff
,
use_ambiguous
=
use_ambiguous
)
distribution
.
append
(
permuted_graph_scores
)
return
score
,
distribution
,
one_sided
(
score
,
distribution
)

Calculates a graph s concordance as well as its statistical probability .


def calculate_concordance_by_annotation(graph, annotation, key, cutoff=None):
    """Returns the concordance scores for each stratified graph based on the given annotation

    :param pybel.BELGraph graph: A BEL graph
    :param str annotation: The annotation to group by.
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :rtype: dict[str,tuple]
    """
    return {
        value: calculate_concordance(subgraph, key, cutoff=cutoff)
        for value, subgraph in get_subgraphs_by_annotation(graph, annotation).items()
    }

def calculate_concordance_by_annotation ( graph , annotation , key , cutoff = None ) : return { value : calculate_concordance ( subgraph , key , cutoff = cutoff ) for value , subgraph in get_subgraphs_by_annotation ( graph , annotation ) . items ( ) }


def calculate_concordance_probability_by_annotation(graph, annotation, key, cutoff=None, permutations=None,
                                                    percentage=None,
                                                    use_ambiguous=False):
    """Returns the results of concordance analysis on each subgraph, stratified by the given annotation.

    :param pybel.BELGraph graph: A BEL graph
    :param str annotation: The annotation to group by.
    :param str key: The node data dictionary key storing the logFC
    :param float cutoff: The optional logFC cutoff for significance
    :param int permutations: The number of random permutations to test. Defaults to 500
    :param float percentage: The percentage of the graph's edges to maintain. Defaults to 0.9
    :param bool use_ambiguous: Compare to ambiguous edges as well
    :rtype: dict[str,tuple]
    """
    result = [
        (value, calculate_concordance_probability(
            subgraph,
            key,
            cutoff=cutoff,
            permutations=permutations,
            percentage=percentage,
            use_ambiguous=use_ambiguous,
        ))
        for value, subgraph in get_subgraphs_by_annotation(graph, annotation).items()
    ]

    return dict(result)

Returns the results of concordance analysis on each subgraph stratified by the given annotation .


def _get_drug_target_interactions(manager: Optional['bio2bel_drugbank.manager'] = None) -> Mapping[str, List[str]]:
    """Get a mapping from drugs to their list of gene."""
    if manager is None:
        import bio2bel_drugbank
        manager = bio2bel_drugbank.Manager()

    if not manager.is_populated():
        manager.populate()

    return manager.get_drug_to_hgnc_symbols()

def _get_drug_target_interactions ( manager : Optional [ 'bio2bel_drugbank.manager' ] = None ) -> Mapping [ str , List [ str ] ] : if manager is None : import bio2bel_drugbank manager = bio2bel_drugbank . Manager ( ) if not manager . is_populated ( ) : manager . populate ( ) return manager . get_drug_to_hgnc_symbols ( )

Get a mapping from drugs to their list of gene .


def multi_run_epicom(graphs: Iterable[BELGraph], path: Union[None, str, TextIO]) -> None:
    """Run EpiCom analysis on many graphs."""
    if isinstance(path, str):
        with open(path, 'w') as file:
            _multi_run_helper_file_wrapper(graphs, file)

    else:
        _multi_run_helper_file_wrapper(graphs, path)

def multi_run_epicom ( graphs : Iterable [ BELGraph ] , path : Union [ None , str , TextIO ] ) -> None : if isinstance ( path , str ) : with open ( path , 'w' ) as file : _multi_run_helper_file_wrapper ( graphs , file ) else : _multi_run_helper_file_wrapper ( graphs , path )


def main():
    """Convert the Alzheimer's and Parkinson's disease NeuroMMSig excel sheets to BEL."""
    logging.basicConfig(level=logging.INFO)
    log.setLevel(logging.INFO)

    bms_base = get_bms_base()
    neurommsig_base = get_neurommsig_base()
    neurommsig_excel_dir = os.path.join(neurommsig_base, 'resources', 'excels', 'neurommsig')

    nift_values = get_nift_values()

    log.info('Starting Alzheimers')

    ad_path = os.path.join(neurommsig_excel_dir, 'alzheimers', 'alzheimers.xlsx')
    ad_df = preprocess(ad_path)
    with open(os.path.join(bms_base, 'aetionomy', 'alzheimers', 'neurommsigdb_ad.bel'), 'w') as ad_file:
        write_neurommsig_bel(ad_file, ad_df, mesh_alzheimer, nift_values)

    log.info('Starting Parkinsons')

    pd_path = os.path.join(neurommsig_excel_dir, 'parkinsons', 'parkinsons.xlsx')
    pd_df = preprocess(pd_path)
    with open(os.path.join(bms_base, 'aetionomy', 'parkinsons', 'neurommsigdb_pd.bel'), 'w') as pd_file:
        write_neurommsig_bel(pd_file, pd_df, mesh_parkinson, nift_values)

def
main
(
)
:
logging
.
basicConfig
(
level
=
logging
.
INFO
)
log
.
setLevel
(
logging
.
INFO
)
bms_base
=
get_bms_base
(
)
neurommsig_base
=
get_neurommsig_base
(
)
neurommsig_excel_dir
=
os
.
path
.
join
(
neurommsig_base
,
'resources'
,
'excels'
,
'neurommsig'
)
nift_values
=
get_nift_values
(
)
log
.
info
(
'Starting Alzheimers'
)
ad_path
=
os
.
path
.
join
(
neurommsig_excel_dir
,
'alzheimers'
,
'alzheimers.xlsx'
)
ad_df
=
preprocess
(
ad_path
)
with
open
(
os
.
path
.
join
(
bms_base
,
'aetionomy'
,
'alzheimers'
,
'neurommsigdb_ad.bel'
)
,
'w'
)
as
ad_file
:
write_neurommsig_bel
(
ad_file
,
ad_df
,
mesh_alzheimer
,
nift_values
)
log
.
info
(
'Starting Parkinsons'
)
pd_path
=
os
.
path
.
join
(
neurommsig_excel_dir
,
'parkinsons'
,
'parkinsons.xlsx'
)
pd_df
=
preprocess
(
pd_path
)
with
open
(
os
.
path
.
join
(
bms_base
,
'aetionomy'
,
'parkinsons'
,
'neurommsigdb_pd.bel'
)
,
'w'
)
as
pd_file
:
write_neurommsig_bel
(
pd_file
,
pd_df
,
mesh_parkinson
,
nift_values
)

Convert the Alzheimer s and Parkinson s disease NeuroMMSig excel sheets to BEL .


def remove_inconsistent_edges(graph: BELGraph) -> None:
    """Remove all edges between node pairs with inconsistent edges.

    This is the all-or-nothing approach. It would be better to do more careful investigation of the evidences during
    curation.
    """
    for u, v in get_inconsistent_edges(graph):
        edges = [(u, v, k) for k in graph[u][v]]
        graph.remove_edges_from(edges)

def remove_inconsistent_edges ( graph : BELGraph ) -> None : for u , v in get_inconsistent_edges ( graph ) : edges = [ ( u , v , k ) for k in graph [ u ] [ v ] ] graph . remove_edges_from ( edges )

Remove all edges between node pairs with inconsistent edges .


def get_walks_exhaustive(graph, node, length):
    """Gets all walks under a given length starting at a given node

    :param networkx.Graph graph: A graph
    :param node: Starting node
    :param int length: The length of walks to get
    :return: A list of paths
    :rtype: list[tuple]
    """
    if 0 == length:
        return (node,),

    return tuple(
        (node, key) + path
        for neighbor in graph.edge[node]
        for path in get_walks_exhaustive(graph, neighbor, length - 1)
        if node not in path
        for key in graph.edge[node][neighbor]
    )

def get_walks_exhaustive ( graph , node , length ) : if 0 == length : return ( node , ) , return tuple ( ( node , key ) + path for neighbor in graph . edge [ node ] for path in get_walks_exhaustive ( graph , neighbor , length - 1 ) if node not in path for key in graph . edge [ node ] [ neighbor ] )


def match_simple_metapath(graph, node, simple_metapath):
    """Matches a simple metapath starting at the given node

    :param pybel.BELGraph graph: A BEL graph
    :param tuple node: A BEL node
    :param list[str] simple_metapath: A list of BEL Functions
    :return: An iterable over paths from the node matching the metapath
    :rtype: iter[tuple]
    """
    if 0 == len(simple_metapath):
        yield node,

    else:
        for neighbor in graph.edges[node]:
            if graph.nodes[neighbor][FUNCTION] == simple_metapath[0]:
                for path in match_simple_metapath(graph, neighbor, simple_metapath[1:]):
                    if node not in path:
                        yield (node,) + path

def match_simple_metapath ( graph , node , simple_metapath ) : if 0 == len ( simple_metapath ) : yield node , else : for neighbor in graph . edges [ node ] : if graph . nodes [ neighbor ] [ FUNCTION ] == simple_metapath [ 0 ] : for path in match_simple_metapath ( graph , neighbor , simple_metapath [ 1 : ] ) : if node not in path : yield ( node , ) + path


def build_database(manager: pybel.Manager, annotation_url: Optional[str] = None) -> None:
    """Build a database of scores for NeuroMMSig annotated graphs.

    1. Get all networks that use the Subgraph annotation
    2. run on each
    """
    annotation_url = annotation_url or NEUROMMSIG_DEFAULT_URL

    annotation = manager.get_namespace_by_url(annotation_url)

    if annotation is None:
        raise RuntimeError('no graphs in database with given annotation')

    networks = get_networks_using_annotation(manager, annotation)

    dtis = ...

    for network in networks:
        graph = network.as_bel()

        scores = epicom_on_graph(graph, dtis)

        for (drug_name, subgraph_name), score in scores.items():
            drug_model = get_drug_model(manager, drug_name)
            subgraph_model = manager.get_annotation_entry(annotation_url, subgraph_name)

            score_model = Score(
                network=network,
                annotation=subgraph_model,
                drug=drug_model,
                score=score
            )

            manager.session.add(score_model)

    t = time.time()
    logger.info('committing scores')
    manager.session.commit()
    logger.info('committed scores in %.2f seconds', time.time() - t)

Build a database of scores for NeuroMMSig annotated graphs .


def calculate_average_scores_on_graph(
        graph: BELGraph,
        key: Optional[str] = None,
        tag: Optional[str] = None,
        default_score: Optional[float] = None,
        runs: Optional[int] = None,
        use_tqdm: bool = False,
):
    """Calculate the scores over all biological processes in the sub-graph.

    As an implementation, it simply computes the sub-graphs then calls :func:`calculate_average_scores_on_subgraphs` as
    described in that function's documentation.

    :param graph: A BEL graph with heats already on the nodes
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of {pybel node tuple: results tuple}
    :rtype: dict[tuple, tuple]

    Suggested usage with :mod:`pandas`:

    >>> import pandas as pd
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_graph
    >>> graph = ...  # load graph and data
    >>> scores = calculate_average_scores_on_graph(graph)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)

    """
    subgraphs = generate_bioprocess_mechanisms(graph, key=key)
    scores = calculate_average_scores_on_subgraphs(
        subgraphs,
        key=key,
        tag=tag,
        default_score=default_score,
        runs=runs,
        use_tqdm=use_tqdm
    )
    return scores

def calculate_average_scores_on_graph ( graph : BELGraph , key : Optional [ str ] = None , tag : Optional [ str ] = None , default_score : Optional [ float ] = None , runs : Optional [ int ] = None , use_tqdm : bool = False , ) : subgraphs = generate_bioprocess_mechanisms ( graph , key = key ) scores = calculate_average_scores_on_subgraphs ( subgraphs , key = key , tag = tag , default_score = default_score , runs = runs , use_tqdm = use_tqdm ) return scores

Calculate the scores over all biological processes in the sub - graph .


def calculate_average_scores_on_subgraphs(
        subgraphs: Mapping[H, BELGraph],
        key: Optional[str] = None,
        tag: Optional[str] = None,
        default_score: Optional[float] = None,
        runs: Optional[int] = None,
        use_tqdm: bool = False,
        tqdm_kwargs: Optional[Mapping[str, Any]] = None,
) -> Mapping[H, Tuple[float, float, float, float, int, int]]:
    """Calculate the scores over precomputed candidate mechanisms.

    :param subgraphs: A dictionary of keys to their corresponding subgraphs
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary of keys to results tuples

    Example Usage:

    >>> import pandas as pd
    >>> from pybel_tools.generation import generate_bioprocess_mechanisms
    >>> from pybel_tools.analysis.heat import calculate_average_scores_on_subgraphs
    >>> # load graph and data
    >>> graph = ...
    >>> candidate_mechanisms = generate_bioprocess_mechanisms(graph)
    >>> scores = calculate_average_scores_on_subgraphs(candidate_mechanisms)
    >>> pd.DataFrame.from_items(scores.items(), orient='index', columns=RESULT_LABELS)
    """
    results = {}

    log.info('calculating results for %d candidate mechanisms using %d permutations', len(subgraphs), runs)

    it = subgraphs.items()

    if use_tqdm:
        _tqdm_kwargs = dict(total=len(subgraphs), desc='Candidate mechanisms')
        if tqdm_kwargs:
            _tqdm_kwargs.update(tqdm_kwargs)
        it = tqdm(it, **_tqdm_kwargs)

    for node, subgraph in it:
        number_first_neighbors = subgraph.in_degree(node)
        number_first_neighbors = 0 if isinstance(number_first_neighbors, dict) else number_first_neighbors
        mechanism_size = subgraph.number_of_nodes()

        runners = workflow(subgraph, node, key=key, tag=tag, default_score=default_score, runs=runs)
        scores = [runner.get_final_score() for runner in runners]

        if 0 == len(scores):
            results[node] = (
                None,
                None,
                None,
                None,
                number_first_neighbors,
                mechanism_size,
            )
            continue

        scores = np.array(scores)

        average_score = np.average(scores)
        score_std = np.std(scores)
        med_score = np.median(scores)
        chi_2_stat, norm_p = stats.normaltest(scores)

        results[node] = (
            average_score,
            score_std,
            norm_p,
            med_score,
            number_first_neighbors,
            mechanism_size,
        )

    return results

Calculate the scores over precomputed candidate mechanisms .


def workflow(
        graph: BELGraph,
        node: BaseEntity,
        key: Optional[str] = None,
        tag: Optional[str] = None,
        default_score: Optional[float] = None,
        runs: Optional[int] = None,
        minimum_nodes: int = 1,
) -> List['Runner']:
    """Generate candidate mechanisms and run the heat diffusion workflow.

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param minimum_nodes: The minimum number of nodes a sub-graph needs to try running heat diffusion
    :return: A list of runners
    """
    subgraph = generate_mechanism(graph, node, key=key)

    if subgraph.number_of_nodes() <= minimum_nodes:
        return []

    runners = multirun(subgraph, node, key=key, tag=tag, default_score=default_score, runs=runs)
    return list(runners)

def workflow ( graph : BELGraph , node : BaseEntity , key : Optional [ str ] = None , tag : Optional [ str ] = None , default_score : Optional [ float ] = None , runs : Optional [ int ] = None , minimum_nodes : int = 1 , ) -> List [ 'Runner' ] : subgraph = generate_mechanism ( graph , node , key = key ) if subgraph . number_of_nodes ( ) <= minimum_nodes : return [ ] runners = multirun ( subgraph , node , key = key , tag = tag , default_score = default_score , runs = runs ) return list ( runners )

Generate candidate mechanisms and run the heat diffusion workflow .


def workflow_aggregate(graph: BELGraph,
                       node: BaseEntity,
                       key: Optional[str] = None,
                       tag: Optional[str] = None,
                       default_score: Optional[float] = None,
                       runs: Optional[int] = None,
                       aggregator: Optional[Callable[[Iterable[float]], float]] = None,
                       ) -> Optional[float]:
    """Get the average score over multiple runs.

    This function is very simple, and can be copied to do more interesting statistics over the :class:`Runner`
    instances. To iterate over the runners themselves, see :func:`workflow`

    :param graph: A BEL graph
    :param node: The BEL node that is the focus of this analysis
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param aggregator: A function that aggregates a list of scores. Defaults to :func:`numpy.average`.
                       Could also use: :func:`numpy.mean`, :func:`numpy.median`, :func:`numpy.min`, :func:`numpy.max`
    :return: The average score for the target node
    """
    runners = workflow(graph, node, key=key, tag=tag, default_score=default_score, runs=runs)
    scores = [runner.get_final_score() for runner in runners]

    if not scores:
        log.warning('Unable to run the heat diffusion workflow for %s', node)
        return

    if aggregator is None:
        return np.average(scores)

    return aggregator(scores)

Get the average score over multiple runs .


def workflow_all(graph: BELGraph,
                 key: Optional[str] = None,
                 tag: Optional[str] = None,
                 default_score: Optional[float] = None,
                 runs: Optional[int] = None,
                 ) -> Mapping[BaseEntity, List[Runner]]:
    """Run the heat diffusion workflow and get runners for every possible candidate mechanism

    1. Get all biological processes
    2. Get candidate mechanism induced two level back from each biological process
    3. Heat diffusion workflow for each candidate mechanism for multiple runs
    4. Return all runner results

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :return: A dictionary of {node: list of runners}
    """
    results = {}

    for node in get_nodes_by_function(graph, BIOPROCESS):
        results[node] = workflow(graph, node, key=key, tag=tag, default_score=default_score, runs=runs)

    return results

def workflow_all ( graph : BELGraph , key : Optional [ str ] = None , tag : Optional [ str ] = None , default_score : Optional [ float ] = None , runs : Optional [ int ] = None , ) -> Mapping [ BaseEntity , List [ Runner ] ] : results = { } for node in get_nodes_by_function ( graph , BIOPROCESS ) : results [ node ] = workflow ( graph , node , key = key , tag = tag , default_score = default_score , runs = runs ) return results


def workflow_all_aggregate(graph: BELGraph,
                           key: Optional[str] = None,
                           tag: Optional[str] = None,
                           default_score: Optional[float] = None,
                           runs: Optional[int] = None,
                           aggregator: Optional[Callable[[Iterable[float]], float]] = None,
                           ):
    """Run the heat diffusion workflow to get average score for every possible candidate mechanism.

    1. Get all biological processes
    2. Get candidate mechanism induced two level back from each biological process
    3. Heat diffusion workflow on each candidate mechanism for multiple runs
    4. Report average scores for each candidate mechanism

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param tag: The key for the nodes' data dictionaries where the scores will be put. Defaults to 'score'
    :param default_score: The initial score for all nodes. This number can go up or down.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param aggregator: A function that aggregates a list of scores. Defaults to :func:`numpy.average`.
                       Could also use: :func:`numpy.mean`, :func:`numpy.median`, :func:`numpy.min`, :func:`numpy.max`
    :return: A dictionary of {node: upstream causal subgraph}
    """
    results = {}

    bioprocess_nodes = list(get_nodes_by_function(graph, BIOPROCESS))

    for bioprocess_node in tqdm(bioprocess_nodes):
        subgraph = generate_mechanism(graph, bioprocess_node, key=key)

        try:
            results[bioprocess_node] = workflow_aggregate(
                graph=subgraph,
                node=bioprocess_node,
                key=key,
                tag=tag,
                default_score=default_score,
                runs=runs,
                aggregator=aggregator
            )
        except Exception:
            log.exception('could not run on %', bioprocess_node)

    return results

def
workflow_all_aggregate
(
graph
:
BELGraph
,
key
:
Optional
[
str
]
=
,
tag
:
Optional
[
str
]
=
,
default_score
:
Optional
[
float
]
=
,
runs
:
Optional
[
int
]
=
,
aggregator
:
Optional
[
Callable
[
[
Iterable
[
float
]
]
,
float
]
]
=
,
)
:
results
=
{
}
bioprocess_nodes
=
list
(
get_nodes_by_function
(
graph
,
BIOPROCESS
)
)
for
bioprocess_node
in
tqdm
(
bioprocess_nodes
)
:
subgraph
=
generate_mechanism
(
graph
,
bioprocess_node
,
key
=
key
)
try
:
results
[
bioprocess_node
]
=
workflow_aggregate
(
graph
=
subgraph
,
node
=
bioprocess_node
,
key
=
key
,
tag
=
tag
,
default_score
=
default_score
,
runs
=
runs
,
aggregator
=
aggregator
)
except
Exception
:
log
.
exception
(
'could not run on %'
,
bioprocess_node
)
return
results

Run the heat diffusion workflow to get average score for every possible candidate mechanism .


def calculate_average_score_by_annotation(
        graph: BELGraph,
        annotation: str,
        key: Optional[str] = None,
        runs: Optional[int] = None,
        use_tqdm: bool = False,
) -> Mapping[str, float]:
    """For each sub-graph induced over the edges matching the annotation, calculate the average score
    for all of the contained biological processes

    Assumes you haven't done anything yet

    1. Generates biological process upstream candidate mechanistic sub-graphs with
       :func:`generate_bioprocess_mechanisms`
    2. Calculates scores for each sub-graph with :func:`calculate_average_scores_on_sub-graphs`
    3. Overlays data with pbt.integration.overlay_data
    4. Calculates averages with pbt.selection.group_nodes.average_node_annotation

    :param graph: A BEL graph
    :param annotation: A BEL annotation
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :param runs: The number of times to run the heat diffusion workflow. Defaults to 100.
    :param use_tqdm: Should there be a progress bar for runners?
    :return: A dictionary from {str annotation value: tuple scores}

    Example Usage:

    >>> import pybel
    >>> from pybel_tools.integration import overlay_data
    >>> from pybel_tools.analysis.heat import calculate_average_score_by_annotation
    >>> graph = pybel.from_path(...)
    >>> scores = calculate_average_score_by_annotation(graph, 'subgraph')
    """
    candidate_mechanisms = generate_bioprocess_mechanisms(graph, key=key)

    #: {bp tuple: list of scores}
    scores: Mapping[BaseEntity, Tuple] = calculate_average_scores_on_subgraphs(
        subgraphs=candidate_mechanisms,
        key=key,
        runs=runs,
        use_tqdm=use_tqdm,
    )

    subgraph_bp: Mapping[str, List[BaseEntity]] = defaultdict(list)
    subgraphs: Mapping[str, BELGraph] = get_subgraphs_by_annotation(graph, annotation)
    for annotation_value, subgraph in subgraphs.items():
        subgraph_bp[annotation_value].extend(get_nodes_by_function(subgraph, BIOPROCESS))

    #: Pick the average by slicing with 0. Refer to :func:`calculate_average_score_on_subgraphs`
    return {
        annotation_value: np.average(scores[bp][0] for bp in bps)
        for annotation_value, bps in subgraph_bp.items()
    }

For each sub - graph induced over the edges matching the annotation calculate the average score for all of the contained biological processes


def iter_leaves(self) -> Iterable[BaseEntity]:
        """Return an iterable over all nodes that are leaves.

        A node is a leaf if either:

         - it doesn't have any predecessors, OR
         - all of its predecessors have a score in their data dictionaries
        """
        for node in self.graph:
            if self.tag in self.graph.nodes[node]:
                continue

            if not any(self.tag not in self.graph.nodes[p] for p in self.graph.predecessors(node)):
                yield node

def iter_leaves ( self ) -> Iterable [ BaseEntity ] : for node in self . graph : if self . tag in self . graph . nodes [ node ] : continue if not any ( self . tag not in self . graph . nodes [ p ] for p in self . graph . predecessors ( node ) ) : yield node

Return an iterable over all nodes that are leaves .


def unscored_nodes_iter(self) -> BaseEntity:
        """Iterate over all nodes without a score."""
        for node, data in self.graph.nodes(data=True):
            if self.tag not in data:
                yield node

def unscored_nodes_iter ( self ) -> BaseEntity : for node , data in self . graph . nodes ( data = True ) : if self . tag not in data : yield node

Iterate over all nodes without a score .


def remove_random_edge_until_has_leaves(self) -> None:
        """Remove random edges until there is at least one leaf node."""
        while True:
            leaves = set(self.iter_leaves())
            if leaves:
                return
            self.remove_random_edge()

def remove_random_edge_until_has_leaves ( self ) -> None : while True : leaves = set ( self . iter_leaves ( ) ) if leaves : return self . remove_random_edge ( )

Remove random edges until there is at least one leaf node .


def score_leaves(self) -> Set[BaseEntity]:
        """Calculate the score for all leaves.

        :return: The set of leaf nodes that were scored
        """
        leaves = set(self.iter_leaves())

        if not leaves:
            log.warning('no leaves.')
            return set()

        for leaf in leaves:
            self.graph.nodes[leaf][self.tag] = self.calculate_score(leaf)
            log.log(5, 'chomping %s', leaf)

        return leaves

def
score_leaves
(
self
)
->
Set
[
BaseEntity
]
:
leaves
=
set
(
self
.
iter_leaves
(
)
)
if
not
leaves
:
log
.
warning
(
'no leaves.'
)
return
set
(
)
for
leaf
in
leaves
:
self
.
graph
.
nodes
[
leaf
]
[
self
.
tag
]
=
self
.
calculate_score
(
leaf
)
log
.
log
(
5
,
'chomping %s'
,
leaf
)
return
leaves


def run_with_graph_transformation(self) -> Iterable[BELGraph]:
        """Calculate scores for all leaves until there are none, removes edges until there are, and repeats until
        all nodes have been scored. Also, yields the current graph at every step so you can make a cool animation
        of how the graph changes throughout the course of the algorithm

        :return: An iterable of BEL graphs
        """
        yield self.get_remaining_graph()
        while not self.done_chomping():
            while not list(self.iter_leaves()):
                self.remove_random_edge()
                yield self.get_remaining_graph()
            self.score_leaves()
            yield self.get_remaining_graph()

def run_with_graph_transformation ( self ) -> Iterable [ BELGraph ] : yield self . get_remaining_graph ( ) while not self . done_chomping ( ) : while not list ( self . iter_leaves ( ) ) : self . remove_random_edge ( ) yield self . get_remaining_graph ( ) self . score_leaves ( ) yield self . get_remaining_graph ( )

Calculate scores for all leaves until there are none removes edges until there are and repeats until all nodes have been scored . Also yields the current graph at every step so you can make a cool animation of how the graph changes throughout the course of the algorithm


def done_chomping(self) -> bool:
        """Determines if the algorithm is complete by checking if the target node of this analysis has been scored
        yet. Because the algorithm removes edges when it gets stuck until it is un-stuck, it is always guaranteed to
        finish.

        :return: Is the algorithm done running?
        """
        return self.tag in self.graph.nodes[self.target_node]

def done_chomping ( self ) -> bool : return self . tag in self . graph . nodes [ self . target_node ]

Determines if the algorithm is complete by checking if the target node of this analysis has been scored yet . Because the algorithm removes edges when it gets stuck until it is un - stuck it is always guaranteed to finish .


def get_final_score(self) -> float:
        """Return the final score for the target node.

        :return: The final score for the target node
        """
        if not self.done_chomping():
            raise ValueError('algorithm has not yet completed')

        return self.graph.nodes[self.target_node][self.tag]

def
get_final_score
(
self
)
->
float
:
if
not
self
.
done_chomping
(
)
:
raise
ValueError
(
'algorithm has not yet completed'
)
return
self
.
graph
.
nodes
[
self
.
target_node
]
[
self
.
tag
]

Return the final score for the target node .


def calculate_score(self, node: BaseEntity) -> float:
        """Calculate the new score of the given node."""
        score = (
            self.graph.nodes[node][self.tag]
            if self.tag in self.graph.nodes[node] else
            self.default_score
        )

        for predecessor, _, d in self.graph.in_edges(node, data=True):
            if d[RELATION] in CAUSAL_INCREASE_RELATIONS:
                score += self.graph.nodes[predecessor][self.tag]
            elif d[RELATION] in CAUSAL_DECREASE_RELATIONS:
                score -= self.graph.nodes[predecessor][self.tag]

        return score

def calculate_score ( self , node : BaseEntity ) -> float : score = ( self . graph . nodes [ node ] [ self . tag ] if self . tag in self . graph . nodes [ node ] else self . default_score ) for predecessor , _ , d in self . graph . in_edges ( node , data = True ) : if d [ RELATION ] in CAUSAL_INCREASE_RELATIONS : score += self . graph . nodes [ predecessor ] [ self . tag ] elif d [ RELATION ] in CAUSAL_DECREASE_RELATIONS : score -= self . graph . nodes [ predecessor ] [ self . tag ] return score

Calculate the new score of the given node .


def microcanonical_statistics_dtype(spanning_cluster=True):
    """
    Return the numpy structured array data type for sample states

    Helper function

    Parameters
    ----------
    spanning_cluster : bool, optional
        Whether to detect a spanning cluster or not.
        Defaults to ``True``.

    Returns
    -------
    ret : list of pairs of str
        A list of tuples of field names and data types to be used as ``dtype``
        argument in numpy ndarray constructors

    See Also
    --------
    http://docs.scipy.org/doc/numpy/user/basics.rec.html
    canonical_statistics_dtype
    """
    fields = list()
    fields.extend([
        ('n', 'uint32'),
        ('edge', 'uint32'),
    ])
    if spanning_cluster:
        fields.extend([
            ('has_spanning_cluster', 'bool'),
        ])
    fields.extend([
        ('max_cluster_size', 'uint32'),
        ('moments', '(5,)uint64'),
    ])
    return _ndarray_dtype(fields)

def microcanonical_statistics_dtype ( spanning_cluster = True ) : fields = list ( ) fields . extend ( [ ( 'n' , 'uint32' ) , ( 'edge' , 'uint32' ) , ] ) if spanning_cluster : fields . extend ( [ ( 'has_spanning_cluster' , 'bool' ) , ] ) fields . extend ( [ ( 'max_cluster_size' , 'uint32' ) , ( 'moments' , '(5,)uint64' ) , ] ) return _ndarray_dtype ( fields )


def canonical_statistics_dtype(spanning_cluster=True):
    """
    The NumPy Structured Array type for canonical statistics

    Helper function

    Parameters
    ----------
    spanning_cluster : bool, optional
        Whether to detect a spanning cluster or not.
        Defaults to ``True``.

    Returns
    -------
    ret : list of pairs of str
        A list of tuples of field names and data types to be used as ``dtype``
        argument in numpy ndarray constructors

    See Also
    --------
    http://docs.scipy.org/doc/numpy/user/basics.rec.html
    microcanoncial_statistics_dtype
    canonical_averages_dtype
    """
    fields = list()
    if spanning_cluster:
        fields.extend([
            ('percolation_probability', 'float64'),
        ])
    fields.extend([
        ('max_cluster_size', 'float64'),
        ('moments', '(5,)float64'),
    ])
    return _ndarray_dtype(fields)

def canonical_statistics_dtype ( spanning_cluster = True ) : fields = list ( ) if spanning_cluster : fields . extend ( [ ( 'percolation_probability' , 'float64' ) , ] ) fields . extend ( [ ( 'max_cluster_size' , 'float64' ) , ( 'moments' , '(5,)float64' ) , ] ) return _ndarray_dtype ( fields )


def bond_canonical_statistics(
    microcanonical_statistics,
    convolution_factors,
    **kwargs
):
    """
    canonical cluster statistics for a single run and a single probability

    Parameters
    ----------

    microcanonical_statistics : ndarray
        Return value of `bond_microcanonical_statistics`

    convolution_factors : 1-D array_like
        The coefficients of the convolution for the given probabilty ``p``
        and for each occupation number ``n``.

    Returns
    -------
    ret : ndarray of size ``1``
        Structured array with dtype as returned by
        `canonical_statistics_dtype`

    ret['percolation_probability'] : ndarray of float
        The "percolation probability" of this run at the value of ``p``.
        Only exists if `microcanonical_statistics` argument has the
        ``has_spanning_cluster`` field.

    ret['max_cluster_size'] : ndarray of int
        Weighted size of the largest cluster (absolute number of sites)

    ret['moments'] : 1-D :py:class:`numpy.ndarray` of float
        Array of size ``5``.
        The ``k``-th entry is the weighted ``k``-th raw moment of the
        (absolute) cluster size distribution, with ``k`` ranging from ``0`` to
        ``4``.

    See Also
    --------

    bond_microcanonical_statistics
    canonical_statistics_dtype

    """
    # initialize return array
    spanning_cluster = (
        'has_spanning_cluster' in microcanonical_statistics.dtype.names
    )
    ret = np.empty(1, dtype=canonical_statistics_dtype(spanning_cluster))

    # compute percolation probability
    if spanning_cluster:
        ret['percolation_probability'] = np.sum(
            convolution_factors *
            microcanonical_statistics['has_spanning_cluster']
        )

    # convolve maximum cluster size
    ret['max_cluster_size'] = np.sum(
        convolution_factors *
        microcanonical_statistics['max_cluster_size']
    )

    # convolve moments
    ret['moments'] = np.sum(
        convolution_factors[:, np.newaxis] *
        microcanonical_statistics['moments'],
        axis=0,
    )

    # return convolved cluster statistics
    return ret

def
bond_canonical_statistics
(
microcanonical_statistics
,
convolution_factors
,
*
*
kwargs
)
:
# initialize return array
spanning_cluster
=
(
'has_spanning_cluster'
in
microcanonical_statistics
.
dtype
.
names
)
ret
=
np
.
empty
(
1
,
dtype
=
canonical_statistics_dtype
(
spanning_cluster
)
)
# compute percolation probability
if
spanning_cluster
:
ret
[
'percolation_probability'
]
=
np
.
sum
(
convolution_factors
*
microcanonical_statistics
[
'has_spanning_cluster'
]
)
# convolve maximum cluster size
ret
[
'max_cluster_size'
]
=
np
.
sum
(
convolution_factors
*
microcanonical_statistics
[
'max_cluster_size'
]
)
# convolve moments
ret
[
'moments'
]
=
np
.
sum
(
convolution_factors
[
:
,
np
.
newaxis
]
*
microcanonical_statistics
[
'moments'
]
,
axis
=
0
,
)
# return convolved cluster statistics
return
ret


def canonical_averages_dtype(spanning_cluster=True):
    """
    The NumPy Structured Array type for canonical averages over several
    runs

    Helper function

    Parameters
    ----------
    spanning_cluster : bool, optional
        Whether to detect a spanning cluster or not.
        Defaults to ``True``.

    Returns
    -------
    ret : list of pairs of str
        A list of tuples of field names and data types to be used as ``dtype``
        argument in numpy ndarray constructors

    See Also
    --------
    http://docs.scipy.org/doc/numpy/user/basics.rec.html
    canonical_statistics_dtype
    finalized_canonical_averages_dtype
    """
    fields = list()
    fields.extend([
        ('number_of_runs', 'uint32'),
    ])
    if spanning_cluster:
        fields.extend([
            ('percolation_probability_mean', 'float64'),
            ('percolation_probability_m2', 'float64'),
        ])
    fields.extend([
        ('max_cluster_size_mean', 'float64'),
        ('max_cluster_size_m2', 'float64'),
        ('moments_mean', '(5,)float64'),
        ('moments_m2', '(5,)float64'),
    ])
    return _ndarray_dtype(fields)

def canonical_averages_dtype ( spanning_cluster = True ) : fields = list ( ) fields . extend ( [ ( 'number_of_runs' , 'uint32' ) , ] ) if spanning_cluster : fields . extend ( [ ( 'percolation_probability_mean' , 'float64' ) , ( 'percolation_probability_m2' , 'float64' ) , ] ) fields . extend ( [ ( 'max_cluster_size_mean' , 'float64' ) , ( 'max_cluster_size_m2' , 'float64' ) , ( 'moments_mean' , '(5,)float64' ) , ( 'moments_m2' , '(5,)float64' ) , ] ) return _ndarray_dtype ( fields )


def bond_initialize_canonical_averages(
    canonical_statistics, **kwargs
):
    """
    Initialize the canonical averages from a single-run cluster statistics

    Parameters
    ----------
    canonical_statistics : 1-D structured ndarray
        Typically contains the canonical statistics for a range of values
        of the occupation probability ``p``.
        The dtype is the result of `canonical_statistics_dtype`.

    Returns
    -------
    ret : structured ndarray
        The dype is the result of `canonical_averages_dtype`.

    ret['number_of_runs'] : 1-D ndarray of int
        Equals ``1`` (initial run).

    ret['percolation_probability_mean'] : 1-D array of float
        Equals ``canonical_statistics['percolation_probability']``
        (if ``percolation_probability`` is present)

    ret['percolation_probability_m2'] : 1-D array of float
        Each entry is ``0.0``

    ret['max_cluster_size_mean'] : 1-D array of float
        Equals ``canonical_statistics['max_cluster_size']``

    ret['max_cluster_size_m2'] : 1-D array of float
        Each entry is ``0.0``

    ret['moments_mean'] : 2-D array of float
        Equals ``canonical_statistics['moments']``

    ret['moments_m2'] : 2-D array of float
        Each entry is ``0.0``

    See Also
    --------
    canonical_averages_dtype
    bond_canonical_statistics

    """
    # initialize return array
    spanning_cluster = (
        'percolation_probability' in canonical_statistics.dtype.names
    )
    # array should have the same size as the input array
    ret = np.empty_like(
        canonical_statistics,
        dtype=canonical_averages_dtype(spanning_cluster=spanning_cluster),
    )
    ret['number_of_runs'] = 1

    # initialize percolation probability mean and sum of squared differences
    if spanning_cluster:
        ret['percolation_probability_mean'] = (
            canonical_statistics['percolation_probability']
        )
        ret['percolation_probability_m2'] = 0.0

    # initialize maximum cluster size mean and sum of squared differences
    ret['max_cluster_size_mean'] = (
        canonical_statistics['max_cluster_size']
    )
    ret['max_cluster_size_m2'] = 0.0

    # initialize moments means and sums of squared differences
    ret['moments_mean'] = canonical_statistics['moments']
    ret['moments_m2'] = 0.0

    return ret

Initialize the canonical averages from a single - run cluster statistics


def bond_reduce(row_a, row_b):
    """
    Reduce the canonical averages over several runs

    This is a "true" reducer.
    It is associative and commutative.

    This is a wrapper around `simoa.stats.online_variance`.

    Parameters
    ----------
    row_a, row_b : structured ndarrays
        Output of this function, or initial input from
        `bond_initialize_canonical_averages`

    Returns
    -------
    ret : structured ndarray
        Array is of dtype as returned by `canonical_averages_dtype`

    See Also
    --------
    bond_initialize_canonical_averages
    canonical_averages_dtype
    simoa.stats.online_variance
    """
    spanning_cluster = (
        'percolation_probability_mean' in row_a.dtype.names and
        'percolation_probability_mean' in row_b.dtype.names and
        'percolation_probability_m2' in row_a.dtype.names and
        'percolation_probability_m2' in row_b.dtype.names
    )

    # initialize return array
    ret = np.empty_like(row_a)

    def _reducer(key, transpose=False):
        mean_key = '{}_mean'.format(key)
        m2_key = '{}_m2'.format(key)
        res = simoa.stats.online_variance(*[
            (
                row['number_of_runs'],
                row[mean_key].T if transpose else row[mean_key],
                row[m2_key].T if transpose else row[m2_key],
            )
            for row in [row_a, row_b]
        ])

        (
            ret[mean_key],
            ret[m2_key],
        ) = (
            res[1].T,
            res[2].T,
        ) if transpose else res[1:]

    if spanning_cluster:
        _reducer('percolation_probability')

    _reducer('max_cluster_size')
    _reducer('moments', transpose=True)

    ret['number_of_runs'] = row_a['number_of_runs'] + row_b['number_of_runs']

    return ret

def
bond_reduce
(
row_a
,
row_b
)
:
spanning_cluster
=
(
'percolation_probability_mean'
in
row_a
.
dtype
.
names
and
'percolation_probability_mean'
in
row_b
.
dtype
.
names
and
'percolation_probability_m2'
in
row_a
.
dtype
.
names
and
'percolation_probability_m2'
in
row_b
.
dtype
.
names
)
# initialize return array
ret
=
np
.
empty_like
(
row_a
)
def
_reducer
(
key
,
transpose
=
False
)
:
mean_key
=
'{}_mean'
.
format
(
key
)
m2_key
=
'{}_m2'
.
format
(
key
)
res
=
simoa
.
stats
.
online_variance
(
*
[
(
row
[
'number_of_runs'
]
,
row
[
mean_key
]
.
T
if
transpose
else
row
[
mean_key
]
,
row
[
m2_key
]
.
T
if
transpose
else
row
[
m2_key
]
,
)
for
row
in
[
row_a
,
row_b
]
]
)
(
ret
[
mean_key
]
,
ret
[
m2_key
]
,
)
=
(
res
[
1
]
.
T
,
res
[
2
]
.
T
,
)
if
transpose
else
res
[
1
:
]
if
spanning_cluster
:
_reducer
(
'percolation_probability'
)
_reducer
(
'max_cluster_size'
)
_reducer
(
'moments'
,
transpose
=
True
)
ret
[
'number_of_runs'
]
=
row_a
[
'number_of_runs'
]
+
row_b
[
'number_of_runs'
]
return
ret


def finalized_canonical_averages_dtype(spanning_cluster=True):
    """
    The NumPy Structured Array type for finalized canonical averages over
    several runs

    Helper function

    Parameters
    ----------
    spanning_cluster : bool, optional
        Whether to detect a spanning cluster or not.
        Defaults to ``True``.

    Returns
    -------
    ret : list of pairs of str
        A list of tuples of field names and data types to be used as ``dtype``
        argument in numpy ndarray constructors

    See Also
    --------
    http://docs.scipy.org/doc/numpy/user/basics.rec.html
    canonical_averages_dtype
    """
    fields = list()
    fields.extend([
        ('number_of_runs', 'uint32'),
        ('p', 'float64'),
        ('alpha', 'float64'),
    ])
    if spanning_cluster:
        fields.extend([
            ('percolation_probability_mean', 'float64'),
            ('percolation_probability_std', 'float64'),
            ('percolation_probability_ci', '(2,)float64'),
        ])
    fields.extend([
        ('percolation_strength_mean', 'float64'),
        ('percolation_strength_std', 'float64'),
        ('percolation_strength_ci', '(2,)float64'),
        ('moments_mean', '(5,)float64'),
        ('moments_std', '(5,)float64'),
        ('moments_ci', '(5,2)float64'),
    ])
    return _ndarray_dtype(fields)

def finalized_canonical_averages_dtype ( spanning_cluster = True ) : fields = list ( ) fields . extend ( [ ( 'number_of_runs' , 'uint32' ) , ( 'p' , 'float64' ) , ( 'alpha' , 'float64' ) , ] ) if spanning_cluster : fields . extend ( [ ( 'percolation_probability_mean' , 'float64' ) , ( 'percolation_probability_std' , 'float64' ) , ( 'percolation_probability_ci' , '(2,)float64' ) , ] ) fields . extend ( [ ( 'percolation_strength_mean' , 'float64' ) , ( 'percolation_strength_std' , 'float64' ) , ( 'percolation_strength_ci' , '(2,)float64' ) , ( 'moments_mean' , '(5,)float64' ) , ( 'moments_std' , '(5,)float64' ) , ( 'moments_ci' , '(5,2)float64' ) , ] ) return _ndarray_dtype ( fields )


def finalize_canonical_averages(
    number_of_nodes, ps, canonical_averages, alpha,
):
    """
    Finalize canonical averages
    """

    spanning_cluster = (
        (
            'percolation_probability_mean' in
            canonical_averages.dtype.names
        ) and
        'percolation_probability_m2' in canonical_averages.dtype.names
    )

    # append values of p as an additional field
    ret = np.empty_like(
        canonical_averages,
        dtype=finalized_canonical_averages_dtype(
            spanning_cluster=spanning_cluster
        ),
    )

    n = canonical_averages['number_of_runs']
    sqrt_n = np.sqrt(canonical_averages['number_of_runs'])

    ret['number_of_runs'] = n
    ret['p'] = ps
    ret['alpha'] = alpha

    def _transform(
        original_key, final_key=None, normalize=False, transpose=False,
    ):
        if final_key is None:
            final_key = original_key
        keys_mean = [
            '{}_mean'.format(key)
            for key in [original_key, final_key]
        ]
        keys_std = [
            '{}_m2'.format(original_key),
            '{}_std'.format(final_key),
        ]
        key_ci = '{}_ci'.format(final_key)

        # calculate sample mean
        ret[keys_mean[1]] = canonical_averages[keys_mean[0]]
        if normalize:
            ret[keys_mean[1]] /= number_of_nodes

        # calculate sample standard deviation
        array = canonical_averages[keys_std[0]]
        result = np.sqrt(
            (array.T if transpose else array) / (n - 1)
        )
        ret[keys_std[1]] = (
            result.T if transpose else result
        )
        if normalize:
            ret[keys_std[1]] /= number_of_nodes

        # calculate standard normal confidence interval
        array = ret[keys_std[1]]
        scale = (array.T if transpose else array) / sqrt_n
        array = ret[keys_mean[1]]
        mean = (array.T if transpose else array)
        result = scipy.stats.t.interval(
            1 - alpha,
            df=n - 1,
            loc=mean,
            scale=scale,
        )
        (
            ret[key_ci][..., 0], ret[key_ci][..., 1]
        ) = ([my_array.T for my_array in result] if transpose else result)

    if spanning_cluster:
        _transform('percolation_probability')

    _transform('max_cluster_size', 'percolation_strength', normalize=True)
    _transform('moments', normalize=True, transpose=True)

    return ret


def compare(graph: BELGraph, annotation: str = 'Subgraph') -> Mapping[str, Mapping[str, float]]:
    """Compare generated mechanisms to actual ones.

    1. Generates candidate mechanisms for each biological process
    2. Gets sub-graphs for all NeuroMMSig signatures
    3. Make tanimoto similarity comparison for all sets

    :return: A dictionary table comparing the canonical subgraphs to generated ones
    """
    canonical_mechanisms = get_subgraphs_by_annotation(graph, annotation)
    canonical_nodes = _transform_graph_dict_to_node_dict(canonical_mechanisms)

    candidate_mechanisms = generate_bioprocess_mechanisms(graph)
    candidate_nodes = _transform_graph_dict_to_node_dict(candidate_mechanisms)

    results: Dict[str, Dict[str, float]] = defaultdict(dict)

    it = itt.product(canonical_nodes.items(), candidate_nodes.items())
    for (canonical_name, canonical_graph), (candidate_bp, candidate_graph) in it:
        tanimoto = tanimoto_set_similarity(candidate_nodes, canonical_nodes)
        results[canonical_name][candidate_bp] = tanimoto

    return dict(results)

def compare ( graph : BELGraph , annotation : str = 'Subgraph' ) -> Mapping [ str , Mapping [ str , float ] ] : canonical_mechanisms = get_subgraphs_by_annotation ( graph , annotation ) canonical_nodes = _transform_graph_dict_to_node_dict ( canonical_mechanisms ) candidate_mechanisms = generate_bioprocess_mechanisms ( graph ) candidate_nodes = _transform_graph_dict_to_node_dict ( candidate_mechanisms ) results : Dict [ str , Dict [ str , float ] ] = defaultdict ( dict ) it = itt . product ( canonical_nodes . items ( ) , candidate_nodes . items ( ) ) for ( canonical_name , canonical_graph ) , ( candidate_bp , candidate_graph ) in it : tanimoto = tanimoto_set_similarity ( candidate_nodes , canonical_nodes ) results [ canonical_name ] [ candidate_bp ] = tanimoto return dict ( results )

Compare generated mechanisms to actual ones .


def summarize_node_filter(graph: BELGraph, node_filters: NodePredicates) -> None:
    """Print a summary of the number of nodes passing a given set of filters.

    :param graph: A BEL graph
    :param node_filters: A node filter or list/tuple of node filters
    """
    passed = count_passed_node_filter(graph, node_filters)
    print('{}/{} nodes passed'.format(passed, graph.number_of_nodes()))

def
summarize_node_filter
(
graph
:
BELGraph
,
node_filters
:
NodePredicates
)
->
:
passed
=
count_passed_node_filter
(
graph
,
node_filters
)
print
(
'{}/{} nodes passed'
.
format
(
passed
,
graph
.
number_of_nodes
(
)
)
)

Print a summary of the number of nodes passing a given set of filters .


def node_inclusion_filter_builder(nodes: Iterable[BaseEntity]) -> NodePredicate:
    """Build a filter that only passes on nodes in the given list.

    :param nodes: An iterable of BEL nodes
    """
    node_set = set(nodes)

    def inclusion_filter(_: BELGraph, node: BaseEntity) -> bool:
        """Pass only for a node that is in the enclosed node list.

        :return: If the node is contained within the enclosed node list
        """
        return node in node_set

    return inclusion_filter

Build a filter that only passes on nodes in the given list .


def node_exclusion_filter_builder(nodes: Iterable[BaseEntity]) -> NodePredicate:
    """Build a filter that fails on nodes in the given list."""
    node_set = set(nodes)

    def exclusion_filter(_: BELGraph, node: BaseEntity) -> bool:
        """Pass only for a node that isn't in the enclosed node list

        :return: If the node isn't contained within the enclosed node list
        """
        return node not in node_set

    return exclusion_filter

Build a filter that fails on nodes in the given list .


def function_namespace_inclusion_builder(func: str, namespace: Strings) -> NodePredicate:
    """Build a filter function for matching the given BEL function with the given namespace or namespaces.

    :param func: A BEL function
    :param namespace: The namespace to search by
    """
    if isinstance(namespace, str):
        def function_namespaces_filter(_: BELGraph, node: BaseEntity) -> bool:
            """Pass only for nodes that have the enclosed function and enclosed namespace."""
            if func != node[FUNCTION]:
                return False
            return NAMESPACE in node and node[NAMESPACE] == namespace

    elif isinstance(namespace, Iterable):
        namespaces = set(namespace)

        def function_namespaces_filter(_: BELGraph, node: BaseEntity) -> bool:
            """Pass only for nodes that have the enclosed function and namespace in the enclose set."""
            if func != node[FUNCTION]:
                return False
            return NAMESPACE in node and node[NAMESPACE] in namespaces

    else:
        raise ValueError('Invalid type for argument: {}'.format(namespace))

    return function_namespaces_filter

Build a filter function for matching the given BEL function with the given namespace or namespaces .


def data_contains_key_builder(key: str) -> NodePredicate:  # noqa: D202
    """Build a filter that passes only on nodes that have the given key in their data dictionary.

    :param key: A key for the node's data dictionary
    """

    def data_contains_key(_: BELGraph, node: BaseEntity) -> bool:
        """Pass only for a node that contains the enclosed key in its data dictionary.

        :return: If the node contains the enclosed key in its data dictionary
        """
        return key in node

    return data_contains_key

Build a filter that passes only on nodes that have the given key in their data dictionary .


def variants_of(
        graph: BELGraph,
        node: Protein,
        modifications: Optional[Set[str]] = None,
) -> Set[Protein]:
    """Returns all variants of the given node."""
    if modifications:
        return _get_filtered_variants_of(graph, node, modifications)

    return {
        v
        for u, v, key, data in graph.edges(keys=True, data=True)
        if (
            u == node
            and data[RELATION] == HAS_VARIANT
            and pybel.struct.has_protein_modification(v)
        )
    }

def variants_of ( graph : BELGraph , node : Protein , modifications : Optional [ Set [ str ] ] = None , ) -> Set [ Protein ] : if modifications : return _get_filtered_variants_of ( graph , node , modifications ) return { v for u , v , key , data in graph . edges ( keys = True , data = True ) if ( u == node and data [ RELATION ] == HAS_VARIANT and pybel . struct . has_protein_modification ( v ) ) }

Returns all variants of the given node .


def get_variants_to_controllers(
        graph: BELGraph,
        node: Protein,
        modifications: Optional[Set[str]] = None,
) -> Mapping[Protein, Set[Protein]]:
    """Get a mapping from variants of the given node to all of its upstream controllers."""
    rv = defaultdict(set)
    variants = variants_of(graph, node, modifications)
    for controller, variant, data in graph.in_edges(variants, data=True):
        if data[RELATION] in CAUSAL_RELATIONS:
            rv[variant].add(controller)
    return rv

def get_variants_to_controllers ( graph : BELGraph , node : Protein , modifications : Optional [ Set [ str ] ] = None , ) -> Mapping [ Protein , Set [ Protein ] ] : rv = defaultdict ( set ) variants = variants_of ( graph , node , modifications ) for controller , variant , data in graph . in_edges ( variants , data = True ) : if data [ RELATION ] in CAUSAL_RELATIONS : rv [ variant ] . add ( controller ) return rv

Get a mapping from variants of the given node to all of its upstream controllers .


def group_dict_set(iterator: Iterable[Tuple[A, B]]) -> Mapping[A, Set[B]]:
    """Make a dict that accumulates the values for each key in an iterator of doubles."""
    d = defaultdict(set)
    for key, value in iterator:
        d[key].add(value)
    return dict(d)

def group_dict_set ( iterator : Iterable [ Tuple [ A , B ] ] ) -> Mapping [ A , Set [ B ] ] : d = defaultdict ( set ) for key , value in iterator : d [ key ] . add ( value ) return dict ( d )

Make a dict that accumulates the values for each key in an iterator of doubles .


def count_unique_relations(graph: BELGraph) -> Counter:
    """Return a histogram of the different types of relations present in a graph.

    Note: this operation only counts each type of edge once for each pair of nodes
    """
    return Counter(itt.chain.from_iterable(get_edge_relations(graph).values()))

def count_unique_relations ( graph : BELGraph ) -> Counter : return Counter ( itt . chain . from_iterable ( get_edge_relations ( graph ) . values ( ) ) )

Return a histogram of the different types of relations present in a graph .


def count_annotation_values(graph: BELGraph, annotation: str) -> Counter:
    """Count in how many edges each annotation appears in a graph

    :param graph: A BEL graph
    :param annotation: The annotation to count
    :return: A Counter from {annotation value: frequency}
    """
    return Counter(iter_annotation_values(graph, annotation))

def count_annotation_values ( graph : BELGraph , annotation : str ) -> Counter : return Counter ( iter_annotation_values ( graph , annotation ) )


def count_annotation_values_filtered(graph: BELGraph,
                                     annotation: str,
                                     source_predicate: Optional[NodePredicate] = None,
                                     target_predicate: Optional[NodePredicate] = None,
                                     ) -> Counter:
    """Count in how many edges each annotation appears in a graph, but filter out source nodes and target nodes.

    See :func:`pybel_tools.utils.keep_node` for a basic filter.

    :param graph: A BEL graph
    :param annotation: The annotation to count
    :param source_predicate: A predicate (graph, node) -> bool for keeping source nodes
    :param target_predicate: A predicate (graph, node) -> bool for keeping target nodes
    :return: A Counter from {annotation value: frequency}
    """
    if source_predicate and target_predicate:
        return Counter(
            data[ANNOTATIONS][annotation]
            for u, v, data in graph.edges(data=True)
            if edge_has_annotation(data, annotation) and source_predicate(graph, u) and target_predicate(graph, v)
        )
    elif source_predicate:
        return Counter(
            data[ANNOTATIONS][annotation]
            for u, v, data in graph.edges(data=True)
            if edge_has_annotation(data, annotation) and source_predicate(graph, u)
        )
    elif target_predicate:
        return Counter(
            data[ANNOTATIONS][annotation]
            for u, v, data in graph.edges(data=True)
            if edge_has_annotation(data, annotation) and target_predicate(graph, u)
        )
    else:
        return Counter(
            data[ANNOTATIONS][annotation]
            for u, v, data in graph.edges(data=True)
            if edge_has_annotation(data, annotation)
        )

def count_annotation_values_filtered ( graph : BELGraph , annotation : str , source_predicate : Optional [ NodePredicate ] = None , target_predicate : Optional [ NodePredicate ] = None , ) -> Counter : if source_predicate and target_predicate : return Counter ( data [ ANNOTATIONS ] [ annotation ] for u , v , data in graph . edges ( data = True ) if edge_has_annotation ( data , annotation ) and source_predicate ( graph , u ) and target_predicate ( graph , v ) ) elif source_predicate : return Counter ( data [ ANNOTATIONS ] [ annotation ] for u , v , data in graph . edges ( data = True ) if edge_has_annotation ( data , annotation ) and source_predicate ( graph , u ) ) elif target_predicate : return Counter ( data [ ANNOTATIONS ] [ annotation ] for u , v , data in graph . edges ( data = True ) if edge_has_annotation ( data , annotation ) and target_predicate ( graph , u ) ) else : return Counter ( data [ ANNOTATIONS ] [ annotation ] for u , v , data in graph . edges ( data = True ) if edge_has_annotation ( data , annotation ) )

Count in how many edges each annotation appears in a graph but filter out source nodes and target nodes .


def pair_is_consistent(graph: BELGraph, u: BaseEntity, v: BaseEntity) -> Optional[str]:
    """Return if the edges between the given nodes are consistent, meaning they all have the same relation.

    :return: If the edges aren't consistent, return false, otherwise return the relation type
    """
    relations = {data[RELATION] for data in graph[u][v].values()}

    if 1 != len(relations):
        return

    return list(relations)[0]

def pair_is_consistent ( graph : BELGraph , u : BaseEntity , v : BaseEntity ) -> Optional [ str ] : relations = { data [ RELATION ] for data in graph [ u ] [ v ] . values ( ) } if 1 != len ( relations ) : return return list ( relations ) [ 0 ]

Return if the edges between the given nodes are consistent meaning they all have the same relation .


def infer_missing_two_way_edges(graph):
    """Add edges to the graph when a two way edge exists, and the opposite direction doesn't exist.

    Use: two way edges from BEL definition and/or axiomatic inverses of membership relations

    :param pybel.BELGraph graph: A BEL graph
    """
    for u, v, k, d in graph.edges(data=True, keys=True):
        if d[RELATION] in TWO_WAY_RELATIONS:
            infer_missing_backwards_edge(graph, u, v, k)

def infer_missing_two_way_edges ( graph ) : for u , v , k , d in graph . edges ( data = True , keys = True ) : if d [ RELATION ] in TWO_WAY_RELATIONS : infer_missing_backwards_edge ( graph , u , v , k )

Add edges to the graph when a two way edge exists and the opposite direction doesn t exist .


def infer_missing_backwards_edge(graph, u, v, k):
    """Add the same edge, but in the opposite direction if not already present.

    :type graph: pybel.BELGraph
    :type u: tuple
    :type v: tuple
    :type k: int
    """
    if u in graph[v]:
        for attr_dict in graph[v][u].values():
            if attr_dict == graph[u][v][k]:
                return

    graph.add_edge(v, u, key=k, **graph[u][v][k])

def infer_missing_backwards_edge ( graph , u , v , k ) : if u in graph [ v ] : for attr_dict in graph [ v ] [ u ] . values ( ) : if attr_dict == graph [ u ] [ v ] [ k ] : return graph . add_edge ( v , u , key = k , * * graph [ u ] [ v ] [ k ] )

Add the same edge but in the opposite direction if not already present .


def enrich_internal_unqualified_edges(graph, subgraph):
    """Add the missing unqualified edges between entities in the subgraph that are contained within the full graph.

    :param pybel.BELGraph graph: The full BEL graph
    :param pybel.BELGraph subgraph: The query BEL subgraph
    """
    for u, v in itt.combinations(subgraph, 2):
        if not graph.has_edge(u, v):
            continue

        for k in graph[u][v]:
            if k < 0:
                subgraph.add_edge(u, v, key=k, **graph[u][v][k])

def enrich_internal_unqualified_edges ( graph , subgraph ) : for u , v in itt . combinations ( subgraph , 2 ) : if not graph . has_edge ( u , v ) : continue for k in graph [ u ] [ v ] : if k < 0 : subgraph . add_edge ( u , v , key = k , * * graph [ u ] [ v ] [ k ] )

Add the missing unqualified edges between entities in the subgraph that are contained within the full graph .


def boilerplate(name, contact, description, pmids, version, copyright, authors, licenses, disclaimer, output):
    """Build a template BEL document with the given PubMed identifiers."""
    from .document_utils import write_boilerplate

    write_boilerplate(
        name=name,
        version=version,
        description=description,
        authors=authors,
        contact=contact,
        copyright=copyright,
        licenses=licenses,
        disclaimer=disclaimer,
        pmids=pmids,
        file=output,
    )

def boilerplate ( name , contact , description , pmids , version , copyright , authors , licenses , disclaimer , output ) : from . document_utils import write_boilerplate write_boilerplate ( name = name , version = version , description = description , authors = authors , contact = contact , copyright = copyright , licenses = licenses , disclaimer = disclaimer , pmids = pmids , file = output , )

Build a template BEL document with the given PubMed identifiers .


def get_pmids(graph: BELGraph, output: TextIO):
    """Output PubMed identifiers from a graph to a stream."""
    for pmid in get_pubmed_identifiers(graph):
        click.echo(pmid, file=output)

def get_pmids ( graph : BELGraph , output : TextIO ) : for pmid in get_pubmed_identifiers ( graph ) : click . echo ( pmid , file = output )

Output PubMed identifiers from a graph to a stream .


def getrowcount(self, window_name, object_name):
        """
        Get count of rows in table object.

        @param window_name: Window name to look for, either full name,
        LDTP's name convention, or a Unix glob.
        @type window_name: string
        @param object_name: Object name to look for, either full name,
        LDTP's name convention, or a Unix glob. Or menu heirarchy
        @type object_name: string

        @return: Number of rows.
        @rtype: integer
        """
        object_handle = self._get_object_handle(window_name, object_name)
        if not object_handle.AXEnabled:
            raise LdtpServerException(u"Object %s state disabled" % object_name)
        return len(object_handle.AXRows)

def
getrowcount
(
self
,
window_name
,
object_name
)
:
object_handle
=
self
.
_get_object_handle
(
window_name
,
object_name
)
if
not
object_handle
.
AXEnabled
:
raise
LdtpServerException
(
u"Object %s state disabled"
%
object_name
)
return
len
(
object_handle
.
AXRows
)