def _aggregations_delete(aggregation_types=None,
                         start_date=None, end_date=None):
    """Delete computed aggregations."""
    aggregation_types = (aggregation_types or
                         list(current_stats.enabled_aggregations))
    for a in aggregation_types:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        aggregator.delete(start_date, end_date)

def _aggregations_delete ( aggregation_types = None , start_date = None , end_date = None ) : aggregation_types = ( aggregation_types or list ( current_stats . enabled_aggregations ) ) for a in aggregation_types : aggr_cfg = current_stats . aggregations [ a ] aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , * * aggr_cfg . aggregator_config ) aggregator . delete ( start_date , end_date )


def _aggregations_list_bookmarks(aggregation_types=None,
                                 start_date=None, end_date=None, limit=None):
    """List aggregation bookmarks."""
    aggregation_types = (aggregation_types or
                         list(current_stats.enabled_aggregations))
    for a in aggregation_types:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        bookmarks = aggregator.list_bookmarks(start_date, end_date, limit)
        click.echo('{}:'.format(a))
        for b in bookmarks:
            click.echo(' - {}'.format(b.date))

def
_aggregations_list_bookmarks
(
aggregation_types
=
,
start_date
=
,
end_date
=
,
limit
=
)
:
aggregation_types
=
(
aggregation_types
or
list
(
current_stats
.
enabled_aggregations
)
)
for
a
in
aggregation_types
:
aggr_cfg
=
current_stats
.
aggregations
[
a
]
aggregator
=
aggr_cfg
.
aggregator_class
(
name
=
aggr_cfg
.
name
,
*
*
aggr_cfg
.
aggregator_config
)
bookmarks
=
aggregator
.
list_bookmarks
(
start_date
,
end_date
,
limit
)
click
.
echo
(
'{}:'
.
format
(
a
)
)
for
b
in
bookmarks
:
click
.
echo
(
' - {}'
.
format
(
b
.
date
)
)


def _events_config(self):
        """Load events configuration."""
        # import iter_entry_points here so that it can be mocked in tests
        result = {}
        for ep in iter_entry_points(
                group=self.entry_point_group_events):
            for cfg in ep.load()():
                if cfg['event_type'] not in self.enabled_events:
                    continue
                elif cfg['event_type'] in result:
                    raise DuplicateEventError(
                        'Duplicate event {0} in entry point '
                        '{1}'.format(cfg['event_type'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_events[cfg['event_type']] or {}
                )
                result[cfg['event_type']] = cfg
        return result


def _aggregations_config(self):
        """Load aggregation configurations."""
        result = {}
        for ep in iter_entry_points(
                group=self.entry_point_group_aggs):
            for cfg in ep.load()():
                if cfg['aggregation_name'] not in self.enabled_aggregations:
                    continue
                elif cfg['aggregation_name'] in result:
                    raise DuplicateAggregationError(
                        'Duplicate aggregation {0} in entry point '
                        '{1}'.format(cfg['event_type'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_aggregations[cfg['aggregation_name']] or {}
                )
                result[cfg['aggregation_name']] = cfg
        return result


def _queries_config(self):
        """Load queries configuration."""
        result = {}
        for ep in iter_entry_points(group=self.entry_point_group_queries):
            for cfg in ep.load()():
                if cfg['query_name'] not in self.enabled_queries:
                    continue
                elif cfg['query_name'] in result:
                    raise DuplicateQueryError(
                        'Duplicate query {0} in entry point '
                        '{1}'.format(cfg['query'], ep.name))
                # Update the default configuration with env/overlay config.
                cfg.update(
                    self.enabled_queries[cfg['query_name']] or {}
                )
                result[cfg['query_name']] = cfg
        return result


def publish(self, event_type, events):
        """Publish events."""
        assert event_type in self.events
        current_queues.queues['stats-{}'.format(event_type)].publish(events)

def publish ( self , event_type , events ) : assert event_type in self . events current_queues . queues [ 'stats-{}' . format ( event_type ) ] . publish ( events )


def consume(self, event_type, no_ack=True, payload=True):
        """Comsume all pending events."""
        assert event_type in self.events
        return current_queues.queues['stats-{}'.format(event_type)].consume(
            payload=payload)

def consume ( self , event_type , no_ack = True , payload = True ) : assert event_type in self . events return current_queues . queues [ 'stats-{}' . format ( event_type ) ] . consume ( payload = payload )


def tell(self, message, sender=no_sender):
        """ Send a message to this actor. Asynchronous fire-and-forget.

        :param message: The message to send.
        :type message: Any

        :param sender: The sender of the message. If provided it will be made
            available to the receiving actor via the :attr:`Actor.sender` attribute.
        :type sender: :class:`Actor`
        """
        if sender is not no_sender and not isinstance(sender, ActorRef):
            raise ValueError("Sender must be actor reference")

        self._cell.send_message(message, sender)

def
tell
(
self
,
message
,
sender
=
no_sender
)
:
if
sender
is
not
no_sender
and
not
isinstance
(
sender
,
ActorRef
)
:
raise
ValueError
(
"Sender must be actor reference"
)
self
.
_cell
.
send_message
(
message
,
sender
)

Send a message to this actor . Asynchronous fire - and - forget .


def get_anonymization_salt(ts):
    """Get the anonymization salt based on the event timestamp's day."""
    salt_key = 'stats:salt:{}'.format(ts.date().isoformat())
    salt = current_cache.get(salt_key)
    if not salt:
        salt_bytes = os.urandom(32)
        salt = b64encode(salt_bytes).decode('utf-8')
        current_cache.set(salt_key, salt, timeout=60 * 60 * 24)
    return salt

def get_anonymization_salt ( ts ) : salt_key = 'stats:salt:{}' . format ( ts . date ( ) . isoformat ( ) ) salt = current_cache . get ( salt_key ) if not salt : salt_bytes = os . urandom ( 32 ) salt = b64encode ( salt_bytes ) . decode ( 'utf-8' ) current_cache . set ( salt_key , salt , timeout = 60 * 60 * 24 ) return salt

Get the anonymization salt based on the event timestamp s day .


def get_geoip(ip):
    """Lookup country for IP address."""
    reader = geolite2.reader()
    ip_data = reader.get(ip) or {}
    return ip_data.get('country', {}).get('iso_code')

def get_geoip ( ip ) : reader = geolite2 . reader ( ) ip_data = reader . get ( ip ) or { } return ip_data . get ( 'country' , { } ) . get ( 'iso_code' )


def get_user():
    """User information.

    .. note::

       **Privacy note** A users IP address, user agent string, and user id
       (if logged in) is sent to a message queue, where it is stored for about
       5 minutes. The information is used to:

       - Detect robot visits from the user agent string.
       - Generate an anonymized visitor id (using a random salt per day).
       - Detect the users host contry based on the IP address.

       The information is then discarded.
    """
    return dict(
        ip_address=request.remote_addr,
        user_agent=request.user_agent.string,
        user_id=(
            current_user.get_id() if current_user.is_authenticated else None
        ),
        session_id=session.get('sid_s')
    )

def get_user ( ) : return dict ( ip_address = request . remote_addr , user_agent = request . user_agent . string , user_id = ( current_user . get_id ( ) if current_user . is_authenticated else None ) , session_id = session . get ( 'sid_s' ) )


def default_permission_factory(query_name, params):
    """Default permission factory.

    It enables by default the statistics if they don't have a dedicated
    permission factory.
    """
    from invenio_stats import current_stats
    if current_stats.queries[query_name].permission_factory is None:
        return AllowAllPermission
    else:
        return current_stats.queries[query_name].permission_factory(
            query_name, params
        )

def default_permission_factory ( query_name , params ) : from invenio_stats import current_stats if current_stats . queries [ query_name ] . permission_factory is None : return AllowAllPermission else : return current_stats . queries [ query_name ] . permission_factory ( query_name , params )


def register_templates():
    """Register elasticsearch templates for events."""
    event_templates = [current_stats._events_config[e]
                       ['templates']
                       for e in
                       current_stats._events_config]
    aggregation_templates = [current_stats._aggregations_config[a]
                             ['templates']
                             for a in
                             current_stats._aggregations_config]
    return event_templates + aggregation_templates

def register_templates ( ) : event_templates = [ current_stats . _events_config [ e ] [ 'templates' ] for e in current_stats . _events_config ] aggregation_templates = [ current_stats . _aggregations_config [ a ] [ 'templates' ] for a in current_stats . _aggregations_config ] return event_templates + aggregation_templates

Register elasticsearch templates for events .


def aggregate_events(aggregations, start_date=None, end_date=None,
                     update_bookmark=True):
    """Aggregate indexed events."""
    start_date = dateutil_parse(start_date) if start_date else None
    end_date = dateutil_parse(end_date) if end_date else None
    results = []
    for a in aggregations:
        aggr_cfg = current_stats.aggregations[a]
        aggregator = aggr_cfg.aggregator_class(
            name=aggr_cfg.name, **aggr_cfg.aggregator_config)
        results.append(aggregator.run(start_date, end_date, update_bookmark))
    return results

def aggregate_events ( aggregations , start_date = None , end_date = None , update_bookmark = True ) : start_date = dateutil_parse ( start_date ) if start_date else None end_date = dateutil_parse ( end_date ) if end_date else None results = [ ] for a in aggregations : aggr_cfg = current_stats . aggregations [ a ] aggregator = aggr_cfg . aggregator_class ( name = aggr_cfg . name , * * aggr_cfg . aggregator_config ) results . append ( aggregator . run ( start_date , end_date , update_bookmark ) ) return results


def _handle_request(self, scheme, netloc, path, headers, body=None, method="GET"):
        """
        Run the actual request
        """
        backend_url = "{}://{}{}".format(scheme, netloc, path)
        try:
            response = self.http_request.request(backend_url, method=method, body=body, headers=dict(headers))
            self._return_response(response)
        except Exception as e:
            body = "Invalid response from backend: '{}' Server might be busy".format(e.message)
            logging.debug(body)
            self.send_error(httplib.SERVICE_UNAVAILABLE, body)


def anonymize_user(doc):
    """Preprocess an event by anonymizing user information.

    The anonymization is done by removing fields that can uniquely identify a
    user, such as the user's ID, session ID, IP address and User Agent, and
    hashing them to produce a ``visitor_id`` and ``unique_session_id``. To
    further secure the method, a randomly generated 32-byte salt is used, that
    expires after 24 hours and is discarded. The salt values are stored in
    Redis (or whichever backend Invenio-Cache uses). The ``unique_session_id``
    is calculated in the same way as the ``visitor_id``, with the only
    difference that it also takes into account the hour of the event . All of
    these rules effectively mean that a user can have a unique ``visitor_id``
    for each day and unique ``unique_session_id`` for each hour of a day.

    This session ID generation process was designed according to the `Project
    COUNTER Code of Practice <https://www.projectcounter.org/code-of-
    practice-sections/general-information/>`_.

    In addition to that the country of the user is extracted from the IP
    address as a ISO 3166-1 alpha-2 two-letter country code (e.g. "CH" for
    Switzerland).
    """
    ip = doc.pop('ip_address', None)
    if ip:
        doc.update({'country': get_geoip(ip)})

    user_id = doc.pop('user_id', '')
    session_id = doc.pop('session_id', '')
    user_agent = doc.pop('user_agent', '')

    # A 'User Session' is defined as activity by a user in a period of
    # one hour. timeslice represents the hour of the day in which
    # the event has been generated and together with user info it determines
    # the 'User Session'
    timestamp = arrow.get(doc.get('timestamp'))
    timeslice = timestamp.strftime('%Y%m%d%H')
    salt = get_anonymization_salt(timestamp)

    visitor_id = hashlib.sha224(salt.encode('utf-8'))
    # TODO: include random salt here, that changes once a day.
    # m.update(random_salt)
    if user_id:
        visitor_id.update(user_id.encode('utf-8'))
    elif session_id:
        visitor_id.update(session_id.encode('utf-8'))
    elif ip and user_agent:
        vid = '{}|{}|{}'.format(ip, user_agent, timeslice)
        visitor_id.update(vid.encode('utf-8'))
    else:
        # TODO: add random data?
        pass

    unique_session_id = hashlib.sha224(salt.encode('utf-8'))
    if user_id:
        sid = '{}|{}'.format(user_id, timeslice)
        unique_session_id.update(sid.encode('utf-8'))
    elif session_id:
        sid = '{}|{}'.format(session_id, timeslice)
        unique_session_id.update(sid.encode('utf-8'))
    elif ip and user_agent:
        sid = '{}|{}|{}'.format(ip, user_agent, timeslice)
        unique_session_id.update(sid.encode('utf-8'))

    doc.update(dict(
        visitor_id=visitor_id.hexdigest(),
        unique_session_id=unique_session_id.hexdigest()
    ))

    return doc

Preprocess an event by anonymizing user information .


def hash_id(iso_timestamp, msg):
    """Generate event id, optimized for ES."""
    return '{0}-{1}'.format(iso_timestamp,
                            hashlib.sha1(
                                msg.get('unique_id').encode('utf-8') +
                                str(msg.get('visitor_id')).
                                encode('utf-8')).
                            hexdigest())

def hash_id ( iso_timestamp , msg ) : return '{0}-{1}' . format ( iso_timestamp , hashlib . sha1 ( msg . get ( 'unique_id' ) . encode ( 'utf-8' ) + str ( msg . get ( 'visitor_id' ) ) . encode ( 'utf-8' ) ) . hexdigest ( ) )


def run(self):
        """Process events queue."""
        return elasticsearch.helpers.bulk(
            self.client,
            self.actionsiter(),
            stats_only=True,
            chunk_size=50
        )

def run ( self ) : return elasticsearch . helpers . bulk ( self . client , self . actionsiter ( ) , stats_only = True , chunk_size = 50 )


def register_events():
    """Register sample events."""
    return [
        dict(
            event_type='file-download',
            templates='invenio_stats.contrib.file_download',
            processor_class=EventsIndexer,
            processor_config=dict(
                preprocessors=[
                    flag_robots,
                    anonymize_user,
                    build_file_unique_id
                ])),
        dict(
            event_type='record-view',
            templates='invenio_stats.contrib.record_view',
            processor_class=EventsIndexer,
            processor_config=dict(
                preprocessors=[
                    flag_robots,
                    anonymize_user,
                    build_record_unique_id
                ]))
    ]

def register_events ( ) : return [ dict ( event_type = 'file-download' , templates = 'invenio_stats.contrib.file_download' , processor_class = EventsIndexer , processor_config = dict ( preprocessors = [ flag_robots , anonymize_user , build_file_unique_id ] ) ) , dict ( event_type = 'record-view' , templates = 'invenio_stats.contrib.record_view' , processor_class = EventsIndexer , processor_config = dict ( preprocessors = [ flag_robots , anonymize_user , build_record_unique_id ] ) ) ]


def register_aggregations():
    """Register sample aggregations."""
    return [dict(
        aggregation_name='file-download-agg',
        templates='invenio_stats.contrib.aggregations.aggr_file_download',
        aggregator_class=StatAggregator,
        aggregator_config=dict(
            client=current_search_client,
            event='file-download',
            aggregation_field='unique_id',
            aggregation_interval='day',
            copy_fields=dict(
                file_key='file_key',
                bucket_id='bucket_id',
                file_id='file_id',
            ),
            metric_aggregation_fields={
                'unique_count': ('cardinality', 'unique_session_id',
                                 {'precision_threshold': 1000}),
                'volume': ('sum', 'size', {}),
            },
        )), dict(
        aggregation_name='record-view-agg',
        templates='invenio_stats.contrib.aggregations.aggr_record_view',
        aggregator_class=StatAggregator,
        aggregator_config=dict(
            client=current_search_client,
            event='record-view',
            aggregation_field='unique_id',
            aggregation_interval='day',
            copy_fields=dict(
                record_id='record_id',
                pid_type='pid_type',
                pid_value='pid_value',
            ),
            metric_aggregation_fields={
                'unique_count': ('cardinality', 'unique_session_id',
                                 {'precision_threshold': 1000}),
            },
        ))]


def register_queries():
    """Register queries."""
    return [
        dict(
            query_name='bucket-file-download-histogram',
            query_class=ESDateHistogramQuery,
            query_config=dict(
                index='stats-file-download',
                doc_type='file-download-day-aggregation',
                copy_fields=dict(
                    bucket_id='bucket_id',
                    file_key='file_key',
                ),
                required_filters=dict(
                    bucket_id='bucket_id',
                    file_key='file_key',
                )
            )
        ),
        dict(
            query_name='bucket-file-download-total',
            query_class=ESTermsQuery,
            query_config=dict(
                index='stats-file-download',
                doc_type='file-download-day-aggregation',
                copy_fields=dict(
                    # bucket_id='bucket_id',
                ),
                required_filters=dict(
                    bucket_id='bucket_id',
                ),
                aggregated_fields=['file_key']
            )
        ),
    ]

def
register_queries
(
)
:
return
[
dict
(
query_name
=
'bucket-file-download-histogram'
,
query_class
=
ESDateHistogramQuery
,
query_config
=
dict
(
index
=
'stats-file-download'
,
doc_type
=
'file-download-day-aggregation'
,
copy_fields
=
dict
(
bucket_id
=
'bucket_id'
,
file_key
=
'file_key'
,
)
,
required_filters
=
dict
(
bucket_id
=
'bucket_id'
,
file_key
=
'file_key'
,
)
)
)
,
dict
(
query_name
=
'bucket-file-download-total'
,
query_class
=
ESTermsQuery
,
query_config
=
dict
(
index
=
'stats-file-download'
,
doc_type
=
'file-download-day-aggregation'
,
copy_fields
=
dict
(
# bucket_id='bucket_id',
)
,
required_filters
=
dict
(
bucket_id
=
'bucket_id'
,
)
,
aggregated_fields
=
[
'file_key'
]
)
)
,
]


def _parse_time(self, tokens):
        """
        Parse the date range for the query

        E.g. WHERE time > now() - 48h AND time < now() - 24h
        would result in DateRange(datetime_start, datetime_end)
        where
        datetime_start would be parsed from now() - 48h
        and
        datetime_end would be parsed from now() - 24h

        :param tokens:
        :return:
        """
        return self.time_parser.parse(self.parse_keyword(Keyword.WHERE, tokens))

def _parse_time ( self , tokens ) : return self . time_parser . parse ( self . parse_keyword ( Keyword . WHERE , tokens ) )


def extract_date(self, date):
        """Extract date from string if necessary.

        :returns: the extracted date.
        """
        if isinstance(date, six.string_types):
            try:
                date = dateutil.parser.parse(date)
            except ValueError:
                raise ValueError(
                    'Invalid date format for statistic {}.'
                ).format(self.query_name)
        if not isinstance(date, datetime):
            raise TypeError(
                'Invalid date type for statistic {}.'
            ).format(self.query_name)
        return date

def
extract_date
(
self
,
date
)
:
if
isinstance
(
date
,
six
.
string_types
)
:
try
:
date
=
dateutil
.
parser
.
parse
(
date
)
except
ValueError
:
raise
ValueError
(
'Invalid date format for statistic {}.'
)
.
format
(
self
.
query_name
)
if
not
isinstance
(
date
,
datetime
)
:
raise
TypeError
(
'Invalid date type for statistic {}.'
)
.
format
(
self
.
query_name
)
return
date


def run(self, start_date=None, end_date=None, **kwargs):
        """Run the query."""
        start_date = self.extract_date(start_date) if start_date else None
        end_date = self.extract_date(end_date) if end_date else None
        self.validate_arguments(start_date, end_date, **kwargs)

        agg_query = self.build_query(start_date, end_date, **kwargs)
        query_result = agg_query.execute().to_dict()
        res = self.process_query_result(query_result, start_date, end_date)
        return res

def run ( self , start_date = None , end_date = None , * * kwargs ) : start_date = self . extract_date ( start_date ) if start_date else None end_date = self . extract_date ( end_date ) if end_date else None self . validate_arguments ( start_date , end_date , * * kwargs ) agg_query = self . build_query ( start_date , end_date , * * kwargs ) query_result = agg_query . execute ( ) . to_dict ( ) res = self . process_query_result ( query_result , start_date , end_date ) return res


def file_download_event_builder(event, sender_app, obj=None, **kwargs):
    """Build a file-download event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        bucket_id=str(obj.bucket_id),
        file_id=str(obj.file_id),
        file_key=obj.key,
        size=obj.file.size,
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event

def
file_download_event_builder
(
event
,
sender_app
,
obj
=
,
*
*
kwargs
)
:
event
.
update
(
dict
(
# When:
timestamp
=
datetime
.
datetime
.
utcnow
(
)
.
isoformat
(
)
,
# What:
bucket_id
=
str
(
obj
.
bucket_id
)
,
file_id
=
str
(
obj
.
file_id
)
,
file_key
=
obj
.
key
,
size
=
obj
.
file
.
size
,
referrer
=
request
.
referrer
,
# Who:
*
*
get_user
(
)
)
)
return
event


def record_view_event_builder(event, sender_app, pid=None, record=None,
                              **kwargs):
    """Build a record-view event."""
    event.update(dict(
        # When:
        timestamp=datetime.datetime.utcnow().isoformat(),
        # What:
        record_id=str(record.id),
        pid_type=pid.pid_type,
        pid_value=str(pid.pid_value),
        referrer=request.referrer,
        # Who:
        **get_user()
    ))
    return event

def
record_view_event_builder
(
event
,
sender_app
,
pid
=
,
record
=
,
*
*
kwargs
)
:
event
.
update
(
dict
(
# When:
timestamp
=
datetime
.
datetime
.
utcnow
(
)
.
isoformat
(
)
,
# What:
record_id
=
str
(
record
.
id
)
,
pid_type
=
pid
.
pid_type
,
pid_value
=
str
(
pid
.
pid_value
)
,
referrer
=
request
.
referrer
,
# Who:
*
*
get_user
(
)
)
)
return
event


def check_write_permissions(file):
    """
    Check if we can write to the given file

    Otherwise since we might detach the process to run in the background
    we might never find out that writing failed and get an ugly
    exit message on startup. For example:
    ERROR: Child exited immediately with non-zero exit code 127

    So we catch this error upfront and print a nicer error message
    with a hint on how to fix it.
    """
    try:
        open(file, 'a')
    except IOError:
        print("Can't open file {}. "
              "Please grant write permissions or change the path in your config".format(file))
        sys.exit(1)


def _is_root():
    """Checks if the user is rooted."""
    import os
    import ctypes
    try:
        return os.geteuid() == 0
    except AttributeError:
        return ctypes.windll.shell32.IsUserAnAdmin() != 0
    return False

def _is_root ( ) : import os import ctypes try : return os . geteuid ( ) == 0 except AttributeError : return ctypes . windll . shell32 . IsUserAnAdmin ( ) != 0 return False


def cmdclass(path, enable=None, user=None):
    """Build nbextension cmdclass dict for the setuptools.setup method.

    Parameters
    ----------
    path: str
        Directory relative to the setup file that the nbextension code
        lives in.
    enable: [str=None]
        Extension to "enable".  Enabling an extension causes it to be loaded
        automatically by the IPython notebook.
    user: [bool=None]
        Whether or not the nbextension should be installed in user mode.
        If this is undefined, the script will install as user mode IF the
        installer is not sudo.

    Usage
    -----
    For automatic loading:
    # Assuming `./extension` is the relative path to the JS files and
    # `./extension/main.js` is the file that you want automatically loaded.
    setup(
        name='extension',
        ...
        cmdclass=cmdclass('extension', 'extension/main'),
    )

    For manual loading:
    # Assuming `./extension` is the relative path to the JS files.
    setup(
        name='extension',
        ...
        cmdclass=cmdclass('extension'),
    )
    """
    import warnings
    from setuptools.command.install import install
    from setuptools.command.develop import develop
    from os.path import dirname, join, exists, realpath
    from traceback import extract_stack

    try:
        # IPython/Jupyter 4.0
        from notebook.nbextensions import install_nbextension
        from notebook.services.config import ConfigManager
    except ImportError:
        # Pre-schism
        try:
            from IPython.html.nbextensions import install_nbextension
            from IPython.html.services.config import ConfigManager
        except ImportError:
            warnings.warn("No jupyter notebook found in your environment. "
                          "Hence jupyter nbextensions were not installed. "
                          "If you would like to have them,"
                          "please issue 'pip install jupyter'.")
            return {}

    # Check if the user flag was set.
    if user is None:
        user = not _is_root()

    # Get the path of the extension
    calling_file = extract_stack()[-2][0]
    fullpath = realpath(calling_file)
    if not exists(fullpath):
        raise Exception('Could not find path of setup file.')
    extension_dir = join(dirname(fullpath), path)

    # Installs the nbextension
    def run_nbextension_install(develop):
        import sys
        sysprefix = hasattr(sys, 'real_prefix')
        if sysprefix:
            install_nbextension(
                extension_dir, symlink=develop, sys_prefix=sysprefix)
        else:
            install_nbextension(extension_dir, symlink=develop, user=user)
        if enable is not None:
            print("Enabling the extension ...")
            cm = ConfigManager()
            cm.update('notebook', {"load_extensions": {enable: True}})

    # Command used for standard installs
    class InstallCommand(install):
        def run(self):
            print("Installing Python module...")
            install.run(self)
            print("Installing nbextension ...")
            run_nbextension_install(False)

    # Command used for development installs (symlinks the JS)
    class DevelopCommand(develop):
        def run(self):
            print("Installing Python module...")
            develop.run(self)
            print("Installing nbextension ...")
            run_nbextension_install(True)

    return {
        'install': InstallCommand,
        'develop': DevelopCommand,
    }

Build nbextension cmdclass dict for the setuptools . setup method .


def count_subgraph_sizes(graph: BELGraph, annotation: str = 'Subgraph') -> Counter[int]:
    """Count the number of nodes in each subgraph induced by an annotation.

    :param annotation: The annotation to group by and compare. Defaults to 'Subgraph'
    :return: A dictionary from {annotation value: number of nodes}
    """
    return count_dict_values(group_nodes_by_annotation(graph, annotation))

def count_subgraph_sizes ( graph : BELGraph , annotation : str = 'Subgraph' ) -> Counter [ int ] : return count_dict_values ( group_nodes_by_annotation ( graph , annotation ) )

Count the number of nodes in each subgraph induced by an annotation .


def calculate_subgraph_edge_overlap(
        graph: BELGraph,
        annotation: str = 'Subgraph'
) -> Tuple[
    Mapping[str, EdgeSet],
    Mapping[str, Mapping[str, EdgeSet]],
    Mapping[str, Mapping[str, EdgeSet]],
    Mapping[str, Mapping[str, float]],
]:
    """Build a DatafFame to show the overlap between different sub-graphs.

    Options:
    1. Total number of edges overlap (intersection)
    2. Percentage overlap (tanimoto similarity)

    :param graph: A BEL graph
    :param annotation: The annotation to group by and compare. Defaults to 'Subgraph'
    :return: {subgraph: set of edges}, {(subgraph 1, subgraph2): set of intersecting edges},
            {(subgraph 1, subgraph2): set of unioned edges}, {(subgraph 1, subgraph2): tanimoto similarity},
    """
    sg2edge = defaultdict(set)

    for u, v, d in graph.edges(data=True):
        if not edge_has_annotation(d, annotation):
            continue
        sg2edge[d[ANNOTATIONS][annotation]].add((u, v))

    subgraph_intersection = defaultdict(dict)
    subgraph_union = defaultdict(dict)
    result = defaultdict(dict)

    for sg1, sg2 in itt.product(sg2edge, repeat=2):
        subgraph_intersection[sg1][sg2] = sg2edge[sg1] & sg2edge[sg2]
        subgraph_union[sg1][sg2] = sg2edge[sg1] | sg2edge[sg2]
        result[sg1][sg2] = len(subgraph_intersection[sg1][sg2]) / len(subgraph_union[sg1][sg2])

    return sg2edge, subgraph_intersection, subgraph_union, result

def calculate_subgraph_edge_overlap ( graph : BELGraph , annotation : str = 'Subgraph' ) -> Tuple [ Mapping [ str , EdgeSet ] , Mapping [ str , Mapping [ str , EdgeSet ] ] , Mapping [ str , Mapping [ str , EdgeSet ] ] , Mapping [ str , Mapping [ str , float ] ] , ] : sg2edge = defaultdict ( set ) for u , v , d in graph . edges ( data = True ) : if not edge_has_annotation ( d , annotation ) : continue sg2edge [ d [ ANNOTATIONS ] [ annotation ] ] . add ( ( u , v ) ) subgraph_intersection = defaultdict ( dict ) subgraph_union = defaultdict ( dict ) result = defaultdict ( dict ) for sg1 , sg2 in itt . product ( sg2edge , repeat = 2 ) : subgraph_intersection [ sg1 ] [ sg2 ] = sg2edge [ sg1 ] & sg2edge [ sg2 ] subgraph_union [ sg1 ] [ sg2 ] = sg2edge [ sg1 ] | sg2edge [ sg2 ] result [ sg1 ] [ sg2 ] = len ( subgraph_intersection [ sg1 ] [ sg2 ] ) / len ( subgraph_union [ sg1 ] [ sg2 ] ) return sg2edge , subgraph_intersection , subgraph_union , result

Build a DatafFame to show the overlap between different sub - graphs .


def summarize_subgraph_node_overlap(graph: BELGraph, node_predicates=None, annotation: str = 'Subgraph'):
    """Calculate the subgraph similarity tanimoto similarity in nodes passing the given filter.

    Provides an alternate view on subgraph similarity, from a more node-centric view
    """
    r1 = group_nodes_by_annotation_filtered(graph, node_predicates=node_predicates, annotation=annotation)
    return calculate_tanimoto_set_distances(r1)

def summarize_subgraph_node_overlap ( graph : BELGraph , node_predicates = None , annotation : str = 'Subgraph' ) : r1 = group_nodes_by_annotation_filtered ( graph , node_predicates = node_predicates , annotation = annotation ) return calculate_tanimoto_set_distances ( r1 )

Calculate the subgraph similarity tanimoto similarity in nodes passing the given filter .


def rank_subgraph_by_node_filter(graph: BELGraph,
                                 node_predicates: Union[NodePredicate, Iterable[NodePredicate]],
                                 annotation: str = 'Subgraph',
                                 reverse: bool = True,
                                 ) -> List[Tuple[str, int]]:
    """Rank sub-graphs by which have the most nodes matching an given filter.

    A use case for this function would be to identify which subgraphs contain the most differentially expressed
    genes.

    >>> from pybel import from_pickle
    >>> from pybel.constants import GENE
    >>> from pybel_tools.integration import overlay_type_data
    >>> from pybel_tools.summary import rank_subgraph_by_node_filter
    >>> import pandas as pd
    >>> graph = from_pickle('~/dev/bms/aetionomy/alzheimers.gpickle')
    >>> df = pd.read_csv('~/dev/bananas/data/alzheimers_dgxp.csv', columns=['Gene', 'log2fc'])
    >>> data = {gene: log2fc for _, gene, log2fc in df.itertuples()}
    >>> overlay_type_data(graph, data, 'log2fc', GENE, 'HGNC', impute=0.0)
    >>> results = rank_subgraph_by_node_filter(graph, lambda g, n: 1.3 < abs(g[n]['log2fc']))
    """
    r1 = group_nodes_by_annotation_filtered(graph, node_predicates=node_predicates, annotation=annotation)
    r2 = count_dict_values(r1)
    # TODO use instead: r2.most_common()
    return sorted(r2.items(), key=itemgetter(1), reverse=reverse)

def
rank_subgraph_by_node_filter
(
graph
:
BELGraph
,
node_predicates
:
Union
[
NodePredicate
,
Iterable
[
NodePredicate
]
]
,
annotation
:
str
=
'Subgraph'
,
reverse
:
bool
=
True
,
)
->
List
[
Tuple
[
str
,
int
]
]
:
r1
=
group_nodes_by_annotation_filtered
(
graph
,
node_predicates
=
node_predicates
,
annotation
=
annotation
)
r2
=
count_dict_values
(
r1
)
# TODO use instead: r2.most_common()
return
sorted
(
r2
.
items
(
)
,
key
=
itemgetter
(
1
)
,
reverse
=
reverse
)

Rank sub - graphs by which have the most nodes matching an given filter .


def to_jupyter(graph: BELGraph, chart: Optional[str] = None) -> Javascript:
    """Render the graph as JavaScript in a Jupyter Notebook."""
    with open(os.path.join(HERE, 'render_with_javascript.js'), 'rt') as f:
        js_template = Template(f.read())

    return Javascript(js_template.render(**_get_context(graph, chart=chart)))

def to_jupyter ( graph : BELGraph , chart : Optional [ str ] = None ) -> Javascript : with open ( os . path . join ( HERE , 'render_with_javascript.js' ) , 'rt' ) as f : js_template = Template ( f . read ( ) ) return Javascript ( js_template . render ( * * _get_context ( graph , chart = chart ) ) )

Render the graph as JavaScript in a Jupyter Notebook .


def prerender(graph: BELGraph) -> Mapping[str, Mapping[str, Any]]:
    """Generate the annotations JSON for Ideogram."""
    import bio2bel_hgnc
    from bio2bel_hgnc.models import HumanGene

    graph: BELGraph = graph.copy()
    enrich_protein_and_rna_origins(graph)
    collapse_all_variants(graph)
    genes: Set[Gene] = get_nodes_by_function(graph, GENE)
    hgnc_symbols = {
        gene.name
        for gene in genes
        if gene.namespace.lower() == 'hgnc'
    }

    result = {}

    hgnc_manager = bio2bel_hgnc.Manager()
    human_genes = (
        hgnc_manager.session
            .query(HumanGene.symbol, HumanGene.location)
            .filter(HumanGene.symbol.in_(hgnc_symbols))
            .all()
    )
    for human_gene in human_genes:
        result[human_gene.symbol] = {
            'name': human_gene.symbol,
            'chr': (
                human_gene.location.split('q')[0]
                if 'q' in human_gene.location else
                human_gene.location.split('p')[0]
            ),
        }

    df = get_df()

    for _, (gene_id, symbol, start, stop) in df[df['Symbol'].isin(hgnc_symbols)].iterrows():
        result[symbol]['start'] = start
        result[symbol]['stop'] = stop

    return result

def prerender ( graph : BELGraph ) -> Mapping [ str , Mapping [ str , Any ] ] : import bio2bel_hgnc from bio2bel_hgnc . models import HumanGene graph : BELGraph = graph . copy ( ) enrich_protein_and_rna_origins ( graph ) collapse_all_variants ( graph ) genes : Set [ Gene ] = get_nodes_by_function ( graph , GENE ) hgnc_symbols = { gene . name for gene in genes if gene . namespace . lower ( ) == 'hgnc' } result = { } hgnc_manager = bio2bel_hgnc . Manager ( ) human_genes = ( hgnc_manager . session . query ( HumanGene . symbol , HumanGene . location ) . filter ( HumanGene . symbol . in_ ( hgnc_symbols ) ) . all ( ) ) for human_gene in human_genes : result [ human_gene . symbol ] = { 'name' : human_gene . symbol , 'chr' : ( human_gene . location . split ( 'q' ) [ 0 ] if 'q' in human_gene . location else human_gene . location . split ( 'p' ) [ 0 ] ) , } df = get_df ( ) for _ , ( gene_id , symbol , start , stop ) in df [ df [ 'Symbol' ] . isin ( hgnc_symbols ) ] . iterrows ( ) : result [ symbol ] [ 'start' ] = start result [ symbol ] [ 'stop' ] = stop return result

Generate the annotations JSON for Ideogram .


def plot_summary_axes(graph: BELGraph, lax, rax, logx=True):
    """Plots your graph summary statistics on the given axes.

    After, you should run :func:`plt.tight_layout` and you must run :func:`plt.show` to view.

    Shows:
    1. Count of nodes, grouped by function type
    2. Count of edges, grouped by relation type

    :param pybel.BELGraph graph: A BEL graph
    :param lax: An axis object from matplotlib
    :param rax: An axis object from matplotlib

    Example usage:

    >>> import matplotlib.pyplot as plt
    >>> from pybel import from_pickle
    >>> from pybel_tools.summary import plot_summary_axes
    >>> graph = from_pickle('~/dev/bms/aetionomy/parkinsons.gpickle')
    >>> fig, axes = plt.subplots(1, 2, figsize=(10, 4))
    >>> plot_summary_axes(graph, axes[0], axes[1])
    >>> plt.tight_layout()
    >>> plt.show()
    """
    ntc = count_functions(graph)
    etc = count_relations(graph)

    df = pd.DataFrame.from_dict(dict(ntc), orient='index')
    df_ec = pd.DataFrame.from_dict(dict(etc), orient='index')

    df.sort_values(0, ascending=True).plot(kind='barh', logx=logx, ax=lax)
    lax.set_title('Number of nodes: {}'.format(graph.number_of_nodes()))

    df_ec.sort_values(0, ascending=True).plot(kind='barh', logx=logx, ax=rax)
    rax.set_title('Number of edges: {}'.format(graph.number_of_edges()))

def
plot_summary_axes
(
graph
:
BELGraph
,
lax
,
rax
,
logx
=
True
)
:
ntc
=
count_functions
(
graph
)
etc
=
count_relations
(
graph
)
df
=
pd
.
DataFrame
.
from_dict
(
dict
(
ntc
)
,
orient
=
'index'
)
df_ec
=
pd
.
DataFrame
.
from_dict
(
dict
(
etc
)
,
orient
=
'index'
)
df
.
sort_values
(
0
,
ascending
=
True
)
.
plot
(
kind
=
'barh'
,
logx
=
logx
,
ax
=
lax
)
lax
.
set_title
(
'Number of nodes: {}'
.
format
(
graph
.
number_of_nodes
(
)
)
)
df_ec
.
sort_values
(
0
,
ascending
=
True
)
.
plot
(
kind
=
'barh'
,
logx
=
logx
,
ax
=
rax
)
rax
.
set_title
(
'Number of edges: {}'
.
format
(
graph
.
number_of_edges
(
)
)
)

Plots your graph summary statistics on the given axes .


def remove_nodes_by_function_namespace(graph: BELGraph, func: str, namespace: Strings) -> None:
    """Remove nodes with the given function and namespace.

    This might be useful to exclude information learned about distant species, such as excluding all information
    from MGI and RGD in diseases where mice and rats don't give much insight to the human disease mechanism.
    """
    remove_filtered_nodes(graph, function_namespace_inclusion_builder(func, namespace))

def remove_nodes_by_function_namespace ( graph : BELGraph , func : str , namespace : Strings ) -> None : remove_filtered_nodes ( graph , function_namespace_inclusion_builder ( func , namespace ) )

Remove nodes with the given function and namespace .


def preprocessing_excel(path):
    """Preprocess the excel sheet

    :param filepath: filepath of the excel data
    :return: df: pandas dataframe with excel data
    :rtype: pandas.DataFrame
    """
    if not os.path.exists(path):
        raise ValueError("Error: %s file not found" % path)

    # Import Models from Excel sheet, independent for AD and PD
    df = pd.read_excel(path, sheetname=0, header=0)

    # Indexes and column name
    # [log.info(str(x)+': '+str((df.columns.values[x]))) for x in range (0,len(df.columns.values))]

    # Starting from 4: Pathway Name

    # Fill Pathway cells that are merged and are 'NaN' after deleting rows where there is no genes
    df.iloc[:, 0] = pd.Series(df.iloc[:, 0]).fillna(method='ffill')

    # Number of gaps
    # log.info(df.ix[:,6].isnull().sum())

    df = df[df.ix[:, 1].notnull()]
    df = df.reset_index(drop=True)

    # Fill NaN to ceros in PubmedID column
    df.ix[:, 2].fillna(0, inplace=True)

    # Number of gaps in the gene column should be already zero
    if (df.ix[:, 1].isnull().sum()) != 0:
        raise ValueError("Error: Empty cells in the gene column")

    # Check current state
    # df.to_csv('out.csv')

    return df


def preprocessing_br_projection_excel(path: str) -> pd.DataFrame:
    """Preprocess the excel file.

    Parameters
    ----------
    path : Filepath of the excel sheet
    """
    if not os.path.exists(path):
        raise ValueError("Error: %s file not found" % path)

    return pd.read_excel(path, sheetname=0, header=0)

def
preprocessing_br_projection_excel
(
path
:
str
)
->
pd
.
DataFrame
:
if
not
os
.
path
.
exists
(
path
)
:
raise
ValueError
(
"Error: %s file not found"
%
path
)
return
pd
.
read_excel
(
path
,
sheetname
=
0
,
header
=
0
)


def get_nift_values() -> Mapping[str, str]:
    """Extract the list of NIFT names from the BEL resource and builds a dictionary mapping from the lowercased version
    to the uppercase version.
    """
    r = get_bel_resource(NIFT)
    return {
        name.lower(): name
        for name in r['Values']
    }

def get_nift_values ( ) -> Mapping [ str , str ] : r = get_bel_resource ( NIFT ) return { name . lower ( ) : name for name in r [ 'Values' ] }

Extract the list of NIFT names from the BEL resource and builds a dictionary mapping from the lowercased version to the uppercase version .


def get_regulatory_pairs(graph: BELGraph) -> Set[NodePair]:
    """Find pairs of nodes that have mutual causal edges that are regulating each other such that ``A -> B`` and
    ``B -| A``.

    :return: A set of pairs of nodes with mutual causal edges
    """
    cg = get_causal_subgraph(graph)

    results = set()

    for u, v, d in cg.edges(data=True):
        if d[RELATION] not in CAUSAL_INCREASE_RELATIONS:
            continue

        if cg.has_edge(v, u) and any(dd[RELATION] in CAUSAL_DECREASE_RELATIONS for dd in cg[v][u].values()):
            results.add((u, v))

    return results

def get_regulatory_pairs ( graph : BELGraph ) -> Set [ NodePair ] : cg = get_causal_subgraph ( graph ) results = set ( ) for u , v , d in cg . edges ( data = True ) : if d [ RELATION ] not in CAUSAL_INCREASE_RELATIONS : continue if cg . has_edge ( v , u ) and any ( dd [ RELATION ] in CAUSAL_DECREASE_RELATIONS for dd in cg [ v ] [ u ] . values ( ) ) : results . add ( ( u , v ) ) return results

Find pairs of nodes that have mutual causal edges that are regulating each other such that A - > B and B - | A .


def get_chaotic_pairs(graph: BELGraph) -> SetOfNodePairs:
    """Find pairs of nodes that have mutual causal edges that are increasing each other such that ``A -> B`` and
    ``B -> A``.

    :return: A set of pairs of nodes with mutual causal edges
    """
    cg = get_causal_subgraph(graph)

    results = set()

    for u, v, d in cg.edges(data=True):
        if d[RELATION] not in CAUSAL_INCREASE_RELATIONS:
            continue

        if cg.has_edge(v, u) and any(dd[RELATION] in CAUSAL_INCREASE_RELATIONS for dd in cg[v][u].values()):
            results.add(tuple(sorted([u, v], key=str)))

    return results

def get_chaotic_pairs ( graph : BELGraph ) -> SetOfNodePairs : cg = get_causal_subgraph ( graph ) results = set ( ) for u , v , d in cg . edges ( data = True ) : if d [ RELATION ] not in CAUSAL_INCREASE_RELATIONS : continue if cg . has_edge ( v , u ) and any ( dd [ RELATION ] in CAUSAL_INCREASE_RELATIONS for dd in cg [ v ] [ u ] . values ( ) ) : results . add ( tuple ( sorted ( [ u , v ] , key = str ) ) ) return results

Find pairs of nodes that have mutual causal edges that are increasing each other such that A - > B and B - > A .


def get_correlation_graph(graph: BELGraph) -> Graph:
    """Extract an undirected graph of only correlative relationships."""
    result = Graph()

    for u, v, d in graph.edges(data=True):
        if d[RELATION] not in CORRELATIVE_RELATIONS:
            continue

        if not result.has_edge(u, v):
            result.add_edge(u, v, **{d[RELATION]: True})

        elif d[RELATION] not in result[u][v]:
            log.log(5, 'broken correlation relation for %s, %s', u, v)
            result[u][v][d[RELATION]] = True
            result[v][u][d[RELATION]] = True

    return result

Extract an undirected graph of only correlative relationships .


def get_correlation_triangles(graph: BELGraph) -> SetOfNodeTriples:
    """Return a set of all triangles pointed by the given node."""
    return {
        tuple(sorted([n, u, v], key=str))
        for n in graph
        for u, v in itt.combinations(graph[n], 2)
        if graph.has_edge(u, v)
    }

def get_correlation_triangles ( graph : BELGraph ) -> SetOfNodeTriples : return { tuple ( sorted ( [ n , u , v ] , key = str ) ) for n in graph for u , v in itt . combinations ( graph [ n ] , 2 ) if graph . has_edge ( u , v ) }

Return a set of all triangles pointed by the given node .


def get_triangles(graph: DiGraph) -> SetOfNodeTriples:
    """Get a set of triples representing the 3-cycles from a directional graph.

    Each 3-cycle is returned once, with nodes in sorted order.
    """
    return {
        tuple(sorted([a, b, c], key=str))
        for a, b in graph.edges()
        for c in graph.successors(b)
        if graph.has_edge(c, a)
    }

def get_triangles ( graph : DiGraph ) -> SetOfNodeTriples : return { tuple ( sorted ( [ a , b , c ] , key = str ) ) for a , b in graph . edges ( ) for c in graph . successors ( b ) if graph . has_edge ( c , a ) }

Get a set of triples representing the 3 - cycles from a directional graph .


def get_separate_unstable_correlation_triples(graph: BELGraph) -> Iterable[NodeTriple]:
    """Yield all triples of nodes A, B, C such that ``A pos B``, ``A pos C``, and ``B neg C``.

    :return: An iterator over triples of unstable graphs, where the second two are negative
    """
    cg = get_correlation_graph(graph)

    for a, b, c in get_correlation_triangles(cg):
        if POSITIVE_CORRELATION in cg[a][b] and POSITIVE_CORRELATION in cg[b][c] and NEGATIVE_CORRELATION in \
                cg[a][c]:
            yield b, a, c
        if POSITIVE_CORRELATION in cg[a][b] and NEGATIVE_CORRELATION in cg[b][c] and POSITIVE_CORRELATION in \
                cg[a][c]:
            yield a, b, c
        if NEGATIVE_CORRELATION in cg[a][b] and POSITIVE_CORRELATION in cg[b][c] and POSITIVE_CORRELATION in \
                cg[a][c]:
            yield c, a, b

Yield all triples of nodes A B C such that A pos B A pos C and B neg C .


def summarize_stability(graph: BELGraph) -> Mapping[str, int]:
    """Summarize the stability of the graph."""
    regulatory_pairs = get_regulatory_pairs(graph)
    chaotic_pairs = get_chaotic_pairs(graph)
    dampened_pairs = get_dampened_pairs(graph)
    contraditory_pairs = get_contradiction_summary(graph)
    separately_unstable_triples = get_separate_unstable_correlation_triples(graph)
    mutually_unstable_triples = get_mutually_unstable_correlation_triples(graph)
    jens_unstable_triples = get_jens_unstable(graph)
    increase_mismatch_triples = get_increase_mismatch_triplets(graph)
    decrease_mismatch_triples = get_decrease_mismatch_triplets(graph)
    chaotic_triples = get_chaotic_triplets(graph)
    dampened_triples = get_dampened_triplets(graph)

    return {
        'Regulatory Pairs': _count_or_len(regulatory_pairs),
        'Chaotic Pairs': _count_or_len(chaotic_pairs),
        'Dampened Pairs': _count_or_len(dampened_pairs),
        'Contradictory Pairs': _count_or_len(contraditory_pairs),
        'Separately Unstable Triples': _count_or_len(separately_unstable_triples),
        'Mutually Unstable Triples': _count_or_len(mutually_unstable_triples),
        'Jens Unstable Triples': _count_or_len(jens_unstable_triples),
        'Increase Mismatch Triples': _count_or_len(increase_mismatch_triples),
        'Decrease Mismatch Triples': _count_or_len(decrease_mismatch_triples),
        'Chaotic Triples': _count_or_len(chaotic_triples),
        'Dampened Triples': _count_or_len(dampened_triples)
    }


def flatten_list_abundance(node: ListAbundance) -> ListAbundance:
    """Flattens the complex or composite abundance."""
    return node.__class__(list(chain.from_iterable(
        (
            flatten_list_abundance(member).members
            if isinstance(member, ListAbundance) else
            [member]
        )
        for member in node.members
    )))

def flatten_list_abundance ( node : ListAbundance ) -> ListAbundance : return node . __class__ ( list ( chain . from_iterable ( ( flatten_list_abundance ( member ) . members if isinstance ( member , ListAbundance ) else [ member ] ) for member in node . members ) ) )

Flattens the complex or composite abundance .


def list_abundance_expansion(graph: BELGraph) -> None:
    """Flatten list abundances."""
    mapping = {
        node: flatten_list_abundance(node)
        for node in graph
        if isinstance(node, ListAbundance)
    }
    relabel_nodes(graph, mapping, copy=False)

def list_abundance_expansion ( graph : BELGraph ) -> None : mapping = { node : flatten_list_abundance ( node ) for node in graph if isinstance ( node , ListAbundance ) } relabel_nodes ( graph , mapping , copy = False )


def list_abundance_cartesian_expansion(graph: BELGraph) -> None:
    """Expand all list abundances to simple subject-predicate-object networks."""
    for u, v, k, d in list(graph.edges(keys=True, data=True)):
        if CITATION not in d:
            continue

        if isinstance(u, ListAbundance) and isinstance(v, ListAbundance):
            for u_member, v_member in itt.product(u.members, v.members):
                graph.add_qualified_edge(
                    u_member, v_member,
                    relation=d[RELATION],
                    citation=d.get(CITATION),
                    evidence=d.get(EVIDENCE),
                    annotations=d.get(ANNOTATIONS),
                )

        elif isinstance(u, ListAbundance):
            for member in u.members:
                graph.add_qualified_edge(
                    member, v,
                    relation=d[RELATION],
                    citation=d.get(CITATION),
                    evidence=d.get(EVIDENCE),
                    annotations=d.get(ANNOTATIONS),
                )

        elif isinstance(v, ListAbundance):
            for member in v.members:
                graph.add_qualified_edge(
                    u, member,
                    relation=d[RELATION],
                    citation=d.get(CITATION),
                    evidence=d.get(EVIDENCE),
                    annotations=d.get(ANNOTATIONS),
                )

    _remove_list_abundance_nodes(graph)

def list_abundance_cartesian_expansion ( graph : BELGraph ) -> None : for u , v , k , d in list ( graph . edges ( keys = True , data = True ) ) : if CITATION not in d : continue if isinstance ( u , ListAbundance ) and isinstance ( v , ListAbundance ) : for u_member , v_member in itt . product ( u . members , v . members ) : graph . add_qualified_edge ( u_member , v_member , relation = d [ RELATION ] , citation = d . get ( CITATION ) , evidence = d . get ( EVIDENCE ) , annotations = d . get ( ANNOTATIONS ) , ) elif isinstance ( u , ListAbundance ) : for member in u . members : graph . add_qualified_edge ( member , v , relation = d [ RELATION ] , citation = d . get ( CITATION ) , evidence = d . get ( EVIDENCE ) , annotations = d . get ( ANNOTATIONS ) , ) elif isinstance ( v , ListAbundance ) : for member in v . members : graph . add_qualified_edge ( u , member , relation = d [ RELATION ] , citation = d . get ( CITATION ) , evidence = d . get ( EVIDENCE ) , annotations = d . get ( ANNOTATIONS ) , ) _remove_list_abundance_nodes ( graph )

Expand all list abundances to simple subject - predicate - object networks .


def _reaction_cartesion_expansion_unqualified_helper(
        graph: BELGraph,
        u: BaseEntity,
        v: BaseEntity,
        d: dict,
) -> None:
    """Helper to deal with cartension expansion in unqualified edges."""
    if isinstance(u, Reaction) and isinstance(v, Reaction):
        enzymes = _get_catalysts_in_reaction(u) | _get_catalysts_in_reaction(v)

        for reactant, product in chain(itt.product(u.reactants, u.products),
                                       itt.product(v.reactants, v.products)):
            if reactant in enzymes or product in enzymes:
                continue

            graph.add_unqualified_edge(
                reactant, product, INCREASES
            )
        for product, reactant in itt.product(u.products, u.reactants):

            if reactant in enzymes or product in enzymes:
                continue

            graph.add_unqualified_edge(
                product, reactant, d[RELATION],
            )

    elif isinstance(u, Reaction):

        enzymes = _get_catalysts_in_reaction(u)

        for product in u.products:

            # Skip create increases edges between enzymes
            if product in enzymes:
                continue

            # Only add edge between v and reaction if the node is not part of the reaction
            # In practice skips hasReactant, hasProduct edges
            if v not in u.products and v not in u.reactants:
                graph.add_unqualified_edge(
                    product, v, INCREASES
                )
            for reactant in u.reactants:
                graph.add_unqualified_edge(
                    reactant, product, INCREASES
                )

    elif isinstance(v, Reaction):

        enzymes = _get_catalysts_in_reaction(v)

        for reactant in v.reactants:

            # Skip create increases edges between enzymes
            if reactant in enzymes:
                continue

            # Only add edge between v and reaction if the node is not part of the reaction
            # In practice skips hasReactant, hasProduct edges
            if u not in v.products and u not in v.reactants:
                graph.add_unqualified_edge(
                    u, reactant, INCREASES
                )
            for product in v.products:
                graph.add_unqualified_edge(
                    reactant, product, INCREASES
                )

Helper to deal with cartension expansion in unqualified edges .


def _get_catalysts_in_reaction(reaction: Reaction) -> Set[BaseAbundance]:
    """Return nodes that are both in reactants and reactions in a reaction."""
    return {
        reactant
        for reactant in reaction.reactants
        if reactant in reaction.products
    }

def _get_catalysts_in_reaction ( reaction : Reaction ) -> Set [ BaseAbundance ] : return { reactant for reactant in reaction . reactants if reactant in reaction . products }

Return nodes that are both in reactants and reactions in a reaction .


def reaction_cartesian_expansion(graph: BELGraph, accept_unqualified_edges: bool = True) -> None:
    """Expand all reactions to simple subject-predicate-object networks."""
    for u, v, d in list(graph.edges(data=True)):
        # Deal with unqualified edges
        if CITATION not in d and accept_unqualified_edges:
            _reaction_cartesion_expansion_unqualified_helper(graph, u, v, d)
            continue

        if isinstance(u, Reaction) and isinstance(v, Reaction):
            catalysts = _get_catalysts_in_reaction(u) | _get_catalysts_in_reaction(v)

            for reactant, product in chain(itt.product(u.reactants, u.products), itt.product(v.reactants, v.products)):
                if reactant in catalysts or product in catalysts:
                    continue
                graph.add_increases(
                    reactant, product,
                    citation=d.get(CITATION),
                    evidence=d.get(EVIDENCE),
                    annotations=d.get(ANNOTATIONS),
                )

            for product, reactant in itt.product(u.products, u.reactants):
                if reactant in catalysts or product in catalysts:
                    continue

                graph.add_qualified_edge(
                    product, reactant,
                    relation=d[RELATION],
                    citation=d.get(CITATION),
                    evidence=d.get(EVIDENCE),
                    annotations=d.get(ANNOTATIONS),
                )

        elif isinstance(u, Reaction):
            catalysts = _get_catalysts_in_reaction(u)

            for product in u.products:
                # Skip create increases edges between enzymes
                if product in catalysts:
                    continue

                # Only add edge between v and reaction if the node is not part of the reaction
                # In practice skips hasReactant, hasProduct edges
                if v not in u.products and v not in u.reactants:
                    graph.add_increases(
                        product, v,
                        citation=d.get(CITATION),
                        evidence=d.get(EVIDENCE),
                        annotations=d.get(ANNOTATIONS),
                    )

                for reactant in u.reactants:
                    graph.add_increases(
                        reactant, product,
                        citation=d.get(CITATION),
                        evidence=d.get(EVIDENCE),
                        annotations=d.get(ANNOTATIONS),
                    )

        elif isinstance(v, Reaction):
            for reactant in v.reactants:
                catalysts = _get_catalysts_in_reaction(v)

                # Skip create increases edges between enzymes
                if reactant in catalysts:
                    continue

                # Only add edge between v and reaction if the node is not part of the reaction
                # In practice skips hasReactant, hasProduct edges
                if u not in v.products and u not in v.reactants:
                    graph.add_increases(
                        u, reactant,
                        citation=d.get(CITATION),
                        evidence=d.get(EVIDENCE),
                        annotations=d.get(ANNOTATIONS),
                    )
                for product in v.products:
                    graph.add_increases(
                        reactant, product,
                        citation=d.get(CITATION),
                        evidence=d.get(EVIDENCE),
                        annotations=d.get(ANNOTATIONS),
                    )

    _remove_reaction_nodes(graph)

Expand all reactions to simple subject - predicate - object networks .


def get_graphs_by_ids(self, network_ids: Iterable[int]) -> List[BELGraph]:
        """Get several graphs by their identifiers."""
        return [
            self.networks[network_id]
            for network_id in network_ids
        ]

def get_graphs_by_ids ( self , network_ids : Iterable [ int ] ) -> List [ BELGraph ] : return [ self . networks [ network_id ] for network_id in network_ids ]

Get several graphs by their identifiers .


def _generate_citation_dict(graph: BELGraph) -> Mapping[str, Mapping[Tuple[BaseEntity, BaseEntity], str]]:
    """Prepare a citation data dictionary from a graph.

    :return: A dictionary of dictionaries {citation type: {(source, target): citation reference}
    """
    results = defaultdict(lambda: defaultdict(set))

    for u, v, data in graph.edges(data=True):
        if CITATION not in data:
            continue
        results[data[CITATION][CITATION_TYPE]][u, v].add(data[CITATION][CITATION_REFERENCE].strip())

    return dict(results)

def _generate_citation_dict ( graph : BELGraph ) -> Mapping [ str , Mapping [ Tuple [ BaseEntity , BaseEntity ] , str ] ] : results = defaultdict ( lambda : defaultdict ( set ) ) for u , v , data in graph . edges ( data = True ) : if CITATION not in data : continue results [ data [ CITATION ] [ CITATION_TYPE ] ] [ u , v ] . add ( data [ CITATION ] [ CITATION_REFERENCE ] . strip ( ) ) return dict ( results )

Prepare a citation data dictionary from a graph .


def count_citations(graph: BELGraph, **annotations) -> Counter:
    """Counts the citations in a graph based on a given filter

    :param graph: A BEL graph
    :param dict annotations: The annotation filters to use
    :return: A counter from {(citation type, citation reference): frequency}
    """
    citations = defaultdict(set)

    annotation_dict_filter = build_edge_data_filter(annotations)

    for u, v, _, d in filter_edges(graph, annotation_dict_filter):
        if CITATION not in d:
            continue

        citations[u, v].add((d[CITATION][CITATION_TYPE], d[CITATION][CITATION_REFERENCE].strip()))

    return Counter(itt.chain.from_iterable(citations.values()))

def count_citations ( graph : BELGraph , * * annotations ) -> Counter : citations = defaultdict ( set ) annotation_dict_filter = build_edge_data_filter ( annotations ) for u , v , _ , d in filter_edges ( graph , annotation_dict_filter ) : if CITATION not in d : continue citations [ u , v ] . add ( ( d [ CITATION ] [ CITATION_TYPE ] , d [ CITATION ] [ CITATION_REFERENCE ] . strip ( ) ) ) return Counter ( itt . chain . from_iterable ( citations . values ( ) ) )


def count_citations_by_annotation(graph: BELGraph, annotation: str) -> Mapping[str, typing.Counter[str]]:
    """Group the citation counters by subgraphs induced by the annotation.

    :param graph: A BEL graph
    :param annotation: The annotation to use to group the graph
    :return: A dictionary of Counters {subgraph name: Counter from {citation: frequency}}
    """
    citations = defaultdict(lambda: defaultdict(set))
    for u, v, data in graph.edges(data=True):
        if not edge_has_annotation(data, annotation) or CITATION not in data:
            continue

        k = data[ANNOTATIONS][annotation]

        citations[k][u, v].add((data[CITATION][CITATION_TYPE], data[CITATION][CITATION_REFERENCE].strip()))

    return {k: Counter(itt.chain.from_iterable(v.values())) for k, v in citations.items()}

def count_citations_by_annotation ( graph : BELGraph , annotation : str ) -> Mapping [ str , typing . Counter [ str ] ] : citations = defaultdict ( lambda : defaultdict ( set ) ) for u , v , data in graph . edges ( data = True ) : if not edge_has_annotation ( data , annotation ) or CITATION not in data : continue k = data [ ANNOTATIONS ] [ annotation ] citations [ k ] [ u , v ] . add ( ( data [ CITATION ] [ CITATION_TYPE ] , data [ CITATION ] [ CITATION_REFERENCE ] . strip ( ) ) ) return { k : Counter ( itt . chain . from_iterable ( v . values ( ) ) ) for k , v in citations . items ( ) }

Group the citation counters by subgraphs induced by the annotation .


def count_author_publications(graph: BELGraph) -> typing.Counter[str]:
    """Count the number of publications of each author to the given graph."""
    authors = group_as_dict(_iter_author_publiations(graph))
    return Counter(count_dict_values(count_defaultdict(authors)))

def count_author_publications ( graph : BELGraph ) -> typing . Counter [ str ] : authors = group_as_dict ( _iter_author_publiations ( graph ) ) return Counter ( count_dict_values ( count_defaultdict ( authors ) ) )

Count the number of publications of each author to the given graph .


def count_authors_by_annotation(graph: BELGraph, annotation: str = 'Subgraph') -> Mapping[str, typing.Counter[str]]:
    """Group the author counters by sub-graphs induced by the annotation.

    :param graph: A BEL graph
    :param annotation: The annotation to use to group the graph
    :return: A dictionary of Counters {subgraph name: Counter from {author: frequency}}
    """
    authors = group_as_dict(_iter_authors_by_annotation(graph, annotation=annotation))
    return count_defaultdict(authors)

def count_authors_by_annotation ( graph : BELGraph , annotation : str = 'Subgraph' ) -> Mapping [ str , typing . Counter [ str ] ] : authors = group_as_dict ( _iter_authors_by_annotation ( graph , annotation = annotation ) ) return count_defaultdict ( authors )

Group the author counters by sub - graphs induced by the annotation .


def get_evidences_by_pmid(graph: BELGraph, pmids: Union[str, Iterable[str]]):
    """Get a dictionary from the given PubMed identifiers to the sets of all evidence strings associated with each
    in the graph.

    :param graph: A BEL graph
    :param pmids: An iterable of PubMed identifiers, as strings. Is consumed and converted to a set.
    :return: A dictionary of {pmid: set of all evidence strings}
    :rtype: dict
    """
    result = defaultdict(set)

    for _, _, _, data in filter_edges(graph, build_pmid_inclusion_filter(pmids)):
        result[data[CITATION][CITATION_REFERENCE]].add(data[EVIDENCE])

    return dict(result)

def get_evidences_by_pmid ( graph : BELGraph , pmids : Union [ str , Iterable [ str ] ] ) : result = defaultdict ( set ) for _ , _ , _ , data in filter_edges ( graph , build_pmid_inclusion_filter ( pmids ) ) : result [ data [ CITATION ] [ CITATION_REFERENCE ] ] . add ( data [ EVIDENCE ] ) return dict ( result )

Get a dictionary from the given PubMed identifiers to the sets of all evidence strings associated with each in the graph .


def count_citation_years(graph: BELGraph) -> typing.Counter[int]:
    """Count the number of citations from each year."""
    result = defaultdict(set)

    for _, _, data in graph.edges(data=True):
        if CITATION not in data or CITATION_DATE not in data[CITATION]:
            continue

        try:
            dt = _ensure_datetime(data[CITATION][CITATION_DATE])
            result[dt.year].add((data[CITATION][CITATION_TYPE], data[CITATION][CITATION_REFERENCE]))
        except Exception:
            continue

    return count_dict_values(result)

def count_citation_years ( graph : BELGraph ) -> typing . Counter [ int ] : result = defaultdict ( set ) for _ , _ , data in graph . edges ( data = True ) : if CITATION not in data or CITATION_DATE not in data [ CITATION ] : continue try : dt = _ensure_datetime ( data [ CITATION ] [ CITATION_DATE ] ) result [ dt . year ] . add ( ( data [ CITATION ] [ CITATION_TYPE ] , data [ CITATION ] [ CITATION_REFERENCE ] ) ) except Exception : continue return count_dict_values ( result )

Count the number of citations from each year .


def get_citation_years(graph: BELGraph) -> List[Tuple[int, int]]:
    """Create a citation timeline counter from the graph."""
    return create_timeline(count_citation_years(graph))

def get_citation_years ( graph : BELGraph ) -> List [ Tuple [ int , int ] ] : return create_timeline ( count_citation_years ( graph ) )

Create a citation timeline counter from the graph .


def create_timeline(year_counter: typing.Counter[int]) -> List[Tuple[int, int]]:
    """Complete the Counter timeline.

    :param Counter year_counter: counter dict for each year
    :return: complete timeline
    """
    if not year_counter:
        return []

    from_year = min(year_counter) - 1
    until_year = datetime.now().year + 1

    return [
        (year, year_counter.get(year, 0))
        for year in range(from_year, until_year)
    ]

def create_timeline ( year_counter : typing . Counter [ int ] ) -> List [ Tuple [ int , int ] ] : if not year_counter : return [ ] from_year = min ( year_counter ) - 1 until_year = datetime . now ( ) . year + 1 return [ ( year , year_counter . get ( year , 0 ) ) for year in range ( from_year , until_year ) ]


def count_confidences(graph: BELGraph) -> typing.Counter[str]:
    """Count the confidences in the graph."""
    return Counter(
        (
            'None'
            if ANNOTATIONS not in data or 'Confidence' not in data[ANNOTATIONS] else
            list(data[ANNOTATIONS]['Confidence'])[0]
        )
        for _, _, data in graph.edges(data=True)
        if CITATION in data  # don't bother with unqualified statements
    )


def enrich_pubmed_citations(graph: BELGraph, manager: Manager) -> Set[str]:
    """Overwrite all PubMed citations with values from NCBI's eUtils lookup service.

    :return: A set of PMIDs for which the eUtils service crashed
    """
    pmids = get_pubmed_identifiers(graph)
    pmid_data, errors = get_citations_by_pmids(manager=manager, pmids=pmids)

    for u, v, k in filter_edges(graph, has_pubmed):
        pmid = graph[u][v][k][CITATION][CITATION_REFERENCE].strip()

        if pmid not in pmid_data:
            log.warning('Missing data for PubMed identifier: %s', pmid)
            errors.add(pmid)
            continue

        graph[u][v][k][CITATION].update(pmid_data[pmid])

    return errors

Overwrite all PubMed citations with values from NCBI s eUtils lookup service .


def update_context(universe: BELGraph, graph: BELGraph):
    """Update the context of a subgraph from the universe of all knowledge."""
    for namespace in get_namespaces(graph):
        if namespace in universe.namespace_url:
            graph.namespace_url[namespace] = universe.namespace_url[namespace]
        elif namespace in universe.namespace_pattern:
            graph.namespace_pattern[namespace] = universe.namespace_pattern[namespace]
        else:
            log.warning('namespace: %s missing from universe', namespace)

    for annotation in get_annotations(graph):
        if annotation in universe.annotation_url:
            graph.annotation_url[annotation] = universe.annotation_url[annotation]
        elif annotation in universe.annotation_pattern:
            graph.annotation_pattern[annotation] = universe.annotation_pattern[annotation]
        elif annotation in universe.annotation_list:
            graph.annotation_list[annotation] = universe.annotation_list[annotation]
        else:
            log.warning('annotation: %s missing from universe', annotation)

def
update_context
(
universe
:
BELGraph
,
graph
:
BELGraph
)
:
for
namespace
in
get_namespaces
(
graph
)
:
if
namespace
in
universe
.
namespace_url
:
graph
.
namespace_url
[
namespace
]
=
universe
.
namespace_url
[
namespace
]
elif
namespace
in
universe
.
namespace_pattern
:
graph
.
namespace_pattern
[
namespace
]
=
universe
.
namespace_pattern
[
namespace
]
else
:
log
.
warning
(
'namespace: %s missing from universe'
,
namespace
)
for
annotation
in
get_annotations
(
graph
)
:
if
annotation
in
universe
.
annotation_url
:
graph
.
annotation_url
[
annotation
]
=
universe
.
annotation_url
[
annotation
]
elif
annotation
in
universe
.
annotation_pattern
:
graph
.
annotation_pattern
[
annotation
]
=
universe
.
annotation_pattern
[
annotation
]
elif
annotation
in
universe
.
annotation_list
:
graph
.
annotation_list
[
annotation
]
=
universe
.
annotation_list
[
annotation
]
else
:
log
.
warning
(
'annotation: %s missing from universe'
,
annotation
)

Update the context of a subgraph from the universe of all knowledge .


def highlight_nodes(graph: BELGraph, nodes: Optional[Iterable[BaseEntity]] = None, color: Optional[str]=None):
    """Adds a highlight tag to the given nodes.

    :param graph: A BEL graph
    :param nodes: The nodes to add a highlight tag on
    :param color: The color to highlight (use something that works with CSS)
    """
    color = color or NODE_HIGHLIGHT_DEFAULT_COLOR
    for node in nodes if nodes is not None else graph:
        graph.node[node][NODE_HIGHLIGHT] = color

def highlight_nodes ( graph : BELGraph , nodes : Optional [ Iterable [ BaseEntity ] ] = None , color : Optional [ str ] = None ) : color = color or NODE_HIGHLIGHT_DEFAULT_COLOR for node in nodes if nodes is not None else graph : graph . node [ node ] [ NODE_HIGHLIGHT ] = color

Adds a highlight tag to the given nodes .


def is_node_highlighted(graph: BELGraph, node: BaseEntity) -> bool:
    """Returns if the given node is highlighted.

    :param graph: A BEL graph
    :param node: A BEL node
    :type node: tuple
    :return: Does the node contain highlight information?
    :rtype: bool
    """
    return NODE_HIGHLIGHT in graph.node[node]

def is_node_highlighted ( graph : BELGraph , node : BaseEntity ) -> bool : return NODE_HIGHLIGHT in graph . node [ node ]

Returns if the given node is highlighted .


def remove_highlight_nodes(graph: BELGraph, nodes: Optional[Iterable[BaseEntity]]=None) -> None:
    """Removes the highlight from the given nodes, or all nodes if none given.

    :param graph: A BEL graph
    :param nodes: The list of nodes to un-highlight
    """
    for node in graph if nodes is None else nodes:
        if is_node_highlighted(graph, node):
            del graph.node[node][NODE_HIGHLIGHT]

def remove_highlight_nodes ( graph : BELGraph , nodes : Optional [ Iterable [ BaseEntity ] ] = None ) -> None : for node in graph if nodes is None else nodes : if is_node_highlighted ( graph , node ) : del graph . node [ node ] [ NODE_HIGHLIGHT ]

Removes the highlight from the given nodes or all nodes if none given .


def highlight_edges(graph: BELGraph, edges=None, color: Optional[str]=None) -> None:
    """Adds a highlight tag to the given edges.

    :param graph: A BEL graph
    :param edges: The edges (4-tuples of u, v, k, d) to add a highlight tag on
    :type edges: iter[tuple]
    :param str color: The color to highlight (use something that works with CSS)
    """
    color = color or EDGE_HIGHLIGHT_DEFAULT_COLOR
    for u, v, k, d in edges if edges is not None else graph.edges(keys=True, data=True):
        graph[u][v][k][EDGE_HIGHLIGHT] = color

def highlight_edges ( graph : BELGraph , edges = None , color : Optional [ str ] = None ) -> None : color = color or EDGE_HIGHLIGHT_DEFAULT_COLOR for u , v , k , d in edges if edges is not None else graph . edges ( keys = True , data = True ) : graph [ u ] [ v ] [ k ] [ EDGE_HIGHLIGHT ] = color

Adds a highlight tag to the given edges .


def remove_highlight_edges(graph: BELGraph, edges=None):
    """Remove the highlight from the given edges, or all edges if none given.

    :param graph: A BEL graph
    :param edges: The edges (4-tuple of u,v,k,d) to remove the highlight from)
    :type edges: iter[tuple]
    """
    for u, v, k, _ in graph.edges(keys=True, data=True) if edges is None else edges:
        if is_edge_highlighted(graph, u, v, k):
            del graph[u][v][k][EDGE_HIGHLIGHT]

def remove_highlight_edges ( graph : BELGraph , edges = None ) : for u , v , k , _ in graph . edges ( keys = True , data = True ) if edges is None else edges : if is_edge_highlighted ( graph , u , v , k ) : del graph [ u ] [ v ] [ k ] [ EDGE_HIGHLIGHT ]

Remove the highlight from the given edges or all edges if none given .


def get_causal_out_edges(
        graph: BELGraph,
        nbunch: Union[BaseEntity, Iterable[BaseEntity]],
) -> Set[Tuple[BaseEntity, BaseEntity]]:
    """Get the out-edges to the given node that are causal.

    :return: A set of (source, target) pairs where the source is the given node
    """
    return {
        (u, v)
        for u, v, k, d in graph.out_edges(nbunch, keys=True, data=True)
        if is_causal_relation(graph, u, v, k, d)
    }

def get_causal_out_edges ( graph : BELGraph , nbunch : Union [ BaseEntity , Iterable [ BaseEntity ] ] , ) -> Set [ Tuple [ BaseEntity , BaseEntity ] ] : return { ( u , v ) for u , v , k , d in graph . out_edges ( nbunch , keys = True , data = True ) if is_causal_relation ( graph , u , v , k , d ) }

Get the out - edges to the given node that are causal .


def get_causal_source_nodes(graph: BELGraph, func: str) -> Set[BaseEntity]:
    """Return a set of all nodes that have an in-degree of 0.

    This likely means that it is an external perturbagen and is not known to have any causal origin from within the
    biological system. These nodes are useful to identify because they generally don't provide any mechanistic insight.
    """
    return {
        node
        for node in graph
        if node.function == func and is_causal_source(graph, node)
    }

def get_causal_source_nodes ( graph : BELGraph , func : str ) -> Set [ BaseEntity ] : return { node for node in graph if node . function == func and is_causal_source ( graph , node ) }

Return a set of all nodes that have an in - degree of 0 .


def get_causal_central_nodes(graph: BELGraph, func: str) -> Set[BaseEntity]:
    """Return a set of all nodes that have both an in-degree > 0 and out-degree > 0.

    This means that they are an integral part of a pathway, since they are both produced and consumed.
    """
    return {
        node
        for node in graph
        if node.function == func and is_causal_central(graph, node)
    }

def get_causal_central_nodes ( graph : BELGraph , func : str ) -> Set [ BaseEntity ] : return { node for node in graph if node . function == func and is_causal_central ( graph , node ) }

Return a set of all nodes that have both an in - degree > 0 and out - degree > 0 .


def get_causal_sink_nodes(graph: BELGraph, func) -> Set[BaseEntity]:
    """Returns a set of all ABUNDANCE nodes that have an causal out-degree of 0.

    This likely means that the knowledge assembly is incomplete, or there is a curation error.
    """
    return {
        node
        for node in graph
        if node.function == func and is_causal_sink(graph, node)
    }

def get_causal_sink_nodes ( graph : BELGraph , func ) -> Set [ BaseEntity ] : return { node for node in graph if node . function == func and is_causal_sink ( graph , node ) }

Returns a set of all ABUNDANCE nodes that have an causal out - degree of 0 .


def count_top_centrality(graph: BELGraph, number: Optional[int] = 30) -> Mapping[BaseEntity, int]:
    """Get top centrality dictionary."""
    dd = nx.betweenness_centrality(graph)
    dc = Counter(dd)
    return dict(dc.most_common(number))

def count_top_centrality ( graph : BELGraph , number : Optional [ int ] = 30 ) -> Mapping [ BaseEntity , int ] : dd = nx . betweenness_centrality ( graph ) dc = Counter ( dd ) return dict ( dc . most_common ( number ) )


def get_modifications_count(graph: BELGraph) -> Mapping[str, int]:
    """Get a modifications count dictionary."""
    return remove_falsy_values({
        'Translocations': len(get_translocated(graph)),
        'Degradations': len(get_degradations(graph)),
        'Molecular Activities': len(get_activities(graph)),
    })

def
get_modifications_count
(
graph
:
BELGraph
)
->
Mapping
[
str
,
int
]
:
return
remove_falsy_values
(
{
'Translocations'
:
len
(
get_translocated
(
graph
)
)
,
'Degradations'
:
len
(
get_degradations
(
graph
)
)
,
'Molecular Activities'
:
len
(
get_activities
(
graph
)
)
,
}
)


def remove_falsy_values(counter: Mapping[Any, int]) -> Mapping[Any, int]:
    """Remove all values that are zero."""
    return {
        label: count
        for label, count in counter.items()
        if count
    }

def remove_falsy_values ( counter : Mapping [ Any , int ] ) -> Mapping [ Any , int ] : return { label : count for label , count in counter . items ( ) if count }


def _collapse_variants_by_function(graph: BELGraph, func: str) -> None:
    """Collapse all of the given functions' variants' edges to their parents, in-place."""
    for parent_node, variant_node, data in graph.edges(data=True):
        if data[RELATION] == HAS_VARIANT and parent_node.function == func:
            collapse_pair(graph, from_node=variant_node, to_node=parent_node)

def _collapse_variants_by_function ( graph : BELGraph , func : str ) -> None : for parent_node , variant_node , data in graph . edges ( data = True ) : if data [ RELATION ] == HAS_VARIANT and parent_node . function == func : collapse_pair ( graph , from_node = variant_node , to_node = parent_node )

Collapse all of the given functions variants edges to their parents in - place .


def _collapse_edge_passing_predicates(graph: BELGraph, edge_predicates: EdgePredicates = None) -> None:
    """Collapse all edges passing the given edge predicates."""
    for u, v, _ in filter_edges(graph, edge_predicates=edge_predicates):
        collapse_pair(graph, survivor=u, victim=v)

def _collapse_edge_passing_predicates ( graph : BELGraph , edge_predicates : EdgePredicates = None ) -> None : for u , v , _ in filter_edges ( graph , edge_predicates = edge_predicates ) : collapse_pair ( graph , survivor = u , victim = v )

Collapse all edges passing the given edge predicates .


def _collapse_edge_by_namespace(graph: BELGraph,
                                victim_namespaces: Strings,
                                survivor_namespaces: str,
                                relations: Strings) -> None:
    """Collapse pairs of nodes with the given namespaces that have the given relationship.

    :param graph: A BEL Graph
    :param victim_namespaces: The namespace(s) of the node to collapse
    :param survivor_namespaces: The namespace of the node to keep
    :param relations: The relation(s) to search
    """
    relation_filter = build_relation_predicate(relations)
    source_namespace_filter = build_source_namespace_filter(victim_namespaces)
    target_namespace_filter = build_target_namespace_filter(survivor_namespaces)

    edge_predicates = [
        relation_filter,
        source_namespace_filter,
        target_namespace_filter
    ]

    _collapse_edge_passing_predicates(graph, edge_predicates=edge_predicates)

def _collapse_edge_by_namespace ( graph : BELGraph , victim_namespaces : Strings , survivor_namespaces : str , relations : Strings ) -> None : relation_filter = build_relation_predicate ( relations ) source_namespace_filter = build_source_namespace_filter ( victim_namespaces ) target_namespace_filter = build_target_namespace_filter ( survivor_namespaces ) edge_predicates = [ relation_filter , source_namespace_filter , target_namespace_filter ] _collapse_edge_passing_predicates ( graph , edge_predicates = edge_predicates )

Collapse pairs of nodes with the given namespaces that have the given relationship .


def collapse_orthologies_by_namespace(graph: BELGraph, victim_namespace: Strings, survivor_namespace: str) -> None:
    """Collapse pairs of nodes with the given namespaces that have orthology relationships.

    :param graph: A BEL Graph
    :param victim_namespace: The namespace(s) of the node to collapse
    :param survivor_namespace: The namespace of the node to keep

    To collapse all MGI nodes to their HGNC orthologs, use:
    >>> collapse_orthologies_by_namespace('MGI', 'HGNC')

    To collapse collapse both MGI and RGD nodes to their HGNC orthologs, use:
    >>> collapse_orthologies_by_namespace(['MGI', 'RGD'], 'HGNC')
    """
    _collapse_edge_by_namespace(graph, victim_namespace, survivor_namespace, ORTHOLOGOUS)

def collapse_orthologies_by_namespace ( graph : BELGraph , victim_namespace : Strings , survivor_namespace : str ) -> None : _collapse_edge_by_namespace ( graph , victim_namespace , survivor_namespace , ORTHOLOGOUS )

Collapse pairs of nodes with the given namespaces that have orthology relationships .


def collapse_entrez_equivalencies(graph: BELGraph):
    """Collapse all equivalence edges away from Entrez. Assumes well formed, 2-way equivalencies."""
    relation_filter = build_relation_predicate(EQUIVALENT_TO)
    source_namespace_filter = build_source_namespace_filter(['EGID', 'EG', 'ENTREZ'])

    edge_predicates = [
        relation_filter,
        source_namespace_filter,
    ]

    _collapse_edge_passing_predicates(graph, edge_predicates=edge_predicates)

def collapse_entrez_equivalencies ( graph : BELGraph ) : relation_filter = build_relation_predicate ( EQUIVALENT_TO ) source_namespace_filter = build_source_namespace_filter ( [ 'EGID' , 'EG' , 'ENTREZ' ] ) edge_predicates = [ relation_filter , source_namespace_filter , ] _collapse_edge_passing_predicates ( graph , edge_predicates = edge_predicates )

Collapse all equivalence edges away from Entrez . Assumes well formed 2 - way equivalencies .


def collapse_consistent_edges(graph: BELGraph):
    """Collapse consistent edges together.

    .. warning:: This operation doesn't preserve evidences or other annotations
    """
    for u, v in graph.edges():
        relation = pair_is_consistent(graph, u, v)

        if not relation:
            continue

        edges = [(u, v, k) for k in graph[u][v]]
        graph.remove_edges_from(edges)
        graph.add_edge(u, v, attr_dict={RELATION: relation})

def collapse_consistent_edges ( graph : BELGraph ) : for u , v in graph . edges ( ) : relation = pair_is_consistent ( graph , u , v ) if not relation : continue edges = [ ( u , v , k ) for k in graph [ u ] [ v ] ] graph . remove_edges_from ( edges ) graph . add_edge ( u , v , attr_dict = { RELATION : relation } )


def collapse_nodes_with_same_names(graph: BELGraph) -> None:
    """Collapse all nodes with the same name, merging namespaces by picking first alphabetical one."""
    survivor_mapping = defaultdict(set) # Collapse mapping dict
    victims = set() # Things already mapped while iterating

    it = tqdm(itt.combinations(graph, r=2), total=graph.number_of_nodes() * (graph.number_of_nodes() - 1) / 2)
    for a, b in it:
        if b in victims:
            continue

        a_name, b_name = a.get(NAME), b.get(NAME)
        if not a_name or not b_name or a_name.lower() != b_name.lower():
            continue

        if a.keys() != b.keys():  # not same version (might have variants)
            continue

        # Ensure that the values in the keys are also the same
        for k in set(a.keys()) - {NAME, NAMESPACE}:
            if a[k] != b[k]:  # something different
                continue

        survivor_mapping[a].add(b)
        # Keep track of things that has been already mapped
        victims.add(b)

    collapse_nodes(graph, survivor_mapping)

Collapse all nodes with the same name merging namespaces by picking first alphabetical one .


def main(output):
    """Output the HBP knowledge graph to the desktop"""
    from hbp_knowledge import get_graph
    graph = get_graph()
    text = to_html(graph)
    print(text, file=output)

def main ( output ) : from hbp_knowledge import get_graph graph = get_graph ( ) text = to_html ( graph ) print ( text , file = output )


def node_is_upstream_leaf(graph: BELGraph, node: BaseEntity) -> bool:
    """Return if the node is an upstream leaf.

    An upstream leaf is defined as a node that has no in-edges, and exactly 1 out-edge.
    """
    return 0 == len(graph.predecessors(node)) and 1 == len(graph.successors(node))

def node_is_upstream_leaf ( graph : BELGraph , node : BaseEntity ) -> bool : return 0 == len ( graph . predecessors ( node ) ) and 1 == len ( graph . successors ( node ) )

Return if the node is an upstream leaf .


def get_unweighted_upstream_leaves(graph: BELGraph, key: Optional[str] = None) -> Iterable[BaseEntity]:
    """Get nodes with no incoming edges, one outgoing edge, and without the given key in its data dictionary.

    .. seealso :: :func:`data_does_not_contain_key_builder`

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    :return: An iterable over leaves (nodes with an in-degree of 0) that don't have the given annotation
    """
    if key is None:
        key = WEIGHT

    return filter_nodes(graph, [node_is_upstream_leaf, data_missing_key_builder(key)])

def get_unweighted_upstream_leaves ( graph : BELGraph , key : Optional [ str ] = None ) -> Iterable [ BaseEntity ] : if key is None : key = WEIGHT return filter_nodes ( graph , [ node_is_upstream_leaf , data_missing_key_builder ( key ) ] )

Get nodes with no incoming edges one outgoing edge and without the given key in its data dictionary .


def is_unweighted_source(graph: BELGraph, node: BaseEntity, key: str) -> bool:
    """Check if the node is both a source and also has an annotation.

    :param graph: A BEL graph
    :param node: A BEL node
    :param key: The key in the node data dictionary representing the experimental data
    """
    return graph.in_degree(node) == 0 and key not in graph.nodes[node]

def is_unweighted_source ( graph : BELGraph , node : BaseEntity , key : str ) -> bool : return graph . in_degree ( node ) == 0 and key not in graph . nodes [ node ]

Check if the node is both a source and also has an annotation .


def get_unweighted_sources(graph: BELGraph, key: Optional[str] = None) -> Iterable[BaseEntity]:
    """Get nodes on the periphery of the sub-graph that do not have a annotation for the given key.

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data
    :return: An iterator over BEL nodes that are unannotated and on the periphery of this subgraph
    """
    if key is None:
        key = WEIGHT

    for node in graph:
        if is_unweighted_source(graph, node, key):
            yield node

def get_unweighted_sources ( graph : BELGraph , key : Optional [ str ] = None ) -> Iterable [ BaseEntity ] : if key is None : key = WEIGHT for node in graph : if is_unweighted_source ( graph , node , key ) : yield node

Get nodes on the periphery of the sub - graph that do not have a annotation for the given key .


def remove_unweighted_sources(graph: BELGraph, key: Optional[str] = None) -> None:
    """Prune unannotated nodes on the periphery of the sub-graph.

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.
    """
    nodes = list(get_unweighted_sources(graph, key=key))
    graph.remove_nodes_from(nodes)

def remove_unweighted_sources ( graph : BELGraph , key : Optional [ str ] = None ) -> None : nodes = list ( get_unweighted_sources ( graph , key = key ) ) graph . remove_nodes_from ( nodes )

Prune unannotated nodes on the periphery of the sub - graph .


def prune_mechanism_by_data(graph, key: Optional[str] = None) -> None:
    """Remove all leaves and source nodes that don't have weights.

    Is a thin wrapper around  :func:`remove_unweighted_leaves` and :func:`remove_unweighted_sources`

    :param graph: A BEL graph
    :param key: The key in the node data dictionary representing the experimental data. Defaults to
     :data:`pybel_tools.constants.WEIGHT`.

    Equivalent to:

    >>> remove_unweighted_leaves(graph)
    >>> remove_unweighted_sources(graph)
    """
    remove_unweighted_leaves(graph, key=key)
    remove_unweighted_sources(graph, key=key)

def prune_mechanism_by_data ( graph , key : Optional [ str ] = None ) -> None : remove_unweighted_leaves ( graph , key = key ) remove_unweighted_sources ( graph , key = key )

Remove all leaves and source nodes that don t have weights .


def generate_mechanism(graph: BELGraph, node: BaseEntity, key: Optional[str] = None) -> BELGraph:
    """Generate a mechanistic sub-graph upstream of the given node.

    :param graph: A BEL graph
    :param node: A BEL node
    :param key: The key in the node data dictionary representing the experimental data.
    :return: A sub-graph grown around the target BEL node
    """
    subgraph = get_upstream_causal_subgraph(graph, node)
    expand_upstream_causal(graph, subgraph)
    remove_inconsistent_edges(subgraph)
    collapse_consistent_edges(subgraph)

    if key is not None:  # FIXME when is it not pruned?
        prune_mechanism_by_data(subgraph, key)

    return subgraph

def
generate_mechanism
(
graph
:
BELGraph
,
node
:
BaseEntity
,
key
:
Optional
[
str
]
=
)
->
BELGraph
:
subgraph
=
get_upstream_causal_subgraph
(
graph
,
node
)
expand_upstream_causal
(
graph
,
subgraph
)
remove_inconsistent_edges
(
subgraph
)
collapse_consistent_edges
(
subgraph
)
if
key
is
not
:
# FIXME when is it not pruned?
prune_mechanism_by_data
(
subgraph
,
key
)
return
subgraph

Generate a mechanistic sub - graph upstream of the given node .


def get_neurommsig_scores(graph: BELGraph,
                          genes: List[Gene],
                          annotation: str = 'Subgraph',
                          ora_weight: Optional[float] = None,
                          hub_weight: Optional[float] = None,
                          top_percent: Optional[float] = None,
                          topology_weight: Optional[float] = None,
                          preprocess: bool = False
                          ) -> Optional[Mapping[str, float]]:
    """Preprocess the graph, stratify by the given annotation, then run the NeuroMMSig algorithm on each.

    :param graph: A BEL graph
    :param genes: A list of gene nodes
    :param annotation: The annotation to use to stratify the graph to subgraphs
    :param ora_weight: The relative weight of the over-enrichment analysis score from
     :py:func:`neurommsig_gene_ora`. Defaults to 1.0.
    :param hub_weight: The relative weight of the hub analysis score from :py:func:`neurommsig_hubs`.
     Defaults to 1.0.
    :param top_percent: The percentage of top genes to use as hubs. Defaults to 5% (0.05).
    :param topology_weight: The relative weight of the topolgical analysis core from
     :py:func:`neurommsig_topology`. Defaults to 1.0.
    :param preprocess: If true, preprocess the graph.
    :return: A dictionary from {annotation value: NeuroMMSig composite score}

    Pre-processing steps:

    1. Infer the central dogma with :func:``
    2. Collapse all proteins, RNAs and miRNAs to genes with :func:``
    3. Collapse variants to genes with :func:``
    """
    if preprocess:
        graph = neurommsig_graph_preprocessor.run(graph)

    if not any(gene in graph for gene in genes):
        logger.debug('no genes mapping to graph')
        return

    subgraphs = get_subgraphs_by_annotation(graph, annotation=annotation)

    return get_neurommsig_scores_prestratified(
        subgraphs=subgraphs,
        genes=genes,
        ora_weight=ora_weight,
        hub_weight=hub_weight,
        top_percent=top_percent,
        topology_weight=topology_weight,
    )

def
get_neurommsig_scores
(
graph
:
BELGraph
,
genes
:
List
[
Gene
]
,
annotation
:
str
=
'Subgraph'
,
ora_weight
:
Optional
[
float
]
=
,
hub_weight
:
Optional
[
float
]
=
,
top_percent
:
Optional
[
float
]
=
,
topology_weight
:
Optional
[
float
]
=
,
preprocess
:
bool
=
False
)
->
Optional
[
Mapping
[
str
,
float
]
]
:
if
preprocess
:
graph
=
neurommsig_graph_preprocessor
.
run
(
graph
)
if
not
any
(
gene
in
graph
for
gene
in
genes
)
:
logger
.
debug
(
'no genes mapping to graph'
)
return
subgraphs
=
get_subgraphs_by_annotation
(
graph
,
annotation
=
annotation
)
return
get_neurommsig_scores_prestratified
(
subgraphs
=
subgraphs
,
genes
=
genes
,
ora_weight
=
ora_weight
,
hub_weight
=
hub_weight
,
top_percent
=
top_percent
,
topology_weight
=
topology_weight
,
)

Preprocess the graph stratify by the given annotation then run the NeuroMMSig algorithm on each .


def get_neurommsig_scores_prestratified(subgraphs: Mapping[str, BELGraph],
                                        genes: List[Gene],
                                        ora_weight: Optional[float] = None,
                                        hub_weight: Optional[float] = None,
                                        top_percent: Optional[float] = None,
                                        topology_weight: Optional[float] = None,
                                        ) -> Optional[Mapping[str, float]]:
    """Takes a graph stratification and runs neurommsig on each

    :param subgraphs: A pre-stratified set of graphs
    :param genes: A list of gene nodes
    :param ora_weight: The relative weight of the over-enrichment analysis score from
     :py:func:`neurommsig_gene_ora`. Defaults to 1.0.
    :param hub_weight: The relative weight of the hub analysis score from :py:func:`neurommsig_hubs`.
     Defaults to 1.0.
    :param top_percent: The percentage of top genes to use as hubs. Defaults to 5% (0.05).
    :param topology_weight: The relative weight of the topolgical analysis core from
     :py:func:`neurommsig_topology`. Defaults to 1.0.
    :return: A dictionary from {annotation value: NeuroMMSig composite score}

    Pre-processing steps:

    1. Infer the central dogma with :func:``
    2. Collapse all proteins, RNAs and miRNAs to genes with :func:``
    3. Collapse variants to genes with :func:``
    """
    return {
        name: get_neurommsig_score(
            graph=subgraph,
            genes=genes,
            ora_weight=ora_weight,
            hub_weight=hub_weight,
            top_percent=top_percent,
            topology_weight=topology_weight,
        )
        for name, subgraph in subgraphs.items()
    }

def get_neurommsig_scores_prestratified ( subgraphs : Mapping [ str , BELGraph ] , genes : List [ Gene ] , ora_weight : Optional [ float ] = None , hub_weight : Optional [ float ] = None , top_percent : Optional [ float ] = None , topology_weight : Optional [ float ] = None , ) -> Optional [ Mapping [ str , float ] ] : return { name : get_neurommsig_score ( graph = subgraph , genes = genes , ora_weight = ora_weight , hub_weight = hub_weight , top_percent = top_percent , topology_weight = topology_weight , ) for name , subgraph in subgraphs . items ( ) }