def issues(self):
        """
        Returns a list of dicts representing issues from a remote service.
        """
        for board in self.get_boards():
            for lst in self.get_lists(board['id']):
                listextra = dict(boardname=board['name'], listname=lst['name'])
                for card in self.get_cards(lst['id']):
                    issue = self.get_issue_for_record(card, extra=listextra)
                    issue.update_extra({"annotations": self.annotations(card)})
                    yield issue

def issues ( self ) : for board in self . get_boards ( ) : for lst in self . get_lists ( board [ 'id' ] ) : listextra = dict ( boardname = board [ 'name' ] , listname = lst [ 'name' ] ) for card in self . get_cards ( lst [ 'id' ] ) : issue = self . get_issue_for_record ( card , extra = listextra ) issue . update_extra ( { "annotations" : self . annotations ( card ) } ) yield issue

Returns a list of dicts representing issues from a remote service .


def annotations(self, card_json):
        """ A wrapper around get_comments that build the taskwarrior
        annotations. """
        comments = self.get_comments(card_json['id'])
        annotations = self.build_annotations(
            ((c['memberCreator']['username'], c['data']['text']) for c in comments),
            card_json["shortUrl"])
        return annotations

def annotations ( self , card_json ) : comments = self . get_comments ( card_json [ 'id' ] ) annotations = self . build_annotations ( ( ( c [ 'memberCreator' ] [ 'username' ] , c [ 'data' ] [ 'text' ] ) for c in comments ) , card_json [ "shortUrl" ] ) return annotations

A wrapper around get_comments that build the taskwarrior annotations .


def get_boards(self):
        """
        Get the list of boards to pull cards from.  If the user gave a value to
        trello.include_boards use that, otherwise ask the Trello API for the
        user's boards.
        """
        if 'include_boards' in self.config:
            for boardid in self.config.get('include_boards', to_type=aslist):
                # Get the board name
                yield self.api_request(
                    "/1/boards/{id}".format(id=boardid), fields='name')
        else:
            boards = self.api_request("/1/members/me/boards", fields='name')
            for board in boards:
                yield board

def
get_boards
(
self
)
:
if
'include_boards'
in
self
.
config
:
for
boardid
in
self
.
config
.
get
(
'include_boards'
,
to_type
=
aslist
)
:
# Get the board name
yield
self
.
api_request
(
"/1/boards/{id}"
.
format
(
id
=
boardid
)
,
fields
=
'name'
)
else
:
boards
=
self
.
api_request
(
"/1/members/me/boards"
,
fields
=
'name'
)
for
board
in
boards
:
yield
board

Get the list of boards to pull cards from . If the user gave a value to trello . include_boards use that otherwise ask the Trello API for the user s boards .


def get_lists(self, board):
        """
        Returns a list of the filtered lists for the given board
        This filters the trello lists according to the configuration values of
        trello.include_lists and trello.exclude_lists.
        """
        lists = self.api_request(
            "/1/boards/{board_id}/lists/open".format(board_id=board),
            fields='name')

        include_lists = self.config.get('include_lists', to_type=aslist)
        if include_lists:
            lists = [l for l in lists if l['name'] in include_lists]

        exclude_lists = self.config.get('exclude_lists', to_type=aslist)
        if exclude_lists:
            lists = [l for l in lists if l['name'] not in exclude_lists]

        return lists

def get_lists ( self , board ) : lists = self . api_request ( "/1/boards/{board_id}/lists/open" . format ( board_id = board ) , fields = 'name' ) include_lists = self . config . get ( 'include_lists' , to_type = aslist ) if include_lists : lists = [ l for l in lists if l [ 'name' ] in include_lists ] exclude_lists = self . config . get ( 'exclude_lists' , to_type = aslist ) if exclude_lists : lists = [ l for l in lists if l [ 'name' ] not in exclude_lists ] return lists

Returns a list of the filtered lists for the given board This filters the trello lists according to the configuration values of trello . include_lists and trello . exclude_lists .


def get_cards(self, list_id):
        """ Returns an iterator for the cards in a given list, filtered
        according to configuration values of trello.only_if_assigned and
        trello.also_unassigned """
        params = {'fields': 'name,idShort,shortLink,shortUrl,url,labels,due'}
        member = self.config.get('only_if_assigned', None)
        unassigned = self.config.get('also_unassigned', False, asbool)
        if member is not None:
            params['members'] = 'true'
            params['member_fields'] = 'username'
        cards = self.api_request(
            "/1/lists/{list_id}/cards/open".format(list_id=list_id),
            **params)
        for card in cards:
            if (member is None
                    or member in [m['username'] for m in card['members']]
                    or (unassigned and not card['members'])):
                yield card

Returns an iterator for the cards in a given list filtered according to configuration values of trello . only_if_assigned and trello . also_unassigned


def get_comments(self, card_id):
        """ Returns an iterator for the comments on a certain card. """
        params = {'filter': 'commentCard', 'memberCreator_fields': 'username'}
        comments = self.api_request(
            "/1/cards/{card_id}/actions".format(card_id=card_id),
            **params)
        for comment in comments:
            assert comment['type'] == 'commentCard'
            yield comment

def get_comments ( self , card_id ) : params = { 'filter' : 'commentCard' , 'memberCreator_fields' : 'username' } comments = self . api_request ( "/1/cards/{card_id}/actions" . format ( card_id = card_id ) , * * params ) for comment in comments : assert comment [ 'type' ] == 'commentCard' yield comment

Returns an iterator for the comments on a certain card .


def _api_url(self, path, **context):
        """ Build the full url to the API endpoint """
        if self.host == 'github.com':
            baseurl = "https://api.github.com"
        else:
            baseurl = "https://{}/api/v3".format(self.host)
        return baseurl + path.format(**context)

def _api_url ( self , path , * * context ) : if self . host == 'github.com' : baseurl = "https://api.github.com" else : baseurl = "https://{}/api/v3" . format ( self . host ) return baseurl + path . format ( * * context )


def _getter(self, url, subkey=None):
        """ Pagination utility.  Obnoxious. """

        kwargs = {}
        if 'basic' in self.auth:
            kwargs['auth'] = self.auth['basic']

        results = []
        link = dict(next=url)

        while 'next' in link:
            response = self.session.get(link['next'], **kwargs)

            # Warn about the mis-leading 404 error code.  See:
            # https://github.com/ralphbean/bugwarrior/issues/374
            if response.status_code == 404 and 'token' in self.auth:
                log.warn("A '404' from github may indicate an auth "
                         "failure. Make sure both that your token is correct "
                         "and that it has 'public_repo' and not 'public "
                         "access' rights.")

            json_res = self.json_response(response)

            if subkey is not None:
                json_res = json_res[subkey]

            results += json_res

            link = self._link_field_to_dict(response.headers.get('link', None))

        return results


def _link_field_to_dict(field):
        """ Utility for ripping apart github's Link header field.
        It's kind of ugly.
        """

        if not field:
            return dict()

        return dict([
            (
                part.split('; ')[1][5:-1],
                part.split('; ')[0][1:-1],
            ) for part in field.split(', ')
        ])

def
_link_field_to_dict
(
field
)
:
if
not
field
:
return
dict
(
)
return
dict
(
[
(
part
.
split
(
'; '
)
[
1
]
[
5
:
1
]
,
part
.
split
(
'; '
)
[
0
]
[
1
:
1
]
,
)
for
part
in
field
.
split
(
', '
)
]
)

Utility for ripping apart github s Link header field . It s kind of ugly .


def get_query(self, query):
        """ Grab all issues matching a github query """
        issues = {}
        for issue in self.client.get_query(query):
            url = issue['html_url']
            try:
                repo = self.get_repository_from_issue(issue)
            except ValueError as e:
                log.critical(e)
            else:
                issues[url] = (repo, issue)
        return issues

def get_query ( self , query ) : issues = { } for issue in self . client . get_query ( query ) : url = issue [ 'html_url' ] try : repo = self . get_repository_from_issue ( issue ) except ValueError as e : log . critical ( e ) else : issues [ url ] = ( repo , issue ) return issues


def _reqs(self, tag):
        """ Grab all the pull requests """
        return [
            (tag, i) for i in
            self.client.get_pulls(*tag.split('/'))
        ]

def _reqs ( self , tag ) : return [ ( tag , i ) for i in self . client . get_pulls ( * tag . split ( '/' ) ) ]


def aggregate_issues(conf, main_section, debug):
    """ Return all issues from every target. """
    log.info("Starting to aggregate remote issues.")

    # Create and call service objects for every target in the config
    targets = aslist(conf.get(main_section, 'targets'))

    queue = multiprocessing.Queue()

    log.info("Spawning %i workers." % len(targets))
    processes = []

    if debug:
        for target in targets:
            _aggregate_issues(
                conf,
                main_section,
                target,
                queue,
                conf.get(target, 'service')
            )
    else:
        for target in targets:
            proc = multiprocessing.Process(
                target=_aggregate_issues,
                args=(conf, main_section, target, queue, conf.get(target, 'service'))
            )
            proc.start()
            processes.append(proc)

            # Sleep for 1 second here to try and avoid a race condition where
            # all N workers start up and ask the gpg-agent process for
            # information at the same time.  This causes gpg-agent to fumble
            # and tell some of our workers some incomplete things.
            time.sleep(1)

    currently_running = len(targets)
    while currently_running > 0:
        issue = queue.get(True)
        if isinstance(issue, tuple):
            completion_type, args = issue
            if completion_type == SERVICE_FINISHED_ERROR:
                target, e = args
                log.info("Terminating workers")
                for process in processes:
                    process.terminate()
                raise RuntimeError(
                    "critical error in target '{}'".format(target))
            currently_running -= 1
            continue
        yield issue

    log.info("Done aggregating remote issues.")


def _get_config_or_default(self, key, default, as_type=lambda x: x):
        """Return a main config value, or default if it does not exist."""

        if self.main_config.has_option(self.main_section, key):
            return as_type(self.main_config.get(self.main_section, key))
        return default

def _get_config_or_default ( self , key , default , as_type = lambda x : x ) : if self . main_config . has_option ( self . main_section , key ) : return as_type ( self . main_config . get ( self . main_section , key ) ) return default

Return a main config value or default if it does not exist .


def get_templates(self):
        """ Get any defined templates for configuration values.

        Users can override the value of any Taskwarrior field using
        this feature on a per-key basis.  The key should be the name of
        the field to you would like to configure the value of, followed
        by '_template', and the value should be a Jinja template
        generating the field's value.  As context variables, all fields
        on the taskwarrior record are available.

        For example, to prefix the returned
        project name for tickets returned by a service with 'workproject_',
        you could add an entry reading:

            project_template = workproject_{{project}}

        Or, if you'd simply like to override the returned project name
        for all tickets incoming from a specific service, you could add
        an entry like:

            project_template = myprojectname

        The above would cause all issues to recieve a project name
        of 'myprojectname', regardless of what the project name of the
        generated issue was.

        """
        templates = {}
        for key in six.iterkeys(Task.FIELDS):
            template_key = '%s_template' % key
            if template_key in self.config:
                templates[key] = self.config.get(template_key)
        return templates

def get_templates ( self ) : templates = { } for key in six . iterkeys ( Task . FIELDS ) : template_key = '%s_template' % key if template_key in self . config : templates [ key ] = self . config . get ( template_key ) return templates

Get any defined templates for configuration values .


def validate_config(cls, service_config, target):
        """ Validate generic options for a particular target """
        if service_config.has_option(target, 'only_if_assigned'):
            die("[%s] has an 'only_if_assigned' option.  Should be "
                "'%s.only_if_assigned'." % (target, cls.CONFIG_PREFIX))
        if service_config.has_option(target, 'also_unassigned'):
            die("[%s] has an 'also_unassigned' option.  Should be "
                "'%s.also_unassigned'." % (target, cls.CONFIG_PREFIX))
        if service_config.has_option(target, 'default_priority'):
            die("[%s] has a 'default_priority' option.  Should be "
                "'%s.default_priority'." % (target, cls.CONFIG_PREFIX))
        if service_config.has_option(target, 'add_tags'):
            die("[%s] has an 'add_tags' option.  Should be "
                "'%s.add_tags'." % (target, cls.CONFIG_PREFIX))


def include(self, issue):
        """ Return true if the issue in question should be included """
        only_if_assigned = self.config.get('only_if_assigned', None)

        if only_if_assigned:
            owner = self.get_owner(issue)
            include_owners = [only_if_assigned]

            if self.config.get('also_unassigned', None, asbool):
                include_owners.append(None)

            return owner in include_owners

        only_if_author = self.config.get('only_if_author', None)

        if only_if_author:
            return self.get_author(issue) == only_if_author

        return True

def include ( self , issue ) : only_if_assigned = self . config . get ( 'only_if_assigned' , None ) if only_if_assigned : owner = self . get_owner ( issue ) include_owners = [ only_if_assigned ] if self . config . get ( 'also_unassigned' , None , asbool ) : include_owners . append ( None ) return owner in include_owners only_if_author = self . config . get ( 'only_if_author' , None ) if only_if_author : return self . get_author ( issue ) == only_if_author return True


def make_table(grid):
    """ Make a RST-compatible table

    From http://stackoverflow.com/a/12539081

    """
    cell_width = 2 + max(
        reduce(
            lambda x, y: x+y, [[len(item) for item in row] for row in grid], []
        )
    )
    num_cols = len(grid[0])
    rst = table_div(num_cols, cell_width, 0)
    header_flag = 1
    for row in grid:
        rst = rst + '| ' + '| '.join(
            [normalize_cell(x, cell_width-1) for x in row]
        ) + '|\n'
        rst = rst + table_div(num_cols, cell_width, header_flag)
        header_flag = 0
    return rst

def
make_table
(
grid
)
:
cell_width
=
2
+
max
(
reduce
(
lambda
x
,
y
:
x
+
y
,
[
[
len
(
item
)
for
item
in
row
]
for
row
in
grid
]
,
[
]
)
)
num_cols
=
len
(
grid
[
0
]
)
rst
=
table_div
(
num_cols
,
cell_width
,
0
)
header_flag
=
1
for
row
in
grid
:
rst
=
rst
+
'| '
+
'| '
.
join
(
[
normalize_cell
(
x
,
cell_width
1
)
for
x
in
row
]
)
+
'|\n'
rst
=
rst
+
table_div
(
num_cols
,
cell_width
,
header_flag
)
header_flag
=
0
return
rst


def oracle_eval(command):
    """ Retrieve password from the given command """
    p = subprocess.Popen(
        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    p.wait()
    if p.returncode == 0:
        return p.stdout.readline().strip().decode('utf-8')
    else:
        die(
            "Error retrieving password: `{command}` returned '{error}'".format(
                command=command, error=p.stderr.read().strip()))


def getint(self, section, option):
        """ Accepts both integers and empty values. """
        try:
            return super(BugwarriorConfigParser, self).getint(section, option)
        except ValueError:
            if self.get(section, option) == u'':
                return None
            else:
                raise ValueError(
                    "{section}.{option} must be an integer or empty.".format(
                        section=section, option=option))

Accepts both integers and empty values .


def pull(dry_run, flavor, interactive, debug):
    """ Pull down tasks from forges and add them to your taskwarrior tasks.

    Relies on configuration in bugwarriorrc
    """

    try:
        main_section = _get_section_name(flavor)
        config = _try_load_config(main_section, interactive)

        lockfile_path = os.path.join(get_data_path(config, main_section),
                                     'bugwarrior.lockfile')
        lockfile = PIDLockFile(lockfile_path)
        lockfile.acquire(timeout=10)
        try:
            # Get all the issues.  This can take a while.
            issue_generator = aggregate_issues(config, main_section, debug)

            # Stuff them in the taskwarrior db as necessary
            synchronize(issue_generator, config, main_section, dry_run)
        finally:
            lockfile.release()
    except LockTimeout:
        log.critical(
            'Your taskrc repository is currently locked. '
            'Remove the file at %s if you are sure no other '
            'bugwarrior processes are currently running.' % (
                lockfile_path
            )
        )
    except RuntimeError as e:
        log.exception("Aborted (%s)" % e)

Pull down tasks from forges and add them to your taskwarrior tasks .


def get_data(self, url):
        """ Perform a request to the fully qualified url and return json. """
        return self.json_response(requests.get(url, **self.requests_kwargs))

def get_data ( self , url ) : return self . json_response ( requests . get ( url , * * self . requests_kwargs ) )

Perform a request to the fully qualified url and return json .


def get_collection(self, url):
        """ Pages through an object collection from the bitbucket API.
        Returns an iterator that lazily goes through all the 'values'
        of all the pages in the collection. """
        url = self.BASE_API2 + url
        while url is not None:
            response = self.get_data(url)
            for value in response['values']:
                yield value
            url = response.get('next', None)

def get_collection ( self , url ) : url = self . BASE_API2 + url while url is not None : response = self . get_data ( url ) for value in response [ 'values' ] : yield value url = response . get ( 'next' , None )

Pages through an object collection from the bitbucket API . Returns an iterator that lazily goes through all the values of all the pages in the collection .


def find_local_uuid(tw, keys, issue, legacy_matching=False):
    """ For a given issue issue, find its local UUID.

    Assembles a list of task IDs existing in taskwarrior
    matching the supplied issue (`issue`) on the combination of any
    set of supplied unique identifiers (`keys`) or, optionally,
    the task's description field (should `legacy_matching` be `True`).

    :params:
    * `tw`: An instance of `taskw.TaskWarriorShellout`
    * `keys`: A list of lists of keys to use for uniquely identifying
      an issue.  To clarify the "list of lists" behavior, assume that
      there are two services, one having a single primary key field
      -- 'serviceAid' -- and another having a pair of fields composing
      its primary key -- 'serviceBproject' and 'serviceBnumber' --, the
      incoming data for this field would be::

        [
            ['serviceAid'],
            ['serviceBproject', 'serviceBnumber'],
        ]

    * `issue`: An instance of a subclass of `bugwarrior.services.Issue`.
    * `legacy_matching`: By default, this is disabled, and it allows
      the matching algorithm to -- in addition to searching by stored
      issue keys -- search using the task's description for a match.
      It is prone to error and should avoided if possible.

    :returns:
    * A single string UUID.

    :raises:
    * `bugwarrior.db.MultipleMatches`: if multiple matches were found.
    * `bugwarrior.db.NotFound`: if an issue was not found.

    """
    if not issue['description']:
        raise ValueError('Issue %s has no description.' % issue)

    possibilities = set([])

    if legacy_matching:
        legacy_description = issue.get_default_description().rsplit('..', 1)[0]
        # Furthermore, we have to kill off any single quotes which break in
        # task-2.4.x, as much as it saddens me.
        legacy_description = legacy_description.split("'")[0]
        results = tw.filter_tasks({
            'description.startswith': legacy_description,
            'or': [
                ('status', 'pending'),
                ('status', 'waiting'),
            ],
        })
        possibilities = possibilities | set([
            task['uuid'] for task in results
        ])

    for service, key_list in six.iteritems(keys):
        if any([key in issue for key in key_list]):
            results = tw.filter_tasks({
                'and': [("%s.is" % key, issue[key]) for key in key_list],
                'or': [
                    ('status', 'pending'),
                    ('status', 'waiting'),
                ],
            })
            possibilities = possibilities | set([
                task['uuid'] for task in results
            ])

    if len(possibilities) == 1:
        return possibilities.pop()

    if len(possibilities) > 1:
        raise MultipleMatches(
            "Issue %s matched multiple IDs: %s" % (
                issue['description'],
                possibilities
            )
        )

    raise NotFound(
        "No issue was found matching %s" % issue
    )

For a given issue issue find its local UUID .


def merge_left(field, local_task, remote_issue, hamming=False):
    """ Merge array field from the remote_issue into local_task

    * Local 'left' entries are preserved without modification
    * Remote 'left' are appended to task if not present in local.

    :param `field`: Task field to merge.
    :param `local_task`: `taskw.task.Task` object into which to merge
        remote changes.
    :param `remote_issue`: `dict` instance from which to merge into
        local task.
    :param `hamming`: (default `False`) If `True`, compare entries by
        truncating to maximum length, and comparing hamming distances.
        Useful generally only for annotations.

    """

    # Ensure that empty defaults are present
    local_field = local_task.get(field, [])
    remote_field = remote_issue.get(field, [])

    # We need to make sure an array exists for this field because
    # we will be appending to it in a moment.
    if field not in local_task:
        local_task[field] = []

    # If a remote does not appear in local, add it to the local task
    new_count = 0
    for remote in remote_field:
        for local in local_field:
            if (
                # For annotations, they don't have to match *exactly*.
                (
                    hamming
                    and get_annotation_hamming_distance(remote, local) == 0
                )
                # But for everything else, they should.
                or (
                    remote == local
                )
            ):
                break
        else:
            log.debug("%s not found in %r" % (remote, local_field))
            local_task[field].append(remote)
            new_count += 1
    if new_count > 0:
        log.debug('Added %s new values to %s (total: %s)' % (
            new_count, field, len(local_task[field]),))


def build_uda_config_overrides(targets):
    """ Returns a list of UDAs defined by given targets

    For all targets in `targets`, build a dictionary of configuration overrides
    representing the UDAs defined by the passed-in services (`targets`).

    Given a hypothetical situation in which you have two services, the first
    of which defining a UDA named 'serviceAid' ("Service A ID", string) and
    a second service defining two UDAs named 'serviceBproject'
    ("Service B Project", string) and 'serviceBnumber'
    ("Service B Number", numeric), this would return the following structure::

        {
            'uda': {
                'serviceAid': {
                    'label': 'Service A ID',
                    'type': 'string',
                },
                'serviceBproject': {
                    'label': 'Service B Project',
                    'type': 'string',
                },
                'serviceBnumber': {
                    'label': 'Service B Number',
                    'type': 'numeric',
                }
            }
        }

    """

    from bugwarrior.services import get_service

    targets_udas = {}
    for target in targets:
        targets_udas.update(get_service(target).ISSUE_CLASS.UDAS)
    return {
        'uda': targets_udas
    }

def build_uda_config_overrides ( targets ) : from bugwarrior . services import get_service targets_udas = { } for target in targets : targets_udas . update ( get_service ( target ) . ISSUE_CLASS . UDAS ) return { 'uda' : targets_udas }


def _parse_sprint_string(sprint):
    """ Parse the big ugly sprint string stored by JIRA.

    They look like:
        com.atlassian.greenhopper.service.sprint.Sprint@4c9c41a5[id=2322,rapid
        ViewId=1173,state=ACTIVE,name=Sprint 1,startDate=2016-09-06T16:08:07.4
        55Z,endDate=2016-09-23T16:08:00.000Z,completeDate=<null>,sequence=2322]
    """
    entries = sprint[sprint.index('[')+1:sprint.index(']')].split('=')
    fields = sum((entry.rsplit(',', 1) for entry in entries), [])
    return dict(zip(fields[::2], fields[1::2]))

def _parse_sprint_string ( sprint ) : entries = sprint [ sprint . index ( '[' ) + 1 : sprint . index ( ']' ) ] . split ( '=' ) fields = sum ( ( entry . rsplit ( ',' , 1 ) for entry in entries ) , [ ] ) return dict ( zip ( fields [ : : 2 ] , fields [ 1 : : 2 ] ) )

Parse the big ugly sprint string stored by JIRA .


def get_credentials(self):
        """Gets valid user credentials from storage.

        If nothing has been stored, or if the stored credentials are invalid,
        the OAuth2 flow is completed to obtain the new credentials.

        Returns:
            Credentials, the obtained credential.
        """
        with self.AUTHENTICATION_LOCK:
            log.info('Starting authentication for %s', self.target)
            store = oauth2client.file.Storage(self.credentials_path)
            credentials = store.get()
            if not credentials or credentials.invalid:
                log.info("No valid login. Starting OAUTH flow.")
                flow = oauth2client.client.flow_from_clientsecrets(self.client_secret_path, self.SCOPES)
                flow.user_agent = self.APPLICATION_NAME
                flags = oauth2client.tools.argparser.parse_args([])
                credentials = oauth2client.tools.run_flow(flow, store, flags)
                log.info('Storing credentials to %r', self.credentials_path)
            return credentials

def
get_credentials
(
self
)
:
with
self
.
AUTHENTICATION_LOCK
:
log
.
info
(
'Starting authentication for %s'
,
self
.
target
)
store
=
oauth2client
.
file
.
Storage
(
self
.
credentials_path
)
credentials
=
store
.
get
(
)
if
not
credentials
or
credentials
.
invalid
:
log
.
info
(
"No valid login. Starting OAUTH flow."
)
flow
=
oauth2client
.
client
.
flow_from_clientsecrets
(
self
.
client_secret_path
,
self
.
SCOPES
)
flow
.
user_agent
=
self
.
APPLICATION_NAME
flags
=
oauth2client
.
tools
.
argparser
.
parse_args
(
[
]
)
credentials
=
oauth2client
.
tools
.
run_flow
(
flow
,
store
,
flags
)
log
.
info
(
'Storing credentials to %r'
,
self
.
credentials_path
)
return
credentials

Gets valid user credentials from storage .


def multi_rouge_n(sequences, scores_ids, n=2):
    """
    Efficient way to compute highly repetitive scoring
    i.e. sequences are involved multiple time

    Args:
        sequences(list[str]): list of sequences (either hyp or ref)
        scores_ids(list[tuple(int)]): list of pairs (hyp_id, ref_id)
            ie. scores[i] = rouge_n(scores_ids[i][0],
                                    scores_ids[i][1])

    Returns:
        scores: list of length `len(scores_ids)` containing rouge `n`
                scores as a dict with 'f', 'r', 'p'
    Raises:
        KeyError: if there's a value of i in scores_ids that is not in
                  [0, len(sequences)[
    """
    ngrams = [_get_word_ngrams(n, sequence) for sequence in sequences]
    counts = [len(ngram) for ngram in ngrams]

    scores = []
    for hyp_id, ref_id in scores_ids:
        evaluated_ngrams = ngrams[hyp_id]
        evaluated_count = counts[hyp_id]

        reference_ngrams = ngrams[ref_id]
        reference_count = counts[ref_id]

        overlapping_ngrams = evaluated_ngrams.intersection(reference_ngrams)
        overlapping_count = len(overlapping_ngrams)

        scores += [f_r_p_rouge_n(evaluated_count,
                                 reference_count, overlapping_count)]
    return scores

def multi_rouge_n ( sequences , scores_ids , n = 2 ) : ngrams = [ _get_word_ngrams ( n , sequence ) for sequence in sequences ] counts = [ len ( ngram ) for ngram in ngrams ] scores = [ ] for hyp_id , ref_id in scores_ids : evaluated_ngrams = ngrams [ hyp_id ] evaluated_count = counts [ hyp_id ] reference_ngrams = ngrams [ ref_id ] reference_count = counts [ ref_id ] overlapping_ngrams = evaluated_ngrams . intersection ( reference_ngrams ) overlapping_count = len ( overlapping_ngrams ) scores += [ f_r_p_rouge_n ( evaluated_count , reference_count , overlapping_count ) ] return scores

Efficient way to compute highly repetitive scoring i . e . sequences are involved multiple time


def calc_pvalues(query, gene_sets, background=20000, **kwargs):
    """ calculate pvalues for all categories in the graph

    :param set query: set of identifiers for which the p value is calculated
    :param dict gene_sets: gmt file dict after background was set
    :param set background: total number of genes in your annotated database.
    :returns: pvalues
              x: overlapped gene number
              n: length of gene_set which belongs to each terms
              hits: overlapped gene names.

    For 2*2 contingency table:
    =============================================================================
                         |   in  query  |  not in query |    row total
    =>      in gene_set  |        a     |       b       |       a+b
    =>  not in gene_set  |        c     |       d       |       c+d
           column total                                 | a+b+c+d = anno database
    =============================================================================
    background genes number = a + b + c + d.

    Then, in R
        x=a     the number of white balls drawn without replacement
                from an urn which contains both black and white balls.
        m=a+b   the number of white balls in the urn
        n=c+d   the number of black balls in the urn
        k=a+c   the number of balls drawn from the urn

    In Scipy:
    for args in scipy.hypergeom.sf(k, M, n, N, loc=0):
        M: the total number of objects,
        n: the total number of Type I objects.
        k: the random variate represents the number of Type I objects in N drawn
           without replacement from the total population.

    Therefore, these two functions are the same when using parameters from 2*2 table:
    R:     >   phyper(x-1, m, n, k, lower.tail=FALSE)
    Scipy: >>> hypergeom.sf(x-1, m+n, m, k)

    """

    # number of genes in your query data
    k = len(query)
    query = set(query)
    vals = []
    # background should be all genes in annotated database
    # such as go, kegg et.al.
    if isinstance(background, set):
        bg = len(background) # total number in your annotated database
        # filter genes that not found in annotated database
        query = query.intersection(background)
    elif isinstance(background, int):
        bg = background
    else:
        raise ValueError("background should be set or int object")
    # pval
    subsets = sorted(gene_sets.keys())
    for s in subsets:
        category = gene_sets.get(s)
        m = len(category)
        hits = query.intersection(set(category))
        x = len(hits)
        if x < 1 : continue
        # pVal = hypergeom.sf(hitCount-1,popTotal,bgHits,queryTotal)
        # p(X >= hitCounts)
        vals.append((s, hypergeom.sf(x-1, bg, m, k), x, m, hits))

    return zip(*vals)


def fdrcorrection(pvals, alpha=0.05):
    """ benjamini hocheberg fdr correction. inspired by statsmodels
    """
    # Implement copy from GOATools.
    pvals = np.asarray(pvals)
    pvals_sortind = np.argsort(pvals)
    pvals_sorted = np.take(pvals, pvals_sortind)

    ecdffactor = _ecdf(pvals_sorted)
    reject = pvals_sorted <= ecdffactor*alpha
    if reject.any():
        rejectmax = max(np.nonzero(reject)[0])
        reject[:rejectmax] = True
    pvals_corrected_raw = pvals_sorted / ecdffactor
    pvals_corrected = np.minimum.accumulate(pvals_corrected_raw[::-1])[::-1]
    del pvals_corrected_raw
    pvals_corrected[pvals_corrected>1] = 1
    pvals_corrected_ = np.empty_like(pvals_corrected)
    pvals_corrected_[pvals_sortind] = pvals_corrected
    del pvals_corrected
    reject_ = np.empty_like(reject)
    reject_[pvals_sortind] = reject
    return reject_, pvals_corrected_

def
fdrcorrection
(
pvals
,
alpha
=
0.05
)
:
# Implement copy from GOATools.
pvals
=
np
.
asarray
(
pvals
)
pvals_sortind
=
np
.
argsort
(
pvals
)
pvals_sorted
=
np
.
take
(
pvals
,
pvals_sortind
)
ecdffactor
=
_ecdf
(
pvals_sorted
)
reject
=
pvals_sorted
<=
ecdffactor
*
alpha
if
reject
.
any
(
)
:
rejectmax
=
max
(
np
.
nonzero
(
reject
)
[
0
]
)
reject
[
:
rejectmax
]
=
True
pvals_corrected_raw
=
pvals_sorted
/
ecdffactor
pvals_corrected
=
np
.
minimum
.
accumulate
(
pvals_corrected_raw
[
:
:
1
]
)
[
:
:
1
]
del
pvals_corrected_raw
pvals_corrected
[
pvals_corrected
>
1
]
=
1
pvals_corrected_
=
np
.
empty_like
(
pvals_corrected
)
pvals_corrected_
[
pvals_sortind
]
=
pvals_corrected
del
pvals_corrected
reject_
=
np
.
empty_like
(
reject
)
reject_
[
pvals_sortind
]
=
reject
return
reject_
,
pvals_corrected_

benjamini hocheberg fdr correction . inspired by statsmodels


def zscore(data2d, axis=0):
    """Standardize the mean and variance of the data axis Parameters.

    :param data2d: DataFrame to normalize.
    :param axis: int, Which axis to normalize across. If 0, normalize across rows,
                  if 1, normalize across columns. If None, don't change data

    :Returns: Normalized DataFrame. Normalized data with a mean of 0 and variance of 1
              across the specified axis.

    """
    if axis is None:
        # normalized to mean and std using entire matrix
        # z_scored = (data2d - data2d.values.mean()) / data2d.values.std(ddof=1)
        return data2d
    assert axis in [0,1]
    # if axis == 1:
    #     z_scored = data2d
    # else:
    #     z_scored = data2d.T

    # z_scored = (z_scored - z_scored.mean()) / z_scored.std(ddof=1)

    # if axis == 1:
    #     return z_scored
    # else:
    #     return z_scored.T
    z_scored = data2d.apply(lambda x: (x-x.mean())/x.std(ddof=1),
                            axis=operator.xor(1, axis))
    return z_scored

Standardize the mean and variance of the data axis Parameters .


def heatmap(df, z_score=None, title='', figsize=(5,5), cmap='RdBu_r',
            xticklabels=True, yticklabels=True, ofname=None, **kwargs):
    """Visualize the dataframe.

    :param df: DataFrame from expression table.
    :param z_score: z_score axis{0, 1}. If None, don't normalize data.
    :param title: gene set name.
    :param outdir: path to save heatmap.
    :param figsize: heatmap figsize.
    :param cmap: matplotlib colormap.
    :param ofname: output file name. If None, don't save figure

    """
    df = zscore(df, axis=z_score)
    df = df.iloc[::-1]
    # Get the positions and used label for the ticks
    ny, nx = df.shape
    xticks = np.arange(0, nx, 1) + .5
    yticks = np.arange(0, ny, 1) + .5

    # If working on commandline, don't show figure
    if hasattr(sys, 'ps1') and (ofname is None):
        fig = plt.figure(figsize=figsize)
    else:
        fig = Figure(figsize=figsize)
        canvas = FigureCanvas(fig)
    ax = fig.add_subplot(111)
    vmin = np.percentile(df.min(), 2)
    vmax =  np.percentile(df.max(), 98)
    matrix = ax.pcolormesh(df.values, cmap=cmap, vmin=vmin, vmax=vmax)
    ax.set_ylim([0,len(df)])
    ax.set(xticks=xticks, yticks=yticks)
    ax.set_xticklabels(df.columns.values if xticklabels else '', fontsize=14, rotation=90)
    ax.set_yticklabels(df.index.values if yticklabels else '',  fontsize=14)
    ax.set_title("%s\nHeatmap of the Analyzed Geneset"%title, fontsize=20)
    ax.tick_params(axis='both', which='both', bottom=False, top=False,
                   right=False, left=False)
    # cax=fig.add_axes([0.93,0.25,0.05,0.20])
    # cbar = fig.colorbar(matrix, cax=cax)
    cbar = colorbar(matrix)
    cbar.ax.tick_params(axis='both', which='both', bottom=False, top=False,
                        right=False, left=False)
    for side in ["top", "right", "left", "bottom"]:
        ax.spines[side].set_visible(False)
        cbar.ax.spines[side].set_visible(False)
    # cbar.ax.set_title('',loc='left')

    if ofname is not None:
        # canvas.print_figure(ofname, bbox_inches='tight', dpi=300)
        fig.savefig(ofname, bbox_inches='tight', dpi=300)
    return


def adjust_spines(ax, spines):
    """function for removing spines and ticks.

    :param ax: axes object
    :param spines: a list of spines names to keep. e.g [left, right, top, bottom]
                    if spines = []. remove all spines and ticks.

    """
    for loc, spine in ax.spines.items():
        if loc in spines:
            # spine.set_position(('outward', 10))  # outward by 10 points
            # spine.set_smart_bounds(True)
            continue
        else:
            spine.set_color('none')  # don't draw spine

    # turn off ticks where there is no spine
    if 'left' in spines:
        ax.yaxis.set_ticks_position('left')
    else:
        # no yaxis ticks
        ax.yaxis.set_ticks([])

    if 'bottom' in spines:
        ax.xaxis.set_ticks_position('bottom')
    else:
        # no xaxis ticks
        ax.xaxis.set_ticks([])

function for removing spines and ticks .


def prepare_argparser():
    """Prepare argparser object. New options will be added in this function first."""
    description = "%(prog)s -- Gene Set Enrichment Analysis in Python"
    epilog = "For command line options of each command, type: %(prog)s COMMAND -h"

    # top-level parser
    argparser = ap.ArgumentParser(description=description, epilog=epilog)
    argparser.add_argument("--version", action="version", version="%(prog)s "+ __version__)
    subparsers = argparser.add_subparsers(dest='subcommand_name') #help="sub-command help")

    # command for 'gsea'
    add_gsea_parser(subparsers)
    # command for 'prerank'
    add_prerank_parser(subparsers)
    # command for 'ssgsea'
    add_singlesample_parser(subparsers)
    # command for 'plot'
    add_plot_parser(subparsers)
    # command for 'enrichr'
    add_enrichr_parser(subparsers)
    # command for 'biomart'
    add_biomart_parser(subparsers)

    return argparser

Prepare argparser object . New options will be added in this function first .


def add_prerank_parser(subparsers):
    """Add function 'prerank' argument parsers."""

    argparser_prerank = subparsers.add_parser("prerank", help="Run GSEApy Prerank tool on preranked gene list.")

    # group for input files
    prerank_input = argparser_prerank.add_argument_group("Input files arguments")
    prerank_input.add_argument("-r", "--rnk", dest="rnk", action="store", type=str, required=True,
                             help="Ranking metric file in .rnk format. Same with GSEA.")
    prerank_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    prerank_input.add_argument("-l", "--label", action='store', nargs=2, dest='label',
                             metavar=('pos', 'neg'), type=str, default=('Pos','Neg'),
                             help="The phenotype label argument need two parameters to define. Default: ('Pos','Neg')")

    # group for output files
    prerank_output = argparser_prerank.add_argument_group("Output arguments")
    add_output_option(prerank_output)

    # group for General options.
    prerank_opt = argparser_prerank.add_argument_group("GSEA advanced arguments")
    prerank_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=1000, metavar='nperm',
                             help="Number of random permutations. For calculating esnulls. Default: 1000")
    prerank_opt.add_argument("--min-size",  dest="mins", action="store", type=int, default=15, metavar='int',
                             help="Min size of input genes presented in Gene Sets. Default: 15")
    prerank_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=500, metavar='int',
                             help="Max size of input genes presented in Gene Sets. Default: 500")
    prerank_opt.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                             help='Weighted_score of rank_metrics. For weighting input genes. Choose from {0, 1, 1.5, 2}. Default: 1',)
    prerank_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                             help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    prerank_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                             help="Number of random seed. Default: None")
    prerank_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return


def add_plot_parser(subparsers):
    """Add function 'plot' argument parsers."""

    argparser_replot = subparsers.add_parser("replot", help="Reproduce GSEA desktop output figures.")

    group_replot = argparser_replot.add_argument_group("Input arguments")

    group_replot.add_argument("-i", "--indir", action="store", dest="indir", required=True, metavar='GSEA_dir',
                              help="The GSEA desktop results directroy that you want to reproduce the figure ")
    add_output_option(group_replot)
    #add_output_group( argparser_plot )
    group_replot.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                              help='Weighted_score of rank_metrics. Please Use the same value in GSEA. Choose from (0, 1, 1.5, 2),default: 1',)

    return


def add_enrichr_parser(subparsers):
    """Add function 'enrichr' argument parsers."""

    argparser_enrichr = subparsers.add_parser("enrichr", help="Using Enrichr API to perform GO analysis.")

    # group for required options.
    enrichr_opt = argparser_enrichr.add_argument_group("Input arguments")
    enrichr_opt.add_argument("-i", "--input-list", action="store", dest="gene_list", type=str, required=True, metavar='IDs',
                              help="Enrichr uses a list of gene names as input.")
    enrichr_opt.add_argument("-g", "--gene-sets", action="store", dest="library", type=str, required=True, metavar='GMT',
                              help="Enrichr library name(s) required. Separate each name by comma.")
    enrichr_opt.add_argument("--org", "--organism", action="store", dest="organism", type=str, default='',
                             help="Enrichr supported organism name. Default: human. See here: https://amp.pharm.mssm.edu/modEnrichr.")
    enrichr_opt.add_argument("--ds", "--description", action="store", dest="descrip", type=str, default='enrichr', metavar='STRING',
                              help="It is recommended to enter a short description for your list so that multiple lists \
                              can be differentiated from each other if you choose to save or share your list.")
    enrichr_opt.add_argument("--cut", "--cut-off", action="store", dest="thresh", metavar='float', type=float, default=0.05,
                              help="Adjust-Pval cutoff, used for generating plots. Default: 0.05.")
    enrichr_opt.add_argument("--bg", "--background", action="store", dest="bg", default='hsapiens_gene_ensembl', metavar='BGNUM',
                              help="BioMart Dataset name or Background total genes number. Default: None")
    enrichr_opt.add_argument("-t", "--top-term", dest="term", action="store", type=int, default=10, metavar='int',
                              help="Numbers of top terms shown in the plot. Default: 10")
    # enrichr_opt.add_argument("--scale", dest = "scale", action="store", type=float, default=0.5, metavar='float',
    #                          help="scatter dot scale in the dotplot. Default: 0.5")
    # enrichr_opt.add_argument("--no-plot", action='store_true', dest='no_plot', default=False,
    #                           help="Suppress the plot output.This is useful only if data are interested. Default: False.")

    enrichr_output = argparser_enrichr.add_argument_group("Output figure arguments")
    add_output_option(enrichr_output)
    return


def enrichment_score(gene_list, correl_vector, gene_set, weighted_score_type=1,
                     nperm=1000, rs=np.random.RandomState(), single=False, scale=False):
    """This is the most important function of GSEApy. It has the same algorithm with GSEA and ssGSEA.

    :param gene_list:       The ordered gene list gene_name_list, rank_metric.index.values
    :param gene_set:        gene_sets in gmt file, please use gsea_gmt_parser to get gene_set.
    :param weighted_score_type:  It's the same with gsea's weighted_score method. Weighting by the correlation
                            is a very reasonable choice that allows significant gene sets with less than perfect coherence.
                            options: 0(classic),1,1.5,2. default:1. if one is interested in penalizing sets for lack of
                            coherence or to discover sets with any type of nonrandom distribution of tags, a value p < 1
                            might be appropriate. On the other hand, if one uses sets with large number of genes and only
                            a small subset of those is expected to be coherent, then one could consider using p > 1.
                            Our recommendation is to use p = 1 and use other settings only if you are very experienced
                            with the method and its behavior.

    :param correl_vector:   A vector with the correlations (e.g. signal to noise scores) corresponding to the genes in
                            the gene list. Or rankings, rank_metric.values
    :param nperm:           Only use this parameter when computing esnull for statistical testing. Set the esnull value
                            equal to the permutation number.
    :param rs:              Random state for initializing gene list shuffling. Default: np.random.RandomState(seed=None)

    :return:

     ES: Enrichment score (real number between -1 and +1)

     ESNULL: Enrichment score calculated from random permutations.

     Hits_Indices: Index of a gene in gene_list, if gene is included in gene_set.

     RES: Numerical vector containing the running enrichment score for all locations in the gene list .

    """

    N = len(gene_list)
    # Test whether each element of a 1-D array is also present in a second array
    # It's more intuitive here than original enrichment_score source code.
    # use .astype to covert bool to integer
    tag_indicator = np.in1d(gene_list, gene_set, assume_unique=True).astype(int)  # notice that the sign is 0 (no tag) or 1 (tag)

    if weighted_score_type == 0 :
        correl_vector = np.repeat(1, N)
    else:
        correl_vector = np.abs(correl_vector)**weighted_score_type

    # get indices of tag_indicator
    hit_ind = np.flatnonzero(tag_indicator).tolist()
    # if used for compute esnull, set esnull equal to permutation number, e.g. 1000
    # else just compute enrichment scores
    # set axis to 1, because we have 2D array
    axis = 1
    tag_indicator = np.tile(tag_indicator, (nperm+1,1))
    correl_vector = np.tile(correl_vector,(nperm+1,1))
    # gene list permutation
    for i in range(nperm): rs.shuffle(tag_indicator[i])
    # np.apply_along_axis(rs.shuffle, 1, tag_indicator)

    Nhint = tag_indicator.sum(axis=axis, keepdims=True)
    sum_correl_tag = np.sum(correl_vector*tag_indicator, axis=axis, keepdims=True)
    # compute ES score, the code below is identical to gsea enrichment_score method.
    no_tag_indicator = 1 - tag_indicator
    Nmiss =  N - Nhint
    norm_tag =  1.0/sum_correl_tag
    norm_no_tag = 1.0/Nmiss

    RES = np.cumsum(tag_indicator * correl_vector * norm_tag - no_tag_indicator * norm_no_tag, axis=axis)

    if scale: RES = RES / N
    if single:
        es_vec = RES.sum(axis=axis)
    else:
        max_ES, min_ES =  RES.max(axis=axis), RES.min(axis=axis)
        es_vec = np.where(np.abs(max_ES) > np.abs(min_ES), max_ES, min_ES)
    # extract values
    es, esnull, RES = es_vec[-1], es_vec[:-1], RES[-1,:]

    return es, esnull, hit_ind, RES

This is the most important function of GSEApy . It has the same algorithm with GSEA and ssGSEA .


def ranking_metric_tensor(exprs, method, permutation_num, pos, neg, classes,
                          ascending, rs=np.random.RandomState()):
    """Build shuffled ranking matrix when permutation_type eq to phenotype.

       :param exprs:   gene_expression DataFrame, gene_name indexed.
       :param str method:  calculate correlation or ranking. methods including:
                           1. 'signal_to_noise'.
                           2. 't_test'.
                           3. 'ratio_of_classes' (also referred to as fold change).
                           4. 'diff_of_classes'.
                           5. 'log2_ratio_of_classes'.
       :param int permuation_num: how many times of classes is being shuffled
       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of
                             dataframe belongs to what class of phenotype.
       :param bool ascending:  bool. Sort ascending vs. descending.

       :return:
                returns two 2d ndarray with shape (nperm, gene_num).

                | cor_mat_indices: the indices of sorted and permutated (exclude last row) ranking matrix.
                | cor_mat: sorted and permutated (exclude last row) ranking matrix.

    """
    # S: samples, G: gene number
    G, S = exprs.shape
    # genes = exprs.index.values
    expr_mat = exprs.values.T
    perm_cor_tensor = np.tile(expr_mat, (permutation_num+1,1,1))
    # random shuffle on the first dim, last matrix is not shuffled
    for arr in perm_cor_tensor[:-1]: rs.shuffle(arr)
    classes = np.array(classes)
    pos = classes == pos
    neg = classes == neg
    pos_cor_mean = perm_cor_tensor[:,pos,:].mean(axis=1)
    neg_cor_mean = perm_cor_tensor[:,neg,:].mean(axis=1)
    pos_cor_std = perm_cor_tensor[:,pos,:].std(axis=1, ddof=1)
    neg_cor_std = perm_cor_tensor[:,neg,:].std(axis=1, ddof=1)

    if method == 'signal_to_noise':
        cor_mat = (pos_cor_mean - neg_cor_mean)/(pos_cor_std + neg_cor_std)
    elif method == 't_test':
        denom = 1.0/G
        cor_mat = (pos_cor_mean - neg_cor_mean)/ np.sqrt(denom*pos_cor_std**2 + denom*neg_cor_std**2)
    elif method == 'ratio_of_classes':
        cor_mat = pos_cor_mean / neg_cor_mean
    elif method == 'diff_of_classes':
        cor_mat  = pos_cor_mean - neg_cor_mean
    elif method == 'log2_ratio_of_classes':
        cor_mat  =  np.log2(pos_cor_mean / neg_cor_mean)
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    # return matix[nperm+1, perm_cors]
    cor_mat_ind = cor_mat.argsort()
    # ndarray: sort in place
    cor_mat.sort()
    # genes_mat = genes.take(cor_mat_ind)
    if ascending: return cor_mat_ind, cor_mat
    # descending order of ranking and genes
    # return genes_mat[:,::-1], cor_mat[:,::-1]
    return cor_mat_ind[:, ::-1], cor_mat[:, ::-1]

Build shuffled ranking matrix when permutation_type eq to phenotype .


def ranking_metric(df, method, pos, neg, classes, ascending):
    """The main function to rank an expression table.

       :param df:      gene_expression DataFrame.
       :param method:  The method used to calculate a correlation or ranking. Default: 'log2_ratio_of_classes'.
                       Others methods are:

                       1. 'signal_to_noise'

                          You must have at least three samples for each phenotype to use this metric.
                          The larger the signal-to-noise ratio, the larger the differences of the means (scaled by the standard deviations);
                          that is, the more distinct the gene expression is in each phenotype and the more the gene acts as a “class marker.”

                       2. 't_test'

                          Uses the difference of means scaled by the standard deviation and number of samples.
                          Note: You must have at least three samples for each phenotype to use this metric.
                          The larger the tTest ratio, the more distinct the gene expression is in each phenotype
                          and the more the gene acts as a “class marker.”

                       3. 'ratio_of_classes' (also referred to as fold change).

                          Uses the ratio of class means to calculate fold change for natural scale data.

                       4. 'diff_of_classes'

                          Uses the difference of class means to calculate fold change for natural scale data

                       5. 'log2_ratio_of_classes'

                          Uses the log2 ratio of class means to calculate fold change for natural scale data.
                          This is the recommended statistic for calculating fold change for log scale data.

       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
       :param bool ascending:  bool or list of bool. Sort ascending vs. descending.
       :return:

            returns a pd.Series of correlation to class of each variable. Gene_name is index, and value is rankings.

            visit here for more docs: http://software.broadinstitute.org/gsea/doc/GSEAUserGuideFrame.html
    """

    # exclude any zero stds.
    df_mean = df.groupby(by=classes, axis=1).mean()
    df_std =  df.groupby(by=classes, axis=1).std()

    if method == 'signal_to_noise':
        ser = (df_mean[pos] - df_mean[neg])/(df_std[pos] + df_std[neg])
    elif method == 't_test':
        ser = (df_mean[pos] - df_mean[neg])/ np.sqrt(df_std[pos]**2/len(df_std)+df_std[neg]**2/len(df_std) )
    elif method == 'ratio_of_classes':
        ser = df_mean[pos] / df_mean[neg]
    elif method == 'diff_of_classes':
        ser  = df_mean[pos] - df_mean[neg]
    elif method == 'log2_ratio_of_classes':
        ser  =  np.log2(df_mean[pos] / df_mean[neg])
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    ser = ser.sort_values(ascending=ascending)

    return ser

def
ranking_metric
(
df
,
method
,
pos
,
neg
,
classes
,
ascending
)
:
# exclude any zero stds.
df_mean
=
df
.
groupby
(
by
=
classes
,
axis
=
1
)
.
mean
(
)
df_std
=
df
.
groupby
(
by
=
classes
,
axis
=
1
)
.
std
(
)
if
method
==
'signal_to_noise'
:
ser
=
(
df_mean
[
pos
]
df_mean
[
neg
]
)
/
(
df_std
[
pos
]
+
df_std
[
neg
]
)
elif
method
==
't_test'
:
ser
=
(
df_mean
[
pos
]
df_mean
[
neg
]
)
/
np
.
sqrt
(
df_std
[
pos
]
**
2
/
len
(
df_std
)
+
df_std
[
neg
]
**
2
/
len
(
df_std
)
)
elif
method
==
'ratio_of_classes'
:
ser
=
df_mean
[
pos
]
/
df_mean
[
neg
]
elif
method
==
'diff_of_classes'
:
ser
=
df_mean
[
pos
]
df_mean
[
neg
]
elif
method
==
'log2_ratio_of_classes'
:
ser
=
np
.
log2
(
df_mean
[
pos
]
/
df_mean
[
neg
]
)
else
:
logging
.
error
(
"Please provide correct method name!!!"
)
sys
.
exit
(
0
)
ser
=
ser
.
sort_values
(
ascending
=
ascending
)
return
ser

The main function to rank an expression table .


def gsea_pval(es, esnull):
    """Compute nominal p-value.

    From article (PNAS):
    estimate nominal p-value for S from esnull by using the positive
    or negative portion of the distribution corresponding to the sign
    of the observed ES(S).
    """

    # to speed up, using numpy function to compute pval in parallel.
    condlist = [ es < 0, es >=0]
    choicelist = [np.sum(esnull < es.reshape(len(es),1), axis=1)/ np.sum(esnull < 0, axis=1),
                  np.sum(esnull >= es.reshape(len(es),1), axis=1)/ np.sum(esnull >= 0, axis=1)]
    pval = np.select(condlist, choicelist)

    return pval


def gsea_significance(enrichment_scores, enrichment_nulls):
    """Compute nominal pvals, normalized ES, and FDR q value.

        For a given NES(S) = NES* >= 0. The FDR is the ratio of the percentage of all (S,pi) with
        NES(S,pi) >= 0, whose NES(S,pi) >= NES*, divided by the percentage of
        observed S wih NES(S) >= 0, whose NES(S) >= NES*, and similarly if NES(S) = NES* <= 0.
    """
    # For a zero by zero division (undetermined, results in a NaN),
    np.seterr(divide='ignore', invalid='ignore')
    # import warnings
    # warnings.simplefilter("ignore")
    es = np.array(enrichment_scores)
    esnull = np.array(enrichment_nulls)
    logging.debug("Start to compute pvals..................................")
    # compute pvals.
    enrichmentPVals = gsea_pval(es, esnull).tolist()

    logging.debug("Compute nes and nesnull.................................")
    # nEnrichmentScores, nEnrichmentNulls = normalize(es, esnull)

    # new normalized enrichment score implementation.
    # this could speed up significantly.
    esnull_pos = (esnull*(esnull>=0)).mean(axis=1)
    esnull_neg = (esnull*(esnull<0)).mean(axis=1)
    nEnrichmentScores  = np.where(es>=0, es/esnull_pos, -es/esnull_neg)
    nEnrichmentNulls = np.where(esnull>=0, esnull/esnull_pos[:,np.newaxis],
                                          -esnull/esnull_neg[:,np.newaxis])

    logging.debug("start to compute fdrs..................................")

    # FDR null distribution histogram
    # create a histogram of all NES(S,pi) over all S and pi
    # Use this null distribution to compute an FDR q value,
    # vals = reduce(lambda x,y: x+y, nEnrichmentNulls, [])
    # nvals = np.array(sorted(vals))
    # or
    nvals = np.sort(nEnrichmentNulls.flatten())
    nnes = np.sort(nEnrichmentScores)
    fdrs = []
    # FDR computation
    for i in range(len(enrichment_scores)):
        nes = nEnrichmentScores[i]
        # use the same pval method to calculate fdr
        if nes >= 0:
            allPos = int(len(nvals) - np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(len(nvals) - np.searchsorted(nvals, nes, side="left"))
            nesPos = len(nnes) - int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = len(nnes) - int(np.searchsorted(nnes, nes, side="left"))
            # allPos = (nvals >= 0).sum()
            # allHigherAndPos = (nvals >= nes).sum()
            # nesPos = (nnes >=0).sum()
            # nesHigherAndPos = (nnes >= nes).sum()
        else:
            allPos = int(np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(np.searchsorted(nvals, nes, side="right"))
            nesPos = int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = int(np.searchsorted(nnes, nes, side="right"))
            # allPos = (nvals < 0).sum()
            # allHigherAndPos = (nvals < nes).sum()
            # nesPos = (nnes < 0).sum()
            # nesHigherAndPos = (nnes < nes).sum()

        try:
            pi_norm = allHigherAndPos/float(allPos)
            pi_obs = nesHigherAndPos/float(nesPos)
            fdr =  pi_norm / pi_obs
            fdrs.append(fdr if fdr < 1 else 1.0)
        except:
            fdrs.append(1000000000.0)

    logging.debug("Statistical testing finished.............................")

    return zip(enrichment_scores, nEnrichmentScores, enrichmentPVals, fdrs)

Compute nominal pvals normalized ES and FDR q value .


def get_marts(self):
        """Get available marts and their names."""

        mart_names = pd.Series(self.names, name="Name")
        mart_descriptions = pd.Series(self.displayNames, name="Description")

        return pd.concat([mart_names, mart_descriptions], axis=1)

def get_marts ( self ) : mart_names = pd . Series ( self . names , name = "Name" ) mart_descriptions = pd . Series ( self . displayNames , name = "Description" ) return pd . concat ( [ mart_names , mart_descriptions ] , axis = 1 )


def get_datasets(self, mart='ENSEMBL_MART_ENSEMBL'):
        """Get available datasets from mart you've selected"""
        datasets = self.datasets(mart, raw=True)
        return pd.read_csv(StringIO(datasets), header=None, usecols=[1, 2],
                            names = ["Name", "Description"],sep="\t")

def get_datasets ( self , mart = 'ENSEMBL_MART_ENSEMBL' ) : datasets = self . datasets ( mart , raw = True ) return pd . read_csv ( StringIO ( datasets ) , header = None , usecols = [ 1 , 2 ] , names = [ "Name" , "Description" ] , sep = "\t" )

Get available datasets from mart you ve selected


def get_attributes(self, dataset):
        """Get available attritbutes from dataset you've selected"""
        attributes = self.attributes(dataset)
        attr_ = [ (k, v[0]) for k, v in attributes.items()]
        return pd.DataFrame(attr_, columns=["Attribute","Description"])

def get_attributes ( self , dataset ) : attributes = self . attributes ( dataset ) attr_ = [ ( k , v [ 0 ] ) for k , v in attributes . items ( ) ] return pd . DataFrame ( attr_ , columns = [ "Attribute" , "Description" ] )

Get available attritbutes from dataset you ve selected


def get_filters(self, dataset):
        """Get available filters from dataset you've selected"""
        filters = self.filters(dataset)
        filt_ = [ (k, v[0]) for k, v in filters.items()]
        return pd.DataFrame(filt_, columns=["Filter", "Description"])

def get_filters ( self , dataset ) : filters = self . filters ( dataset ) filt_ = [ ( k , v [ 0 ] ) for k , v in filters . items ( ) ] return pd . DataFrame ( filt_ , columns = [ "Filter" , "Description" ] )

Get available filters from dataset you ve selected


def query(self, dataset='hsapiens_gene_ensembl', attributes=[],
              filters={}, filename=None):
        """mapping ids using BioMart.

        :param dataset: str, default: 'hsapiens_gene_ensembl'
        :param attributes: str, list, tuple
        :param filters: dict, {'filter name': list(filter value)}
        :param host: www.ensembl.org, asia.ensembl.org, useast.ensembl.org
        :return: a dataframe contains all attributes you selected.

        **Note**: it will take a couple of minutes to get the results.
        A xml template for querying biomart. (see https://gist.github.com/keithshep/7776579)

        exampleTaxonomy = "mmusculus_gene_ensembl"
        exampleGene = "ENSMUSG00000086981,ENSMUSG00000086982,ENSMUSG00000086983"
        urlTemplate = \
        '''http://ensembl.org/biomart/martservice?query=''' \
        '''<?xml version="1.0" encoding="UTF-8"?>''' \
        '''<!DOCTYPE Query>''' \
        '''<Query virtualSchemaName="default" formatter="CSV" header="0" uniqueRows="0" count="" datasetConfigVersion="0.6">''' \
        '''<Dataset name="%s" interface="default"><Filter name="ensembl_gene_id" value="%s"/>''' \
        '''<Attribute name="ensembl_gene_id"/><Attribute name="ensembl_transcript_id"/>''' \
        '''<Attribute name="transcript_start"/><Attribute name="transcript_end"/>''' \
        '''<Attribute name="exon_chrom_start"/><Attribute name="exon_chrom_end"/>''' \
        '''</Dataset>''' \
        '''</Query>'''

        exampleURL = urlTemplate % (exampleTaxonomy, exampleGene)
        req = requests.get(exampleURL, stream=True)

        """
        if not attributes:
            attributes = ['ensembl_gene_id', 'external_gene_name', 'entrezgene', 'go_id']
        # i=0
        # while (self.host is None) and (i < 3):
        #     self.host = self.ghosts[i]
        #     i +=1
        self.new_query()
        # 'mmusculus_gene_ensembl'
        self.add_dataset_to_xml(dataset)
        for at in attributes:
            self.add_attribute_to_xml(at)
        # add filters
        if filters:
            for k, v in filters.items():
                if isinstance(v, list): v = ",".join(v)
                self.add_filter_to_xml(k, v)

        xml_query = self.get_xml()
        results = super(Biomart, self).query(xml_query)
        df = pd.read_csv(StringIO(results), header=None, sep="\t",
                         names=attributes, index_col=None)
        # save file to cache path.
        if filename is None:
            mkdirs(DEFAULT_CACHE_PATH)
            filename = os.path.join(DEFAULT_CACHE_PATH, "{}.background.genes.txt".format(dataset))
        df.to_csv(filename, sep="\t", index=False)

        return df


def gsea(data, gene_sets, cls, outdir='GSEA_', min_size=15, max_size=500, permutation_num=1000,
          weighted_score_type=1,permutation_type='gene_set', method='log2_ratio_of_classes',
	      ascending=False, processes=1, figsize=(6.5,6), format='pdf',
          graph_num=20, no_plot=False, seed=None, verbose=False):
    """ Run Gene Set Enrichment Analysis.

    :param data: Gene expression data table, Pandas DataFrame, gct file.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param cls: A list or a .cls file format required for GSEA.
    :param str outdir: Results output directory.
    :param int permutation_num: Number of permutations for significance computation. Default: 1000.
    :param str permutation_type: Permutation type, "phenotype" for phenotypes, "gene_set" for genes.
    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Default: 500.
    :param float weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:1.
    :param method:  The method used to calculate a correlation or ranking. Default: 'log2_ratio_of_classes'.
                   Others methods are:

                   1. 'signal_to_noise'

                      You must have at least three samples for each phenotype to use this metric.
                      The larger the signal-to-noise ratio, the larger the differences of the means (scaled by the standard deviations);
                      that is, the more distinct the gene expression is in each phenotype and the more the gene acts as a “class marker.”

                   2. 't_test'

                      Uses the difference of means scaled by the standard deviation and number of samples.
                      Note: You must have at least three samples for each phenotype to use this metric.
                      The larger the tTest ratio, the more distinct the gene expression is in each phenotype
                      and the more the gene acts as a “class marker.”

                   3. 'ratio_of_classes' (also referred to as fold change).

                      Uses the ratio of class means to calculate fold change for natural scale data.

                   4. 'diff_of_classes'

                      Uses the difference of class means to calculate fold change for nature scale data

                   5. 'log2_ratio_of_classes'

                      Uses the log2 ratio of class means to calculate fold change for natural scale data.
                      This is the recommended statistic for calculating fold change for log scale data.

    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [6.5,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a GSEA obj. All results store to a dictionary, obj.results,
             where contains::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes}

    """
    gs = GSEA(data, gene_sets, cls, outdir, min_size, max_size, permutation_num,
              weighted_score_type, permutation_type, method, ascending, processes,
               figsize, format, graph_num, no_plot, seed, verbose)
    gs.run()

    return gs

def gsea ( data , gene_sets , cls , outdir = 'GSEA_' , min_size = 15 , max_size = 500 , permutation_num = 1000 , weighted_score_type = 1 , permutation_type = 'gene_set' , method = 'log2_ratio_of_classes' , ascending = False , processes = 1 , figsize = ( 6.5 , 6 ) , format = 'pdf' , graph_num = 20 , no_plot = False , seed = None , verbose = False ) : gs = GSEA ( data , gene_sets , cls , outdir , min_size , max_size , permutation_num , weighted_score_type , permutation_type , method , ascending , processes , figsize , format , graph_num , no_plot , seed , verbose ) gs . run ( ) return gs


def ssgsea(data, gene_sets, outdir="ssGSEA_", sample_norm_method='rank', min_size=15, max_size=2000,
           permutation_num=0, weighted_score_type=0.25, scale=True, ascending=False, processes=1,
           figsize=(7,6), format='pdf', graph_num=20, no_plot=False, seed=None, verbose=False):
    """Run Gene Set Enrichment Analysis with single sample GSEA tool

    :param data: Expression table, pd.Series, pd.DataFrame, GCT file, or .rnk file format.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param outdir: Results output directory.
    :param str sample_norm_method: "Sample normalization method. Choose from {'rank', 'log', 'log_rank'}. Default: rank.

               1. 'rank': Rank your expression data, and transform by 10000*rank_dat/gene_numbers
               2. 'log' : Do not rank, but transform data by log(data + exp(1)), while data = data[data<1] =1.
               3. 'log_rank': Rank your expression data, and transform by log(10000*rank_dat/gene_numbers+ exp(1))
               4. 'custom': Do nothing, and use your own rank value to calculate enrichment score.

    see here: https://github.com/GSEA-MSigDB/ssGSEAProjection-gpmodule/blob/master/src/ssGSEAProjection.Library.R, line 86

    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Default: 2000.
    :param int permutation_num: Number of permutations for significance computation. Default: 0.
    :param str weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:0.25.
    :param bool scale: If True, normalize the scores by number of genes in the gene sets.
    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [7,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a ssGSEA obj.
             All results store to  a dictionary, access enrichment score by obj.resultsOnSamples,
             and normalized enrichment score by obj.res2d.
             if permutation_num > 0, additional results contain::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes, if permutation_num >0}

    """

    ss = SingleSampleGSEA(data, gene_sets, outdir, sample_norm_method, min_size, max_size,
                          permutation_num, weighted_score_type, scale, ascending,
                          processes, figsize, format, graph_num, no_plot, seed, verbose)
    ss.run()
    return ss

def ssgsea ( data , gene_sets , outdir = "ssGSEA_" , sample_norm_method = 'rank' , min_size = 15 , max_size = 2000 , permutation_num = 0 , weighted_score_type = 0.25 , scale = True , ascending = False , processes = 1 , figsize = ( 7 , 6 ) , format = 'pdf' , graph_num = 20 , no_plot = False , seed = None , verbose = False ) : ss = SingleSampleGSEA ( data , gene_sets , outdir , sample_norm_method , min_size , max_size , permutation_num , weighted_score_type , scale , ascending , processes , figsize , format , graph_num , no_plot , seed , verbose ) ss . run ( ) return ss


def prerank(rnk, gene_sets, outdir='GSEA_Prerank', pheno_pos='Pos', pheno_neg='Neg',
            min_size=15, max_size=500, permutation_num=1000, weighted_score_type=1,
            ascending=False, processes=1, figsize=(6.5,6), format='pdf',
            graph_num=20, no_plot=False, seed=None, verbose=False):
    """ Run Gene Set Enrichment Analysis with pre-ranked correlation defined by user.

    :param rnk: pre-ranked correlation table or pandas DataFrame. Same input with ``GSEA`` .rnk file.
    :param gene_sets: Enrichr Library name or .gmt gene sets file or dict of gene sets. Same input with GSEA.
    :param outdir: results output directory.
    :param int permutation_num: Number of permutations for significance computation. Default: 1000.
    :param int min_size: Minimum allowed number of genes from gene set also the data set. Default: 15.
    :param int max_size: Maximum allowed number of genes from gene set also the data set. Defaults: 500.
    :param str weighted_score_type: Refer to :func:`algorithm.enrichment_score`. Default:1.
    :param bool ascending: Sorting order of rankings. Default: False.
    :param int processes: Number of Processes you are going to use. Default: 1.
    :param list figsize: Matplotlib figsize, accept a tuple or list, e.g. [width,height]. Default: [6.5,6].
    :param str format: Matplotlib figure format. Default: 'pdf'.
    :param int graph_num: Plot graphs for top sets of each phenotype.
    :param bool no_plot: If equals to True, no figure will be drawn. Default: False.
    :param seed: Random seed. expect an integer. Default:None.
    :param bool verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Return a Prerank obj. All results store to  a dictionary, obj.results,
             where contains::

                 | {es: enrichment score,
                 |  nes: normalized enrichment score,
                 |  p: P-value,
                 |  fdr: FDR,
                 |  size: gene set size,
                 |  matched_size: genes matched to the data,
                 |  genes: gene names from the data set
                 |  ledge_genes: leading edge genes}

    """
    pre = Prerank(rnk, gene_sets, outdir, pheno_pos, pheno_neg,
                  min_size, max_size, permutation_num, weighted_score_type,
                  ascending, processes, figsize, format, graph_num, no_plot, seed, verbose)
    pre.run()
    return pre

def prerank ( rnk , gene_sets , outdir = 'GSEA_Prerank' , pheno_pos = 'Pos' , pheno_neg = 'Neg' , min_size = 15 , max_size = 500 , permutation_num = 1000 , weighted_score_type = 1 , ascending = False , processes = 1 , figsize = ( 6.5 , 6 ) , format = 'pdf' , graph_num = 20 , no_plot = False , seed = None , verbose = False ) : pre = Prerank ( rnk , gene_sets , outdir , pheno_pos , pheno_neg , min_size , max_size , permutation_num , weighted_score_type , ascending , processes , figsize , format , graph_num , no_plot , seed , verbose ) pre . run ( ) return pre

Run Gene Set Enrichment Analysis with pre - ranked correlation defined by user .


def replot(indir, outdir='GSEA_Replot', weighted_score_type=1,
           min_size=3, max_size=1000, figsize=(6.5,6), graph_num=20, format='pdf', verbose=False):
    """The main function to reproduce GSEA desktop outputs.

    :param indir: GSEA desktop results directory. In the sub folder, you must contain edb file folder.
    :param outdir: Output directory.
    :param float weighted_score_type: weighted score type. choose from {0,1,1.5,2}. Default: 1.
    :param list figsize: Matplotlib output figure figsize. Default: [6.5,6].
    :param str format: Matplotlib output figure format. Default: 'pdf'.
    :param int min_size: Min size of input genes presented in Gene Sets. Default: 3.
    :param int max_size: Max size of input genes presented in Gene Sets. Default: 5000.
                     You are not encouraged to use min_size, or max_size argument in :func:`replot` function.
                     Because gmt file has already been filtered.
    :param verbose: Bool, increase output verbosity, print out progress of your job, Default: False.

    :return: Generate new figures with selected figure format. Default: 'pdf'.

    """
    rep = Replot(indir, outdir, weighted_score_type,
                 min_size, max_size, figsize, graph_num, format, verbose)
    rep.run()

    return

def replot ( indir , outdir = 'GSEA_Replot' , weighted_score_type = 1 , min_size = 3 , max_size = 1000 , figsize = ( 6.5 , 6 ) , graph_num = 20 , format = 'pdf' , verbose = False ) : rep = Replot ( indir , outdir , weighted_score_type , min_size , max_size , figsize , graph_num , format , verbose ) rep . run ( ) return

The main function to reproduce GSEA desktop outputs .


def _set_cores(self):
        """set cpu numbers to be used"""

        cpu_num = cpu_count()-1
        if self._processes > cpu_num:
            cores = cpu_num
        elif self._processes < 1:
            cores = 1
        else:
            cores = self._processes
        # have to be int if user input is float
        self._processes = int(cores)

def
_set_cores
(
self
)
:
cpu_num
=
cpu_count
(
)
1
if
self
.
_processes
>
cpu_num
:
cores
=
cpu_num
elif
self
.
_processes
<
1
:
cores
=
1
else
:
cores
=
self
.
_processes
# have to be int if user input is float
self
.
_processes
=
int
(
cores
)


def load_gmt(self, gene_list, gmt):
        """load gene set dict"""

        if isinstance(gmt, dict):
            genesets_dict = gmt
        elif isinstance(gmt, str):
            genesets_dict = self.parse_gmt(gmt)
        else:
            raise Exception("Error parsing gmt parameter for gene sets")

        subsets = list(genesets_dict.keys())
        self.n_genesets = len(subsets)
        for subset in subsets:
            subset_list = genesets_dict.get(subset)
            if isinstance(subset_list, set):
                subset_list = list(subset_list)
                genesets_dict[subset] = subset_list
            tag_indicator = np.in1d(gene_list, subset_list, assume_unique=True)
            tag_len = tag_indicator.sum()
            if  self.min_size <= tag_len <= self.max_size: continue
            del genesets_dict[subset]

        filsets_num = len(subsets) - len(genesets_dict)
        self._logger.info("%04d gene_sets have been filtered out when max_size=%s and min_size=%s"%(filsets_num, self.max_size, self.min_size))

        if filsets_num == len(subsets):
            self._logger.error("No gene sets passed through filtering condition!!!, try new parameters again!\n" +\
                               "Note: check gene name, gmt file format, or filtering size." )
            sys.exit(0)

        self._gmtdct=genesets_dict
        return genesets_dict


def get_libraries(self, database=''):
        """return active enrichr library name.Offical API """

        lib_url='http://amp.pharm.mssm.edu/%sEnrichr/datasetStatistics'%database
        libs_json = json.loads(requests.get(lib_url).text)
        libs = [lib['libraryName'] for lib in libs_json['statistics']]
        return sorted(libs)

return active enrichr library name . Offical API


def _download_libraries(self, libname):
        """ download enrichr libraries."""
        self._logger.info("Downloading and generating Enrichr library gene sets......")
        s = retry(5)
        # queery string
        ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary'
        query_string = '?mode=text&libraryName=%s'
        # get
        response = s.get( ENRICHR_URL + query_string % libname, timeout=None)
        if not response.ok:
            raise Exception('Error fetching enrichment results, check internet connection first.')
        # reformat to dict and save to disk
        mkdirs(DEFAULT_CACHE_PATH)
        genesets_dict = {}
        outname = "enrichr.%s.gmt"%libname
        gmtout = open(os.path.join(DEFAULT_CACHE_PATH, outname), "w")
        for line in response.iter_lines(chunk_size=1024, decode_unicode='utf-8'):
            line=line.strip()
            k = line.split("\t")[0]
            v = list(map(lambda x: x.split(",")[0], line.split("\t")[2:]))
            genesets_dict.update({ k: v})
            outline = "%s\t\t%s\n"%(k, "\t".join(v))
            gmtout.write(outline)
        gmtout.close()

        return genesets_dict


def _heatmat(self, df, classes, pheno_pos, pheno_neg):
        """only use for gsea heatmap"""
        width = len(classes) if len(classes) >= 6 else  5
        cls_booA =list(map(lambda x: True if x == pheno_pos else False, classes))
        cls_booB =list(map(lambda x: True if x == pheno_neg else False, classes))
        datA = df.loc[:, cls_booA]
        datB = df.loc[:, cls_booB]
        datAB=pd.concat([datA,datB], axis=1)
        self._width = width
        self.heatmat = datAB
        return

def _heatmat ( self , df , classes , pheno_pos , pheno_neg ) : width = len ( classes ) if len ( classes ) >= 6 else 5 cls_booA = list ( map ( lambda x : True if x == pheno_pos else False , classes ) ) cls_booB = list ( map ( lambda x : True if x == pheno_neg else False , classes ) ) datA = df . loc [ : , cls_booA ] datB = df . loc [ : , cls_booB ] datAB = pd . concat ( [ datA , datB ] , axis = 1 ) self . _width = width self . heatmat = datAB return


def _save_results(self, zipdata, outdir, module, gmt, rank_metric, permutation_type):
        """reformat gsea results, and save to txt"""

        res = OrderedDict()
        for gs, gseale, ind, RES in zipdata:
            rdict = OrderedDict()
            rdict['es'] = gseale[0]
            rdict['nes'] = gseale[1]
            rdict['pval'] = gseale[2]
            rdict['fdr'] = gseale[3]
            rdict['geneset_size'] = len(gmt[gs])
            rdict['matched_size'] = len(ind)
            #reformat gene list.
            _genes = rank_metric.index.values[ind]
            rdict['genes'] = ";".join([ str(g).strip() for g in _genes ])

            if self.module != 'ssgsea':
                # extract leading edge genes
                if rdict['es'] > 0:
                    # RES -> ndarray, ind -> list
                    idx = RES.argmax()
                    ldg_pos = list(filter(lambda x: x<= idx, ind))
                elif rdict['es'] < 0:
                    idx = RES.argmin()
                    ldg_pos = list(filter(lambda x: x >= idx, ind))
                else:
                    ldg_pos = ind # es == 0 ?
                rdict['ledge_genes'] = ';'.join(list(map(str,rank_metric.iloc[ldg_pos].index)))

            rdict['RES'] = RES
            rdict['hits_indices'] = ind
            # save to one odict
            res[gs] = rdict
        # save
        self.results  = res
        # save to dataframe
        res_df = pd.DataFrame.from_dict(res, orient='index')
        res_df.index.name = 'Term'
        res_df.drop(['RES','hits_indices'], axis=1, inplace=True)
        res_df.sort_values(by=['fdr','pval'], inplace=True)
        self.res2d = res_df

        if self._outdir is None: return
        out = os.path.join(outdir,'gseapy.{b}.{c}.report.csv'.format(b=module, c=permutation_type))
        if self.module == 'ssgsea':
            out = out.replace(".csv",".txt")
            with open(out, 'a') as f:
                f.write('# normalize enrichment scores by random permutation procedure (GSEA method)\n')
                f.write("# might not proper for publication\n")
                res_df.to_csv(f, sep='\t')
        else:
            res_df.to_csv(out)

        return


def load_data(self, cls_vec):
        """pre-processed the data frame.new filtering methods will be implement here.
        """
        # read data in
        if isinstance(self.data, pd.DataFrame) :
            exprs = self.data.copy()
            # handle index is gene_names
            if exprs.index.dtype == 'O':
                exprs = exprs.reset_index()
        elif os.path.isfile(self.data) :
            # GCT input format?
            if self.data.endswith("gct"):
                exprs = pd.read_csv(self.data, skiprows=1, comment='#',sep="\t")
            else:
                exprs = pd.read_csv(self.data, comment='#',sep="\t")
        else:
            raise Exception('Error parsing gene expression DataFrame!')

        #drop duplicated gene names
        if exprs.iloc[:,0].duplicated().sum() > 0:
            self._logger.warning("Warning: dropping duplicated gene names, only keep the first values")
            exprs.drop_duplicates(subset=exprs.columns[0], inplace=True) #drop duplicate gene_names.
        if exprs.isnull().any().sum() > 0:
            self._logger.warning("Warning: Input data contains NA, filled NA with 0")
            exprs.dropna(how='all', inplace=True) #drop rows with all NAs
            exprs = exprs.fillna(0)
        # set gene name as index
        exprs.set_index(keys=exprs.columns[0], inplace=True)
        # select numberic columns
        df = exprs.select_dtypes(include=[np.number])
        # drop any genes which std ==0
        df_std =  df.groupby(by=cls_vec, axis=1).std()
        df =  df[~df_std.isin([0]).any(axis=1)]
        df = df + 0.00001 # we don't like zeros!!!
        return df

pre - processed the data frame . new filtering methods will be implement here .


def run(self):
        """GSEA main procedure"""

        assert self.permutation_type in ["phenotype", "gene_set"]
        assert self.min_size <= self.max_size

        # Start Analysis
        self._logger.info("Parsing data files for GSEA.............................")
        # phenotype labels parsing
        phenoPos, phenoNeg, cls_vector = gsea_cls_parser(self.classes)
        # select correct expression genes and values.
        dat = self.load_data(cls_vector)
        # data frame must have length > 1
        assert len(dat) > 1
        # ranking metrics calculation.
        dat2 = ranking_metric(df=dat, method=self.method, pos=phenoPos, neg=phenoNeg,
                              classes=cls_vector, ascending=self.ascending)
        self.ranking = dat2
        # filtering out gene sets and build gene sets dictionary
        gmt = self.load_gmt(gene_list=dat2.index.values, gmt=self.gene_sets)

        self._logger.info("%04d gene_sets used for further statistical testing....."% len(gmt))
        self._logger.info("Start to run GSEA...Might take a while..................")
        # cpu numbers
        self._set_cores()
        # compute ES, NES, pval, FDR, RES
        dataset = dat if self.permutation_type =='phenotype' else dat2
        gsea_results,hit_ind,rank_ES, subsets = gsea_compute_tensor(data=dataset, gmt=gmt, n=self.permutation_num,
                                                             weighted_score_type=self.weighted_score_type,
                                                             permutation_type=self.permutation_type,
                                                             method=self.method,
                                                             pheno_pos=phenoPos, pheno_neg=phenoNeg,
                                                             classes=cls_vector, ascending=self.ascending,
                                                             processes=self._processes, seed=self.seed)

        self._logger.info("Start to generate GSEApy reports and figures............")
        res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
        self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                   gmt=gmt, rank_metric=dat2, permutation_type=self.permutation_type)

        # reorder datarame for heatmap
        self._heatmat(df=dat.loc[dat2.index], classes=cls_vector,
                      pheno_pos=phenoPos, pheno_neg=phenoNeg)
        # Plotting
        if not self._noplot:
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format,
                           pheno_pos=phenoPos, pheno_neg=phenoNeg)

        self._logger.info("Congratulations. GSEApy ran successfully.................\n")
        if self._outdir is None:
            self._tmpdir.cleanup()

        return


def run(self):
        """GSEA prerank workflow"""

        assert self.min_size <= self.max_size

        # parsing rankings
        dat2 = self._load_ranking(self.rnk)
        assert len(dat2) > 1

        # cpu numbers
        self._set_cores()
        # Start Analysis
        self._logger.info("Parsing data files for GSEA.............................")
        # filtering out gene sets and build gene sets dictionary
        gmt = self.load_gmt(gene_list=dat2.index.values, gmt=self.gene_sets)

        self._logger.info("%04d gene_sets used for further statistical testing....."% len(gmt))
        self._logger.info("Start to run GSEA...Might take a while..................")
        # compute ES, NES, pval, FDR, RES
        gsea_results, hit_ind,rank_ES, subsets = gsea_compute(data=dat2, n=self.permutation_num, gmt=gmt,
                                                              weighted_score_type=self.weighted_score_type,
                                                              permutation_type='gene_set', method=None,
                                                              pheno_pos=self.pheno_pos, pheno_neg=self.pheno_neg,
                                                              classes=None, ascending=self.ascending,
                                                              processes=self._processes, seed=self.seed)
        self._logger.info("Start to generate gseapy reports, and produce figures...")
        res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
        self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                   gmt=gmt, rank_metric=dat2, permutation_type="gene_sets")

        # Plotting
        if not self._noplot:
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format,
                           pheno_pos=self.pheno_pos, pheno_neg=self.pheno_neg)

        self._logger.info("Congratulations. GSEApy runs successfully................\n")
        if self._outdir is None:
            self._tmpdir.cleanup()

        return


def runSamplesPermu(self, df, gmt=None):
        """Single Sample GSEA workflow with permutation procedure"""

        assert self.min_size <= self.max_size
        mkdirs(self.outdir)
        self.resultsOnSamples = OrderedDict()
        outdir = self.outdir
        # iter throught each sample
        for name, ser in df.iteritems():
            self.outdir = os.path.join(outdir, str(name))
            self._logger.info("Run Sample: %s " % name)
            mkdirs(self.outdir)
            # sort ranking values from high to low or reverse
            dat2 = ser.sort_values(ascending=self.ascending)
            # reset integer index, or caused unwanted problems
            # df.reset_index(drop=True, inplace=True)

            # compute ES, NES, pval, FDR, RES
            gsea_results, hit_ind,rank_ES, subsets = gsea_compute(data=dat2, n=self.permutation_num, gmt=gmt,
                                                                  weighted_score_type=self.weighted_score_type,
                                                                  permutation_type='gene_set', method=None,
                                                                  pheno_pos='', pheno_neg='',
                                                                  classes=None, ascending=self.ascending,
                                                                  processes=self._processes,
                                                                  seed=self.seed, single=True, scale=self.scale)

            # write file
            res_zip = zip(subsets, list(gsea_results), hit_ind, rank_ES)
            self._save_results(zipdata=res_zip, outdir=self.outdir, module=self.module,
                                       gmt=gmt, rank_metric=dat2, permutation_type="gene_sets")
            self.resultsOnSamples[name] = self.res2d.es
            # plotting
            if self._noplot: continue
            self._logger.info("Plotting Sample: %s \n" % name)
            self._plotting(rank_metric=dat2, results=self.results,
                           graph_num=self.graph_num, outdir=self.outdir,
                           figsize=self.figsize, format=self.format)

        # save es, nes to file
        self._save(outdir)

        return


def runSamples(self, df, gmt=None):
        """Single Sample GSEA workflow.
           multiprocessing utility on samples.
        """

        # df.index.values are gene_names
        # Save each sample results to odict
        self.resultsOnSamples = OrderedDict()
        outdir = self.outdir
        # run ssgsea for gct expression matrix
        #multi-threading
        subsets = sorted(gmt.keys())
        tempes=[]
        names=[]
        rankings=[]
        pool = Pool(processes=self._processes)
        for name, ser in df.iteritems():
            #prepare input
            dat = ser.sort_values(ascending=self.ascending)
            rankings.append(dat)
            names.append(name)
            genes_sorted, cor_vec = dat.index.values, dat.values
            rs = np.random.RandomState(self.seed)
            # apply_async
            tempes.append(pool.apply_async(enrichment_score_tensor,
                                           args=(genes_sorted, cor_vec, gmt,
                                               self.weighted_score_type,
                                               self.permutation_num, rs, True,
                                               self.scale)))
        pool.close()
        pool.join()
        # save results and plotting
        for i, temp in enumerate(tempes):
            name, rnk = names[i], rankings[i]
            self._logger.info("Calculate Enrichment Score for Sample: %s "%name)
            es, esnull, hit_ind, RES = temp.get()
            # create results subdir
            self.outdir= os.path.join(outdir, str(name))
            mkdirs(self.outdir)
            # save results
            self.resultsOnSamples[name] = pd.Series(data=es, index=subsets, name=name)
            # plotting
            if self._noplot: continue
            self._logger.info("Plotting Sample: %s \n" % name)
            for i, term in enumerate(subsets):
                term = term.replace('/','_').replace(":","_")
                outfile = '{0}/{1}.{2}.{3}'.format(self.outdir, term, self.module, self.format)
                gseaplot(rank_metric=rnk, term=term,
                         hits_indices=hit_ind[i], nes=es[i], pval=1, fdr=1,
                         RES=RES[i], pheno_pos='', pheno_neg='',
                         figsize=self.figsize, ofname=outfile)
        # save es, nes to file
        self._save(outdir)

        return

Single Sample GSEA workflow . multiprocessing utility on samples .