import os, sys, logging
import argparse as ap

# ------------------------------------
# Main function
# ------------------------------------

# there is a bug in add_argument(required=True), for hacking, don't set metavar='' when required=True,
# or args = argparser.parse_args() will throw bugs!!!

__version__ = '0.9.13'

def main():
    """The Main function/pipeline for GSEApy."""

    # Parse options...
    argparser = prepare_argparser()
    args = argparser.parse_args()
    subcommand = args.subcommand_name

    if subcommand == "replot":
        # reproduce plots using GSEAPY
        from .gsea import Replot
        rep = Replot(indir=args.indir, outdir=args.outdir, weighted_score_type=args.weight,
                     figsize=args.figsize, graph_num=args.graph,
                     format=args.format, verbose=args.verbose)
        rep.run()

    elif subcommand == "gsea":
        # compute using GSEAPY
        from .gsea import GSEA

        gs = GSEA(args.data, args.gmt, args.cls, args.outdir,
                  args.mins, args.maxs, args.n, args.weight,
                  args.type, args.method, args.ascending, args.threads,
                  args.figsize, args.format, args.graph, args.noplot, args.seed, args.verbose)
        gs.run()
    elif subcommand == "prerank":
        from .gsea import Prerank

        pre = Prerank(args.rnk, args.gmt, args.outdir, args.label[0], args.label[1],
                      args.mins, args.maxs, args.n, args.weight, args.ascending, args.threads,
                      args.figsize, args.format, args.graph, args.noplot, args.seed, args.verbose)
        pre.run()

    elif subcommand == "ssgsea":
        from .gsea import SingleSampleGSEA
        ss = SingleSampleGSEA(data=args.data, gene_sets=args.gmt, outdir=args.outdir,
                              sample_norm_method=args.norm,
                              min_size=args.mins, max_size=args.maxs, permutation_num=args.n,
                              weighted_score_type=args.weight, scale=args.scale,
                              ascending=args.ascending, processes=args.threads,
                              figsize=args.figsize, format=args.format, graph_num=args.graph,
                              no_plot=args.noplot, seed=args.seed, verbose=args.verbose)
        ss.run()

    elif subcommand == "enrichr":
        # calling enrichr API
        from .enrichr import Enrichr
        enr = Enrichr(gene_list=args.gene_list, descriptions=args.descrip,
                      gene_sets=args.library, organism=args.organism,
                      outdir=args.outdir, format=args.format, cutoff=args.thresh,
                      background=args.bg, figsize=args.figsize,
                      top_term=args.term, no_plot=args.noplot, verbose=args.verbose)
        enr.run()
    elif subcommand == "biomart":
        from .parser import Biomart
        # read input file or a argument
        name, value = args.filter
        if os.path.isfile(value):
            with open(value, 'r') as val:
                lines = val.readlines()
            value = [ l.strip() for l in lines]
        # run query
        bm = Biomart(host=args.host, verbose=args.verbose)
        bm.query(dataset=args.bg, attributes=args.attrs.split(","),
                 filters={name : value}, filename=args.ofile)
    else:
        argparser.print_help()
        sys.exit(0)

def prepare_argparser():
    """Prepare argparser object. New options will be added in this function first."""
    description = "%(prog)s -- Gene Set Enrichment Analysis in Python"
    epilog = "For command line options of each command, type: %(prog)s COMMAND -h"

    # top-level parser
    argparser = ap.ArgumentParser(description=description, epilog=epilog)
    argparser.add_argument("--version", action="version", version="%(prog)s "+ __version__)
    subparsers = argparser.add_subparsers(dest='subcommand_name') #help="sub-command help")

    # command for 'gsea'
    add_gsea_parser(subparsers)
    # command for 'prerank'
    add_prerank_parser(subparsers)
    # command for 'ssgsea'
    add_singlesample_parser(subparsers)
    # command for 'plot'
    add_plot_parser(subparsers)
    # command for 'enrichr'
    add_enrichr_parser(subparsers)
    # command for 'biomart'
    add_biomart_parser(subparsers)

    return argparser

def add_output_option(parser):
    """output option"""

    parser.add_argument("-o", "--outdir", dest="outdir", type=str, default='GSEApy_reports',
                        metavar='', action="store",
                        help="The GSEApy output directory. Default: the current working directory")
    parser.add_argument("-f", "--format", dest="format", type=str, metavar='', action="store",
                        choices=("pdf", "png", "jpeg", "eps", "svg"), default="pdf",
                        help="File extensions supported by Matplotlib active backend,\
                              choose from {'pdf', 'png', 'jpeg','ps', 'eps','svg'}. Default: 'pdf'.")
    parser.add_argument("--fs", "--figsize", action='store', nargs=2, dest='figsize',
                        metavar=('width', 'height'),type=float, default=(6.5, 6),
                        help="The figsize keyword argument need two parameters to define. Default: (6.5, 6)")
    parser.add_argument("--graph", dest = "graph", action="store", type=int, default=20, metavar='int',
                        help="Numbers of top graphs produced. Default: 20")
    parser.add_argument("--no-plot", action='store_true', dest='noplot', default=False,
                        help="Speed up computing by suppressing the plot output."+\
                             "This is useful only if data are interested. Default: False.")
    parser.add_argument("-v", "--verbose", action="store_true", default=False, dest='verbose',
                        help="Increase output verbosity, print out progress of your job", )

def add_output_group(parser, required=True):
    """output group"""

    output_group = parser.add_mutually_exclusive_group(required=required)
    output_group.add_argument("-o", "--ofile", dest="ofile", type=str, default='GSEApy_reports',
                              help="Output file name. Mutually exclusive with --o-prefix.")
    output_group.add_argument("--o-prefix", dest="ofile", type=str, default='GSEApy_reports',
                              help="Output file prefix. Mutually exclusive with -o/--ofile.")

def add_gsea_parser(subparsers):
    """Add main function 'gsea' argument parsers."""

    argparser_gsea = subparsers.add_parser("gsea", help="Main GSEApy Function: run GSEApy instead of GSEA.")

    # group for input files
    group_input = argparser_gsea.add_argument_group("Input files arguments")
    group_input.add_argument("-d", "--data", dest="data", action="store", type=str, required=True,
                             help="Input gene expression dataset file in txt format.Same with GSEA.")
    group_input.add_argument("-c", "--cls", dest="cls", action="store", type=str, required=True,
                             help="Input class vector (phenotype) file in CLS format. Same with GSEA.")
    group_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    group_input.add_argument("-t", "--permu-type", action="store", dest="type", type=str, metavar='perType',
                             choices=("gene_set", "phenotype"), default="gene_set",
                             help="Permutation type. Same with GSEA, choose from {'gene_set', 'phenotype'}")

    # group for output files
    group_output = argparser_gsea.add_argument_group("Output arguments")
    add_output_option(group_output)

    # group for General options.
    group_opt = argparser_gsea.add_argument_group("GSEA advanced arguments")
    group_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=1000, metavar='nperm',
                           help="Number of random permutations. For calculating esnulls. Default: 1000")
    group_opt.add_argument("--min-size",  dest="mins", action="store", type=int, default=15, metavar='int',
                           help="Min size of input genes presented in Gene Sets. Default: 15")
    group_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=500, metavar='int',
                           help="Max size of input genes presented in Gene Sets. Default: 500")
    group_opt.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                           help='Weighted_score of rank_metrics. For weighting input genes. Choose from {0, 1, 1.5, 2}. Default: 1',)
    group_opt.add_argument("-m", "--method", action="store", dest="method", type=str, metavar='',
                           choices=("signal_to_noise", "t_test", "ratio_of_classes", "diff_of_classes", "log2_ratio_of_classes"),
                           default="log2_ratio_of_classes",
                           help="Methods to calculate correlations of ranking metrics. \
                           Choose from {'signal_to_noise', 't_test', 'ratio_of_classes', 'diff_of_classes','log2_ratio_of_classes'}.\
                           Default: 'log2_ratio_of_classes'")
    group_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                           help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    group_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                           help="Number of random seed. Default: None")
    group_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return

def add_prerank_parser(subparsers):
    """Add function 'prerank' argument parsers."""

    argparser_prerank = subparsers.add_parser("prerank", help="Run GSEApy Prerank tool on preranked gene list.")

    # group for input files
    prerank_input = argparser_prerank.add_argument_group("Input files arguments")
    prerank_input.add_argument("-r", "--rnk", dest="rnk", action="store", type=str, required=True,
                             help="Ranking metric file in .rnk format. Same with GSEA.")
    prerank_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    prerank_input.add_argument("-l", "--label", action='store', nargs=2, dest='label',
                             metavar=('pos', 'neg'), type=str, default=('Pos','Neg'),
                             help="The phenotype label argument need two parameters to define. Default: ('Pos','Neg')")

    # group for output files
    prerank_output = argparser_prerank.add_argument_group("Output arguments")
    add_output_option(prerank_output)

    # group for General options.
    prerank_opt = argparser_prerank.add_argument_group("GSEA advanced arguments")
    prerank_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=1000, metavar='nperm',
                             help="Number of random permutations. For calculating esnulls. Default: 1000")
    prerank_opt.add_argument("--min-size",  dest="mins", action="store", type=int, default=15, metavar='int',
                             help="Min size of input genes presented in Gene Sets. Default: 15")
    prerank_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=500, metavar='int',
                             help="Max size of input genes presented in Gene Sets. Default: 500")
    prerank_opt.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                             help='Weighted_score of rank_metrics. For weighting input genes. Choose from {0, 1, 1.5, 2}. Default: 1',)
    prerank_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                             help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    prerank_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                             help="Number of random seed. Default: None")
    prerank_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return

def add_singlesample_parser(subparsers):
    """Add function 'singlesample' argument parsers."""

    argparser_gsea = subparsers.add_parser("ssgsea", help="Run Single Sample GSEA.")

    # group for input files
    group_input = argparser_gsea.add_argument_group("Input files arguments")
    group_input.add_argument("-d", "--data", dest="data", action="store", type=str, required=True,
                             help="Input gene expression dataset file in txt format. Same with GSEA.")
    group_input.add_argument("-g", "--gmt", dest="gmt", action="store", type=str, required=True,
                             help="Gene set database in GMT format. Same with GSEA.")
    # group for output files
    group_output = argparser_gsea.add_argument_group("Output arguments")
    add_output_option(group_output)

    # group for General options.
    group_opt = argparser_gsea.add_argument_group("Single Sample GSEA advanced arguments")
    group_opt.add_argument("--sn", "--sample-norm", dest = "norm", action="store", type=str,
                           default='rank', metavar='normalize',
                           choices=("rank", "log", "log_rank", "custom"),
                           help="Sample normalization method. Choose from {'rank', 'log', 'log_rank','custom'}. Default: rank")
    group_opt.add_argument("--ns", "--no-scale", action='store_false', dest='scale', default=True,
                           help="If the flag was set, don't normalize the enrichment scores by number of genes.")
    group_opt.add_argument("-n", "--permu-num", dest = "n", action="store", type=int, default=0, metavar='nperm',
                           help="Number of random permutations. For calculating esnulls. Default: 0")
    group_opt.add_argument("--min-size", dest="mins", action="store", type=int, default=15, metavar='int',
                           help="Min size of input genes presented in Gene Sets. Default: 15")
    group_opt.add_argument("--max-size", dest = "maxs", action="store", type=int, default=2000,metavar='int',
                           help="Max size of input genes presented in Gene Sets. Default: 2000")
    group_opt.add_argument("-w", "--weight", action='store', dest='weight', default=0.25, type=float, metavar='weight',
                           help='Weighted_score of rank_metrics. For weighting input genes. Default: 0.25',)
    group_opt.add_argument("-a", "--ascending", action='store_true', dest='ascending', default=False,
                           help='Rank metric sorting order. If the -a flag was chosen, then ascending equals to True. Default: False.')
    group_opt.add_argument("-s", "--seed", dest = "seed", action="store", type=int, default=None, metavar='',
                           help="Number of random seed. Default: None")
    group_opt.add_argument("-p", "--threads", dest = "threads", action="store", type=int, default=1, metavar='procs',
                           help="Number of Processes you are going to use. Default: 1")

    return

def add_plot_parser(subparsers):
    """Add function 'plot' argument parsers."""

    argparser_replot = subparsers.add_parser("replot", help="Reproduce GSEA desktop output figures.")

    group_replot = argparser_replot.add_argument_group("Input arguments")

    group_replot.add_argument("-i", "--indir", action="store", dest="indir", required=True, metavar='GSEA_dir',
                              help="The GSEA desktop results directroy that you want to reproduce the figure ")
    add_output_option(group_replot)
    #add_output_group( argparser_plot )
    group_replot.add_argument("-w", "--weight", action='store', dest='weight', default=1.0, type=float, metavar='float',
                              help='Weighted_score of rank_metrics. Please Use the same value in GSEA. Choose from (0, 1, 1.5, 2),default: 1',)

    return

def add_enrichr_parser(subparsers):
    """Add function 'enrichr' argument parsers."""

    argparser_enrichr = subparsers.add_parser("enrichr", help="Using Enrichr API to perform GO analysis.")

    # group for required options.
    enrichr_opt = argparser_enrichr.add_argument_group("Input arguments")
    enrichr_opt.add_argument("-i", "--input-list", action="store", dest="gene_list", type=str, required=True, metavar='IDs',
                              help="Enrichr uses a list of gene names as input.")
    enrichr_opt.add_argument("-g", "--gene-sets", action="store", dest="library", type=str, required=True, metavar='GMT',
                              help="Enrichr library name(s) required. Separate each name by comma.")
    enrichr_opt.add_argument("--org", "--organism", action="store", dest="organism", type=str, default='',
                             help="Enrichr supported organism name. Default: human. See here: https://amp.pharm.mssm.edu/modEnrichr.")
    enrichr_opt.add_argument("--ds", "--description", action="store", dest="descrip", type=str, default='enrichr', metavar='STRING',
                              help="It is recommended to enter a short description for your list so that multiple lists \
                              can be differentiated from each other if you choose to save or share your list.")
    enrichr_opt.add_argument("--cut", "--cut-off", action="store", dest="thresh", metavar='float', type=float, default=0.05,
                              help="Adjust-Pval cutoff, used for generating plots. Default: 0.05.")
    enrichr_opt.add_argument("--bg", "--background", action="store", dest="bg", default='hsapiens_gene_ensembl', metavar='BGNUM',
                              help="BioMart Dataset name or Background total genes number. Default: None")
    enrichr_opt.add_argument("-t", "--top-term", dest="term", action="store", type=int, default=10, metavar='int',
                              help="Numbers of top terms shown in the plot. Default: 10")
    # enrichr_opt.add_argument("--scale", dest = "scale", action="store", type=float, default=0.5, metavar='float',
    #                          help="scatter dot scale in the dotplot. Default: 0.5")
    # enrichr_opt.add_argument("--no-plot", action='store_true', dest='no_plot', default=False,
    #                           help="Suppress the plot output.This is useful only if data are interested. Default: False.")

    enrichr_output = argparser_enrichr.add_argument_group("Output figure arguments")
    add_output_option(enrichr_output)
    return

def add_biomart_parser(subparsers):
    """Add function 'biomart' argument parsers."""

    argparser_biomart = subparsers.add_parser("biomart", help="Using BioMart API to convert gene ids.")

    # group for required options.
    biomart_opt = argparser_biomart.add_argument_group("Input arguments")
    biomart_opt.add_argument("-f", "--filter", action='store', nargs=2, dest='filter',
                             required=True, metavar=('NAME', 'VALUE'),
                             help="""Which filter to use. Input filter name, and value.
                                     If multi-value required, separate each value by comma.
                                     If value is a txt file, then one ID per row, exclude header.""")
    biomart_opt.add_argument("-a", "--attributes", action="store", dest="attrs", type=str, required=True, metavar='ATTR',
                             help="Which attribute(s) to retrieve. Separate each attr by comma.")
    biomart_opt.add_argument("-o", "--ofile", dest="ofile", type=str, required=True, help="Output file name")
    biomart_opt.add_argument("-d", "--dataset", action="store", dest="bg", type=str, default='hsapiens_gene_ensembl', metavar='DATA',
                             help="Which dataset to use. Default: hsapiens_gene_ensembl")
    biomart_opt.add_argument("--host", action="store", dest="host", type=str, default='www.ensembl.org', metavar='HOST',
                             help="Which host to use. Select from {'www.ensembl.org', 'asia.ensembl.org', 'useast.ensembl.org'}.")
    biomart_opt.add_argument("-m", "--mart", action="store", dest="mart", type=str, metavar='MART',
                             default="ENSEMBL_MART_ENSEMBL", help="Which mart to use. Default: ENSEMBL_MART_ENSEMBL.")
    biomart_opt.add_argument("-v", "--verbose", action="store_true", default=False, dest='verbose',
                             help="Increase output verbosity, print out progress of your job", )

if __name__ == '__main__':
    try:
        main()
    except KeyboardInterrupt:
        sys.stderr.write("User interrupted me! ;-) Bye!\n")
        sys.exit(0)


# -*- coding: utf-8 -*-

import sys, logging
import numpy as np
from functools import reduce
from multiprocessing import Pool
from math import ceil

def enrichment_score(gene_list, correl_vector, gene_set, weighted_score_type=1,
                     nperm=1000, rs=np.random.RandomState(), single=False, scale=False):
    """This is the most important function of GSEApy. It has the same algorithm with GSEA and ssGSEA.

    :param gene_list:       The ordered gene list gene_name_list, rank_metric.index.values
    :param gene_set:        gene_sets in gmt file, please use gsea_gmt_parser to get gene_set.
    :param weighted_score_type:  It's the same with gsea's weighted_score method. Weighting by the correlation
                            is a very reasonable choice that allows significant gene sets with less than perfect coherence.
                            options: 0(classic),1,1.5,2. default:1. if one is interested in penalizing sets for lack of
                            coherence or to discover sets with any type of nonrandom distribution of tags, a value p < 1
                            might be appropriate. On the other hand, if one uses sets with large number of genes and only
                            a small subset of those is expected to be coherent, then one could consider using p > 1.
                            Our recommendation is to use p = 1 and use other settings only if you are very experienced
                            with the method and its behavior.

    :param correl_vector:   A vector with the correlations (e.g. signal to noise scores) corresponding to the genes in
                            the gene list. Or rankings, rank_metric.values
    :param nperm:           Only use this parameter when computing esnull for statistical testing. Set the esnull value
                            equal to the permutation number.
    :param rs:              Random state for initializing gene list shuffling. Default: np.random.RandomState(seed=None)

    :return:

     ES: Enrichment score (real number between -1 and +1)

     ESNULL: Enrichment score calculated from random permutations.

     Hits_Indices: Index of a gene in gene_list, if gene is included in gene_set.

     RES: Numerical vector containing the running enrichment score for all locations in the gene list .

    """

    N = len(gene_list)
    # Test whether each element of a 1-D array is also present in a second array
    # It's more intuitive here than original enrichment_score source code.
    # use .astype to covert bool to integer
    tag_indicator = np.in1d(gene_list, gene_set, assume_unique=True).astype(int)  # notice that the sign is 0 (no tag) or 1 (tag)

    if weighted_score_type == 0 :
        correl_vector = np.repeat(1, N)
    else:
        correl_vector = np.abs(correl_vector)**weighted_score_type

    # get indices of tag_indicator
    hit_ind = np.flatnonzero(tag_indicator).tolist()
    # if used for compute esnull, set esnull equal to permutation number, e.g. 1000
    # else just compute enrichment scores
    # set axis to 1, because we have 2D array
    axis = 1
    tag_indicator = np.tile(tag_indicator, (nperm+1,1))
    correl_vector = np.tile(correl_vector,(nperm+1,1))
    # gene list permutation
    for i in range(nperm): rs.shuffle(tag_indicator[i])
    # np.apply_along_axis(rs.shuffle, 1, tag_indicator)

    Nhint = tag_indicator.sum(axis=axis, keepdims=True)
    sum_correl_tag = np.sum(correl_vector*tag_indicator, axis=axis, keepdims=True)
    # compute ES score, the code below is identical to gsea enrichment_score method.
    no_tag_indicator = 1 - tag_indicator
    Nmiss =  N - Nhint
    norm_tag =  1.0/sum_correl_tag
    norm_no_tag = 1.0/Nmiss

    RES = np.cumsum(tag_indicator * correl_vector * norm_tag - no_tag_indicator * norm_no_tag, axis=axis)

    if scale: RES = RES / N
    if single:
        es_vec = RES.sum(axis=axis)
    else:
        max_ES, min_ES =  RES.max(axis=axis), RES.min(axis=axis)
        es_vec = np.where(np.abs(max_ES) > np.abs(min_ES), max_ES, min_ES)
    # extract values
    es, esnull, RES = es_vec[-1], es_vec[:-1], RES[-1,:]

    return es, esnull, hit_ind, RES

def enrichment_score_tensor(gene_mat, cor_mat, gene_sets, weighted_score_type, nperm=1000,
                            rs=np.random.RandomState(), single=False, scale=False):
    """Next generation algorithm of GSEA and ssGSEA.

        :param gene_mat:        the ordered gene list(vector) with or without gene indices matrix.
        :param cor_mat:         correlation vector or matrix  (e.g. signal to noise scores)
                                corresponding to the genes in the gene list or matrix.
        :param dict gene_sets:  gmt file dict.
        :param float weighted_score_type:     weighting by the correlation.
                                options: 0(classic), 1, 1.5, 2. default:1 for GSEA and 0.25 for ssGSEA.
        :param int nperm:       permutation times.
        :param bool scale:      If True, normalize the scores by number of genes_mat.
        :param bool single:     If True, use ssGSEA algorithm, otherwise use GSEA.
        :param rs:              Random state for initialize gene list shuffling.
                                Default: np.random.RandomState(seed=None)
        :return: a tuple contains::

                 | ES: Enrichment score (real number between -1 and +1), for ssGSEA, set scale eq to True.
                 | ESNULL: Enrichment score calculated from random permutation.
                 | Hits_Indices: Indices of genes if genes are included in gene_set.
                 | RES: The running enrichment score for all locations in the gene list.

    """
    # gene_mat -> 1d: prerank, ssSSEA or 2d: GSEA
    keys = sorted(gene_sets.keys())

    if weighted_score_type == 0:
        # don't bother doing calcuation, just set to 1
        cor_mat = np.ones(cor_mat.shape)
    elif weighted_score_type > 0:
        pass
    else:
        logging.error("Using negative values of weighted_score_type, not allowed")
        sys.exit(0)

    cor_mat = np.abs(cor_mat)
    if cor_mat.ndim ==1:
        # ssGSEA or Prerank
        # genestes->M, genes->N, perm-> axis=2
        N, M = len(gene_mat), len(keys)
        # generate gene hits matrix
        # for 1d ndarray of gene_mat, set assume_unique=True,
        # means the input arrays are both assumed to be unique,
        # which can speed up the calculation.
        tag_indicator = np.vstack([np.in1d(gene_mat, gene_sets[key], assume_unique=True) for key in keys])
        tag_indicator = tag_indicator.astype(int)
        # index of hits
        hit_ind = [ np.flatnonzero(tag).tolist() for tag in tag_indicator ]
        # generate permutated hits matrix
        perm_tag_tensor = np.repeat(tag_indicator, nperm+1).reshape((M,N,nperm+1))
        # shuffle matrix, last matrix is not shuffled when nperm > 0
        if nperm: np.apply_along_axis(lambda x: np.apply_along_axis(rs.shuffle,0,x),1, perm_tag_tensor[:,:,:-1])
        # missing hits
        no_tag_tensor = 1 - perm_tag_tensor
        # calculate numerator, denominator of each gene hits
        rank_alpha = (perm_tag_tensor*cor_mat[np.newaxis,:,np.newaxis])** weighted_score_type

    elif cor_mat.ndim == 2:
        # GSEA
        # 2d ndarray, gene_mat and cor_mat are shuffled already
        # reshape matrix
        cor_mat = cor_mat.T
        # gene_mat is a tuple contains (gene_name, permuate_gene_name_indices)
        genes, genes_ind = gene_mat
        # genestes->M, genes->N, perm-> axis=2
        # don't use assume_unique=True in 2d array when use np.isin().
        # elements in gene_mat are not unique, or will cause unwanted results
        tag_indicator = np.vstack([np.in1d(genes, gene_sets[key], assume_unique=True) for key in keys])
        tag_indicator = tag_indicator.astype(int)
        perm_tag_tensor = np.stack([tag.take(genes_ind).T for tag in tag_indicator], axis=0)
        #index of hits
        hit_ind = [ np.flatnonzero(tag).tolist() for tag in perm_tag_tensor[:,:,-1] ]
        # nohits
        no_tag_tensor = 1 - perm_tag_tensor
        # calculate numerator, denominator of each gene hits
        rank_alpha = (perm_tag_tensor*cor_mat[np.newaxis,:,:])** weighted_score_type
    else:
        logging.error("Program die because of unsupported input")
        sys.exit(0)

    # Nhint = tag_indicator.sum(1)
    # Nmiss =  N - Nhint
    axis=1
    P_GW_denominator = np.sum(rank_alpha, axis=axis, keepdims=True)
    P_NG_denominator = np.sum(no_tag_tensor, axis=axis, keepdims=True)
    REStensor = np.cumsum(rank_alpha / P_GW_denominator - no_tag_tensor / P_NG_denominator, axis=axis)
    # ssGSEA: scale es by gene numbers ?
    # https://gist.github.com/gaoce/39e0907146c752c127728ad74e123b33
    if scale: REStensor = REStensor / len(gene_mat)
    if single:
        #ssGSEA
        esmatrix = REStensor.sum(axis=axis)
    else:
        #GSEA
        esmax, esmin = REStensor.max(axis=axis), REStensor.min(axis=axis)
        esmatrix = np.where(np.abs(esmax)>np.abs(esmin), esmax, esmin)

    es, esnull, RES = esmatrix[:,-1], esmatrix[:,:-1], REStensor[:,:,-1]

    return es, esnull, hit_ind, RES

def ranking_metric_tensor(exprs, method, permutation_num, pos, neg, classes,
                          ascending, rs=np.random.RandomState()):
    """Build shuffled ranking matrix when permutation_type eq to phenotype.

       :param exprs:   gene_expression DataFrame, gene_name indexed.
       :param str method:  calculate correlation or ranking. methods including:
                           1. 'signal_to_noise'.
                           2. 't_test'.
                           3. 'ratio_of_classes' (also referred to as fold change).
                           4. 'diff_of_classes'.
                           5. 'log2_ratio_of_classes'.
       :param int permuation_num: how many times of classes is being shuffled
       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of
                             dataframe belongs to what class of phenotype.
       :param bool ascending:  bool. Sort ascending vs. descending.

       :return:
                returns two 2d ndarray with shape (nperm, gene_num).

                | cor_mat_indices: the indices of sorted and permutated (exclude last row) ranking matrix.
                | cor_mat: sorted and permutated (exclude last row) ranking matrix.

    """
    # S: samples, G: gene number
    G, S = exprs.shape
    # genes = exprs.index.values
    expr_mat = exprs.values.T
    perm_cor_tensor = np.tile(expr_mat, (permutation_num+1,1,1))
    # random shuffle on the first dim, last matrix is not shuffled
    for arr in perm_cor_tensor[:-1]: rs.shuffle(arr)
    classes = np.array(classes)
    pos = classes == pos
    neg = classes == neg
    pos_cor_mean = perm_cor_tensor[:,pos,:].mean(axis=1)
    neg_cor_mean = perm_cor_tensor[:,neg,:].mean(axis=1)
    pos_cor_std = perm_cor_tensor[:,pos,:].std(axis=1, ddof=1)
    neg_cor_std = perm_cor_tensor[:,neg,:].std(axis=1, ddof=1)

    if method == 'signal_to_noise':
        cor_mat = (pos_cor_mean - neg_cor_mean)/(pos_cor_std + neg_cor_std)
    elif method == 't_test':
        denom = 1.0/G
        cor_mat = (pos_cor_mean - neg_cor_mean)/ np.sqrt(denom*pos_cor_std**2 + denom*neg_cor_std**2)
    elif method == 'ratio_of_classes':
        cor_mat = pos_cor_mean / neg_cor_mean
    elif method == 'diff_of_classes':
        cor_mat  = pos_cor_mean - neg_cor_mean
    elif method == 'log2_ratio_of_classes':
        cor_mat  =  np.log2(pos_cor_mean / neg_cor_mean)
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    # return matix[nperm+1, perm_cors]
    cor_mat_ind = cor_mat.argsort()
    # ndarray: sort in place
    cor_mat.sort()
    # genes_mat = genes.take(cor_mat_ind)
    if ascending: return cor_mat_ind, cor_mat
    # descending order of ranking and genes
    # return genes_mat[:,::-1], cor_mat[:,::-1]
    return cor_mat_ind[:, ::-1], cor_mat[:, ::-1]

def ranking_metric(df, method, pos, neg, classes, ascending):
    """The main function to rank an expression table.

       :param df:      gene_expression DataFrame.
       :param method:  The method used to calculate a correlation or ranking. Default: 'log2_ratio_of_classes'.
                       Others methods are:

                       1. 'signal_to_noise'

                          You must have at least three samples for each phenotype to use this metric.
                          The larger the signal-to-noise ratio, the larger the differences of the means (scaled by the standard deviations);
                          that is, the more distinct the gene expression is in each phenotype and the more the gene acts as a “class marker.”

                       2. 't_test'

                          Uses the difference of means scaled by the standard deviation and number of samples.
                          Note: You must have at least three samples for each phenotype to use this metric.
                          The larger the tTest ratio, the more distinct the gene expression is in each phenotype
                          and the more the gene acts as a “class marker.”

                       3. 'ratio_of_classes' (also referred to as fold change).

                          Uses the ratio of class means to calculate fold change for natural scale data.

                       4. 'diff_of_classes'

                          Uses the difference of class means to calculate fold change for natural scale data

                       5. 'log2_ratio_of_classes'

                          Uses the log2 ratio of class means to calculate fold change for natural scale data.
                          This is the recommended statistic for calculating fold change for log scale data.

       :param str pos: one of labels of phenotype's names.
       :param str neg: one of labels of phenotype's names.
       :param list classes:  a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
       :param bool ascending:  bool or list of bool. Sort ascending vs. descending.
       :return:

            returns a pd.Series of correlation to class of each variable. Gene_name is index, and value is rankings.

            visit here for more docs: http://software.broadinstitute.org/gsea/doc/GSEAUserGuideFrame.html
    """

    # exclude any zero stds.
    df_mean = df.groupby(by=classes, axis=1).mean()
    df_std =  df.groupby(by=classes, axis=1).std()

    if method == 'signal_to_noise':
        ser = (df_mean[pos] - df_mean[neg])/(df_std[pos] + df_std[neg])
    elif method == 't_test':
        ser = (df_mean[pos] - df_mean[neg])/ np.sqrt(df_std[pos]**2/len(df_std)+df_std[neg]**2/len(df_std) )
    elif method == 'ratio_of_classes':
        ser = df_mean[pos] / df_mean[neg]
    elif method == 'diff_of_classes':
        ser  = df_mean[pos] - df_mean[neg]
    elif method == 'log2_ratio_of_classes':
        ser  =  np.log2(df_mean[pos] / df_mean[neg])
    else:
        logging.error("Please provide correct method name!!!")
        sys.exit(0)
    ser = ser.sort_values(ascending=ascending)

    return ser

def gsea_compute_tensor(data, gmt, n, weighted_score_type, permutation_type,
                 method, pheno_pos, pheno_neg, classes, ascending,
                 processes=1, seed=None, single=False, scale=False):
    """compute enrichment scores and enrichment nulls.

        :param data: preprocessed expression dataframe or a pre-ranked file if prerank=True.
        :param dict gmt: all gene sets in .gmt file. need to call load_gmt() to get results.
        :param int n: permutation number. default: 1000.
        :param str method: ranking_metric method. see above.
        :param str pheno_pos: one of labels of phenotype's names.
        :param str pheno_neg: one of labels of phenotype's names.
        :param list classes: a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
        :param float weighted_score_type: default:1
        :param bool ascending: sorting order of rankings. Default: False.
        :param seed: random seed. Default: np.random.RandomState()
        :param bool scale: if true, scale es by gene number.

        :return: a tuple contains::

                | zipped results of es, nes, pval, fdr.
                | nested list of hit indices of input gene_list.
                | nested list of ranked enrichment score of each input gene_sets.
                | list of enriched terms

    """
    w = weighted_score_type
    subsets = sorted(gmt.keys())
    rs = np.random.RandomState(seed)
    genes_mat, cor_mat = data.index.values, data.values
    base = 5 if data.shape[0] >= 5000 else 10
    block = ceil(len(subsets) / base)

    if permutation_type == "phenotype":
        # shuffling classes and generate random correlation rankings
        logging.debug("Start to permutate classes..............................")
        genes_ind = []
        cor_mat = []
        temp_rnk = []
        pool_rnk = Pool(processes=processes)
        # split large array into smaller blocks to avoid memory overflow
        i=1
        while i <= block:
            rs = np.random.RandomState(seed)
            temp_rnk.append(pool_rnk.apply_async(ranking_metric_tensor,
                                         args=(data, method, base, pheno_pos, pheno_neg, classes,
                                                ascending, rs)))
            i +=1
        pool_rnk.close()
        pool_rnk.join()

        for k, temp in enumerate(temp_rnk):
            gi, cor = temp.get()
            if k+1 == block:
               genes_ind.append(gi)
               cor_mat.append(cor)
            else:
                genes_ind.append(gi[:-1])
                cor_mat.append(cor[:-1])

        genes_ind, cor_mat = np.vstack(genes_ind), np.vstack(cor_mat)
        # convert to tuple
        genes_mat = (data.index.values, genes_ind)

    logging.debug("Start to compute es and esnulls........................")
    # Prerank, ssGSEA, GSEA
    es = []
    RES = []
    hit_ind = []
    esnull = []
    temp_esnu = []
    pool_esnu = Pool(processes=processes)
    # split large array into smaller blocks to avoid memory overflow
    i, m = 1, 0
    while i <= block:
        # you have to reseed, or all your processes are sharing the same seed value
        rs = np.random.RandomState(seed)
        gmtrim = {k: gmt.get(k) for k in subsets[m:base * i]}
        temp_esnu.append(pool_esnu.apply_async(enrichment_score_tensor,
                                               args=(genes_mat, cor_mat,
                                                     gmtrim, w, n, rs,
                                                     single, scale)))
        m = base * i
        i += 1
    pool_esnu.close()
    pool_esnu.join()
    # esn is a list, don't need to use append method.
    for si, temp in enumerate(temp_esnu):
        e, enu, hit, rune = temp.get()
        esnull.append(enu)
        es.append(e)
        RES.append(rune)
        hit_ind += hit
    # concate results
    es, esnull, RES = np.hstack(es), np.vstack(esnull), np.vstack(RES)

    return gsea_significance(es, esnull), hit_ind, RES, subsets

def gsea_compute(data, gmt, n, weighted_score_type, permutation_type,
                 method, pheno_pos, pheno_neg, classes, ascending,
                 processes=1, seed=None, single=False, scale=False):
    """compute enrichment scores and enrichment nulls.

        :param data: preprocessed expression dataframe or a pre-ranked file if prerank=True.
        :param dict gmt: all gene sets in .gmt file. need to call load_gmt() to get results.
        :param int n: permutation number. default: 1000.
        :param str method: ranking_metric method. see above.
        :param str pheno_pos: one of labels of phenotype's names.
        :param str pheno_neg: one of labels of phenotype's names.
        :param list classes: a list of phenotype labels, to specify which column of dataframe belongs to what category of phenotype.
        :param float weighted_score_type: default:1
        :param bool ascending: sorting order of rankings. Default: False.
        :param seed: random seed. Default: np.random.RandomState()
        :param bool scale: if true, scale es by gene number.

        :return: a tuple contains::

                | zipped results of es, nes, pval, fdr.
                | nested list of hit indices of input gene_list.
                | nested list of ranked enrichment score of each input gene_sets.
                | list of enriched terms

    """

    w = weighted_score_type
    subsets = sorted(gmt.keys())
    es = []
    RES=[]
    hit_ind=[]
    esnull = [ [] for a in range(len(subsets)) ]

    logging.debug("Start to compute enrichment scores......................")

    if permutation_type == "phenotype":
        logging.debug("Start to permutate classes..............................")
        # shuffling classes and generate random correlation rankings
        rs = np.random.RandomState(seed)
        genes_mat, cor_mat = ranking_metric_tensor(exprs=data, method=method,
                                                   permutation_num=n,
                                                   pos=pheno_pos, neg=pheno_neg,
                                                   classes=classes,
                                                   ascending=ascending, rs=rs)

        # compute es, esnulls. hits, RES
        logging.debug("Start to compute enrichment nulls.......................")
        es, esnull, hit_ind, RES = enrichment_score_tensor(gene_mat=genes_mat,
                                                           cor_mat=cor_mat,
                                                           gene_sets=gmt,
                                                           weighted_score_type=w,
                                                           nperm=n, rs=rs,
                                                           single=False, scale=False,)

    else:
        # Prerank, ssGSEA, GSEA with gene_set permutation
        gl, cor_vec = data.index.values, data.values
        logging.debug("Start to compute es and esnulls........................")
        # es, esnull, hit_ind, RES = enrichment_score_tensor(gene_mat=gl,
        #                                                    cor_mat=cor_mat,
        #                                                    gene_sets=gmt,
        #                                                    weighted_score_type=w,
        #                                                    nperm=n, rs=rs
        #                                                    single=single, scale=scale)

        # split large array into smaller blocks to avoid memory overflow
        temp_esnu=[]
        pool_esnu = Pool(processes=processes)
        for subset in subsets:
            # you have to reseed, or all your processes are sharing the same seed value
            rs = np.random.RandomState(seed)
            temp_esnu.append(pool_esnu.apply_async(enrichment_score,
                                                   args=(gl, cor_vec, gmt.get(subset), w,
                                                         n, rs, single, scale)))

        pool_esnu.close()
        pool_esnu.join()
        # esn is a list, don't need to use append method.
        for si, temp in enumerate(temp_esnu):
            e, enu, hit, rune = temp.get()
            esnull[si] = enu
            es.append(e)
            RES.append(rune)
            hit_ind.append(hit)

    return gsea_significance(es, esnull), hit_ind, RES, subsets

def gsea_pval(es, esnull):
    """Compute nominal p-value.

    From article (PNAS):
    estimate nominal p-value for S from esnull by using the positive
    or negative portion of the distribution corresponding to the sign
    of the observed ES(S).
    """

    # to speed up, using numpy function to compute pval in parallel.
    condlist = [ es < 0, es >=0]
    choicelist = [np.sum(esnull < es.reshape(len(es),1), axis=1)/ np.sum(esnull < 0, axis=1),
                  np.sum(esnull >= es.reshape(len(es),1), axis=1)/ np.sum(esnull >= 0, axis=1)]
    pval = np.select(condlist, choicelist)

    return pval

def normalize(es, esnull):
    """normalize the ES(S,pi) and the observed ES(S), separately rescaling
       the positive and negative scores by dividing the mean of the ES(S,pi).

       return: NES, NESnull
    """

    nEnrichmentScores =np.zeros(es.shape)
    nEnrichmentNulls=np.zeros(esnull.shape)

    esnull_pos = (esnull * (esnull >= 0)).mean(axis=1)
    esnull_neg = (esnull * (esnull < 0)).mean(axis=1)
    # calculate nESnulls
    for i in range(esnull.shape[0]):
        # NES
        if es[i] >= 0:
            nEnrichmentScores[i] = es[i] / esnull_pos[i]
        else:
            nEnrichmentScores[i] = - es[i] / esnull_neg[i]

        # NESnull
        for j in range(esnull.shape[1]):
            if esnull[i,j] >= 0:
                nEnrichmentNulls[i,j] = esnull[i,j] / esnull_pos[i]
            else:
                nEnrichmentNulls[i,j] = - esnull[i,j] / esnull_neg[i]

    return nEnrichmentScores, nEnrichmentNulls

def gsea_significance(enrichment_scores, enrichment_nulls):
    """Compute nominal pvals, normalized ES, and FDR q value.

        For a given NES(S) = NES* >= 0. The FDR is the ratio of the percentage of all (S,pi) with
        NES(S,pi) >= 0, whose NES(S,pi) >= NES*, divided by the percentage of
        observed S wih NES(S) >= 0, whose NES(S) >= NES*, and similarly if NES(S) = NES* <= 0.
    """
    # For a zero by zero division (undetermined, results in a NaN),
    np.seterr(divide='ignore', invalid='ignore')
    # import warnings
    # warnings.simplefilter("ignore")
    es = np.array(enrichment_scores)
    esnull = np.array(enrichment_nulls)
    logging.debug("Start to compute pvals..................................")
    # compute pvals.
    enrichmentPVals = gsea_pval(es, esnull).tolist()

    logging.debug("Compute nes and nesnull.................................")
    # nEnrichmentScores, nEnrichmentNulls = normalize(es, esnull)

    # new normalized enrichment score implementation.
    # this could speed up significantly.
    esnull_pos = (esnull*(esnull>=0)).mean(axis=1)
    esnull_neg = (esnull*(esnull<0)).mean(axis=1)
    nEnrichmentScores  = np.where(es>=0, es/esnull_pos, -es/esnull_neg)
    nEnrichmentNulls = np.where(esnull>=0, esnull/esnull_pos[:,np.newaxis],
                                          -esnull/esnull_neg[:,np.newaxis])

    logging.debug("start to compute fdrs..................................")

    # FDR null distribution histogram
    # create a histogram of all NES(S,pi) over all S and pi
    # Use this null distribution to compute an FDR q value,
    # vals = reduce(lambda x,y: x+y, nEnrichmentNulls, [])
    # nvals = np.array(sorted(vals))
    # or
    nvals = np.sort(nEnrichmentNulls.flatten())
    nnes = np.sort(nEnrichmentScores)
    fdrs = []
    # FDR computation
    for i in range(len(enrichment_scores)):
        nes = nEnrichmentScores[i]
        # use the same pval method to calculate fdr
        if nes >= 0:
            allPos = int(len(nvals) - np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(len(nvals) - np.searchsorted(nvals, nes, side="left"))
            nesPos = len(nnes) - int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = len(nnes) - int(np.searchsorted(nnes, nes, side="left"))
            # allPos = (nvals >= 0).sum()
            # allHigherAndPos = (nvals >= nes).sum()
            # nesPos = (nnes >=0).sum()
            # nesHigherAndPos = (nnes >= nes).sum()
        else:
            allPos = int(np.searchsorted(nvals, 0, side="left"))
            allHigherAndPos = int(np.searchsorted(nvals, nes, side="right"))
            nesPos = int(np.searchsorted(nnes, 0, side="left"))
            nesHigherAndPos = int(np.searchsorted(nnes, nes, side="right"))
            # allPos = (nvals < 0).sum()
            # allHigherAndPos = (nvals < nes).sum()
            # nesPos = (nnes < 0).sum()
            # nesHigherAndPos = (nnes < nes).sum()

        try:
            pi_norm = allHigherAndPos/float(allPos)
            pi_obs = nesHigherAndPos/float(nesPos)
            fdr =  pi_norm / pi_obs
            fdrs.append(fdr if fdr < 1 else 1.0)
        except:
            fdrs.append(1000000000.0)

    logging.debug("Statistical testing finished.............................")

    return zip(enrichment_scores, nEnrichmentScores, enrichmentPVals, fdrs)


import os, errno, logging
import requests
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from os.path import expanduser

DEFAULT_CACHE_PATH = os.path.join(expanduser("~"), ".gseapy")

def unique(seq):
    """Remove duplicates from a list in Python while preserving order.

    :param seq: a python list object.
    :return: a list without duplicates while preserving order.

    """

    seen = set()
    seen_add = seen.add
    """
    The fastest way to sovle this problem is here
    Python is a dynamic language, and resolving seen.add each iteration
    is more costly than resolving a local variable. seen.add could have
    changed between iterations, and the runtime isn't smart enough to rule
    that out. To play it safe, it has to check the object each time.
    """

    return [x for x in seq if x not in seen and not seen_add(x)]

def mkdirs(outdir):

    try:
        os.makedirs(outdir)
    except OSError as exc:
        if exc.errno != errno.EEXIST:
            raise exc
        pass

def log_init(outlog, log_level=logging.INFO):
    """logging start"""

    # clear old root logger handlers
    logging.getLogger("gseapy").handlers = []
    # init a root logger
    logging.basicConfig(level    = logging.DEBUG,
                        format   = 'LINE %(lineno)-4d: %(asctime)s [%(levelname)-8s] %(message)s',
                        filename = outlog,
                        filemode = 'w')

    # define a Handler which writes INFO messages or higher to the sys.stderr
    console = logging.StreamHandler()
    console.setLevel(log_level)
    # set a format which is simpler for console use
    formatter = logging.Formatter('%(asctime)s %(message)s')
    # tell the handler to use this format
    console.setFormatter(formatter)
    # add handlers
    logging.getLogger("gseapy").addHandler(console)
    logger = logging.getLogger("gseapy")
    # logger.setLevel(log_level)
    return logger

def log_stop(logger):
    """log stop"""

    handlers = logger.handlers[:]
    for handler in handlers:
        handler.close()
        logger.removeHandler(handler)

def retry(num=5):
    """"retry connection.

        define max tries num
        if the backoff_factor is 0.1, then sleep() will sleep for
        [0.1s, 0.2s, 0.4s, ...] between retries.
        It will also force a retry if the status code returned is 500, 502, 503 or 504.

    """
    s = requests.Session()
    retries = Retry(total=num, backoff_factor=0.1,
                    status_forcelist=[500, 502, 503, 504])
    s.mount('http://', HTTPAdapter(max_retries=retries))

    return s

# CONSTANT
DEFAULT_LIBRARY=['GO_Biological_Process_2013',
				 'GO_Biological_Process_2015',
				 'GO_Cellular_Component_2013',
				 'GO_Cellular_Component_2015',
				 'GO_Molecular_Function_2013',
				 'GO_Molecular_Function_2015',
				 'GeneSigDB',
				 'HumanCyc_2015',
				 'Human_Gene_Atlas',
				 'Human_Phenotype_Ontology',
				 'Humancyc_2016',
				 'KEGG_2013',
				 'KEGG_2015',
				 'KEGG_2016',
				 'MGI_Mammalian_Phenotype_2013',
				 'MGI_Mammalian_Phenotype_Level_3',
				 'MGI_Mammalian_Phenotype_Level_4',
				 'MSigDB_Computational',
				 'MSigDB_Oncogenic_Signatures',
				 'Mouse_Gene_Atlas',
				 'Panther_2015',
				 'Panther_2016',
				 'Reactome_2013',
				 'Reactome_2015',
				 'Reactome_2016',
				 'WikiPathways_2013',
				 'WikiPathways_2015',
				 'WikiPathways_2016']


# -*- coding: utf-8 -*-

import sys, logging, json, os
import requests
import pandas as pd
from io import StringIO
from numpy import in1d
from requests.packages.urllib3.util.retry import Retry
from requests.adapters import HTTPAdapter
from bioservices import BioMart, BioServicesError
from gseapy.utils import unique, DEFAULT_LIBRARY, DEFAULT_CACHE_PATH, mkdirs

def gsea_cls_parser(cls):
    """Extract class(phenotype) name from .cls file.

    :param cls: the a class list instance or .cls file which is identical to GSEA input .
    :return: phenotype name and a list of class vector.
    """

    if isinstance(cls, list) :
        classes = cls
        sample_name= unique(classes)
    elif isinstance(cls, str) :
        with open(cls) as c:
            file = c.readlines()
        classes = file[2].strip('\n').split(" ")
        sample_name = file[1].lstrip("# ").strip('\n').split(" ")
    else:
        raise Exception('Error parsing sample name!')

    return sample_name[0], sample_name[1], classes

def gsea_edb_parser(results_path, index=0):
    """Parse results.edb file stored under **edb** file folder.

    :param results_path: the .results file located inside edb folder.
    :param index: gene_set index of gmt database, used for iterating items.
    :return: enrichment_term, hit_index,nes, pval, fdr.
    """
    from bs4 import BeautifulSoup

    soup = BeautifulSoup(open(results_path), features='xml')
    tag = soup.findAll('DTG')
    term = dict(tag[index].attrs)
    # dict_keys(['RANKED_LIST', 'GENESET', 'FWER', 'ES_PROFILE',
    # 'HIT_INDICES', 'ES', 'NES', 'TEMPLATE', 'RND_ES', 'RANK_SCORE_AT_ES',
    # 'NP', 'RANK_AT_ES', 'FDR'])
    enrich_term = term.get('GENESET').split("#")[1]
    es_profile = term.get('ES_PROFILE').split(" ")
    # rank_es = term.get('RND_ES').split(" ")
    hit_ind =term.get('HIT_INDICES').split(" ")
    es_profile = [float(i) for i in es_profile ]
    hit_ind = [float(i) for i in hit_ind ]
    #r ank_es = [float(i) for i in rank_es ]
    nes = term.get('NES')
    pval = term.get('NP')
    fdr =  term.get('FDR')
    # fwer = term.get('FWER')
    # index_range = len(tag)-1
    logging.debug("Enriched Gene set is: "+ enrich_term)

    return enrich_term, hit_ind, nes, pval, fdr

def gsea_gmt_parser(gmt, min_size = 3, max_size = 1000, gene_list=None):
    """Parse gene_sets.gmt(gene set database) file or download from enrichr server.

    :param gmt: the gene_sets.gmt file of GSEA input or an enrichr library name.
                checkout full enrichr library name here: http://amp.pharm.mssm.edu/Enrichr/#stats

    :param min_size: Minimum allowed number of genes from gene set also the data set. Default: 3.
    :param max_size: Maximum allowed number of genes from gene set also the data set. Default: 5000.
    :param gene_list: Used for filtering gene set. Only used this argument for :func:`call` method.
    :return: Return a new filtered gene set database dictionary.

    **DO NOT** filter gene sets, when use :func:`replot`. Because ``GSEA`` Desktop have already
    done this for you.

    """

    if gmt.lower().endswith(".gmt"):
        logging.info("User Defined gene sets is given.......continue..........")
        with open(gmt) as genesets:
             genesets_dict = { line.strip().split("\t")[0]: line.strip().split("\t")[2:]
                              for line in genesets.readlines()}
    else:
        logging.info("Downloading and generating Enrichr library gene sets...")
        if gmt in DEFAULT_LIBRARY:
            names = DEFAULT_LIBRARY
        else:
            names = get_library_name()
        if gmt in names:
            """
            define max tries num
            if the backoff_factor is 0.1, then sleep() will sleep for
            [0.1s, 0.2s, 0.4s, ...] between retries.
            It will also force a retry if the status code returned is 500, 502, 503 or 504.
            """
            s = requests.Session()
            retries = Retry(total=5, backoff_factor=0.1,
                            status_forcelist=[ 500, 502, 503, 504 ])
            s.mount('http://', HTTPAdapter(max_retries=retries))
            # queery string
            ENRICHR_URL = 'http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary'
            query_string = '?mode=text&libraryName=%s'
            # get
            response = s.get( ENRICHR_URL + query_string % gmt, timeout=None)
        else:
            raise Exception("gene_set files(.gmt) not found")
        if not response.ok:
            raise Exception('Error fetching enrichment results, check internet connection first.')

        genesets_dict = { line.strip().split("\t")[0]:
                          list(map(lambda x: x.split(",")[0], line.strip().split("\t")[2:]))
                          for line in response.iter_lines(chunk_size=1024, decode_unicode='utf-8')}

    # filtering dict
    if sys.version_info[0] >= 3 :
        genesets_filter =  {k: v for k, v in genesets_dict.items() if len(v) >= min_size and len(v) <= max_size}
    elif sys.version_info[0] == 2:
        genesets_filter =  {k: v for k, v in genesets_dict.iteritems() if len(v) >= min_size and len(v) <= max_size}
    else:
        logging.error("System failure. Please Provide correct input files")
        sys.exit(1)
    if gene_list is not None:
        subsets = sorted(genesets_filter.keys())
        for subset in subsets:
            tag_indicator = in1d(gene_list, genesets_filter.get(subset), assume_unique=True)
            tag_len = sum(tag_indicator)
            if tag_len <= min_size or tag_len >= max_size:
                del genesets_filter[subset]
            else:
                continue
    # some_dict = {key: value for key, value in some_dict.items() if value != value_to_remove}
    # use np.intersect1d() may be faster???
    filsets_num = len(genesets_dict) - len(genesets_filter)
    logging.info("%04d gene_sets have been filtered out when max_size=%s and min_size=%s"%(filsets_num, max_size, min_size))

    if filsets_num == len(genesets_dict):
        logging.error("No gene sets passed throught filtering condition!!!, try new paramters again!\n" +\
                         "Note: Gene names for gseapy is case sensitive." )
        sys.exit(1)
    else:
        return genesets_filter

def get_library_name(database='Human'):
    """return enrichr active enrichr library name.
    :param str database: Select one from { 'Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm' }

    """

    # make a get request to get the gmt names and meta data from Enrichr

    # old code
    # response = requests.get('http://amp.pharm.mssm.edu/Enrichr/geneSetLibrary?mode=meta')
    # gmt_data = response.json()
    # # generate list of lib names
    # libs = []
    # # get library names
    # for inst_gmt in gmt_data['libraries']:
    #     # only include active gmts
    #     if inst_gmt['isActive'] == True:
    #         libs.append(inst_gmt['libraryName'])

    if database not in ['Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm']:
        sys.stderr.write("""No supported database. Please input one of these:
                            'Human', 'Mouse', 'Yeast', 'Fly', 'Fish', 'Worm' """)
        return
    if database in ['Human', 'Mouse']: database=''
    lib_url='http://amp.pharm.mssm.edu/%sEnrichr/datasetStatistics'%database
    libs_json = json.loads(requests.get(lib_url).text)
    libs = [lib['libraryName'] for lib in libs_json['statistics']]

    return sorted(libs)

class Biomart(BioMart):
    """query from BioMart"""
    def __init__(self, host="www.ensembl.org", verbose=False):
        """A wrapper of BioMart() from bioseverices.

        How to query validated dataset, attributes, filters:
        example:
        >>> from gseapy.parser import Biomart
        >>> bm = Biomart(verbose=False, host="asia.ensembl.org")
        >>> ## view validated marts
        >>> marts = bm.get_marts()
        >>> ## view validated dataset
        >>> datasets = bm.get_datasets(mart='ENSEMBL_MART_ENSEMBL')
        >>> ## view validated attributes
        >>> attrs = bm.get_attributes(dataset='hsapiens_gene_ensembl')
        >>> ## view validated filters
        >>> filters = bm.get_filters(dataset='hsapiens_gene_ensembl')
        >>> ## query results
        >>> results = bm.query(dataset='hsapiens_gene_ensembl',
                               attributes=['entrezgene', ‘go_id'],
                               filters={'ensembl_gene_id': [your input list]}
                              )
        """
        super(Biomart, self).__init__(host=host, verbose=verbose)
        hosts=["www.ensembl.org", "asia.ensembl.org", "useast.ensembl.org"]
        # if host not work, select next
        i=0
        while (self.host is None) and (i < 3):
            self.host = hosts[i]
            i +=1

    def get_marts(self):
        """Get available marts and their names."""

        mart_names = pd.Series(self.names, name="Name")
        mart_descriptions = pd.Series(self.displayNames, name="Description")

        return pd.concat([mart_names, mart_descriptions], axis=1)

    def get_datasets(self, mart='ENSEMBL_MART_ENSEMBL'):
        """Get available datasets from mart you've selected"""
        datasets = self.datasets(mart, raw=True)
        return pd.read_csv(StringIO(datasets), header=None, usecols=[1, 2],
                            names = ["Name", "Description"],sep="\t")

    def get_attributes(self, dataset):
        """Get available attritbutes from dataset you've selected"""
        attributes = self.attributes(dataset)
        attr_ = [ (k, v[0]) for k, v in attributes.items()]
        return pd.DataFrame(attr_, columns=["Attribute","Description"])

    def get_filters(self, dataset):
        """Get available filters from dataset you've selected"""
        filters = self.filters(dataset)
        filt_ = [ (k, v[0]) for k, v in filters.items()]
        return pd.DataFrame(filt_, columns=["Filter", "Description"])

    def query(self, dataset='hsapiens_gene_ensembl', attributes=[],
              filters={}, filename=None):
        """mapping ids using BioMart.

        :param dataset: str, default: 'hsapiens_gene_ensembl'
        :param attributes: str, list, tuple
        :param filters: dict, {'filter name': list(filter value)}
        :param host: www.ensembl.org, asia.ensembl.org, useast.ensembl.org
        :return: a dataframe contains all attributes you selected.

        **Note**: it will take a couple of minutes to get the results.
        A xml template for querying biomart. (see https://gist.github.com/keithshep/7776579)

        exampleTaxonomy = "mmusculus_gene_ensembl"
        exampleGene = "ENSMUSG00000086981,ENSMUSG00000086982,ENSMUSG00000086983"
        urlTemplate = \
        '''http://ensembl.org/biomart/martservice?query=''' \
        '''<?xml version="1.0" encoding="UTF-8"?>''' \
        '''<!DOCTYPE Query>''' \
        '''<Query virtualSchemaName="default" formatter="CSV" header="0" uniqueRows="0" count="" datasetConfigVersion="0.6">''' \
        '''<Dataset name="%s" interface="default"><Filter name="ensembl_gene_id" value="%s"/>''' \
        '''<Attribute name="ensembl_gene_id"/><Attribute name="ensembl_transcript_id"/>''' \
        '''<Attribute name="transcript_start"/><Attribute name="transcript_end"/>''' \
        '''<Attribute name="exon_chrom_start"/><Attribute name="exon_chrom_end"/>''' \
        '''</Dataset>''' \
        '''</Query>'''

        exampleURL = urlTemplate % (exampleTaxonomy, exampleGene)
        req = requests.get(exampleURL, stream=True)

        """
        if not attributes:
            attributes = ['ensembl_gene_id', 'external_gene_name', 'entrezgene', 'go_id']
        # i=0
        # while (self.host is None) and (i < 3):
        #     self.host = self.ghosts[i]
        #     i +=1
        self.new_query()
        # 'mmusculus_gene_ensembl'
        self.add_dataset_to_xml(dataset)
        for at in attributes:
            self.add_attribute_to_xml(at)
        # add filters
        if filters:
            for k, v in filters.items():
                if isinstance(v, list): v = ",".join(v)
                self.add_filter_to_xml(k, v)

        xml_query = self.get_xml()
        results = super(Biomart, self).query(xml_query)
        df = pd.read_csv(StringIO(results), header=None, sep="\t",
                         names=attributes, index_col=None)
        # save file to cache path.
        if filename is None:
            mkdirs(DEFAULT_CACHE_PATH)
            filename = os.path.join(DEFAULT_CACHE_PATH, "{}.background.genes.txt".format(dataset))
        df.to_csv(filename, sep="\t", index=False)

        return df