Looks for all KEGG annotatios of an organism in biomart and the respective pathways in KEGG. It can also retrieve links to pathways figures with red labeled genes provided in a dataframe.

    :param organism: a KEGG organism identifier
    :param dataset: a biomaRt dataset
    :param query_attributes: biomaRt query attributes, the name can change but the output should stay in the same order ie. 'ensembl_gene_id','kegg_enzyme'
    :param host: biomaRt_host
    :param links: if True, returns df_links
    :param dfexp: a Pandas dataframe with the following columns: 'ensembl_gene_id', 'log2FC'
    :param kegg_db: a KEGG database as recovered by the databasesKEGG function
    :param database: a biomaRt database, depecrated, default=None.
    :returns df: a Pandas dataframe with the 'KEGGid','pathsIDs','pathName','ensembl_gene_id','kegg_enzyme'
    :returns df_: a matrix with a column for each KEGG pathway for a given organism and the expression values in the respective dfexp in parameter
    :returns fullmatrix: a matrix with a column for each KEGG pathway for a given organism
    :returns df_links: a dataframe with links for each pathway and the links in the dfexp highlighted red (if df_links.

def KEGGmatrix(organism, dataset, query_attributes=['ensembl_gene_id','kegg_enzyme'], host=biomart_host,links=True,dfexp=None,kegg_db=None, database=None ):
    """
    Looks for all KEGG annotatios of an organism in biomart and the respective pathways in KEGG. It can also retrieve links to pathways figures with red labeled genes provided in a dataframe.

    :param organism: a KEGG organism identifier
    :param dataset: a biomaRt dataset
    :param query_attributes: biomaRt query attributes, the name can change but the output should stay in the same order ie. 'ensembl_gene_id','kegg_enzyme'
    :param host: biomaRt_host
    :param links: if True, returns df_links
    :param dfexp: a Pandas dataframe with the following columns: 'ensembl_gene_id', 'log2FC'
    :param kegg_db: a KEGG database as recovered by the databasesKEGG function
    :param database: a biomaRt database, depecrated, default=None.
    :returns df: a Pandas dataframe with the 'KEGGid','pathsIDs','pathName','ensembl_gene_id','kegg_enzyme'
    :returns df_: a matrix with a column for each KEGG pathway for a given organism and the expression values in the respective dfexp in parameter
    :returns fullmatrix: a matrix with a column for each KEGG pathway for a given organism
    :returns df_links: a dataframe with links for each pathway and the links in the dfexp highlighted red (if df_links.
    """
    try:
        # Get all ensembl gene ids and keeg enzyme labels from biomaRt
        #biomaRt = importr("biomaRt")
        #ensemblMart=biomaRt.useMart(database, host=host)
        #ensembl=biomaRt.useDataset(dataset, mart=ensemblMart)
        #biomaRt_output=biomaRt.getBM(attributes=query_attributes,mart=ensembl)
        #biomaRt_output = [tuple([biomaRt_output[j][i] for j in range(biomaRt_output.ncol)]) for i in range(biomaRt_output.nrow)]
        #biomaRt_output = pd.DataFrame(biomaRt_output)
        #biomaRt_output.columns = ['ensembl_gene_id','kegg_enzyme']
        def QueryBioMart(dataset,attributes,host=host):
            server = BiomartServer( host )
            organism=server.datasets[dataset]
            response=organism.search({'attributes':attributes})
            response=response.content.split("\n")
            response=[s.split("\t") for s in response ]
            response=pd.DataFrame(response,columns=attributes)
            return response
        biomaRt_output=QueryBioMart(dataset,query_attributes,host=host)
        biomaRt_output = biomaRt_output[biomaRt_output['kegg_enzyme']!='']
        biomaRt_output.reset_index(inplace=True,drop=True)
        biomaRt_output=biomaRtTOkegg(biomaRt_output)
    except:
        # Do it wiht KEGG
        ec_KEGGid=ecs_idsKEGG(organism)
        KEGGid_ENSid=ensembl_to_kegg(organism,kegg_db)
        biomaRt_output=pd.merge(ec_KEGGid,KEGGid_ENSid,on=['KEGGid'],how="outer")
        biomaRt_output=biomaRt_output.drop(['KEGGid'],axis=1)
        biomaRt_output=biomaRt_output[['ENSid','ec']]
        biomaRt_output.columns=['ensembl_gene_id','kegg_enzyme']


    # Gett all pathways
    df, df_=pathwaysKEGG(organism)
    fullmatrix=df_.copy()

    # Get all KEGG ecs from KEGG
    ecs=ecs_idsKEGG(organism)

    biomaRt_output=pd.merge(biomaRt_output,ecs,left_on=['kegg_enzyme'],right_on=['ec'],how="outer")
    biomaRt_output=biomaRt_output.drop(['ec'],axis=1)

    df=pd.merge(df, biomaRt_output, how="outer",on=['KEGGid']).drop_duplicates()
    df=df[df['KEGGid'].astype(str)!="nan"]
    df=df.sort_values(by=['ensembl_gene_id'])
    #print df.head()
    df=df.drop_duplicates(subset=['KEGGid','pathIDs','pathName','kegg_enzyme' ])
    df=df.reset_index()
    if not isinstance(dfexp, pd.DataFrame):
        print("Returning df and fullmatrix")
        sys.stdout.flush()
        return df, fullmatrix
    else:
        expDic=dfexp[['ensembl_gene_id','log2FC']].dropna()
        expDic=expDic.set_index(['ensembl_gene_id'])
        expDic=expDic.to_dict().get("log2FC")

        dfexp=pd.merge(biomaRt_output, dfexp, how="right",on=['ensembl_gene_id'])

        #expDic=dfexp[['KEGGid','log2FC']].dropna()
        #expDic=expDic.set_index(['KEGGid'])
        #expDic=expDic.to_dict().get("log2FC")

        cols=df_.columns.tolist()
        cols=[ s for s in cols if "KEGGid" not in s ]
        #for c in cols:
        #    df_[c]=df_[c].apply(lambda x: expDic.get(x))

        df_=pd.merge(dfexp,df_,on=['KEGGid'],how="left")
        df_=df_.dropna(subset=['KEGGid','ensembl_gene_id'])
        df_=df_.sort_values(by=cols)

        def get_expression(df_,col,expDic=expDic):
            v=df_[col]
            if str(v) != "nan":
                eID=df_['ensembl_gene_id']
                ex=expDic.get(eID)
            else:
                ex=v
            return ex

        for c in cols:
            df_[c]=df_.apply(get_expression, args=(c,),axis=1)

        if links==True:
            df_links=pd.DataFrame()
            for p in cols:
                dfT=df_.dropna(subset=[p])
                dfT=dfT.dropna(subset=['kegg_enzyme'])
                print(dfT.head())
                dfT=dfT.drop_duplicates(subset=['kegg_enzyme'])
                if len(dfT) > 0:
                    pathway=p.split(":")[1]
                    URL="http://www.kegg.jp/kegg-bin/show_pathway?"+pathway
                    for i in dfT['kegg_enzyme'].tolist():
                        gKEGG=i
                        color="red"
                        text="/"+gKEGG+"%09"+color
                        URL=URL+text
                    print(URL)
                    d={"pathway":pathway, "URL":URL}
                    d=pd.DataFrame(d,index=[0])
                    df_links=pd.concat([df_links,d])
            df_links.reset_index(inplace=True, drop=True)

            return df, df_, fullmatrix, df_links

        else:
            return df, df_, fullmatrix