import os
import sys
import logging

def init_log(logger, filename=None, loglevel=None):
    """
    Initializes the log file in the proper format.

    Arguments:

        filename (str): Path to a file. Or None if logging is to
                         be disabled.
        loglevel (str): Determines the level of the log output.
    """
    template = '[%(asctime)s] %(levelname)-8s: %(name)-25s: %(message)s'
    formatter = logging.Formatter(template)

    if loglevel:
        logger.setLevel(getattr(logging, loglevel))

    # We will always print warnings and higher to stderr
    console = logging.StreamHandler()
    console.setLevel('WARNING')
    console.setFormatter(formatter)

    if filename:
        file_handler = logging.FileHandler(filename, encoding='utf-8')
        if loglevel:
            file_handler.setLevel(getattr(logging, loglevel))
        file_handler.setFormatter(formatter)
        logger.addHandler(file_handler)
    # If no logfile is provided we print all log messages that the user has
    # defined to stderr
    else:
        if loglevel:
            console.setLevel(getattr(logging, loglevel))

    logger.addHandler(console)

def get_log_stream(logger):
    """
    Returns a stream to the root log file.
    If there is no logfile return the stderr log stream

    Returns:
        A stream to the root log file or stderr stream.
    """

    file_stream = None
    log_stream = None
    for handler in logger.handlers:
        if isinstance(handler, logging.FileHandler):
            file_stream = handler.stream
        else:
            log_stream = handler.stream

    if file_stream:
        return file_stream

    return log_stream


import re
import click
import logging

from pprint import pprint as pp

mimnr_pattern = re.compile("[0-9]{6,6}")
entry_pattern = re.compile("\([1,2,3,4]\)")

mim_inheritance_terms = [
        'Autosomal recessive',
        'Autosomal dominant',
        'Digenic recessive',
        'X-linked dominant',
        'X-linked recessive',
        'Y-linked',
        'Mitochondrial'
    ]

TERMS_MAPPER = {
     'Autosomal recessive': 'AR',
     'Autosomal dominant': 'AD',
     'Digenic recessive': 'DR',
     'X-linked dominant': 'XD',
     'X-linked recessive': 'XR',
     'Y-linked': 'Y',
     'Mitochondrial': 'MT'
   }

OMIM_STATUS_MAP = {
    '[':'nondisease',
    '{':'susceptibility',
    '?':'provisional',
}

LOG = logging.getLogger(__name__)

def parse_omim_line(line, header):
    """docstring for parse_omim_2_line"""
    omim_info = dict(zip(header, line.split('\t')))
    return omim_info

def parse_genemap2(lines):
    """Parse the omim source file called genemap2.txt

    Explanation of Phenotype field:
    Brackets, "[ ]", indicate "nondiseases," mainly genetic variations that
    lead to apparently abnormal laboratory test values.

    Braces, "{ }", indicate mutations that contribute to susceptibility to
    multifactorial disorders (e.g., diabetes, asthma) or to susceptibility
    to infection (e.g., malaria).

    A question mark, "?", before the phenotype name indicates that the
    relationship between the phenotype and gene is provisional.
    More details about this relationship are provided in the comment
    field of the map and in the gene and phenotype OMIM entries.

    The number in parentheses after the name of each disorder indicates
    the following:
        (1) the disorder was positioned by mapping of the wildtype gene;
        (2) the disease phenotype itself was mapped;
        (3) the molecular basis of the disorder is known;
        (4) the disorder is a chromosome deletion or duplication syndrome.

    Args:
        lines(iterable(str))

    Yields:
        parsed_entry(dict)
    """
    LOG.info("Parsing the omim genemap2")
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if line.startswith('#'):
            if i < 10:
                if line.startswith('# Chromosome'):
                    header = line[2:].split('\t')
            continue

        if len(line) < 5:
            continue

        parsed_entry = parse_omim_line(line, header)
        parsed_entry['mim_number'] = int(parsed_entry['Mim Number'])
        parsed_entry['raw'] = line

        # This is the approved symbol for the entry
        hgnc_symbol = parsed_entry.get("Approved Symbol")

        # If no approved symbol could be found choose the first of
        # the gene symbols
        gene_symbols = []
        if parsed_entry.get('Gene Symbols'):
            gene_symbols = [symbol.strip() for symbol in parsed_entry['Gene Symbols'].split(',')]
        parsed_entry['hgnc_symbols'] = gene_symbols

        if not hgnc_symbol and gene_symbols:
            hgnc_symbol = gene_symbols[0]

        parsed_entry['hgnc_symbol'] = hgnc_symbol

        # Gene inheritance is a construct. It is the union of all inheritance
        # patterns found in the associated phenotypes
        gene_inheritance = set()

        parsed_phenotypes = []
        # Information about the related phenotypes
        # Each related phenotype is separated by ';'
        for phenotype_info in parsed_entry.get('Phenotypes', '').split(';'):
            if not phenotype_info:
                continue

            phenotype_info = phenotype_info.lstrip()

            # First symbol in description indicates phenotype status
            # If no special symbol is used the phenotype is 'established'
            phenotype_status = OMIM_STATUS_MAP.get(phenotype_info[0], 'established')

            # Skip phenotype entries that not associated to disease
            if phenotype_status == 'nondisease':
                continue

            phenotype_description = ""

            # We will try to save the description
            splitted_info = phenotype_info.split(',')
            for i, text in enumerate(splitted_info):
                # Everything before ([1,2,3])
                # We check if we are in the part where the mim number exists
                match = entry_pattern.search(text)
                if not match:
                    phenotype_description += text
                else:
                    # If we find the end of the entry
                    mimnr_match = mimnr_pattern.search(phenotype_info)
                    # Then if the entry have a mim number we choose that
                    if mimnr_match:
                        phenotype_mim = int(mimnr_match.group())
                    else:
                        phenotype_mim = parsed_entry['mim_number']
                        phenotype_description += text[:-4]
                    break
            # Find the inheritance
            inheritance = set()
            inheritance_text = ','.join(splitted_info[i:])
            for term in mim_inheritance_terms:
                if term in inheritance_text:

                    inheritance.add(TERMS_MAPPER[term])
                    gene_inheritance.add(TERMS_MAPPER[term])

            parsed_phenotypes.append(
                {
                    'mim_number':phenotype_mim,
                    'inheritance': inheritance,
                    'description': phenotype_description.strip('?\{\}'),
                    'status': phenotype_status,
                }
            )

        parsed_entry['phenotypes'] = parsed_phenotypes
        parsed_entry['inheritance'] = gene_inheritance

        yield parsed_entry

def parse_mim2gene(lines):
    """Parse the file called mim2gene

    This file describes what type(s) the different mim numbers have.
    The different entry types are: 'gene', 'gene/phenotype', 'moved/removed',
    'phenotype', 'predominantly phenotypes'
    Where:
        gene: Is a gene entry
        gene/phenotype: This entry describes both a phenotype and a gene
        moved/removed: No explanation needed
        phenotype: Describes a phenotype
        predominantly phenotype: Not clearly established (probably phenotype)

    Args:
        lines(iterable(str)): The mim2gene lines

    Yields:
        parsed_entry(dict)

        {
            "mim_number": int,
            "entry_type": str,
            "entrez_gene_id": int,
            "hgnc_symbol": str,
            "ensembl_gene_id": str,
            "ensembl_transcript_id": str,
        }

    """
    LOG.info("Parsing mim2gene")
    header = ["mim_number", "entry_type", "entrez_gene_id", "hgnc_symbol", "ensembl_gene_id"]
    for i, line in enumerate(lines):
        if line.startswith('#'):
            continue

        if not len(line) > 0:
            continue

        line = line.rstrip()
        parsed_entry = parse_omim_line(line, header)
        parsed_entry['mim_number'] = int(parsed_entry['mim_number'])
        parsed_entry['raw'] = line

        if 'hgnc_symbol' in parsed_entry:
            parsed_entry['hgnc_symbol'] = parsed_entry['hgnc_symbol']

        if parsed_entry.get('entrez_gene_id'):
            parsed_entry['entrez_gene_id'] = int(parsed_entry['entrez_gene_id'])

        if parsed_entry.get('ensembl_gene_id'):
            ensembl_info = parsed_entry['ensembl_gene_id'].split(',')
            parsed_entry['ensembl_gene_id'] = ensembl_info[0].strip()
            if len(ensembl_info) > 1:
                parsed_entry['ensembl_transcript_id'] = ensembl_info[1].strip()

        yield parsed_entry

def parse_omim_morbid(lines):
    """docstring for parse_omim_morbid"""
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if line.startswith('#'):
            if i < 10:
                if line.startswith('# Phenotype'):
                    header = line[2:].split('\t')
        else:
            yield parse_omim_line(line, header)

def parse_mim_titles(lines):
    """Parse the mimTitles.txt file

    This file hold information about the description for each entry in omim.
    There is not information about entry type.
    parse_mim_titles collects the preferred title and maps it to the mim number.

    Args:
        lines(iterable): lines from mimTitles file

    Yields:
        parsed_entry(dict)

        {
        'mim_number': int, # The mim number for entry
        'preferred_title': str, # the preferred title for a entry
        }

    """
    header = ['prefix', 'mim_number', 'preferred_title', 'alternative_title', 'included_title']
    for i,line in enumerate(lines):
        line = line.rstrip()
        if not line.startswith('#'):
            parsed_entry = parse_omim_line(line, header)
            parsed_entry['mim_number'] = int(parsed_entry['mim_number'])
            parsed_entry['preferred_title'] = parsed_entry['preferred_title'].split(';')[0]
            yield parsed_entry

def get_mim_genes(genemap_lines, mim2gene_lines):
    """Get a dictionary with genes and their omim information

    Args:
        genemap_lines(iterable(str))
        mim2gene_lines(iterable(str))

    Returns.
        hgnc_genes(dict): A dictionary with hgnc_symbol as keys

    """
    LOG.info("Get the mim genes")

    genes = {}
    hgnc_genes = {}

    gene_nr = 0
    no_hgnc = 0

    for entry in parse_mim2gene(mim2gene_lines):
        if 'gene' in entry['entry_type']:
            mim_nr = entry['mim_number']
            gene_nr += 1
            if not 'hgnc_symbol' in entry:
                no_hgnc += 1
            else:
                genes[mim_nr] = entry

    LOG.info("Number of genes without hgnc symbol %s", str(no_hgnc))

    for entry in parse_genemap2(genemap_lines):
        mim_number = entry['mim_number']
        inheritance = entry['inheritance']
        phenotype_info = entry['phenotypes']
        hgnc_symbol = entry['hgnc_symbol']
        hgnc_symbols = entry['hgnc_symbols']
        if mim_number in genes:
            genes[mim_number]['inheritance'] = inheritance
            genes[mim_number]['phenotypes'] = phenotype_info
            genes[mim_number]['hgnc_symbols'] = hgnc_symbols

    for mim_nr in genes:
        gene_info = genes[mim_nr]
        hgnc_symbol = gene_info['hgnc_symbol']

        if hgnc_symbol in hgnc_genes:
            existing_info = hgnc_genes[hgnc_symbol]
            if not existing_info['phenotypes']:
                hgnc_genes[hgnc_symbol] = gene_info

        else:
            hgnc_genes[hgnc_symbol] = gene_info

    return hgnc_genes

def get_mim_phenotypes(genemap_lines):
    """Get a dictionary with phenotypes

    Use the mim numbers for phenotypes as keys and phenotype information as
    values.

    Args:
        genemap_lines(iterable(str))

    Returns:
        phenotypes_found(dict): A dictionary with mim_numbers as keys and
        dictionaries with phenotype information as values.

        {
             'description': str, # Description of the phenotype
             'hgnc_symbols': set(), # Associated hgnc symbols
             'inheritance': set(),  # Associated phenotypes
             'mim_number': int, # mim number of phenotype
        }
    """
    # Set with all omim numbers that are phenotypes
    # Parsed from mim2gene.txt
    phenotype_mims = set()

    phenotypes_found = {}

    # Genemap is a file with one entry per gene.
    # Each line hold a lot of information and in specific it
    # has information about the phenotypes that a gene is associated with
    # From this source we collect inheritane patterns and what hgnc symbols
    # a phenotype is associated with
    for entry in parse_genemap2(genemap_lines):
        hgnc_symbol = entry['hgnc_symbol']
        for phenotype in entry['phenotypes']:
            mim_nr = phenotype['mim_number']
            if mim_nr in phenotypes_found:
                phenotype_entry = phenotypes_found[mim_nr]
                phenotype_entry['inheritance'] = phenotype_entry['inheritance'].union(phenotype['inheritance'])
                phenotype_entry['hgnc_symbols'].add(hgnc_symbol)
            else:
                phenotype['hgnc_symbols'] = set([hgnc_symbol])
                phenotypes_found[mim_nr] = phenotype

    return phenotypes_found

@click.command()
@click.option('--morbid', type=click.Path(exists=True))
@click.option('--genemap', type=click.Path(exists=True))
@click.option('--mim2gene', type=click.Path(exists=True))
@click.option('--mim_titles', type=click.Path(exists=True))
@click.option('--phenotypes', is_flag=True)
@click.pass_context
def cli(context, morbid, genemap, mim2gene, mim_titles, phenotypes):
    """Parse the omim files"""
    # if not (morbid and genemap and mim2gene, mim_titles):
    #     print("Please provide all files")
    #     context.abort()

    from scout.utils.handle import get_file_handle
    from pprint import pprint as pp

    print("Morbid file: %s" % morbid)
    print("Genemap file: %s" % genemap)
    print("mim2gene file: %s" % mim2gene)
    print("MimTitles file: %s" % mim_titles)

    if morbid:
        morbid_handle = get_file_handle(morbid)
    if genemap:
        genemap_handle = get_file_handle(genemap)
    if mim2gene:
        mim2gene_handle = get_file_handle(mim2gene)
    if mim_titles:
        mimtitles_handle = get_file_handle(mim_titles)

    mim_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for entry in mim_genes:
        if entry == 'C10orf11':
            pp(mim_genes[entry])

    context.abort()
    if phenotypes:
        if not genemap:
            click.echo("Please provide the genemap file")
            context.abort()
        phenotypes = get_mim_phenotypes(genemap_handle)
        for i,mim_term in enumerate(phenotypes):
            # pp(phenotypes[mim_term])
            pass

    print("Number of phenotypes found: %s" % i)

    context.abort()
    # hgnc_genes = get_mim_genes(genemap_handle, mim2gene_handle)
    # for hgnc_symbol in hgnc_genes:
    #     pp(hgnc_genes[hgnc_symbol])
    # phenotypes = get_mim_phenotypes(genemap_handle, mim2gene_handle, mimtitles_handle)
    # for mim_nr in phenotypes:
    #     pp(phenotypes[mim_nr])

    genes = get_mim_genes(genemap_handle, mim2gene_handle)
    for hgnc_symbol in genes:
        if hgnc_symbol == 'OPA1':
            print(genes[hgnc_symbol])

if __name__ == '__main__':
    cli()


def isfloat(x):
    try:
        a = float(x)
    except ValueError:
        return False
    else:
        return True

def isint(x):
    try:
        a = float(x)
        b = int(a)
    except ValueError:
        return False
    else:
        return a == b

def make_bool(string):
    """Convert a string to boolean"""
    if str(string).lower() in ['yes', 'true', '1', 't']:
        return True
    return False

def convert_number(string):
    """Convert a string to number
    If int convert to int otherwise float

    If not possible return None
    """
    res = None
    if isint(string):
        res = int(string)
    elif isfloat(string):
        res = float(string)
    return res


import logging
from pprint import pprint as pp

import pymongo
import click

LOG = logging.getLogger(__name__)

@click.command('case', short_help='Update a case')
@click.argument('case_id', required=False)
@click.option('--case-name', '-n',
                help="Add/update the display name of case",
)
@click.option('--institute', '-i',
                help="Update what institutes that has access to case",
)
@click.option('--collaborator', '-c',
                help="Add a collaborator to the case",
)
@click.option('--vcf', type=click.Path(exists=True),
              help='path to clinical VCF file to be added')
@click.option('--vcf-sv', type=click.Path(exists=True),
              help='path to clinical SV VCF file to be added')
@click.option('--vcf-cancer', type=click.Path(exists=True),
              help='path to clinical cancer VCF file to be added')
@click.option('--vcf-research', type=click.Path(exists=True),
              help='path to research VCF file to be added')
@click.option('--vcf-sv-research', type=click.Path(exists=True),
              help='path to research VCF with SV variants to be added')
@click.option('--vcf-cancer-research', type=click.Path(exists=True),
              help='path to research VCF with cancer variants to be added')
@click.option('--peddy-ped', type=click.Path(exists=True),
              help='path to outfile .peddy.ped from peddy')
@click.option('--reupload-sv', is_flag=True,
              help='Remove all SVs and re upload from existing files')
@click.option('--rankscore-treshold',
              help='Set a new rank score treshold if desired')
@click.option('--rankmodel-version',
              help='Update the rank model version')
@click.pass_context
def case(context, case_id, case_name, institute, collaborator, vcf, vcf_sv,
         vcf_cancer, vcf_research, vcf_sv_research, vcf_cancer_research, peddy_ped,
         reupload_sv, rankscore_treshold, rankmodel_version):
    """
    Update a case in the database
    """
    adapter = context.obj['adapter']
    if not case_id:
        if not (case_name and institute):
            LOG.info("Please specify which case to update.")
            context.abort
        case_id = "{0}-{1}".format(institute, case_name)
    # Check if the case exists
    case_obj = adapter.case(case_id)

    if not case_obj:
        LOG.warning("Case %s could not be found", case_id)
        context.abort()

    case_changed = False
    if collaborator:
        if not adapter.institute(collaborator):
            LOG.warning("Institute %s could not be found", collaborator)
            context.abort()
        if not collaborator in case_obj['collaborators']:
            case_changed = True
            case_obj['collaborators'].append(collaborator)
            LOG.info("Adding collaborator %s", collaborator)

    if vcf:
        LOG.info("Updating 'vcf_snv' to %s", vcf)
        case_obj['vcf_files']['vcf_snv'] = vcf
        case_changed = True
    if vcf_sv:
        LOG.info("Updating 'vcf_sv' to %s", vcf_sv)
        case_obj['vcf_files']['vcf_sv'] = vcf_sv
        case_changed = True
    if vcf_cancer:
        LOG.info("Updating 'vcf_cancer' to %s", vcf_cancer)
        case_obj['vcf_files']['vcf_cancer'] = vcf_cancer
        case_changed = True
    if vcf_research:
        LOG.info("Updating 'vcf_research' to %s", vcf_research)
        case_obj['vcf_files']['vcf_research'] = vcf_research
        case_changed = True
    if vcf_sv_research:
        LOG.info("Updating 'vcf_sv_research' to %s", vcf_sv_research)
        case_obj['vcf_files']['vcf_sv_research'] = vcf_sv_research
        case_changed = True
    if vcf_cancer_research:
        LOG.info("Updating 'vcf_cancer_research' to %s", vcf_cancer_research)
        case_obj['vcf_files']['vcf_cancer_research'] = vcf_cancer_research
        case_changed = True

    if case_changed:
        adapter.update_case(case_obj)

    if reupload_sv:
        LOG.info("Set needs_check to True for case %s", case_id)
        updates = {'needs_check': True}
        if rankscore_treshold:
            updates['sv_rank_model_version'] = rankmodel_version
        if vcf_sv:
            updates['vcf_files.vcf_sv'] = vcf_sv
        if vcf_sv:
            updates['vcf_files.vcf_sv_research'] = vcf_sv_research

        updated_case = adapter.case_collection.find_one_and_update(
            {'_id':case_id},
            {'$set': updates
            },
            return_document=pymongo.ReturnDocument.AFTER
        )
        rankscore_treshold = rankscore_treshold or updated_case.get("rank_score_threshold", 5)
        # Delete and reload the clinical SV variants
        if updated_case['vcf_files'].get('vcf_sv'):
            adapter.delete_variants(case_id, variant_type='clinical', category='sv')
            adapter.load_variants(updated_case, variant_type='clinical',
                                  category='sv', rank_threshold=rankscore_treshold)
        # Delete and reload research SV variants
        if updated_case['vcf_files'].get('vcf_sv_research'):
            adapter.delete_variants(case_id, variant_type='research', category='sv')
            if updated_case.get('is_research'):
                adapter.load_variants(updated_case, variant_type='research',
                                      category='sv', rank_threshold=rankscore_treshold)


"""scout/load/setup.py

Sets up a scout database.
This means add a default institute, a user and the internal definitions such as gene objects,
transcripts, hpo terms etc

"""
import logging
import datetime
import yaml

from pprint import pprint as pp

from pymongo.errors import (ConnectionFailure, ServerSelectionTimeoutError)

### Import demo files ###
# Resources
from scout.demo.resources import (
    hgnc_reduced_path, exac_reduced_path, transcripts37_reduced_path, transcripts38_reduced_path,
    mim2gene_reduced_path, genemap2_reduced_path, hpogenes_reduced_path, hpo_to_genes_reduced_path,
    hpoterms_reduced_path, hpo_phenotype_to_terms_reduced_path, madeline_path,
    genes37_reduced_path, genes38_reduced_path)

# Gene panel
from scout.demo import panel_path

# Case files
from scout.demo import load_path

# Import the functions to setup scout
from scout.parse.panel import parse_gene_panel

from scout.build import build_institute

from scout.load import (load_hgnc_genes, load_hpo, load_transcripts)

from scout.utils.handle import get_file_handle

from scout.utils.requests import (fetch_mim_files, fetch_hpo_genes, fetch_ensembl_genes,
                                  fetch_ensembl_transcripts, fetch_hgnc, fetch_exac_constraint)

LOG = logging.getLogger(__name__)

def setup_scout(adapter, institute_id='cust000', user_name='Clark Kent',
                user_mail='clark.kent@mail.com', api_key=None, demo=False):
    """docstring for setup_scout"""
    ########################## Delete previous information ##########################
    LOG.info("Deleting previous database")
    for collection_name in adapter.db.collection_names():
        if not collection_name.startswith('system'):
            LOG.info("Deleting collection %s", collection_name)
            adapter.db.drop_collection(collection_name)
    LOG.info("Database deleted")

    ########################## Add a institute ##########################
    #####################################################################
    # Build a institute with id institute_name
    institute_obj = build_institute(
        internal_id=institute_id,
        display_name=institute_id,
        sanger_recipients=[user_mail]
    )

    # Add the institute to database
    adapter.add_institute(institute_obj)

    ########################## Add a User ###############################
    #####################################################################
    # Build a user obj
    user_obj = dict(
                _id=user_mail,
                email=user_mail,
                name=user_name,
                roles=['admin'],
                institutes=[institute_id]
            )

    adapter.add_user(user_obj)

    ### Get the mim information ###

    if not demo:
        # Fetch the mim files
        try:
            mim_files = fetch_mim_files(api_key, mim2genes=True, morbidmap=True, genemap2=True)
        except Exception as err:
            LOG.warning(err)
            raise err
        mim2gene_lines = mim_files['mim2genes']
        genemap_lines = mim_files['genemap2']

        # Fetch the genes to hpo information
        hpo_gene_lines = fetch_hpo_genes()
        # Fetch the latest version of the hgnc information
        hgnc_lines = fetch_hgnc()
        # Fetch the latest exac pli score information
        exac_lines = fetch_exac_constraint()

    else:
        mim2gene_lines = [line for line in get_file_handle(mim2gene_reduced_path)]
        genemap_lines = [line for line in get_file_handle(genemap2_reduced_path)]

        # Fetch the genes to hpo information
        hpo_gene_lines = [line for line in get_file_handle(hpogenes_reduced_path)]
        # Fetch the reduced hgnc information
        hgnc_lines = [line for line in get_file_handle(hgnc_reduced_path)]
        # Fetch the latest exac pli score information
        exac_lines = [line for line in get_file_handle(exac_reduced_path)]

    builds = ['37', '38']
    ################## Load Genes and transcripts #######################
    #####################################################################
    for build in builds:
        # Fetch the ensembl information
        if not demo:
            ensembl_genes = fetch_ensembl_genes(build=build)
        else:
            ensembl_genes = get_file_handle(genes37_reduced_path)
        # load the genes
        hgnc_genes = load_hgnc_genes(
            adapter=adapter,
            ensembl_lines=ensembl_genes,
            hgnc_lines=hgnc_lines,
            exac_lines=exac_lines,
            mim2gene_lines=mim2gene_lines,
            genemap_lines=genemap_lines,
            hpo_lines=hpo_gene_lines,
            build=build,
        )

        # Create a map from ensembl ids to gene objects
        ensembl_genes = {}
        for gene_obj in hgnc_genes:
            ensembl_id = gene_obj['ensembl_id']
            ensembl_genes[ensembl_id] = gene_obj

        # Fetch the transcripts from ensembl
        if not demo:
            ensembl_transcripts = fetch_ensembl_transcripts(build=build)
        else:
            ensembl_transcripts = get_file_handle(transcripts37_reduced_path)
        # Load the transcripts for a certain build
        transcripts = load_transcripts(adapter, ensembl_transcripts, build, ensembl_genes)

    hpo_terms_handle = None
    hpo_to_genes_handle = None
    hpo_disease_handle = None
    if demo:
        hpo_terms_handle = get_file_handle(hpoterms_reduced_path)
        hpo_to_genes_handle = get_file_handle(hpo_to_genes_reduced_path)
        hpo_disease_handle = get_file_handle(hpo_phenotype_to_terms_reduced_path)

    load_hpo(
        adapter=adapter,
        hpo_lines=hpo_terms_handle,
        hpo_gene_lines=hpo_to_genes_handle,
        disease_lines=genemap_lines,
        hpo_disease_lines=hpo_disease_handle
    )

    # If demo we load a gene panel and some case information
    if demo:
        parsed_panel = parse_gene_panel(
            path=panel_path,
            institute='cust000',
            panel_id='panel1',
            version=1.0,
            display_name='Test panel'
        )
        adapter.load_panel(parsed_panel)

        case_handle = get_file_handle(load_path)
        case_data = yaml.load(case_handle)

        adapter.load_case(case_data)

    LOG.info("Creating indexes")
    adapter.load_indexes()
    LOG.info("Scout instance setup successful")


import logging

LOG = logging.getLogger(__name__)

def export_transcripts(adapter, build='37'):
    """Export all transcripts from the database

    Args:
        adapter(scout.adapter.MongoAdapter)
        build(str)

    Yields:
        transcript(scout.models.Transcript)
    """
    LOG.info("Exporting all transcripts")

    for tx_obj in adapter.transcripts(build=build):
        yield tx_obj


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

# python lib:
from calendar import LocaleHTMLCalendar, HTMLCalendar, month_name
from datetime import date
import sys

# django:
from django.conf import settings
from django.core.urlresolvers import reverse
from django.template.loader import render_to_string
from django.utils.dates import WEEKDAYS, WEEKDAYS_ABBR
from django.utils.html import mark_safe

# thirdparties:
import six

# happenings:
from .common import get_now, get_next_and_prev

URL = getattr(settings, "CALENDAR_URL", 'calendar')
URLS_NAMESPACE = getattr(settings, "CALENDAR_URLS_NAMESPACE", 'calendar')

CALENDAR_LOCALE = getattr(settings, 'CALENDAR_LOCALE', 'en_US.utf8')

LEGACY_CALENDAR_HOUR_FORMAT = LEGACY_CALENDAR_TIME_FORMAT = getattr(settings, 'CALENDAR_LEGACY_TIME_FORMAT', "%I:%M%p")

if '%p' in LEGACY_CALENDAR_TIME_FORMAT:
    LEGACY_CALENDAR_HOUR_FORMAT = LEGACY_CALENDAR_TIME_FORMAT.replace('%M', '')

CALENDAR_TIME_FORMAT = getattr(settings, 'CALENDAR_TIME_FORMAT', "TIME_FORMAT")

CALENDAR_HOUR_FORMAT = getattr(settings, 'CALENDAR_HOUR_FORMAT', '')

if not CALENDAR_HOUR_FORMAT:
    CALENDAR_HOUR_FORMAT = CALENDAR_TIME_FORMAT
    if not CALENDAR_HOUR_FORMAT.endswith('_FORMAT'):
        for char in ('i', 's', 'u'):
            CALENDAR_HOUR_FORMAT = CALENDAR_HOUR_FORMAT.replace(':'+char, '')
            CALENDAR_HOUR_FORMAT = CALENDAR_HOUR_FORMAT.replace('.'+char, '')
            CALENDAR_HOUR_FORMAT = CALENDAR_HOUR_FORMAT.replace(char, '')
        CALENDAR_HOUR_FORMAT = CALENDAR_HOUR_FORMAT.strip(':')

class GenericCalendar(HTMLCalendar):
    def __init__(self, year, month, count, all_month_events, firstweekday=0, request=None, base_context=None, *args, **kwargs):
        super(GenericCalendar, self).__init__(firstweekday)
        self.yr = year
        self.mo = month
        self.count = count  # defaultdict in {date:[(title1, pk1), (title2, pk2),]} format
        self.events = all_month_events
        self.request = request
        self.base_context = base_context or {}
        self._context = None

#    def add_occurrence(self):
#        try:
#            self.event.occurrence.append(self.day)
#        except AttributeError:
#            self.event.occurrence = []
#            self.event.occurrence.append(self.day)

    def get_context(self, day=None):
        if self._context is None:
            now = get_now()
            context = dict(self.base_context)
            context.update({
                'URLS_NAMESPACE': URLS_NAMESPACE,
                'CALENDAR_TIME_FORMAT': CALENDAR_TIME_FORMAT,
                'CALENDAR_HOUR_FORMAT': CALENDAR_HOUR_FORMAT,
                'calendar': self,
                'is_current_day': False,
                'now': now,
                'request': self.request,
            })
            self._context = context
        return dict(self._context)

    def check_if_cancelled(self):
        d = date(self.yr, self.mo, self.day)
        is_cancelled = self.event.check_if_cancelled(d)
        if is_cancelled:
            self.title += " (CANCELLED)"
        return is_cancelled

    def get_day_url(self, day):
        if URLS_NAMESPACE:
            url_name = '%s:day_list' % (URLS_NAMESPACE)
        else:
            url_name = 'day_list'
        return reverse(url_name, args=(self.yr, self.mo, day))

    def formatmonth(self, theyear, themonth, withyear=True, net=None, qs=None, template='happenings/partials/calendar/month_table.html'):
        """Return a formatted month as a table."""
        context = self.get_context()
        context['month_start_date'] = date(self.yr, self.mo, 1)
        context['week_rows'] = []
        for week in self.monthdays2calendar(theyear, themonth):
            week_row = []
            for day, weekday in week:
                week_row.append(self.formatday(day, weekday))
            context['week_rows'].append(week_row)

        nxt, prev = get_next_and_prev(net)
        extra_qs = ('&' + '&'.join(qs)) if qs else ''
        context['prev_qs'] = mark_safe('?cal_prev=%d%s' % (prev, extra_qs))
        context['next_qs'] = mark_safe('?cal_next=%d%s' % (nxt, extra_qs))
        context['withyear'] = withyear
        return render_to_string(template, context)

class EventCalendar(GenericCalendar):

    def popover_helper(self):
        self.when = ''
        self.where = ''
        self.desc = ''
        self.title2 = ''

    def formatday(
            self, day, weekday,
            day_template='happenings/partials/calendar/day_cell.html',
            noday_template='happenings/partials/calendar/day_noday_cell.html',
            popover_template='happenings/partials/calendar/popover.html',
            ):
        """Return a day as a table cell."""
        super(EventCalendar, self).formatday(day, weekday)
        now = get_now()
        context = self.get_context()
        context['events'] = []
        context['day'] = day
        context['day_url'] = self.get_day_url(day)
        context['month_start_date'] = date(self.yr, self.mo, 1)
        context['weekday'] = weekday
        context['cssclass'] = self.cssclasses[weekday]
        context['popover_template'] = popover_template
        context['num_events'] = len(self.count.get(day, [])),
        try:
            processed_date = date(self.yr, self.mo, day)
        except ValueError:
            # day is out of range for month
            processed_date = None

        context['month_start_date'] = date(self.yr, self.mo, 1)

        if day == 0:
            template = noday_template
        else:
            template = day_template

        if now.date() == processed_date:
            context['is_current_day'] = True

        if processed_date and (day in self.count):
            for item in self.count[day]:
                self.pk = item[1]
                self.title = item[0]
                for event in self.events:
                    if event.pk == self.pk:
                        event.check_if_cancelled(processed_date)
                        # allow to use event.last_check_if_cancelled and populate event.title.extra

                        context['events'].append(event)

        return render_to_string(template, context)

class MiniEventCalendar(EventCalendar):
    def __init__(self, *args, **kwargs):
        super(MiniEventCalendar, self).__init__(*args, **kwargs)
        # Change count from a defaultdict to a regular dict, so that when we
        # try and check if there are days in count, they won't be added if they
        # aren't there.
        self.count = dict(self.count)

    def formatday(self, day, weekday):
        """Return a day as a table cell."""
        return super(MiniEventCalendar, self).formatday(
            day, weekday,
            day_template='happenings/partials/calendar/mini_day_cell.html',
            popover_template='happenings/partials/calendar/mini_popover.html',
        )

class LegacyGenericCalendar(LocaleHTMLCalendar):
    def __init__(self, year, month, count, all_month_events, *args):
        if len(args) < 2:
            args = args + (CALENDAR_LOCALE, )
        super(GenericCalendar, self).__init__(*args)
        if isinstance(self.locale, tuple):
            if len(self.locale) == 2:
                self.encoding = self.locale[1]
        elif self.locale.find('.') > 0:
            self.encoding = self.locale.split('.')[1]
        else:
            self.encoding = None
        self.yr = year
        self.mo = month
        self.count = count  # defaultdict in {date:[(title1, pk1), (title2, pk2),]} format
        self.events = all_month_events

#    def add_occurrence(self):
#        try:
#            self.event.occurrence.append(self.day)
#        except AttributeError:
#            self.event.occurrence = []
#            self.event.occurrence.append(self.day)

    def check_if_cancelled(self):
        d = date(self.yr, self.mo, self.day)
        is_cancelled = self.event.check_if_cancelled(d)
        if is_cancelled:
            self.title += " (CANCELLED)"
        return is_cancelled

    def formatday(self, day, weekday):
        """Set some commonly used variables."""
        self.wkday_not_today = '
' % (
            self.cssclasses[weekday])

        self.wkday_today = (
            '
' % (
                self.cssclasses[weekday])
        )
        if URLS_NAMESPACE:
            url_name = '%s:day_list' % (URLS_NAMESPACE)
        else:
            url_name = 'day_list'

        self.day_url = reverse(url_name, args=(self.yr, self.mo, day))
        self.day = day

        self.anch = '%d' % (
            self.day_url, day
        )

        self.end = '
	'

    def get_display_month(self, themonth):
        display_month = month_name[themonth]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode(self.encoding)
        return display_month

    def formatmonthname(self, theyear, themonth, withyear=True):
        """
        Change colspan to "5", add "today" button, and return a month
        name as a table row.
        """
        display_month = month_name[themonth]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode(self.encoding)

        if withyear:
            s = u'%s %s' % (display_month, theyear)
        else:
            s = u'%s' % display_month
        return ('
'
                '<button id="cal-today-btn" class="btn btn-small">'
                'Today</button> %s
' % s)

class LegacyEventCalendar(LegacyGenericCalendar):
    def popover_helper(self):
        """Populate variables used to build popovers."""
        # when
        display_month = month_name[self.mo]

        if isinstance(display_month, six.binary_type) and self.encoding:
            display_month = display_month.decode('utf-8')

        self.when = ('
When: ' + display_month + ' ' +
                     str(self.day) + ', ' + self.event.l_start_date.strftime(
                         LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') + ' - ' +
                     self.event.l_end_date.strftime(LEGACY_CALENDAR_TIME_FORMAT).lstrip('0') +
                     '
')

        if self.event.location.exists():  # where
            self.where = '
Where: '
            for l in self.event.location.all():
                self.where += l.name
            self.where += '
'
        else:
            self.where = ''

        # description
        self.desc = '
Description: ' + self.event.description[:100]
        self.desc += ('...
' if len(self.event.description) > 100
                      else '
')

        self.event_url = self.event.get_absolute_url()  # url
        t = LEGACY_CALENDAR_TIME_FORMAT if self.event.l_start_date.minute else LEGACY_CALENDAR_HOUR_FORMAT
        self.title2 = (self.event.l_start_date.strftime(t).lstrip('0') +
                       ' ' + self.title)

    def formatday(self, day, weekday):
        """Return a day as a table cell."""
        super(EventCalendar, self).formatday(day, weekday)
        now = get_now()
        self.day = day
        out = ''

        if day == 0:
            return ' 	'  # day outside month
        elif now.month == self.mo and now.year == self.yr and day == now.day:
            if day in self.count:
                # don't return just yet
                out = self.wkday_today + self.anch
            else:
                return self.wkday_today + self.anch + self.end
        elif day in self.count:
            # don't return just yet
            out = self.wkday_not_today + self.anch
        else:
            return self.wkday_not_today + self.anch + self.end

        detail = "%s%s%s
View details"
        extras = ('
')
        common = ' style=background:%s;color:%s;'

        # inject style and extras into calendar html
        for item in self.count[day]:
            self.pk = item[1]
            self.title = item[0]
            for event in self.events:
                if event.pk == self.pk:
                    self.event = event
                    self.check_if_cancelled()
                    # self.add_occurrence
                    self.popover_helper()
                    bg, fnt = self.event.get_colors()
            out += ('' +
                    extras % (
                        self.title,
                        detail % (
                            self.when, self.where, self.desc, self.event_url
                        ),
                        common % (bg, fnt)
                    ) +
                    self.title2 + '
')

        return out + self.end

class LegacyMiniEventCalendar(LegacyGenericCalendar):
    def __init__(self, *args):
        super(MiniEventCalendar, self).__init__(*args)
        # Change count from a defaultdict to a regular dict, so that when we
        # try and check if there are days in count, they won't be added if they
        # aren't there.
        self.count = dict(self.count)

    def popover_helper(self):
        num_events = len(self.count[self.day])
        titles = ''
        for item in self.count[self.day]:
            pk = item[1]
            for event in self.events:
                if event.pk == pk:
                    self.event = event
                    t = LEGACY_CALENDAR_TIME_FORMAT if event.l_start_date.minute else LEGACY_CALENDAR_HOUR_FORMAT
                    self.title = event.l_start_date.strftime(t).lstrip('0') + \
                        ' - ' + event.title
                    self.check_if_cancelled()
                    titles += "
%s
" % (
                        event.get_absolute_url(), self.title
                    )
        self.cal_event = '%s
' % (
            titles, num_events)

    def formatday(self, day, weekday):
        """Return a day as a table cell."""
        super(MiniEventCalendar, self).formatday(day, weekday)
        now = get_now()
        self.day = day

        if day == 0:
            return ' 	'  # day outside month
        elif now.month == self.mo and now.year == self.yr and day == now.day:
            if day in self.count:
                self.popover_helper()
                return self.wkday_today + self.anch + self.cal_event + self.end
            else:
                return self.wkday_today + self.anch + self.end
        elif day in self.count:
            self.popover_helper()
            return self.wkday_not_today + self.anch + self.cal_event + self.end
        else:
            return self.wkday_not_today + self.anch + self.end


# -*- coding: utf-8 -*-
import logging

from pprint import pprint as pp
from datetime import datetime

from scout.utils.date import get_date
from scout.utils.handle import get_file_handle
from scout.utils.link import get_correct_ids

from .omim import get_mim_genes

LOG = logging.getLogger(__name__)

VALID_MODELS = ('AR','AD','MT','XD','XR','X','Y')

MODELS_MAP = {
    'monoallelic_not_imprinted': ['AD'],
    'monoallelic_maternally_imprinted': ['AD'],
    'monoallelic_paternally_imprinted': ['AD'],
    'monoallelic': ['AD'],
    'biallelic': ['AR'],
    'monoallelic_and_biallelic': ['AD','AR'] ,
    'monoallelic_and_more_severe_biallelic': ['AD','AR'],
    'xlinked_biallelic': ['XR'],
    'xlinked_monoallelic': ['XD'],
    'mitochondrial': ['MT'],
    'unknown': [],
}

INCOMPLETE_PENETRANCE_MAP = {
    'unknown': None,
    'Complete': None,
    'Incomplete': True
}

def get_panel_info(panel_lines=None, panel_id=None, institute=None, version=None, date=None,
                   display_name=None):
    """Parse metadata for a gene panel

    For historical reasons it is possible to include all information about a gene panel in the
    header of a panel file. This function parses the header.

    Args:
        panel_lines(iterable(str))

    Returns:
        panel_info(dict): Dictionary with panel information
    """
    panel_info = {
        'panel_id': panel_id,
        'institute': institute,
        'version': version,
        'date': date,
        'display_name': display_name,
    }

    if panel_lines:
        for line in panel_lines:
            line = line.rstrip()
            if not line.startswith('##'):
                break

            info = line[2:].split('=')
            field = info[0]
            value = info[1]

            if not panel_info.get(field):
                panel_info[field] = value

    panel_info['date'] = get_date(panel_info['date'])

    return panel_info

def parse_gene(gene_info):
    """Parse a gene line with information from a panel file

        Args:
            gene_info(dict): dictionary with gene info

        Returns:
            gene(dict): A dictionary with the gene information
                {
                'hgnc_id': int,
                'hgnc_symbol': str,
                'disease_associated_transcripts': list(str),
                'inheritance_models': list(str),
                'mosaicism': bool,
                'reduced_penetrance': bool,
                'database_entry_version': str,
                }

    """
    gene = {}
    # This is either hgnc id or hgnc symbol
    identifier = None

    hgnc_id = None
    try:
        if 'hgnc_id' in gene_info:
            hgnc_id = int(gene_info['hgnc_id'])
        elif 'hgnc_idnumber' in gene_info:
            hgnc_id = int(gene_info['hgnc_idnumber'])
        elif 'hgncid' in gene_info:
            hgnc_id = int(gene_info['hgncid'])
    except ValueError as e:
        raise SyntaxError("Invalid hgnc id: {0}".format(hgnc_id))

    gene['hgnc_id'] = hgnc_id
    identifier = hgnc_id

    hgnc_symbol = None
    if 'hgnc_symbol' in gene_info:
        hgnc_symbol = gene_info['hgnc_symbol']
    elif 'hgncsymbol' in gene_info:
        hgnc_symbol = gene_info['hgncsymbol']
    elif 'symbol' in gene_info:
        hgnc_symbol = gene_info['symbol']

    gene['hgnc_symbol'] = hgnc_symbol

    if not identifier:
        if hgnc_symbol:
            identifier = hgnc_symbol
        else:
            raise SyntaxError("No gene identifier could be found")
    gene['identifier'] = identifier
    # Disease associated transcripts is a ','-separated list of
    # manually curated transcripts
    transcripts = ""
    if 'disease_associated_transcripts' in gene_info:
        transcripts = gene_info['disease_associated_transcripts']
    elif 'disease_associated_transcript' in gene_info:
        transcripts = gene_info['disease_associated_transcript']
    elif 'transcripts' in gene_info:
        transcripts = gene_info['transcripts']

    gene['transcripts'] = [
            transcript.strip() for transcript in
            transcripts.split(',') if transcript
        ]

    # Genetic disease models is a ','-separated list of manually curated
    # inheritance patterns that are followed for a gene
    models = ""
    if 'genetic_disease_models' in gene_info:
        models = gene_info['genetic_disease_models']
    elif 'genetic_disease_model' in gene_info:
        models = gene_info['genetic_disease_model']
    elif 'inheritance_models' in gene_info:
        models = gene_info['inheritance_models']
    elif 'genetic_inheritance_models' in gene_info:
        models = gene_info['genetic_inheritance_models']

    gene['inheritance_models'] = [
        model.strip() for model in models.split(',')
        if model.strip() in VALID_MODELS
    ]

    # If a gene is known to be associated with mosaicism this is annotated
    gene['mosaicism'] = True if gene_info.get('mosaicism') else False

    # If a gene is known to have reduced penetrance this is annotated
    gene['reduced_penetrance'] = True if gene_info.get('reduced_penetrance') else False

    # The database entry version is a way to track when a a gene was added or
    # modified, optional
    gene['database_entry_version'] = gene_info.get('database_entry_version')

    return gene

def parse_genes(gene_lines):
    """Parse a file with genes and return the hgnc ids

    Args:
        gene_lines(iterable(str)): Stream with genes

    Returns:
        genes(list(dict)): Dictionaries with relevant gene info
    """
    genes = []
    header = []
    hgnc_identifiers = set()
    delimiter = '\t'
    # This can be '\t' or ';'
    delimiters = ['\t', ' ', ';']

    # There are files that have '#' to indicate headers
    # There are some files that start with a header line without
    # any special symbol
    for i,line in enumerate(gene_lines):
        line = line.rstrip()
        if not len(line) > 0:
            continue
        if line.startswith('#'):
            if not line.startswith('##'):
                # We need to try delimiters
                # We prefer ';' or '\t' byt should accept ' '
                line_length = 0
                delimiter = None
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                header = [word.lower() for word in line[1:].split(delimiter)]
        else:
            # If no header symbol(#) assume first line is header
            if i == 0:
                line_length = 0
                for alt in delimiters:
                    head_line = line.split(alt)
                    if len(head_line) > line_length:
                        line_length = len(head_line)
                        delimiter = alt

                if ('hgnc' in line or 'HGNC' in line):
                    header = [word.lower() for word in line.split(delimiter)]
                    continue
                # If first line is not a header try to sniff what the first
                # columns holds
                if line.split(delimiter)[0].isdigit():
                    header = ['hgnc_id']
                else:
                    header = ['hgnc_symbol']

            splitted_line = line.split(delimiter)
            gene_info = dict(zip(header, splitted_line))

            # There are cases when excel exports empty lines that looks like
            # ;;;;;;;. This is a exception to handle these
            info_found = False
            for key in gene_info:
                if gene_info[key]:
                    info_found = True
                    break
            # If no info was found we skip that line
            if not info_found:
                continue

            try:
                gene = parse_gene(gene_info)
            except Exception as e:
                LOG.warning(e)
                raise SyntaxError("Line {0} is malformed".format(i + 1))

            identifier = gene.pop('identifier')

            if not identifier in hgnc_identifiers:
                hgnc_identifiers.add(identifier)
                genes.append(gene)

    return genes

def parse_gene_panel(path, institute='cust000', panel_id='test', panel_type='clinical', date=datetime.now(),
                     version=1.0, display_name=None, genes = None):
    """Parse the panel info and return a gene panel

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

        Returns:
            gene_panel(dict)
    """
    LOG.info("Parsing gene panel %s", panel_id)
    gene_panel = {}

    gene_panel['path'] = path
    gene_panel['type'] = panel_type
    gene_panel['date'] = date
    gene_panel['panel_id'] = panel_id
    gene_panel['institute'] = institute
    version = version or 1.0
    gene_panel['version'] = float(version)
    gene_panel['display_name'] = display_name or panel_id

    if not path:
        panel_handle = genes
    else:
        panel_handle = get_file_handle(gene_panel['path'])
    gene_panel['genes'] = parse_genes(gene_lines=panel_handle)

    return gene_panel

def parse_panel_app_gene(app_gene, hgnc_map):
    """Parse a panel app formated gene

    Args:
        app_gene(dict): Dict with panel app info
        hgnc_map(dict): Map from hgnc_symbol to hgnc_id

    Returns:
        gene_info(dict): Scout infromation
    """
    gene_info = {}
    confidence_level = app_gene['LevelOfConfidence']
    # Return empty gene if not confident gene
    if not confidence_level == 'HighEvidence':
        return gene_info

    hgnc_symbol = app_gene['GeneSymbol']
    # Returns a set of hgnc ids
    hgnc_ids = get_correct_ids(hgnc_symbol, hgnc_map)
    if not hgnc_ids:
        LOG.warning("Gene %s does not exist in database. Skipping gene...", hgnc_symbol)
        return gene_info

    if len(hgnc_ids) > 1:
        LOG.warning("Gene %s has unclear identifier. Choose random id", hgnc_symbol)

    gene_info['hgnc_symbol'] = hgnc_symbol
    for hgnc_id in hgnc_ids:
        gene_info['hgnc_id'] = hgnc_id

    gene_info['reduced_penetrance'] = INCOMPLETE_PENETRANCE_MAP.get(app_gene['Penetrance'])

    inheritance_models = []
    for model in MODELS_MAP.get(app_gene['ModeOfInheritance'],[]):
        inheritance_models.append(model)

    gene_info['inheritance_models'] = inheritance_models

    return gene_info

def parse_panel_app_panel(panel_info, hgnc_map, institute='cust000', panel_type='clinical'):
    """Parse a PanelApp panel

    Args:
        panel_info(dict)
        hgnc_map(dict): Map from symbol to hgnc ids
        institute(str)
        panel_type(str)

    Returns:
        gene_panel(dict)
    """
    date_format = "%Y-%m-%dT%H:%M:%S.%f"

    gene_panel = {}
    gene_panel['version'] = float(panel_info['version'])
    gene_panel['date'] = get_date(panel_info['Created'][:-1], date_format=date_format)
    gene_panel['display_name'] = panel_info['SpecificDiseaseName']
    gene_panel['institute'] = institute
    gene_panel['panel_type'] = panel_type

    LOG.info("Parsing panel %s", gene_panel['display_name'])

    gene_panel['genes'] = []

    nr_low_confidence = 1
    nr_genes = 0
    for nr_genes, gene in enumerate(panel_info['Genes'],1):
        gene_info = parse_panel_app_gene(gene, hgnc_map)
        if not gene_info:
            nr_low_confidence += 1
            continue
        gene_panel['genes'].append(gene_info)

    LOG.info("Number of genes in panel %s", nr_genes)
    LOG.info("Number of low confidence genes in panel %s", nr_low_confidence)

    return gene_panel

def get_omim_panel_genes(genemap2_lines, mim2gene_lines, alias_genes):
    """Return all genes that should be included in the OMIM-AUTO panel
    Return the hgnc symbols

    Genes that have at least one 'established' or 'provisional' phenotype connection
    are included in the gene panel

    Args:
        genemap2_lines(iterable)
        mim2gene_lines(iterable)
        alias_genes(dict): A dictionary that maps hgnc_symbol to hgnc_id

    Yields:
        hgnc_symbol(str)
    """
    parsed_genes = get_mim_genes(genemap2_lines, mim2gene_lines)

    STATUS_TO_ADD = set(['established', 'provisional'])

    for hgnc_symbol in parsed_genes:
        try:
            gene = parsed_genes[hgnc_symbol]
            keep = False
            for phenotype_info in gene.get('phenotypes',[]):
                if phenotype_info['status'] in STATUS_TO_ADD:
                    keep = True
                    break
            if keep:
                hgnc_id_info = alias_genes.get(hgnc_symbol)
                if not hgnc_id_info:
                    for symbol in gene.get('hgnc_symbols', []):
                        if symbol in alias_genes:
                            hgnc_id_info = alias_genes[symbol]
                            break

                if hgnc_id_info:
                    yield {
                    'hgnc_id': hgnc_id_info['true'],
                    'hgnc_symbol': hgnc_symbol,
                    }
                else:
                    LOG.warning("Gene symbol %s does not exist", hgnc_symbol)

        except KeyError:
            pass


import logging
import click

LOG = logging.getLogger(__name__)

@click.command('diseases', short_help='Display all diseases')
@click.pass_context
def diseases(context):
    """Show all diseases in the database"""
    LOG.info("Running scout view diseases")
    adapter = context.obj['adapter']

    disease_objs = adapter.disease_terms()

    nr_diseases = disease_objs.count()
    if nr_diseases == 0:
        click.echo("No diseases found")
    else:
        click.echo("Disease")
        for disease_obj in adapter.disease_terms():
            click.echo("{0}".format(disease_obj['_id']))
        LOG.info("{0} diseases found".format(nr_diseases))


import logging
import click
import urllib.request

from pprint import pprint as pp

from scout.load.hpo import load_hpo_terms
from scout.utils.requests import (fetch_hpo_terms, fetch_hpo_to_genes)

from scout.commands.utils import abort_if_false

LOG = logging.getLogger(__name__)

@click.command('hpo', short_help='Update hpo terms')
@click.option('--yes', is_flag=True, callback=abort_if_false,
              expose_value=False,
              prompt='Are you sure you want to drop the hpo terms?')
@click.pass_context
def hpo(context):
    """
    Update the hpo terms in the database. Fetch the latest release and update terms.
    """
    LOG.info("Running scout update hpo")
    adapter = context.obj['adapter']

    LOG.info("Dropping HPO terms")
    adapter.hpo_term_collection.drop()
    LOG.debug("HPO terms dropped")

    load_hpo_terms(adapter)


# Used to generate upcoming events

from __future__ import unicode_literals

from datetime import timedelta, date, datetime

from django.utils.timezone import make_aware, get_default_timezone
from django.utils.six.moves import xrange

from happenings.utils.common import inc_month

class UpcomingEvents(object):
    def __init__(self, event, now, finish, num=5):
        self.event = event
        self.finish = finish  # datetime.datetime
        self.now = now        # datetime.datetime
        self.num = num
        self.events = []

    def get_upcoming_events(self):
        """
        Repeats an event and returns 'num' (or fewer)
        upcoming events from 'now'.
        """
        if self.event.repeats('NEVER'):
            has_ended = False
            now_gt_start = self.now > self.event.l_start_date
            now_gt_end = self.now > self.event.end_date
            if now_gt_end or now_gt_start:
                has_ended = True
            has_not_started = self.event.l_start_date > self.finish
            if has_ended or has_not_started:
                return self.events
            self.events.append((self.event.l_start_date, self.event))
            return self.events
        if self.event.repeats('WEEKDAY'):
            self._weekday()
        elif self.event.repeats('MONTHLY'):
            self._monthly()
        elif self.event.repeats('YEARLY'):
            self._yearly()
        else:
            self._others()
        return self.events

    def we_should_stop(self, start, start_):
        """
        Checks 'start' to see if we should stop collecting upcoming events.
        'start' should be a datetime.datetime, 'start_' should be the same
        as 'start', but it should be a datetime.date to allow comparison
        w/ end_repeat.
        """
        if start > self.finish or \
                self.event.end_repeat is not None and \
                start_ > self.event.end_repeat:
            return True
        else:
            return False

    def _yearly(self):
        num = self.num
        year = self.now.year
        if self.event.l_start_date > self.now:  # event starts in the future
            year = self.event.l_start_date.year
        elif self.now.month > self.event.l_start_date.month:
            year += 1
        # The event occurs this month, so check the day to see if passed
        elif self.now.month == self.event.l_start_date.month:
            # if the current day is > event day, or if the event was today but
            # finished, increment the year.
            if self.now.day > self.event.l_start_date.day or \
                self.now.day == self.event.l_start_date.day and \
                    self.now.time() > self.event.l_start_date.time():
                year += 1
        month = self.event.l_start_date.month
        day = self.event.l_start_date.day
        while num:
            try:
                start = make_aware(
                    datetime(year, month, day), get_default_timezone()
                )
                # change to date() so we can compare to event.end_repeat
                start_ = date(year, month, day)
            # value error most likely means that the event's start date doesn't
            # appear this calendar month
            except ValueError:
                year += 1
                continue
            if self.we_should_stop(start, start_):
                return
            self.events.append((start, self.event))
            year += 1
            num -= 1

    def _monthly(self):
        num = self.num
        start = self.event.l_start_date
        year = self.now.year
        month = self.now.month
        day = self.event.l_start_date.day
        if self.event.l_start_date > self.now:  # event starts in the future
            year = self.event.l_start_date.year
            month = self.event.l_start_date.month
        while num:
            try:
                start = start.replace(year=year, month=month)
                if self.now > start:  # if event already happened
                    month, year = inc_month(month, year)
                    continue
                # change to date() so we can compare to event.end_repeat
                start_ = date(year, month, day)
            # value error most likely means that the event's start date doesn't
            # appear this calendar month
            except ValueError:
                month, year = inc_month(month, year)
                continue
            if self.we_should_stop(start, start_):
                return
            self.events.append((start, self.event))
            month, year = inc_month(month, year)
            num -= 1

    def _weekday(self):
        start = self.event.l_start_date
        if not self.event.l_start_date > self.now:
            start = start.replace(
                year=self.now.year, month=self.now.month, day=self.now.day
            )
        while start.weekday() > 4:
            start += timedelta(days=1)
        if start < self.now:
            start += timedelta(days=1)
            while start.weekday() > 4:
                start += timedelta(days=1)
        for i in xrange(self.num):
            # change to date() so we can compare to event.end_repeat
            start_ = date(start.year, start.month, start.day)
            if self.we_should_stop(start, start_):
                return
            while start.weekday() > 4:
                start += timedelta(days=1)
            self.events.append((start, self.event))
            start += timedelta(days=1)

    def _others(self):
        repeat = {'WEEKLY': 7, 'BIWEEKLY': 14, 'DAILY': 1}
        if self.event.repeats('DAILY'):
            if self.event.l_start_date > self.now:
                start = self.event.l_start_date
            elif self.event.l_start_date.time() < self.now.time():
                # has it already begun? Then no longer upcoming, so ++day
                start = self.now
                start += timedelta(days=1)
            else:
                start = self.now
        else:
            start = self.event.l_start_date
            end = self.event.l_end_date
            while start < self.now or end <= self.now:
                start += timedelta(days=repeat[self.event.repeat])
                end += timedelta(days=repeat[self.event.repeat])
        for i in xrange(self.num):
            # change to date() so we can compare to event.end_repeat
            start_ = date(start.year, start.month, start.day)
            if self.we_should_stop(start, start_):
                return
            self.events.append((start, self.event))
            start += timedelta(days=repeat[self.event.repeat])


# -*- coding: utf-8 -*-

def event_rank(count):
    """Determine event ranking."""
    if count < 10:
        return 'constable'
    elif count < 100:
        return 'sergeant'
    elif count < 250:
        return 'inspector'
    elif count < 500:
        return 'superintendent'
    elif count < 1000:
        return 'commander'
    else:
        return 'commissioner'

def users(store):
    """Display a list of all users and which institutes they belong to."""
    user_objs = list(store.users())
    total_events = store.user_events().count()
    for user_obj in user_objs:
        if user_obj.get('institutes'):
            user_obj['institutes'] = [store.institute(inst_id) for inst_id in user_obj.get('institutes')]
        else:
            user_obj['institutes'] = []
        user_obj['events'] = store.user_events(user_obj).count()
        user_obj['events_rank'] = event_rank(user_obj['events'])
    return dict(
        users=sorted(user_objs, key=lambda user: -user['events']),
        total_events=total_events,
    )


from scout.constants import CONSERVATION
import numbers

def parse_conservations(variant):
    """Parse the conservation predictors

        Args:
            variant(dict): A variant dictionary

        Returns:
            conservations(dict): A dictionary with the conservations
    """
    conservations = {}

    conservations['gerp'] = parse_conservation(
                                            variant,
                                            'dbNSFP_GERP___RS'
                                        )
    conservations['phast'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phastCons100way_vertebrate'
                                        )
    conservations['phylop'] = parse_conservation(
                                            variant,
                                            'dbNSFP_phyloP100way_vertebrate'
                                        )
    return conservations

def parse_conservation(variant, info_key):
    """Get the conservation prediction

        Args:
            variant(dict): A variant dictionary
            info_key(str)

        Returns:
            conservations(list): List of censervation terms
    """
    raw_score = variant.INFO.get(info_key)
    conservations = []

    if raw_score:
        if isinstance(raw_score, numbers.Number):
            raw_score = (raw_score,)

        for score in raw_score:
            if score >= CONSERVATION[info_key]['conserved_min']:
                conservations.append('Conserved')
            else:
                conservations.append('NotConserved')

    return conservations


from __future__ import unicode_literals

from datetime import date

from .common import check_weekday, inc_month
from .handlers import CountHandler

def get_next_event(event, now):
    """
    Returns the next occurrence of a given event, relative to 'now'.
    The 'event' arg should be an iterable containing one element,
    namely the event we'd like to find the occurrence of.
    The reason for this is b/c the get_count() function of CountHandler,
    which this func makes use of, expects an iterable.
    CHANGED: The 'now' arg must be an instance of datetime.datetime()
    to allow time comparison (used to accept datetime.date() as well)
    """
    year = now.year
    month = now.month
    day = now.day
    e_day = event[0].l_start_date.day
    e_end_day = event[0].l_end_date.day
    good_today = True if event[0].l_start_date.time() >= now.time() else False
    if event[0].starts_same_year_month_as(year, month) and \
            e_day <= now.day <= e_end_day:
        occurrences = CountHandler(year, month, event).get_count()
        future_dates = (x for x in occurrences if x >= now.day)
        day = min(future_dates, key=lambda x: abs(x - now.day))
    else:
        e_year = event[0].l_start_date.year
        e_month = event[0].l_start_date.month
        # convert to datetime.date() to be sure we can make a comparison
        if date(e_year, e_month, e_day) > date(now.year, now.month, now.day):
            # if the event hasn't started yet, then its next occurrence will
            # be on its start date, so return that.
            year = e_year
            month = e_month
            day = e_day
        else:
            occurrences = CountHandler(year, month, event).get_count()
            future_dates = [x for x in occurrences if x >= now.day]
            e_end_month = event[0].l_end_date.month
            if future_dates and future_dates[0] is day and not good_today:
                future_dates.pop(0)
            while not future_dates:
                month, year = inc_month(month, year)
                if event[0].repeats('YEARLY') and \
                        (month != e_month or month != e_end_month):
                    continue
                occurrences = CountHandler(year, month, event).get_count()
                # we don't check for now.day here, b/c we're in a month past
                # whatever now is. As an example, if we checked for now.day
                # we'd get stuck in an infinite loop if this were a
                # monthly repeating event and our 'now' was on a day after the
                # event's l_end_date.day
                future_dates = [x for x in occurrences]
            day = min(future_dates)

    if event[0].repeats('WEEKDAY'):
        return check_weekday(year, month, day)

    return year, month, day


import logging
from flask_login import current_user

LOG = logging.getLogger(__name__)

def get_dashboard_info(adapter, institute_id=None, slice_query=None):
    """Returns cases with phenotype

        If phenotypes are provided search for only those

    Args:
        adapter(adapter.MongoAdapter)
        institute_id(str): an institute _id
        slice_query(str):  query to filter cases to obtain statistics for.

    Returns:
        data(dict): Dictionary with relevant information
    """

    LOG.debug("General query with institute_id {}.".format(institute_id))

    # if institute_id == 'None' or None, all cases and general stats will be returned
    if institute_id == 'None':
        institute_id = None

    # If a slice_query is present then numbers in "General statistics" and "Case statistics" will
    # reflect the data available for the query
    general_sliced_info = get_general_case_info(adapter, institute_id=institute_id,
                                                    slice_query=slice_query)
    total_sliced_cases = general_sliced_info['total_cases']

    data = {'total_cases': total_sliced_cases}
    if total_sliced_cases == 0:
        return data

    data['pedigree'] = []
    for ped_info in general_sliced_info['pedigree'].values():
        ped_info['percent'] = ped_info['count'] / total_sliced_cases
        data['pedigree'].append(ped_info)

    data['cases'] = get_case_groups(adapter, total_sliced_cases,
                        institute_id=institute_id, slice_query=slice_query)

    data['analysis_types'] = get_analysis_types(adapter, total_sliced_cases,
                                institute_id=institute_id, slice_query=slice_query)

    overview = [
        {
            'title': 'Phenotype terms',
            'count': general_sliced_info['phenotype_cases'],
            'percent': general_sliced_info['phenotype_cases'] / total_sliced_cases,
        },
        {
            'title': 'Causative variants',
            'count': general_sliced_info['causative_cases'],
            'percent': general_sliced_info['causative_cases'] / total_sliced_cases,
        },
        {
            'title': 'Pinned variants',
            'count': general_sliced_info['pinned_cases'],
            'percent': general_sliced_info['pinned_cases'] / total_sliced_cases,
        },
        {
            'title': 'Cohort tag',
            'count': general_sliced_info['cohort_cases'],
            'percent': general_sliced_info['cohort_cases'] / total_sliced_cases,
        }
    ]

    # Data from "Variant statistics tab" is not filtered by slice_query and numbers will
    # reflect verified variants in all available cases for an institute
    general_info = get_general_case_info(adapter, institute_id=institute_id)
    total_cases = general_info['total_cases']
    sliced_case_ids = general_sliced_info['case_ids']
    verified_query = {
        'verb' : 'validate',
    }
    if institute_id: # filter by institute if users wishes so
        verified_query['institute'] =  institute_id

    # Case level information
    sliced_validation_cases = set()
    sliced_validated_cases = set()

    # Variant level information
    validated_tp = set()
    validated_fp = set()
    var_valid_orders = 0 # use this counter to count 'True Positive', 'False positive' and 'Not validated' vars

    validate_events = adapter.event_collection.find(verified_query)
    for validate_event in list(validate_events):
        case_id = validate_event.get('case')
        var_obj = adapter.variant(case_id=case_id, document_id=validate_event['variant_id'])

        if var_obj: # Don't take into account variants which have been removed from db
            var_valid_orders += 1
            if case_id in sliced_case_ids:
                sliced_validation_cases.add(case_id) # add to the set. Can't add same id twice since it'a a set

            validation = var_obj.get('validation')
            if validation and validation in ['True positive', 'False positive']:
                if case_id in sliced_case_ids:
                    sliced_validated_cases.add(case_id)
                if validation == 'True positive':
                    validated_tp.add(var_obj['_id'])
                elif validation == 'False positive':
                    validated_fp.add(var_obj['_id'])

    n_validation_cases = len(sliced_validation_cases)
    n_validated_cases = len(sliced_validated_cases)

    # append
    overview.append(
    {
        'title': 'Validation ordered',
        'count': n_validation_cases,
        'percent': n_validation_cases / total_sliced_cases,
    })
    overview.append(
    {
        'title': 'Validated cases (TP + FP)',
        'count': n_validated_cases,
        'percent': n_validated_cases / total_sliced_cases,
    })

    data['overview'] = overview

    variants = []
    nr_validated = len(validated_tp) + len(validated_fp)
    variants.append(
        {
            'title': 'Validation ordered',
            'count': var_valid_orders,
            'percent': 1
        }
    )

    # taking into account that var_valid_orders might be 0:
    percent_validated_tp = 0
    percent_validated_fp = 0

    if var_valid_orders:
        percent_validated_tp = len(validated_tp) / var_valid_orders
        percent_validated_fp = len(validated_fp) / var_valid_orders

    variants.append(
        {
            'title': 'Validated True Positive',
            'count': len(validated_tp),
            'percent': percent_validated_tp,
        }
    )

    variants.append(
        {
            'title': 'Validated False Positive',
            'count': len(validated_fp),
            'percent': percent_validated_fp,
        }
    )

    data['variants'] = variants
    return data

def get_general_case_info(adapter, institute_id=None, slice_query=None):
    """Return general information about cases

    Args:
        adapter(adapter.MongoAdapter)
        institute_id(str)
        slice_query(str):   Query to filter cases to obtain statistics for.

    Returns:
        general(dict)
    """
    general = {}

    # Potentially sensitive slice queries are assumed allowed if we have got this far
    name_query = slice_query

    cases = adapter.cases(owner=institute_id, name_query=name_query)

    phenotype_cases = 0
    causative_cases = 0
    pinned_cases = 0
    cohort_cases = 0

    pedigree = {
        1: {
            'title': 'Single',
            'count': 0
        },
        2: {
            'title': 'Duo',
            'count': 0
        },
        3: {
            'title': 'Trio',
            'count': 0
        },
        'many': {
            'title': 'Many',
            'count': 0
        },
    }

    case_ids = set()

    total_cases = 0
    for total_cases,case in enumerate(cases,1):
        # If only looking at one institute we need to save the case ids
        if institute_id:
            case_ids.add(case['_id'])
        if case.get('phenotype_terms'):
            phenotype_cases += 1
        if case.get('causatives'):
            causative_cases += 1
        if case.get('suspects'):
            pinned_cases += 1
        if case.get('cohorts'):
            cohort_cases += 1

        nr_individuals = len(case.get('individuals',[]))
        if nr_individuals == 0:
            continue
        if nr_individuals > 3:
            pedigree['many']['count'] += 1
        else:
            pedigree[nr_individuals]['count'] += 1

    general['total_cases'] = total_cases
    general['phenotype_cases'] = phenotype_cases
    general['causative_cases'] = causative_cases
    general['pinned_cases'] = pinned_cases
    general['cohort_cases'] = cohort_cases
    general['pedigree'] = pedigree
    general['case_ids'] = case_ids

    return general

def get_case_groups(adapter, total_cases, institute_id=None, slice_query=None):
    """Return the information about case groups

    Args:
        store(adapter.MongoAdapter)
        total_cases(int): Total number of cases
        slice_query(str): Query to filter cases to obtain statistics for.

    Returns:
        cases(dict):
    """
    # Create a group with all cases in the database
    cases = [{'status': 'all', 'count': total_cases, 'percent': 1}]
    # Group the cases based on their status
    pipeline = []
    group = {'$group' : {'_id': '$status', 'count': {'$sum': 1}}}

    subquery = {}
    if institute_id and slice_query:
        subquery = adapter.cases(owner=institute_id, name_query=slice_query,
                              yield_query=True)
    elif institute_id:
        subquery = adapter.cases(owner=institute_id, yield_query=True)
    elif slice_query:
        subquery = adapter.cases(name_query=slice_query, yield_query=True)

    query = {'$match': subquery} if subquery else {}

    if query:
        pipeline.append(query)

    pipeline.append(group)
    res = adapter.case_collection.aggregate(pipeline)

    for status_group in res:
        cases.append({'status': status_group['_id'],
                      'count': status_group['count'],
                      'percent': status_group['count'] / total_cases})

    return cases

def get_analysis_types(adapter, total_cases, institute_id=None, slice_query=None):
    """ Return information about analysis types.
        Group cases based on analysis type for the individuals.
    Args:
        adapter(adapter.MongoAdapter)
        total_cases(int): Total number of cases
        institute_id(str)
        slice_query(str): Query to filter cases to obtain statistics for.
    Returns:
        analysis_types array of hashes with name: analysis_type(str), count: count(int)

    """
    # Group cases based on analysis type of the individuals
    query = {}

    subquery = {}
    if institute_id and slice_query:
        subquery = adapter.cases(owner=institute_id, name_query=slice_query,
                              yield_query=True)
    elif institute_id:
        subquery = adapter.cases(owner=institute_id, yield_query=True)
    elif slice_query:
        subquery = adapter.cases(name_query=slice_query, yield_query=True)

    query = {'$match': subquery}

    pipeline = []
    if query:
        pipeline.append(query)

    pipeline.append({'$unwind': '$individuals'})
    pipeline.append({'$group': {'_id': '$individuals.analysis_type', 'count': {'$sum': 1}}})
    analysis_query = adapter.case_collection.aggregate(pipeline)
    analysis_types = [{'name': group['_id'], 'count': group['count']} for group in analysis_query]

    return analysis_types


from __future__ import unicode_literals

from datetime import datetime
from json import dumps, loads

from django.core import serializers
from django.http import HttpResponse
from django.utils.encoding import force_text
from django.utils.functional import Promise

class JSONResponseMixin(object):
    """
    A mixin that can be used to render a JSON response. See:
    https://docs.djangoproject.com/en/dev/topics/class-based-views/mixins/
    """
    def render_to_json_response(self, context, **kwargs):
        """
        Returns a JSON response, transforming 'context' to make the payload.
        """
        return HttpResponse(
            self.convert_context_to_json(context),
            content_type='application/json',
            **kwargs
        )

    def convert_context_to_json(self, context):
        """
        Get what we want out of the context dict and convert that to a JSON
        object. Note that this does no object serialization b/c we're
        not sending any objects.
        """
        if 'month/shift' in self.request.path:  # month calendar
            return dumps(self.get_month_calendar_dict(context))
        elif 'event-list/shift' in self.request.path:  # month event list
            return dumps(self.get_month_event_list_dict(context))
        elif 'cal-and-list/shift' in self.request.path:
            cal = self.get_month_calendar_dict(context)
            l = self.get_month_event_list_dict(context)
            cal.update(l)
            return dumps(cal)
        else:  # day list view
            for key, val in context.items():
                if isinstance(val, Promise):
                    context[key] = force_text(val)

            return dumps(self.get_day_context_dict(context))

    @staticmethod
    def get_month_calendar_dict(context):
        return dict(
            calendar=context['calendar'],
            month_and_year=context['month_and_year']
        )

    @staticmethod
    def get_month_event_list_dict(context):
        m = int(context['current']['month_num'])
        y = int(context['current']['year'])
        remove_these = (
            'all_day', 'description', 'categories', 'tags', 'created_by',
            'location',
        )

        for k, v in context['events'].items():
            context['events'][k] = loads(serializers.serialize("json", v))
            for event in context['events'][k]:
                # remove unnec. fields to shorten the json response
                for item in remove_these:
                    del event['fields'][item]
                event['weekday'] = datetime(  # add weekday e.g Fri, Mon, etc.
                    y, m, k).strftime("%A")[:3]
        return dict(
            month=context['current']['month'], events=context['events']
        )

    @staticmethod
    def get_day_context_dict(context):
        events = loads(serializers.serialize("json", context['events']))
        return dict(
            events=events,
            year=context['year'],
            month=context['month_num'],
            month_name=context['month'],
            day=context['day'],
            nxt=context['next'],
            prev=context['prev'],
        )


# -*- coding: utf-8 -*-
from __future__ import unicode_literals

# python lib:
from datetime import date, timedelta

# django:
from django.views.generic import ListView, DetailView
from django.conf import settings
from django.shortcuts import get_object_or_404
from django.utils.dates import MONTHS_ALT

# thirdparties:
import six

# happenings:
from .models import Event
from happenings.utils.displays import month_display, day_display
from happenings.utils.next_event import get_next_event
from happenings.utils.mixins import JSONResponseMixin
from happenings.utils import common as c

CALENDAR_LOCALE = getattr(settings, 'CALENDAR_LOCALE', 'en_US.utf8')

class GenericEventView(JSONResponseMixin, ListView):
    model = Event

    def render_to_response(self, context, **kwargs):
        if self.request.is_ajax():
            return self.render_to_json_response(context, **kwargs)
        return super(GenericEventView, self).render_to_response(
            context, **kwargs
        )

    def get_context_data(self, **kwargs):
        context = super(GenericEventView, self).get_context_data(**kwargs)

        self.net, self.category, self.tag = c.get_net_category_tag(
            self.request
        )

        if self.category is not None:
            context['cal_category'] = self.category
        if self.tag is not None:
            context['cal_tag'] = self.tag
        return context

class EventMonthView(GenericEventView):
    template_name = 'happenings/event_month_list.html'

    def get_year_and_month(self, net, qs, **kwargs):
        """
        Get the year and month. First tries from kwargs, then from
        querystrings. If none, or if cal_ignore qs is specified,
        sets year and month to this year and this month.
        """
        now = c.get_now()
        year = now.year
        month = now.month + net
        month_orig = None

        if 'cal_ignore=true' not in qs:
            if 'year' and 'month' in self.kwargs:  # try kwargs
                year, month_orig = map(
                    int, (self.kwargs['year'], self.kwargs['month'])
                )
                month = month_orig + net
            else:
                try:  # try querystring
                    year = int(self.request.GET['cal_year'])
                    month_orig = int(self.request.GET['cal_month'])
                    month = month_orig + net
                except Exception:
                    pass
        # return the year and month, and any errors that may have occurred do
        # to an invalid month/year being given.
        return c.clean_year_month(year, month, month_orig)

    def get_month_events(self, *args, **kwargs):
        return Event.objects.all_month_events(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(EventMonthView, self).get_context_data(**kwargs)

        qs = self.request.META['QUERY_STRING']

        year, month, error = self.get_year_and_month(self.net, qs)

        mini = True if 'cal_mini=true' in qs else False

        # get any querystrings that are not next/prev/year/month
        if qs:
            qs = c.get_qs(qs)

        # add a dict containing the year, month, and month name to the context
        current = dict(
            year=year, month_num=month, month=MONTHS_ALT[month][:3]
        )
        context['current'] = current

        display_month = MONTHS_ALT[month]

        if isinstance(display_month, six.binary_type):
            display_month = display_month.decode('utf-8')

        context['month_and_year'] = u"%(month)s, %(year)d" % (
            {'month': display_month, 'year': year}
        )

        if error:  # send any year/month errors
            context['cal_error'] = error

        # List enables sorting. As far as I can tell, .order_by() can't be used
        # here because we need it ordered by l_start_date.hour (simply ordering
        # by start_date won't work). The only alternative I've found is to use
        # extra(), but this would likely require different statements for
        # different databases...
        all_month_events = list(self.get_month_events(
            year, month, self.category, self.tag, loc=True, cncl=True
        ))

        all_month_events.sort(key=lambda x: x.l_start_date.hour)

        start_day = getattr(settings, "CALENDAR_START_DAY", 0)
        context['calendar'] = month_display(
            year, month, all_month_events, start_day, self.net, qs, mini,
            request=self.request,
        )

        context['show_events'] = False
        if getattr(settings, "CALENDAR_SHOW_LIST", False):
            context['show_events'] = True
            context['events'] = c.order_events(all_month_events, d=True) \
                if self.request.is_ajax() else c.order_events(all_month_events)

        return context

class EventDayView(GenericEventView):
    template_name = 'happenings/event_day_list.html'

    def check_for_cancelled_events(self, d):
        """Check if any events are cancelled on the given date 'd'."""
        for event in self.events:
            for cn in event.cancellations.all():
                if cn.date == d:
                    event.title += ' (CANCELLED)'

    def get_month_events(self, *args, **kwargs):
        return Event.objects.all_month_events(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(EventDayView, self).get_context_data(**kwargs)

        kw = self.kwargs
        y, m, d = map(int, (kw['year'], kw['month'], kw['day']))
        year, month, day, error = c.clean_year_month_day(y, m, d, self.net)

        if error:
            context['cal_error'] = error

        # Note that we don't prefetch 'cancellations' because they will be
        # prefetched later (in day_display in displays.py)
        all_month_events = self.get_month_events(
            year, month, self.category, self.tag
        )

        self.events = day_display(
            year, month, all_month_events, day
        )

        self.check_for_cancelled_events(d=date(year, month, day))
        context['events'] = self.events

        display_month = MONTHS_ALT[month]
        if isinstance(display_month, six.binary_type):
            display_month = display_month.decode('utf-8')

        context['month'] = display_month
        context['month_num'] = month
        context['year'] = year
        context['day'] = day
        context['month_day_year'] = u"%(month)s %(day)d, %(year)d" % (
            {'month': display_month, 'day': day, 'year': year}
        )

        # for use in the template to build next & prev querystrings
        context['next'], context['prev'] = c.get_next_and_prev(self.net)
        return context

class EventDetailView(DetailView):
    model = Event
    context_object_name = 'event'

    def get_object(self):
        return get_object_or_404(
            Event.objects.prefetch_related(
                'location', 'categories', 'tags', 'cancellations'
            ),
            pk=self.kwargs['pk']
        )

    def get_cncl_days(self):
        now = c.get_now()
        cncl = self.object.cancellations.all()
        return [(x.date, x.reason) for x in cncl if x.date >= now.date()]

    def check_cncl(self, d):
        cncl = self.object.cancellations.all()
        return True if [x for x in cncl if x.date == d] else False

    def get_context_data(self, **kwargs):
        now = c.get_now()
        context = super(EventDetailView, self).get_context_data(**kwargs)
        e = self.object

        for choice in Event.REPEAT_CHOICES:
            if choice[0] == e.repeat:
                context['repeat'] = choice[1]

        context['cncl_days'] = self.get_cncl_days()

        event = [e]  # event needs to be an iterable, see get_next_event()
        if not e.repeats('NEVER'):  # event is ongoing; get next occurrence
            if e.will_occur(now):
                year, month, day = get_next_event(event, now)
                next_event = date(year, month, day)
                context['next_event'] = date(year, month, day)
                context['next_or_prev_cncl'] = self.check_cncl(next_event)
            else:  # event is finished repeating; get last occurrence
                end = e.end_repeat
                last_event = end
                if e.repeats('WEEKDAY'):
                    year, month, day = c.check_weekday(
                        end.year, end.month, end.day, reverse=True
                    )
                    last_event = date(year, month, day)
                context['last_event'] = last_event
                context['next_or_prev_cncl'] = self.check_cncl(last_event)
        else:
            if e.is_chunk():
                # list of days for single-day event chunk
                context['event_days'] = (  # list comp
                    (e.l_start_date + timedelta(days=x))
                    for x in range(e.start_end_diff + 1)
                )
            else:
                # let template know if this single-day, non-repeating event is
                # cancelled
                context['this_cncl'] = self.check_cncl(e.l_start_date.date())
        return context


# -*- coding: utf-8 -*-
import logging

import operator

from pymongo.errors import (DuplicateKeyError, BulkWriteError)
from pymongo import (ASCENDING)

from scout.exceptions import IntegrityError

LOG = logging.getLogger(__name__)

class HpoHandler(object):

    def load_hpo_term(self, hpo_obj):
        """Add a hpo object

        Arguments:
            hpo_obj(dict)

        """
        LOG.debug("Loading hpo term %s into database", hpo_obj['_id'])
        try:
            self.hpo_term_collection.insert_one(hpo_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Hpo term %s already exists in database".format(hpo_obj['_id']))
        LOG.debug("Hpo term saved")

    def load_hpo_bulk(self, hpo_bulk):
        """Add a hpo object

        Arguments:
            hpo_bulk(list(scout.models.HpoTerm))

        Returns:
            result: pymongo bulkwrite result

        """
        LOG.debug("Loading hpo bulk")

        try:
            result = self.hpo_term_collection.insert_many(hpo_bulk)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)
        return result

    def hpo_term(self, hpo_id):
        """Fetch a hpo term

        Args:
            hpo_id(str)

        Returns:
            hpo_obj(dict)
        """
        LOG.debug("Fetching hpo term %s", hpo_id)

        return self.hpo_term_collection.find_one({'_id': hpo_id})

    def hpo_terms(self, query=None, hpo_term=None, text=None, limit=None):
        """Return all HPO terms

        If a query is sent hpo_terms will try to match with regex on term or
        description.

        Args:
            query(str): Part of a hpoterm or description
            hpo_term(str): Search for a specific hpo term
            limit(int): the number of desired results

        Returns:
            result(pymongo.Cursor): A cursor with hpo terms
        """
        query_dict = {}
        search_term = None
        if query:
            query_dict = {'$or':
                [
                    {'hpo_id': {'$regex': query, '$options':'i'}},
                    {'description': {'$regex': query, '$options':'i'}},
                ]
            }
            search_term = query
        elif text:
            new_string = ''
            for i,word in enumerate(text.split(' ')):
                if i == 0:
                    new_string += word
                else:
                    new_string += ' \"{0}\"'.format(word)
            LOG.info("Search HPO terms with %s", new_string)
            query_dict['$text'] = {'$search': new_string}
            search_term = text
        elif hpo_term:
            query_dict['hpo_id'] = hpo_term
            search_term = hpo_term

        limit = limit or int(10e10)
        res = self.hpo_term_collection.find(query_dict).limit(limit).sort('hpo_number',ASCENDING)

        LOG.info("Found {0} terms with search word {1}".format(res.count(), search_term))
        return res

    def disease_term(self, disease_identifier):
        """Return a disease term

        Checks if the identifier is a disease number or a id

        Args:
            disease_identifier(str)

        Returns:
            disease_obj(dict)
        """
        query = {}
        try:
            disease_identifier = int(disease_identifier)
            query['disease_nr'] = disease_identifier
        except ValueError:
            query['_id'] = disease_identifier

        return self.disease_term_collection.find_one(query)

    def disease_terms(self, hgnc_id=None):
        """Return all disease terms that overlaps a gene

            If no gene, return all disease terms

        Args:
            hgnc_id(int)

        Returns:
            iterable(dict): A list with all disease terms that match
        """
        query = {}
        if hgnc_id:
            LOG.debug("Fetching all diseases for gene %s", hgnc_id)
            query['genes'] = hgnc_id
        else:
            LOG.info("Fetching all disease terms")

        return list(self.disease_term_collection.find(query))

    def load_disease_term(self, disease_obj):
        """Load a disease term into the database

        Args:
            disease_obj(dict)
        """
        LOG.debug("Loading disease term %s into database", disease_obj['_id'])
        try:
            self.disease_term_collection.insert_one(disease_obj)
        except DuplicateKeyError as err:
            raise IntegrityError("Disease term %s already exists in database".format(disease_obj['_id']))

        LOG.debug("Disease term saved")

    def generate_hpo_gene_list(self, *hpo_terms):
        """Generate a sorted list with namedtuples of hpogenes

            Each namedtuple of the list looks like (hgnc_id, count)

            Args:
                hpo_terms(iterable(str))

            Returns:
                hpo_genes(list(HpoGene))
        """
        genes = {}
        for term in hpo_terms:
            hpo_obj = self.hpo_term(term)
            if hpo_obj:
                for hgnc_id in hpo_obj['genes']:
                    if hgnc_id in genes:
                        genes[hgnc_id] += 1
                    else:
                        genes[hgnc_id] = 1
            else:
                LOG.warning("Term %s could not be found", term)

        sorted_genes = sorted(genes.items(), key=operator.itemgetter(1), reverse=True)
        return sorted_genes