import logging

from datetime import datetime

from click import progressbar

from scout.parse.hpo import (parse_hpo_phenotypes, parse_hpo_diseases, parse_hpo_obo, parse_hpo_to_genes)
from scout.utils.requests import (fetch_hpo_terms, fetch_hpo_to_genes, fetch_hpo_phenotype_to_terms)

from scout.parse.omim import get_mim_phenotypes
from scout.build.hpo import build_hpo_term
from scout.build.disease import build_disease_term

from pprint import pprint as pp

LOG = logging.getLogger(__name__)

def load_hpo(adapter, disease_lines, hpo_disease_lines=None, hpo_lines=None, hpo_gene_lines=None):
    """Load the hpo terms and hpo diseases into database

    Args:
        adapter(MongoAdapter)
        disease_lines(iterable(str)): These are the omim genemap2 information
        hpo_lines(iterable(str))
        disease_lines(iterable(str))
        hpo_gene_lines(iterable(str))
    """
    # Create a map from gene aliases to gene objects
    alias_genes = adapter.genes_by_alias()

    # Fetch the hpo terms if no file
    if not hpo_lines:
        hpo_lines = fetch_hpo_terms()

    # Fetch the hpo gene information if no file
    if not hpo_gene_lines:
        hpo_gene_lines = fetch_hpo_to_genes()

    # Fetch the hpo phenotype information if no file
    if not hpo_disease_lines:
        hpo_disease_lines = fetch_hpo_phenotype_to_terms()

    load_hpo_terms(adapter, hpo_lines, hpo_gene_lines, alias_genes)

    load_disease_terms(adapter, disease_lines, alias_genes, hpo_disease_lines)

def load_hpo_terms(adapter, hpo_lines=None, hpo_gene_lines=None, alias_genes=None):
    """Load the hpo terms into the database

    Parse the hpo lines, build the objects and add them to the database

    Args:
        adapter(MongoAdapter)
        hpo_lines(iterable(str))
        hpo_gene_lines(iterable(str))
    """

    # Store the hpo terms
    hpo_terms = {}

    # Fetch the hpo terms if no file
    if not hpo_lines:
        hpo_lines = fetch_hpo_terms()

    # Fetch the hpo gene information if no file
    if not hpo_gene_lines:
        hpo_gene_lines = fetch_hpo_to_genes()

    # Parse the terms
    # This will yield dictionaries with information about the terms
    LOG.info("Parsing hpo terms")
    for term in parse_hpo_obo(hpo_lines):
        hpo_terms[term['hpo_id']] = term

    # Get a map with hgnc symbols to hgnc ids from scout
    if not alias_genes:
        alias_genes = adapter.genes_by_alias()

    LOG.info("Adding gene information to hpo terms ...")
    for hpo_to_symbol in parse_hpo_to_genes(hpo_gene_lines):
        hgnc_symbol = hpo_to_symbol['hgnc_symbol']
        hpo_id = hpo_to_symbol['hpo_id']

        # Fetch gene info to get correct hgnc id
        gene_info = alias_genes.get(hgnc_symbol)
        if not gene_info:
            continue

        hgnc_id = gene_info['true']

        if hpo_id not in hpo_terms:
            continue

        hpo_term = hpo_terms[hpo_id]

        if not 'genes' in hpo_term:
            hpo_term['genes'] = set()

        hpo_term['genes'].add(hgnc_id)

    start_time = datetime.now()

    LOG.info("Loading the hpo terms...")
    nr_terms = len(hpo_terms)
    hpo_bulk = []
    with progressbar(hpo_terms.values(), label="Loading hpo terms", length=nr_terms) as bar:

        for hpo_info in bar:
            hpo_bulk.append(build_hpo_term(hpo_info))

        if len(hpo_bulk) > 10000:
            adapter.load_hpo_bulk(hpo_bulk)
            hpo_bulk = []

    if hpo_bulk:
        adapter.load_hpo_bulk(hpo_bulk)

    LOG.info("Loading done. Nr of terms loaded {0}".format(nr_terms))
    LOG.info("Time to load terms: {0}".format(datetime.now() - start_time))

def load_disease_terms(adapter, genemap_lines, genes=None, hpo_disease_lines=None):
    """Load the omim phenotypes into the database

    Parse the phenotypes from genemap2.txt and find the associated hpo terms
    from ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt.

    Args:
        adapter(MongoAdapter)
        genemap_lines(iterable(str))
        genes(dict): Dictionary with all genes found in database
        hpo_disease_lines(iterable(str))

    """
    # Get a map with hgnc symbols to hgnc ids from scout
    if not genes:
        genes = adapter.genes_by_alias()

    # Fetch the disease terms from omim
    disease_terms = get_mim_phenotypes(genemap_lines=genemap_lines)

    if not hpo_disease_lines:
        hpo_disease_lines = fetch_hpo_phenotype_to_terms()
    hpo_diseases = parse_hpo_diseases(hpo_disease_lines)

    start_time = datetime.now()
    nr_diseases = None

    LOG.info("Loading the hpo disease...")
    for nr_diseases, disease_number in enumerate(disease_terms):
        disease_info = disease_terms[disease_number]
        disease_id = "OMIM:{0}".format(disease_number)

        if disease_id in hpo_diseases:
            hpo_terms = hpo_diseases[disease_id]['hpo_terms']
            if hpo_terms:
                disease_info['hpo_terms'] = hpo_terms
        disease_obj = build_disease_term(disease_info, genes)

        adapter.load_disease_term(disease_obj)

    LOG.info("Loading done. Nr of diseases loaded {0}".format(nr_diseases))
    LOG.info("Time to load diseases: {0}".format(datetime.now() - start_time))


def parse_frequencies(variant, transcripts):
    """Add the frequencies to a variant

    Frequencies are parsed either directly from keys in info fieds or from the
    transcripts is they are annotated there.

    Args:
        variant(cyvcf2.Variant): A parsed vcf variant
        transcripts(iterable(dict)): Parsed transcripts

    Returns:
        frequencies(dict): A dictionary with the relevant frequencies
    """
    frequencies = {}
    # These lists could be extended...
    thousand_genomes_keys = ['1000GAF']
    thousand_genomes_max_keys = ['1000G_MAX_AF']

    exac_keys = ['EXACAF']
    exac_max_keys = ['ExAC_MAX_AF', 'EXAC_MAX_AF']

    gnomad_keys = ['GNOMADAF', 'GNOMAD_AF']
    gnomad_max_keys = ['GNOMADAF_POPMAX', 'GNOMADAF_MAX']

    for test_key in thousand_genomes_keys:
        thousand_g = parse_frequency(variant, test_key)
        if thousand_g:
            frequencies['thousand_g'] = thousand_g
            break

    for test_key in thousand_genomes_max_keys:
        thousand_g_max = parse_frequency(variant, test_key)
        if thousand_g_max:
            frequencies['thousand_g_max'] = thousand_g_max
            break

    for test_key in exac_keys:
        exac = parse_frequency(variant, test_key)
        if exac:
            frequencies['exac'] = exac
            break

    for test_key in exac_max_keys:
        exac_max = parse_frequency(variant, test_key)
        if exac_max:
            frequencies['exac_max'] = exac_max
            break

    for test_key in gnomad_keys:
        gnomad = parse_frequency(variant, test_key)
        if gnomad:
            frequencies['gnomad'] = gnomad
            break

    for test_key in gnomad_max_keys:
        gnomad_max = parse_frequency(variant, test_key)
        if gnomad_max:
            frequencies['gnomad_max'] = gnomad_max
            break

    # Search transcripts if not found in VCF
    if not frequencies:
        for transcript in transcripts:
            exac = transcript.get('exac_maf')
            exac_max = transcript.get('exac_max')

            thousand_g = transcript.get('thousand_g_maf')
            thousandg_max = transcript.get('thousandg_max')

            gnomad = transcript.get('gnomad_maf')
            gnomad_max = transcript.get('gnomad_max')
            if exac:
                frequencies['exac'] = exac
            if exac_max:
                frequencies['exac_max'] = exac_max
            if thousand_g:
                frequencies['thousand_g'] = thousand_g
            if thousandg_max:
                frequencies['thousand_g_max'] = thousandg_max
            if gnomad:
                frequencies['gnomad'] = gnomad
            if gnomad_max:
                frequencies['gnomad_max'] = gnomad_max

    #These are SV-specific frequencies
    thousand_g_left = parse_frequency(variant, 'left_1000GAF')
    if thousand_g_left:
        frequencies['thousand_g_left'] = thousand_g_left

    thousand_g_right = parse_frequency(variant, 'right_1000GAF')
    if thousand_g_right:
        frequencies['thousand_g_right'] = thousand_g_right

    return frequencies

def parse_frequency(variant, info_key):
    """Parse any frequency from the info dict

    Args:
        variant(cyvcf2.Variant)
        info_key(str)

    Returns:
        frequency(float): or None if frequency does not exist
    """
    raw_annotation = variant.INFO.get(info_key)
    raw_annotation = None if raw_annotation == '.' else raw_annotation
    frequency = float(raw_annotation) if raw_annotation else None
    return frequency

def parse_sv_frequencies(variant):
    """Parsing of some custom sv frequencies

    These are very specific at the moment, this will hopefully get better over time when the
    field of structural variants is more developed.

    Args:
        variant(cyvcf2.Variant)

    Returns:
        sv_frequencies(dict)
    """
    frequency_keys = [
        'clingen_cgh_benignAF',
        'clingen_cgh_benign',
        'clingen_cgh_pathogenicAF',
        'clingen_cgh_pathogenic',
        'clingen_ngi',
        'clingen_ngiAF',
        'swegen',
        'swegenAF',
        'decipherAF',
        'decipher'
    ]
    sv_frequencies = {}

    for key in frequency_keys:
        value = variant.INFO.get(key, 0)
        if 'AF' in key:
            value = float(value)
        else:
            value = int(value)
        if value > 0:
            sv_frequencies[key] = value
    return sv_frequencies


import logging
import click

LOG = logging.getLogger(__name__)

@click.command('users', short_help='Display users')
@click.pass_context
def users(context):
    """Show all users in the database"""
    LOG.info("Running scout view users")
    adapter = context.obj['adapter']

    user_objs = adapter.users()
    if user_objs.count() == 0:
        LOG.info("No users found")
        context.abort()

    click.echo("#name\temail\troles\tinstitutes")
    for user_obj in user_objs:
        click.echo("{0}\t{1}\t{2}\t{3}\t".format(
            user_obj['name'],
            user_obj.get('mail', user_obj['_id']),
            ', '.join(user_obj.get('roles', [])),
            ', '.join(user_obj.get('institutes', [])),
        )
        )


# -*- coding: utf-8 -*-
import logging

from scout.exceptions import IntegrityError

logger = logging.getLogger(__name__)

def load_case(adapter, case_obj, update=False):
    """Load a case into the database

    If the case already exists the function will exit.
    If the user want to load a case that is already in the database
    'update' has to be 'True'

    Args:
        adapter (MongoAdapter): connection to the database
        case_obj (dict): case object to persist to the database
        update(bool): If existing case should be updated

    Returns:
        case_obj(dict): A dictionary with the builded case
    """
    logger.info('Loading case {} into database'.format(case_obj['display_name']))

    # Check if case exists in database
    existing_case = adapter.case(case_obj['_id'])

    if existing_case:
        if update:
            adapter.update_case(case_obj)
        else:
            raise IntegrityError("Case {0} already exists in database".format(case_obj['_id']))
    else:
        adapter.add_case(case_obj)
    return case_obj


import logging

from scout.models.hgnc_map import HgncGene

LOG = logging.getLogger(__name__)

def build_phenotype(phenotype_info):
    phenotype_obj = {}
    phenotype_obj['mim_number'] = phenotype_info['mim_number']
    phenotype_obj['description'] = phenotype_info['description']
    phenotype_obj['inheritance_models'] = list(phenotype_info.get('inheritance', set()))
    phenotype_obj['status'] = phenotype_info['status']

    return phenotype_obj

def build_hgnc_gene(gene_info, build='37'):
    """Build a hgnc_gene object

        Args:
            gene_info(dict): Gene information

        Returns:
            gene_obj(dict)

            {
                '_id': ObjectId(),
                # This is the hgnc id, required:
                'hgnc_id': int,
                # The primary symbol, required
                'hgnc_symbol': str,
                'ensembl_id': str, # required
                'build': str, # '37' or '38', defaults to '37', required

                'chromosome': str, # required
                'start': int, # required
                'end': int, # required

                'description': str, # Gene description
                'aliases': list(), # Gene symbol aliases, includes hgnc_symbol, str
                'entrez_id': int,
                'omim_id': int,
                'pli_score': float,
                'primary_transcripts': list(), # List of refseq transcripts (str)
                'ucsc_id': str,
                'uniprot_ids': list(), # List of str
                'vega_id': str,
                'transcripts': list(), # List of hgnc_transcript

                # Inheritance information
                'inheritance_models': list(), # List of model names
                'incomplete_penetrance': bool, # Acquired from HPO

                # Phenotype information
                'phenotypes': list(), # List of dictionaries with phenotype information
            }
    """
    try:
        hgnc_id = int(gene_info['hgnc_id'])
    except KeyError as err:
        raise KeyError("Gene has to have a hgnc_id")
    except ValueError as err:
        raise ValueError("hgnc_id has to be integer")

    try:
        hgnc_symbol = gene_info['hgnc_symbol']
    except KeyError as err:
        raise KeyError("Gene has to have a hgnc_symbol")

    try:
        ensembl_id = gene_info['ensembl_gene_id']
    except KeyError as err:
        raise KeyError("Gene has to have a ensembl_id")

    try:
        chromosome = gene_info['chromosome']
    except KeyError as err:
        raise KeyError("Gene has to have a chromosome")

    try:
        start = int(gene_info['start'])
    except KeyError as err:
        raise KeyError("Gene has to have a start position")
    except TypeError as err:
        raise TypeError("Gene start has to be a integer")

    try:
        end = int(gene_info['end'])
    except KeyError as err:
        raise KeyError("Gene has to have a end position")
    except TypeError as err:
        raise TypeError("Gene end has to be a integer")

    gene_obj = HgncGene(
        hgnc_id=hgnc_id,
        hgnc_symbol=hgnc_symbol,
        ensembl_id=ensembl_id,
        chrom=chromosome,
        start=start,
        end=end,
        build=build,
    )

    if gene_info.get('description'):
        gene_obj['description'] = gene_info['description']
        # LOG.debug("Adding info %s", gene_info['description'])

    if gene_info.get('previous_symbols'):
        gene_obj['aliases'] = gene_info['previous_symbols']

    if gene_info.get('entrez_id'):
        gene_obj['entrez_id'] = int(gene_info['entrez_id'])

    if gene_info.get('omim_id'):
        gene_obj['omim_id'] = int(gene_info['omim_id'])

    if gene_info.get('pli_score'):
        gene_obj['pli_score'] = float(gene_info['pli_score'])

    if gene_info.get('ref_seq'):
        gene_obj['primary_transcripts'] = gene_info['ref_seq']

    if gene_info.get('ucsc_id'):
        gene_obj['ucsc_id'] = gene_info['ucsc_id']

    if gene_info.get('uniprot_ids'):
        gene_obj['uniprot_ids'] = gene_info['uniprot_ids']

    if gene_info.get('vega_id'):
        gene_obj['vega_id'] = gene_info['vega_id']

    if gene_info.get('incomplete_penetrance'):
        gene_obj['incomplete_penetrance'] = True

    if gene_info.get('inheritance_models'):
        gene_obj['inheritance_models'] = gene_info['inheritance_models']

    phenotype_objs = []
    for phenotype_info in gene_info.get('phenotypes', []):
        phenotype_objs.append(build_phenotype(phenotype_info))

    if phenotype_objs:
        gene_obj['phenotypes'] = phenotype_objs

    for key in list(gene_obj):
        if gene_obj[key] is None:
            gene_obj.pop(key)

    return gene_obj


import logging
import math

from pprint import pprint as pp
from copy import deepcopy

import datetime as dt

import pymongo
from bson import ObjectId

from scout.parse.panel import (parse_gene_panel, get_omim_panel_genes)
from scout.build import build_panel
from scout.utils.requests import fetch_mim_files
from scout.utils.date import get_date

from scout.exceptions import IntegrityError

LOG = logging.getLogger(__name__)

class PanelHandler(object):

    def load_panel(self, parsed_panel):
        """Load a gene panel based on the info sent
        A panel object is built and integrity checks are made.
        The panel object is then loaded into the database.

        Args:
            path(str): Path to panel file
            institute(str): Name of institute that owns the panel
            panel_id(str): Panel id
            date(datetime.datetime): Date of creation
            version(float)
            full_name(str): Option to have a long name

            panel_info(dict): {
                'file': <path to panel file>(str),
                'institute': <institute>(str),
                'type': <panel type>(str),
                'date': date,
                'version': version,
                'panel_name': panel_id,
                'full_name': name,
            }
        """
        panel_obj = build_panel(parsed_panel, self)

        self.add_gene_panel(panel_obj)

    def load_omim_panel(self, api_key, institute=None):
        """Create and load the OMIM-AUTO panel"""
        existing_panel = self.gene_panel(panel_id='OMIM-AUTO')
        if not existing_panel:
            LOG.warning("OMIM-AUTO does not exists in database")
            LOG.info('Creating a first version')
            version = 1.0

        if existing_panel:
            version = float(math.floor(existing_panel['version']) + 1)

        LOG.info("Setting version to %s", version)

        try:
            mim_files = fetch_mim_files(api_key=api_key, genemap2=True, mim2genes=True)
        except Exception as err:
            raise err

        date_string = None
        # Get the correct date when omim files where released
        for line in mim_files['genemap2']:
            if 'Generated' in line:
                date_string = line.split(':')[-1].lstrip().rstrip()
        date_obj = get_date(date_string)

        if existing_panel:
            if existing_panel['date'] == date_obj:
                LOG.warning("There is no new version of OMIM")
                return

        panel_data = {}
        panel_data['path'] = None
        panel_data['type'] = 'clinical'
        panel_data['date'] = date_obj
        panel_data['panel_id'] = 'OMIM-AUTO'
        panel_data['institute'] = institute or 'cust002'
        panel_data['version'] = version
        panel_data['display_name'] = 'OMIM-AUTO'
        panel_data['genes'] = []

        alias_genes = self.genes_by_alias()

        genes = get_omim_panel_genes(
            genemap2_lines = mim_files['genemap2'],
            mim2gene_lines = mim_files['mim2genes'],
            alias_genes = alias_genes,
        )

        for gene in genes:
            panel_data['genes'].append(gene)

        panel_obj = build_panel(panel_data, self)

        if existing_panel:

            new_genes = self.compare_mim_panels(existing_panel, panel_obj)
            if new_genes:
                self.update_mim_version(new_genes, panel_obj, old_version=existing_panel['version'])
            else:
                LOG.info("The new version of omim does not differ from the old one")
                LOG.info("No update is added")
                return

        self.add_gene_panel(panel_obj)

    def compare_mim_panels(self, existing_panel, new_panel):
        """Check if the latest version of OMIM differs from the most recent in database
           Return all genes that where not in the previous version.

        Args:
            existing_panel(dict)
            new_panel(dict)

        Returns:
            new_genes(set(str))
        """
        existing_genes = set([gene['hgnc_id'] for gene in existing_panel['genes']])
        new_genes = set([gene['hgnc_id'] for gene in new_panel['genes']])

        return new_genes.difference(existing_genes)

    def update_mim_version(self, new_genes, new_panel, old_version):
        """Set the correct version for each gene
        Loop over the genes in the new panel

        Args:
            new_genes(set(str)): Set with the new gene symbols
            new_panel(dict)

        """
        LOG.info('Updating versions for new genes')
        version = new_panel['version']
        for gene in new_panel['genes']:
            gene_symbol = gene['hgnc_id']
            # If the gene is new we add the version
            if gene_symbol in new_genes:
                gene['database_entry_version'] = version
                continue
            # If the gene is old it will have the previous version
            gene['database_entry_version'] = old_version

        return

    def add_gene_panel(self, panel_obj):
        """Add a gene panel to the database

            Args:
                panel_obj(dict)
        """
        panel_name = panel_obj['panel_name']
        panel_version = panel_obj['version']
        display_name = panel_obj.get('display_name', panel_name)

        if self.gene_panel(panel_name, panel_version):
            raise IntegrityError("Panel {0} with version {1} already"
                                 " exist in database".format(panel_name, panel_version))
        LOG.info("loading panel {0}, version {1} to database".format(
            display_name, panel_version
        ))
        result = self.panel_collection.insert_one(panel_obj)
        LOG.debug("Panel saved")
        return result.inserted_id

    def panel(self, panel_id):
        """Fetch a gene panel by '_id'.

        Args:
            panel_id (str, ObjectId): str or ObjectId of document ObjectId

        Returns:
            dict: panel object or `None` if panel not found
        """
        if not isinstance(panel_id, ObjectId):
            panel_id = ObjectId(panel_id)
        panel_obj = self.panel_collection.find_one({'_id': panel_id})
        return panel_obj

    def delete_panel(self, panel_obj):
        """Delete a panel by '_id'.

        Args:
            panel_obj(dict)

        Returns:
            res(pymongo.DeleteResult)
        """
        res = self.panel_collection.delete_one({'_id': panel_obj['_id']})
        LOG.warning("Deleting panel %s, version %s" % (panel_obj['panel_name'], panel_obj['version']))
        return res

    def gene_panel(self, panel_id, version=None):
        """Fetch a gene panel.

        If no panel is sent return all panels

        Args:
            panel_id (str): unique id for the panel
            version (str): version of the panel. If 'None' latest version will be returned

        Returns:
            gene_panel: gene panel object
        """
        query = {'panel_name': panel_id}
        if version:
            LOG.info("Fetch gene panel {0}, version {1} from database".format(
                panel_id, version
            ))
            query['version'] = version
            return self.panel_collection.find_one(query)
        else:
            LOG.info("Fetching gene panels %s from database", panel_id)
            res = self.panel_collection.find(query).sort('version', -1)
            if res.count() > 0:
                return res[0]
            else:
                LOG.info("No gene panel found")
                return None

    def gene_panels(self, panel_id=None, institute_id=None, version=None):
        """Return all gene panels

        If panel_id return all versions of panels by that panel name

        Args:
            panel_id(str)

        Returns:
            cursor(pymongo.cursor)
        """
        query = {}
        if panel_id:
            query['panel_name'] = panel_id
            if version:
                query['version'] = version
        if institute_id:
            query['institute'] = institute_id

        return self.panel_collection.find(query)

    def hgnc_to_panels(self, hgnc_id):
        """Get a list of gene panel objects for a hgnc_id

            Args:
                hgnc_id(int)

            Returns:
                hgnc_panels(dict): A dictionary with hgnc as keys and lists of
                                   gene panel objects as values
        """
        return self.panel_collection.find({'genes.hgnc_id' : hgnc_id })

    def gene_to_panels(self, case_obj):
        """Fetch all gene panels and group them by gene

            Args:
                case_obj(scout.models.Case)
            Returns:
                gene_dict(dict): A dictionary with gene as keys and a set of
                                 panel names as value
        """
        LOG.info("Building gene to panels")
        gene_dict = {}

        for panel_info in case_obj.get('panels', []):
            panel_name = panel_info['panel_name']
            panel_version = panel_info['version']
            panel_obj = self.gene_panel(panel_name, version=panel_version)
            if not panel_obj:
                ## Raise exception here???
                LOG.warning("Panel: {0}, version {1} does not exist in database".format(panel_name, panel_version))

            for gene in panel_obj['genes']:
                hgnc_id = gene['hgnc_id']

                if hgnc_id not in gene_dict:
                    gene_dict[hgnc_id] = set([panel_name])
                    continue

                gene_dict[hgnc_id].add(panel_name)

        LOG.info("Gene to panels done")

        return gene_dict

    def update_panel(self, panel_obj, version=None, date_obj=None):
        """Replace a existing gene panel with a new one

        Keeps the object id

        Args:
            panel_obj(dict)
            version(float)
            date_obj(datetime.datetime)

        Returns:
            updated_panel(dict)
        """
        LOG.info("Updating panel %s", panel_obj['panel_name'])
        # update date of panel to "today"
        date = panel_obj['date']
        if version:
            LOG.info("Updating version from {0} to version {1}".format(
                panel_obj['version'], version))
            panel_obj['version'] = version
            # Updating version should not update date
            if date_obj:
                date = date_obj
        else:
            date = date_obj or dt.datetime.now()
        panel_obj['date'] = date

        updated_panel = self.panel_collection.find_one_and_replace(
            {'_id': panel_obj['_id']},
            panel_obj,
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_panel

    def add_pending(self, panel_obj, hgnc_gene, action, info=None):
        """Add a pending action to a gene panel

        Store the pending actions in panel.pending

        Args:
            panel_obj(dict): The panel that is about to be updated
            hgnc_gene(dict)
            action(str): choices=['add','delete','edit']
            info(dict): additional gene info (disease_associated_transcripts,
                        reduced_penetrance, mosaicism, database_entry_version ,
                        inheritance_models, comment)

        Returns:
            updated_panel(dict):

        """

        valid_actions = ['add', 'delete', 'edit']
        if action not in valid_actions:
            raise ValueError("Invalid action {0}".format(action))

        info = info or {}
        pending_action = {
            'hgnc_id': hgnc_gene['hgnc_id'],
            'action': action,
            'info': info,
            'symbol': hgnc_gene['hgnc_symbol'],
        }

        updated_panel = self.panel_collection.find_one_and_update(
            {'_id': panel_obj['_id']},
            {
                '$addToSet': {
                    'pending': pending_action
                }
            },
            return_document=pymongo.ReturnDocument.AFTER
        )

        return updated_panel

    def apply_pending(self, panel_obj, version):
        """Apply the pending changes to an existing gene panel or create a new version of the same panel.

        Args:
            panel_obj(dict): panel in database to update
            version(double): panel version to update

        Returns:
            inserted_id(str): id of updated panel or the new one
        """

        updates = {}
        new_panel = deepcopy(panel_obj)
        new_panel['pending'] = []
        new_panel['date'] = dt.datetime.now()
        info_fields = ['disease_associated_transcripts', 'inheritance_models', 'reduced_penetrance',
            'mosaicism', 'database_entry_version', 'comment']
        new_genes = []

        for update in panel_obj.get('pending', []):
            hgnc_id = update['hgnc_id']

            # If action is add we create a new gene object
            if update['action'] != 'add':
                updates[hgnc_id] = update
                continue
            info = update.get('info', {})
            gene_obj = {
                'hgnc_id': hgnc_id,
                'symbol': update['symbol']
            }

            for field in info_fields:
                if field in info:
                    gene_obj[field] = info[field]
            new_genes.append(gene_obj)

        for gene in panel_obj['genes']:
            hgnc_id = gene['hgnc_id']

            if hgnc_id not in updates:
                new_genes.append(gene)
                continue

            current_update = updates[hgnc_id]
            action = current_update['action']
            info = current_update['info']

            # If action is delete we do not add the gene to new genes
            if action == 'delete':
                continue

            elif action == 'edit':
                for field in info_fields:
                    if field in info:
                        gene[field] = info[field]
                new_genes.append(gene)

        new_panel['genes'] = new_genes
        new_panel['version'] = float(version)

        inserted_id = None
        # if the same version of the panel should be updated
        if new_panel['version'] == panel_obj['version']:
            # replace panel_obj with new_panel
            result = self.panel_collection.find_one_and_replace(
                {'_id':panel_obj['_id']},
                new_panel,
                return_document=pymongo.ReturnDocument.AFTER
            )
            inserted_id = result['_id']
        else: # create a new version of the same panel
            new_panel.pop('_id')

            # archive the old panel
            panel_obj['is_archived'] = True
            self.update_panel(panel_obj=panel_obj, date_obj=panel_obj['date'])

            # insert the new panel
            inserted_id = self.panel_collection.insert_one(new_panel).inserted_id

        return inserted_id

    def latest_panels(self, institute_id):
        """Return the latest version of each panel."""
        panel_names = self.gene_panels(institute_id=institute_id).distinct('panel_name')
        for panel_name in panel_names:
            panel_obj = self.gene_panel(panel_name)
            yield panel_obj

    def clinical_symbols(self, case_obj):
        """Return all the clinical gene symbols for a case."""
        panel_ids = [panel['panel_id'] for panel in case_obj['panels']]
        query = self.panel_collection.aggregate([
            {'$match': {'_id': {'$in': panel_ids}}},
            {'$unwind': '$genes'},
            {'$group': {'_id': '$genes.symbol'}}
        ])
        return set(item['_id'] for item in query)


import logging
import json

from pprint import pprint as pp

from bson.json_util import dumps

import click

from scout.constants import CASE_STATUSES
from .utils import json_option

LOG = logging.getLogger(__name__)

@click.command('cases', short_help='Fetch cases')
@click.option('--case-id',
              help='Case id to search for'
)
@click.option('-i', '--institute',
              help='institute id of related cases'
)
@click.option('-r', '--reruns',
              is_flag=True,
              help='requested to be rerun'
)
@click.option('-f', '--finished',
              is_flag=True,
              help='archived or solved'
)
@click.option('--causatives',
              is_flag=True,
              help='Has causative variants'
)
@click.option('--research-requested',
              is_flag=True,
              help='If research is requested'
)
@click.option('--is-research',
              is_flag=True,
              help='If case is in research mode'
)
@click.option('-s', '--status',
              type=click.Choice(CASE_STATUSES),
              help='Specify what status to look for'
)
@json_option
@click.pass_context
def cases(context, case_id, institute, reruns, finished, causatives, research_requested,
          is_research, status, json):
    """Interact with cases existing in the database."""
    adapter = context.obj['adapter']

    models = []
    if case_id:
        case_obj = adapter.case(case_id=case_id)
        if case_obj:
            models.append(case_obj)
        else:
            LOG.info("No case with id {}".format(case_id))

    else:
        models = adapter.cases(collaborator=institute, reruns=reruns,
                           finished=finished, has_causatives=causatives,
                           research_requested=research_requested,
                           is_research=is_research, status=status)
        models = [case_obj for case_obj in models]
        if len(models) == 0:
            LOG.info("No cases could be found")

    if json:
        click.echo(dumps(models))
        return

    for model in models:
        pp(model)


# -*- coding: utf-8 -*-
import logging
import logging.handlers

class TlsSMTPHandler(logging.handlers.SMTPHandler):
    def emit(self, record):
        """Emit a record.
        Format the record and send it to the specified addressees.
        """
        try:
            import smtplib
            try:
                from email.utils import formatdate
            except ImportError:
                formatdate = self.date_time

            port = self.mailport
            if not port:
                port = smtplib.SMTP_PORT

            smtp = smtplib.SMTP(self.mailhost, port)
            msg = self.format(record)
            msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\nDate: %s\r\n\r\n%s" % (
                self.fromaddr,
                ','.join(self.toaddrs),
                self.getSubject(record),
                formatdate(), msg
            )

            if self.username:
                smtp.ehlo()  # For 'tls', add this line
                smtp.starttls()  # For 'tls', add this line
                smtp.ehlo()  # For 'tls', add this line
                smtp.login(self.username, self.password)

            smtp.sendmail(self.fromaddr, self.toaddrs, msg)
            smtp.quit()

        except (KeyboardInterrupt, SystemExit):
            raise

        except:
            self.handleError(record)


# -*- coding: utf-8 -*-
import logging

from scout.constants import INDEXES

LOG = logging.getLogger(__name__)

class IndexHandler(object):

    def indexes(self, collection=None):
        """Return a list with the current indexes

        Skip the mandatory _id_ indexes

        Args:
            collection(str)

        Returns:
            indexes(list)
        """

        indexes = []

        for collection_name in self.collections():
            if collection and collection != collection_name:
                continue
            for index_name in self.db[collection_name].index_information():
                if index_name != '_id_':
                    indexes.append(index_name)
        return indexes

    def load_indexes(self):
        """Add the proper indexes to the scout instance.

        All indexes are specified in scout/constants/indexes.py

        If this method is utilised when new indexes are defined those should be added

        """
        for collection_name in INDEXES:
            existing_indexes = self.indexes(collection_name)
            indexes = INDEXES[collection_name]
            for index in indexes:
                index_name = index.document.get('name')
                if index_name in existing_indexes:
                    LOG.info("Deleting old index: %s" % index_name)
                    self.db[collection_name].drop_index(index_name)
            LOG.info("creating indexes for {0} collection: {1}".format(
                collection_name,
                ', '.join([index.document.get('name') for index in indexes])
                ))
            self.db[collection_name].create_indexes(indexes)

    def update_indexes(self):
        """Update the indexes

        If there are any indexes that are not added to the database, add those.

        """
        LOG.info("Updating indexes...")
        nr_updated = 0
        for collection_name in INDEXES:
            existing_indexes = self.indexes(collection_name)
            indexes = INDEXES[collection_name]
            for index in indexes:
                index_name = index.document.get('name')
                if index_name not in existing_indexes:
                    nr_updated += 1
                    LOG.info("Adding index : %s" % index_name)
                    self.db[collection_name].create_indexes(indexes)
        if nr_updated == 0:
            LOG.info("All indexes in place")

    def drop_indexes(self):
        """Delete all indexes for the database"""
        LOG.warning("Dropping all indexe")
        for collection_name in INDEXES:
            LOG.warning("Dropping all indexes for collection name %s", collection_name)
            self.db[collection_name].drop_indexes()


import logging
import re
from scout.constants import FUNDAMENTAL_CRITERIA, PRIMARY_CRITERIA, SECONDARY_CRITERIA

LOG = logging.getLogger(__name__)

from scout.constants import (SPIDEX_HUMAN, CLINSIG_MAP)

class QueryHandler(object):

    def build_variant_query(self, query=None, category='snv', variant_type=['clinical']):
        """Build a mongo query across multiple cases.
        Translate query options from a form into a complete mongo query dictionary.

        Beware that unindexed queries against a large variant collection will
        be extremely slow.

        Currently indexed query options:
            hgnc_symbols
            rank_score
            variant_type
            category

        Args:
            query(dict): A query dictionary for the database, from a query form.
            category(str): 'snv', 'sv', 'str' or 'cancer'
            variant_type(str): 'clinical' or 'research'

        Returns:
            mongo_query : A dictionary in the mongo query format.
        """

        query = query or {}
        mongo_variant_query = {}

        LOG.debug("Building a mongo query for %s" % query)

        if query.get('hgnc_symbols'):
            mongo_variant_query['hgnc_symbols'] = {'$in': query['hgnc_symbols']}

        mongo_variant_query['variant_type'] = {'$in': variant_type}

        mongo_variant_query['category'] = category

        rank_score = query.get('rank_score') or 15

        mongo_variant_query['rank_score'] = {'$gte': rank_score}
        LOG.debug("Querying %s" % mongo_variant_query)

        return mongo_variant_query

    def build_query(self, case_id, query=None, variant_ids=None, category='snv'):
        """Build a mongo query

        These are the different query options:
            {
                'genetic_models': list,
                'chrom': str,
                'thousand_genomes_frequency': float,
                'exac_frequency': float,
                'clingen_ngi': int,
                'cadd_score': float,
                'cadd_inclusive": boolean,
                'genetic_models': list(str),
                'hgnc_symbols': list,
                'region_annotations': list,
                'functional_annotations': list,
                'clinsig': list,
                'clinsig_confident_always_returned': boolean,
                'variant_type': str(('research', 'clinical')),
                'chrom': str,
                'start': int,
                'end': int,
                'svtype': list,
                'size': int,
                'size_shorter': boolean,
                'gene_panels': list(str),
                'mvl_tag": boolean,
                'decipher": boolean,
            }

        Arguments:
            case_id(str)
            query(dict): a dictionary of query filters specified by the users
            variant_ids(list(str)): A list of md5 variant ids

        Returns:
            mongo_query : A dictionary in the mongo query format

        """
        query = query or {}
        mongo_query = {}
        gene_query = None

        ##### Base query params

        # set up the fundamental query params: case_id, category, type and
        # restrict to list of variants (if var list is provided)
        for criterion in FUNDAMENTAL_CRITERIA:
            if criterion == 'case_id':
                LOG.debug("Building a mongo query for %s" % case_id)
                mongo_query['case_id'] = case_id

            elif criterion == 'variant_ids' and variant_ids:
                LOG.debug("Adding variant_ids %s to query" % ', '.join(variant_ids))
                mongo_query['variant_id'] = {'$in': variant_ids}

            elif criterion == 'category':
                LOG.debug("Querying category %s" % category)
                mongo_query['category'] = category

            elif criterion == 'variant_type':
                mongo_query['variant_type'] = query.get('variant_type', 'clinical')
                LOG.debug("Set variant type to %s", mongo_query['variant_type'])

            # Requests to filter based on gene panels, hgnc_symbols or
            # coordinate ranges must always be honored. They are always added to
            # query as top level, implicit '$and'. When both hgnc_symbols and a
            # panel is used, addition of this is delayed until after the rest of
            # the query content is clear.

            elif criterion in ['hgnc_symbols', 'gene_panels'] and gene_query is None:
                gene_query = self.gene_filter(query, mongo_query)

            elif criterion == 'chrom' and query.get('chrom'): # filter by coordinates
                self.coordinate_filter(query, mongo_query)

            elif criterion == 'variant_ids' and variant_ids:
                LOG.debug("Adding variant_ids %s to query" % ', '.join(variant_ids))
                mongo_query['variant_id'] = {'$in': variant_ids}

            ##### end of fundamental query params

        ##### start of the custom query params
        # there is only 'clinsig' criterion among the primary terms right now
        primary_terms = False

        # gnomad_frequency, local_obs, clingen_ngi, swegen, spidex_human, cadd_score, genetic_models, mvl_tag
        # functional_annotations, region_annotations, size, svtype, decipher, depth, alt_count, control_frequency
        secondary_terms = False

        # check if any of the primary criteria was specified in the query
        for term in PRIMARY_CRITERIA:
            if query.get(term):
                primary_terms = True

        # check if any of the secondary criteria was specified in the query:
        for term in SECONDARY_CRITERIA:
            if query.get(term):
                secondary_terms = True

        if primary_terms is True:
            clinsign_filter = self.clinsig_query(query, mongo_query)

        # Secondary, excluding filter criteria will hide variants in general,
        # but can be overridden by an including, major filter criteria
        # such as a Pathogenic ClinSig.
        if secondary_terms is True:
            secondary_filter = self.secondary_query(query, mongo_query)

            # If there are no primary criteria given, all secondary criteria are added as a
            # top level '$and' to the query.
            if primary_terms is False:
                if gene_query:
                    mongo_query['$and'] = [ {'$or': gene_query}, {'$and': secondary_filter}]
                else:
                    mongo_query['$and'] = secondary_filter

            # If there is only one primary criterion given without any secondary, it will also be
            # added as a top level '$and'.
            # Otherwise, primary criteria are added as a high level '$or' and all secondary criteria
            # are joined together with them as a single lower level '$and'.
            if primary_terms is True: # clinsig is specified

                # Given a request to always return confident clinical variants,
                # add the clnsig query as a major criteria, but only
                # trust clnsig entries with trusted revstat levels.
                if query.get('clinsig_confident_always_returned') == True:
                    if gene_query:
                        mongo_query['$and'] = [
                            {'$or': gene_query},
                            {
                                '$or': [
                                    {'$and': secondary_filter}, clinsign_filter
                                ]
                            }
                        ]
                    else:
                        mongo_query['$or'] = [ {'$and': secondary_filter}, clinsign_filter ]
                else: # clisig terms are provided but no need for trusted revstat levels
                    secondary_filter.append(clinsign_filter)
                    if gene_query:
                        mongo_query['$and'] = [ {'$or': gene_query}, {'$and': secondary_filter}]
                    else:
                        mongo_query['$and'] = secondary_filter

        elif primary_terms is True: # clisig is provided without secondary terms query
            # use implicit and
            mongo_query['clnsig'] = clinsign_filter['clnsig']
            if gene_query:
                mongo_query['$and'] = [{ '$or': gene_query }]

        elif gene_query: # no primary or secondary filters provided
            mongo_query['$and'] = [{ '$or': gene_query }]

        LOG.info("mongo query: %s", mongo_query)

        return mongo_query

    def clinsig_query(self, query, mongo_query):
        """ Add clinsig filter values to the mongo query object

            Args:
                query(dict): a dictionary of query filters specified by the users
                mongo_query(dict): the query that is going to be submitted to the database

            Returns:
                clinsig_query(dict): a dictionary with clinsig key-values

        """
        LOG.debug('clinsig is a query parameter')
        trusted_revision_level = ['mult', 'single', 'exp', 'guideline']
        rank = []
        str_rank = []
        clnsig_query = {}

        for item in query['clinsig']:
            rank.append(int(item))
            # search for human readable clinsig values in newer cases
            rank.append(CLINSIG_MAP[int(item)])
            str_rank.append(CLINSIG_MAP[int(item)])

        if query.get('clinsig_confident_always_returned') == True:
            LOG.debug("add CLINSIG filter with trusted_revision_level")

            clnsig_query = { "clnsig":
                        {
                            '$elemMatch': {
                                '$or' : [
                                    {
                                        '$and' : [
                                             {'value' : { '$in': rank }},
                                             {'revstat': { '$in': trusted_revision_level }}
                                        ]
                                    },
                                    {
                                        '$and': [
                                            {'value' : re.compile('|'.join(str_rank))},
                                            {'revstat' : re.compile('|'.join(trusted_revision_level))}
                                        ]
                                    }
                                ]
                            }
                         }
                    }
        else:
            LOG.debug("add CLINSIG filter for rank: %s" %
                             ', '.join(str(query['clinsig'])))

            clnsig_query = {
                    "clnsig":
                        {
                            '$elemMatch': {
                                '$or' : [
                                    { 'value' : { '$in': rank }},
                                    { 'value' : re.compile('|'.join(str_rank)) }
                                ]
                            }
                        }
                }

        return clnsig_query

    def coordinate_filter(self, query, mongo_query):
        """ Adds genomic coordinated-related filters to the query object

        Args:
            query(dict): a dictionary of query filters specified by the users
            mongo_query(dict): the query that is going to be submitted to the database

        Returns:
            mongo_query(dict): returned object contains coordinate filters

        """
        LOG.debug('Adding genomic coordinates to the query')
        chromosome = query['chrom']
        mongo_query['chromosome'] = chromosome

        if (query.get('start') and query.get('end')):
            mongo_query['position'] = {'$lte': int(query['end'])}
            mongo_query['end'] = {'$gte': int(query['start'])}

        return mongo_query

    def gene_filter(self, query, mongo_query):
        """ Adds gene-related filters to the query object

        Args:
            query(dict): a dictionary of query filters specified by the users
            mongo_query(dict): the query that is going to be submitted to the database

        Returns:
            mongo_query(dict): returned object contains gene and panel-related filters

        """
        LOG.debug('Adding panel and genes-related parameters to the query')

        gene_query = []

        if query.get('hgnc_symbols') and query.get('gene_panels'):
            gene_query.append({'hgnc_symbols': {'$in': query['hgnc_symbols']}})
            gene_query.append({'panels': {'$in': query['gene_panels']}})
            mongo_query['$or']=gene_query
        else:
            if query.get('hgnc_symbols'):
                hgnc_symbols = query['hgnc_symbols']
                mongo_query['hgnc_symbols'] = {'$in': hgnc_symbols}
                LOG.debug("Adding hgnc_symbols: %s to query" %
                             ', '.join(hgnc_symbols))

            if query.get('gene_panels'):
                gene_panels = query['gene_panels']
                mongo_query['panels'] = {'$in': gene_panels}

        return gene_query

    def secondary_query(self, query, mongo_query, secondary_filter=None):
        """Creates a secondary query object based on secondary parameters specified by user

            Args:
                query(dict): a dictionary of query filters specified by the users
                mongo_query(dict): the query that is going to be submitted to the database

            Returns:
                mongo_secondary_query(list): a dictionary with secondary query parameters

        """
        LOG.debug('Creating a query object with secondary parameters')

        mongo_secondary_query = []

        # loop over secondary query criteria
        for criterion in SECONDARY_CRITERIA:
            if not query.get(criterion):
                continue

            if criterion == 'gnomad_frequency':
                gnomad = query.get('gnomad_frequency')
                if gnomad == '-1':
                    # -1 means to exclude all variants that exists in gnomad
                    mongo_query['gnomad_frequency'] = {'$exists': False}
                else:
                    # Replace comma with dot
                    mongo_secondary_query.append(
                        {
                            '$or': [
                                {
                                    'gnomad_frequency': {'$lt': float(gnomad)}
                                },
                                {
                                    'gnomad_frequency': {'$exists': False}
                                }
                            ]
                        }
                    )
                LOG.debug("Adding gnomad_frequency to query")

            if criterion == 'local_obs':
                local_obs = query.get('local_obs')
                mongo_secondary_query.append({
                    '$or': [
                        {'local_obs_old': None},
                        {'local_obs_old': {'$lt': local_obs + 1}},
                    ]
                })

            if criterion in ['clingen_ngi', 'swegen']:
                mongo_secondary_query.append({
                    '$or': [
                        { criterion : {'$exists': False}},
                        { criterion : {'$lt': query[criterion] + 1}},
                    ]
                })

            if criterion == 'spidex_human':
                # construct spidex query. Build the or part starting with empty SPIDEX values
                spidex_human = query['spidex_human']

                spidex_query_or_part = []
                if ( 'not_reported' in spidex_human):
                    spidex_query_or_part.append({'spidex': {'$exists': False}})

                for spidex_level in SPIDEX_HUMAN:
                    if ( spidex_level in spidex_human ):
                        spidex_query_or_part.append({'$or': [
                                    {'$and': [{'spidex': {'$gt': SPIDEX_HUMAN[spidex_level]['neg'][0]}},
                                              {'spidex': {'$lt': SPIDEX_HUMAN[spidex_level]['neg'][1]}}]},
                                    {'$and': [{'spidex': {'$gt': SPIDEX_HUMAN[spidex_level]['pos'][0]}},
                                              {'spidex': {'$lt': SPIDEX_HUMAN[spidex_level]['pos'][1]}} ]} ]})

                mongo_secondary_query.append({'$or': spidex_query_or_part })

            if criterion == 'cadd_score':
                cadd = query['cadd_score']
                cadd_query = {'cadd_score': {'$gt': float(cadd)}}
                LOG.debug("Adding cadd_score: %s to query", cadd)

                if query.get('cadd_inclusive') is True:
                    cadd_query = {
                        '$or': [
                            cadd_query,
                            {'cadd_score': {'$exists': False}}
                            ]}
                    LOG.debug("Adding cadd inclusive to query")

                mongo_secondary_query.append(cadd_query)

            if criterion in ['genetic_models', 'functional_annotations', 'region_annotations']:
                criterion_values = query[criterion]
                if criterion == 'genetic_models':
                    mongo_secondary_query.append({criterion: {'$in': criterion_values}})
                else:
                    # filter key will be genes.[criterion (minus final char)]
                    mongo_secondary_query.append({ '.'.join(['genes', criterion[:-1]]) : {'$in': criterion_values}})

                LOG.debug("Adding {0}: {1} to query".format(criterion, ', '.join(criterion_values)))

            if criterion == 'size':
                size = query['size']
                size_query = {'length': {'$gt': int(size)}}
                LOG.debug("Adding length: %s to query" % size)

                if query.get('size_shorter'):
                    size_query = {
                        '$or': [
                            {'length': {'$lt': int(size)}},
                            {'length': {'$exists': False}}
                        ]}
                    LOG.debug("Adding size less than, undef inclusive to query.")

                mongo_secondary_query.append(size_query)

            if criterion == 'svtype':
                svtype = query['svtype']
                mongo_secondary_query.append({'sub_category': {'$in': svtype}})
                LOG.debug("Adding SV_type %s to query" %
                             ', '.join(svtype))

            if criterion == 'decipher':
                mongo_query['decipher'] = {'$exists': True}
                LOG.debug("Adding decipher to query")

            if criterion == 'depth':
                LOG.debug("add depth filter")
                mongo_secondary_query.append({
                    'tumor.read_depth': {
                        '$gt': query.get('depth'),
                    }
                })

            if criterion == 'alt_count':
                LOG.debug("add min alt count filter")
                mongo_secondary_query.append({
                    'tumor.alt_depth': {
                        '$gt': query.get('alt_count'),
                    }
                })

            if criterion == 'control_frequency':
                LOG.debug("add minimum control frequency filter")
                mongo_secondary_query.append({
                    'normal.alt_freq': {
                        '$lt': float(query.get('control_frequency')),
                    }
                })

            if criterion == 'mvl_tag':
                LOG.debug("add managed variant list filter")
                mongo_secondary_query.append({
                    'mvl_tag': {
                        '$exists': True,
                    }
                })

        return mongo_secondary_query


#!/usr/bin/env python
# encoding: utf-8
"""
wipe_database.py

Cli to clean the database.

Created by Måns Magnusson on 2015-01-14.
Copyright (c) 2015 __MoonsoInc__. All rights reserved.

"""

import logging
import click

LOG = logging.getLogger(__name__)

def abort_if_false(ctx, param, value):
    if not value:
        ctx.abort()

@click.command('wipe', short_help="Wipe a scout instance")
@click.option("--yes",
    is_flag=True,
    callback=abort_if_false,
    expose_value=False,
    prompt="Are you sure you want to drop the db?"
)
@click.pass_context
def wipe(ctx):
    """Drop the mongo database given."""
    LOG.info("Running scout wipe")
    db_name = ctx.obj['mongodb']
    LOG.info("Dropping database %s", db_name)
    try:
        ctx.obj['client'].drop_database(db_name)
    except Exception as err:
        LOG.warning(err)
        ctx.abort()
    LOG.info("Dropped whole database")


# -*- coding: utf-8 -*-
import csv

def parse_panel(csv_stream):
    """Parse user submitted panel."""
    reader = csv.DictReader(csv_stream, delimiter=';', quoting=csv.QUOTE_NONE)
    genes = []
    for gene_row in reader:
        if not gene_row['HGNC_IDnumber'].strip().isdigit():
            continue
        transcripts_raw = gene_row.get('Disease_associated_transcript')
        if transcripts_raw:
            transcripts_list = [tx.split(':', 1)[-1].strip() for tx in transcripts_raw.split(',')]
        else:
            transcripts_list = []

        models_raw = gene_row.get('Genetic_disease_model')
        models_list = [model.strip() for model in models_raw.split(',')] if models_raw else []
        panel_gene = dict(
            symbol=gene_row['HGNC_symbol'].strip() if gene_row.get('HGNC_symbol') else None,
            hgnc_id=int(gene_row['HGNC_IDnumber'].strip()),
            disease_associated_transcripts=transcripts_list,
            reduced_penetrance=True if gene_row.get('Reduced_penetrance') else None,
            mosaicism=True if gene_row.get('Mosaicism') else None,
            inheritance_models=models_list,
            database_entry_version=gene_row.get('Database_entry_version'),
        )
        genes.append(panel_gene)

    return genes


def build_clnsig(clnsig_info):
    """docstring for build_clnsig"""
    clnsig_obj = dict(
        value = clnsig_info['value'],
        accession = clnsig_info.get('accession'),
        revstat = clnsig_info.get('revstat')
    )

    return clnsig_obj


import logging
from pprint import pprint as pp
import intervaltree

from scout.build.genes.exon import build_exon
from pymongo.errors import (DuplicateKeyError, BulkWriteError)

from scout.exceptions import IntegrityError

LOG = logging.getLogger(__name__)

class GeneHandler(object):

    def load_hgnc_gene(self, gene_obj):
        """Add a gene object with transcripts to the database

        Arguments:
            gene_obj(dict)

        """
        #LOG.debug("Loading gene %s, build %s into database" %
        #             (gene_obj['hgnc_symbol'], gene_obj['build']))
        res = self.hgnc_collection.insert_one(gene_obj)
        #LOG.debug("Gene saved")
        return res

    def load_hgnc_bulk(self, gene_objs):
        """Load a bulk of hgnc gene objects

        Raises IntegrityError if there are any write concerns

        Args:
            gene_objs(iterable(scout.models.hgnc_gene))

        Returns:
            result (pymongo.results.InsertManyResult)
        """

        LOG.info("Loading gene bulk with length %s", len(gene_objs))
        try:
            result = self.hgnc_collection.insert_many(gene_objs)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)

        return result

    def load_hgnc_transcript(self, transcript_obj):
        """Add a transcript object to the database

        Arguments:
            transcript_obj(dict)

        """
        res = self.transcript_collection.insert_one(transcript_obj)
        return res

    def load_transcript_bulk(self, transcript_objs):
        """Load a bulk of transcript objects to the database

        Arguments:
            transcript_objs(iterable(scout.models.hgnc_transcript))

        """
        LOG.info("Loading transcript bulk")
        try:
            result = self.transcript_collection.insert_many(transcript_objs)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)

        return result

    def load_exon(self, exon_obj):
        """Add a exon object to the database

        Arguments:
            exon_obj(dict)

        """
        res = self.exon_collection.insert_one(exon_obj)
        return res

    def load_exon_bulk(self, exon_objs):
        """Load a bulk of exon objects to the database

        Arguments:
            exon_objs(iterable(scout.models.hgnc_exon))

        """
        try:
            result = self.exon_collection.insert_many(transcript_objs)
        except (DuplicateKeyError, BulkWriteError) as err:
            raise IntegrityError(err)

        return result

    def hgnc_gene(self, hgnc_identifier, build='37'):
        """Fetch a hgnc gene

            Args:
                hgnc_identifier(int)

            Returns:
                gene_obj(HgncGene)
        """
        if not build in ['37', '38']:
            build = '37'
        query = {}
        try:
            # If the identifier is a integer we search for hgnc_id
            hgnc_identifier = int(hgnc_identifier)
            query['hgnc_id'] = hgnc_identifier
        except ValueError:
            # Else we seach for a hgnc_symbol
            query['hgnc_symbol'] = hgnc_identifier

        query['build'] = build
        LOG.debug("Fetching gene %s" % hgnc_identifier)
        gene_obj = self.hgnc_collection.find_one(query)
        if not gene_obj:
            return None

        # Add the transcripts:
        transcripts = []
        tx_objs = self.transcripts(build=build, hgnc_id=gene_obj['hgnc_id'])
        if tx_objs.count() > 0:
            for tx in tx_objs:
                transcripts.append(tx)
        gene_obj['transcripts'] = transcripts

        return gene_obj

    def hgnc_id(self, hgnc_symbol, build='37'):
        """Query the genes with a hgnc symbol and return the hgnc id

        Args:
            hgnc_symbol(str)
            build(str)

        Returns:
            hgnc_id(int)
        """
        #LOG.debug("Fetching gene %s", hgnc_symbol)
        query = {'hgnc_symbol':hgnc_symbol, 'build':build}
        projection = {'hgnc_id':1, '_id':0}
        res = self.hgnc_collection.find(query, projection)

        if res.count() > 0:
            return res[0]['hgnc_id']
        else:
            return None

    def hgnc_genes(self, hgnc_symbol, build='37', search=False):
        """Fetch all hgnc genes that match a hgnc symbol

            Check both hgnc_symbol and aliases

            Args:
                hgnc_symbol(str)
                build(str): The build in which to search
                search(bool): if partial searching should be used

            Returns:
                result()
        """
        LOG.debug("Fetching genes with symbol %s" % hgnc_symbol)
        if search:
            # first search for a full match
            full_query = self.hgnc_collection.find({
                '$or': [
                    {'aliases': hgnc_symbol},
                    {'hgnc_id': int(hgnc_symbol) if hgnc_symbol.isdigit() else None},
                ],
                'build': build
            })
            if full_query.count() != 0:
                return full_query

            return self.hgnc_collection.find({
                'aliases': {'$regex': hgnc_symbol, '$options': 'i'},
                'build': build
            })

        return self.hgnc_collection.find({'build': build, 'aliases': hgnc_symbol})

    def all_genes(self, build='37'):
        """Fetch all hgnc genes

            Returns:
                result()
        """
        LOG.info("Fetching all genes")
        return self.hgnc_collection.find({'build': build}).sort('chromosome', 1)

    def nr_genes(self, build=None):
        """Return the number of hgnc genes in collection

        If build is used, return the number of genes of a certain build

            Returns:
                result()
        """
        if build:
            LOG.info("Fetching all genes from build %s",  build)
        else:
            LOG.info("Fetching all genes")

        return self.hgnc_collection.find({'build':build}).count()

    def drop_genes(self, build=None):
        """Delete the genes collection"""
        if build:
            LOG.info("Dropping the hgnc_gene collection, build %s", build)
            self.hgnc_collection.delete_many({'build': build})
        else:
            LOG.info("Dropping the hgnc_gene collection")
            self.hgnc_collection.drop()

    def drop_transcripts(self, build=None):
        """Delete the transcripts collection"""
        if build:
            LOG.info("Dropping the transcripts collection, build %s", build)
            self.transcript_collection.delete_many({'build': build})
        else:
            LOG.info("Dropping the transcripts collection")
            self.transcript_collection.drop()

    def drop_exons(self, build=None):
        """Delete the exons collection"""
        if build:
            LOG.info("Dropping the exons collection, build %s", build)
            self.exon_collection.delete_many({'build': build})
        else:
            LOG.info("Dropping the exons collection")
            self.exon_collection.drop()

    def ensembl_transcripts(self, build='37'):
        """Return a dictionary with ensembl ids as keys and transcripts as value.

        Args:
            build(str)

        Returns:
            ensembl_transcripts(dict): {<enst_id>: transcripts_obj, ...}
        """
        ensembl_transcripts = {}
        LOG.info("Fetching all transcripts")
        for transcript_obj in self.transcript_collection.find({'build':build}):
            enst_id = transcript_obj['transcript_id']
            ensembl_transcripts[enst_id] = transcript_obj
        LOG.info("Ensembl transcripts fetched")

        return ensembl_transcripts

    def hgncid_to_gene(self, build='37', genes=None):
        """Return a dictionary with hgnc_id as key and gene_obj as value

        The result will have ONE entry for each gene in the database.
        (For a specific build)

        Args:
            build(str):
            genes(iterable(scout.models.HgncGene)):

        Returns:
            hgnc_dict(dict): {<hgnc_id(int)>: <gene(dict)>}

        """
        hgnc_dict = {}
        LOG.info("Building hgncid_to_gene")
        if not genes:
            genes = self.hgnc_collection.find({'build':build})

        for gene_obj in genes:
            hgnc_dict[gene_obj['hgnc_id']] = gene_obj

        return hgnc_dict

    def hgncsymbol_to_gene(self, build='37', genes=None):
        """Return a dictionary with hgnc_symbol as key and gene_obj as value

        The result will have ONE entry for each gene in the database.
        (For a specific build)

        Args:
            build(str)
            genes(iterable(scout.models.HgncGene)):

        Returns:
            hgnc_dict(dict): {<hgnc_symbol(str)>: <gene(dict)>}

        """
        hgnc_dict = {}
        LOG.info("Building hgncsymbol_to_gene")
        if not genes:
            genes = self.hgnc_collection.find({'build':build})

        for gene_obj in genes:
            hgnc_dict[gene_obj['hgnc_symbol']] = gene_obj
        LOG.info("All genes fetched")
        return hgnc_dict

    def gene_by_alias(self, symbol, build='37'):
        """Return a iterable with hgnc_genes.

        If the gene symbol is listed as primary the iterable will only have
        one result. If not the iterable will include all hgnc genes that have
        the symbol as an alias.

        Args:
            symbol(str)
            build(str)

        Returns:
            res(pymongo.Cursor(dict))
        """
        res = self.hgnc_collection.find({'hgnc_symbol': symbol, 'build':build})
        if res.count() == 0:
            res = self.hgnc_collection.find({'aliases': symbol, 'build':build})

        return res

    def genes_by_alias(self, build='37', genes=None):
        """Return a dictionary with hgnc symbols as keys and a list of hgnc ids
             as value.

        If a gene symbol is listed as primary the list of ids will only consist
        of that entry if not the gene can not be determined so the result is a list
        of hgnc_ids

        Args:
            build(str)
            genes(iterable(scout.models.HgncGene)):

        Returns:
            alias_genes(dict): {<hgnc_alias>: {'true': <hgnc_id>, 'ids': {<hgnc_id_1>, <hgnc_id_2>, ...}}}
        """
        LOG.info("Fetching all genes by alias")
        # Collect one entry for each alias symbol that exists
        alias_genes = {}
        # Loop over all genes
        if not genes:
            genes = self.hgnc_collection.find({'build':build})

        for gene in genes:
            # Collect the hgnc_id
            hgnc_id = gene['hgnc_id']
            # Collect the true symbol given by hgnc
            hgnc_symbol = gene['hgnc_symbol']
            # Loop aver all aliases
            for alias in gene['aliases']:
                true_id = None
                # If the alias is the same as hgnc symbol we know the true id
                if alias == hgnc_symbol:
                    true_id = hgnc_id
                # If the alias is already in the list we add the id
                if alias in alias_genes:
                    alias_genes[alias]['ids'].add(hgnc_id)
                    if true_id:
                        alias_genes[alias]['true'] = hgnc_id
                else:
                    alias_genes[alias] = {
                        'true': hgnc_id,
                        'ids': set([hgnc_id])
                    }

        return alias_genes

    def get_id_transcripts(self, hgnc_id, build='37'):
        """Return a set with identifier transcript(s)

        Choose all refseq transcripts with NM symbols, if none where found choose ONE with NR,
        if no NR choose ONE with XM. If there are no RefSeq transcripts identifiers choose the
        longest ensembl transcript.

        Args:
            hgnc_id(int)
            build(str)

        Returns:
            identifier_transcripts(set)

        """
        transcripts = self.transcripts(build=build, hgnc_id=hgnc_id)

        identifier_transcripts = set()
        longest = None
        nr = []
        xm = []
        for tx in transcripts:
            enst_id = tx['transcript_id']
            # Should we not check if it is longest?
            if not longest:
                longest = enst_id
            refseq_id = tx.get('refseq_id')
            if not refseq_id:
                continue

            if 'NM' in refseq_id:
                identifier_transcripts.add(enst_id)
            elif 'NR' in refseq_id:
                nr.append(enst_id)
            elif 'XM' in refseq_id:
                xm.append(enst_id)

        if identifier_transcripts:
            return identifier_transcripts

        if nr:
            return set([nr[0]])

        if xm:
            return set([xm[0]])

        return set([longest])

    def transcripts_by_gene(self, build='37'):
        """Return a dictionary with hgnc_id as keys and a list of transcripts as value

        Args:
            build(str)

        Returns:
            hgnc_transcripts(dict)

        """
        hgnc_transcripts = {}
        LOG.info("Fetching all transcripts")
        for transcript in self.transcript_collection.find({'build':build}):
            hgnc_id = transcript['hgnc_id']
            if not hgnc_id in hgnc_transcripts:
                hgnc_transcripts[hgnc_id] = []

            hgnc_transcripts[hgnc_id].append(transcript)

        return hgnc_transcripts

    def id_transcripts_by_gene(self, build='37'):
        """Return a dictionary with hgnc_id as keys and a set of id transcripts as value

        Args:
            build(str)

        Returns:
            hgnc_id_transcripts(dict)
        """
        hgnc_id_transcripts = {}
        LOG.info("Fetching all id transcripts")
        for gene_obj in self.hgnc_collection.find({'build': build}):
            hgnc_id = gene_obj['hgnc_id']
            id_transcripts = self.get_id_transcripts(hgnc_id=hgnc_id, build=build)
            hgnc_id_transcripts[hgnc_id] = id_transcripts

        return hgnc_id_transcripts

    def ensembl_genes(self, build='37'):
        """Return a dictionary with ensembl ids as keys and gene objects as value.

        Args:
            build(str)

        Returns:
            genes(dict): {<ensg_id>: gene_obj, ...}
        """
        genes = {}

        LOG.info("Fetching all genes")
        for gene_obj in self.hgnc_collection.find({'build':build}):
            ensg_id = gene_obj['ensembl_id']
            hgnc_id = gene_obj['hgnc_id']

            genes[ensg_id] = gene_obj

        LOG.info("Ensembl genes fetched")

        return genes

    def transcripts(self, build='37', hgnc_id=None):
        """Return all transcripts.

            If a gene is specified return all transcripts for the gene

        Args:
            build(str)
            hgnc_id(int)

        Returns:
            iterable(transcript)
        """

        query = {'build': build}
        if hgnc_id:
            query['hgnc_id'] = hgnc_id

        return self.transcript_collection.find(query)

    def to_hgnc(self, hgnc_alias, build='37'):
        """Check if a hgnc symbol is an alias

            Return the correct hgnc symbol, if not existing return None

            Args:
                hgnc_alias(str)

            Returns:
                hgnc_symbol(str)
        """
        result = self.hgnc_genes(hgnc_symbol=hgnc_alias, build=build)
        if result:
            for gene in result:
                return gene['hgnc_symbol']
        else:
            return None

    def add_hgnc_id(self, genes):
        """Add the correct hgnc id to a set of genes with hgnc symbols

        Args:
            genes(list(dict)): A set of genes with hgnc symbols only

        """
        genes_by_alias = self.genes_by_alias()

        for gene in genes:
            id_info = genes_by_alias.get(gene['hgnc_symbol'])
            if not id_info:
                LOG.warning("Gene %s does not exist in scout", gene['hgnc_symbol'])
                continue
            gene['hgnc_id'] = id_info['true']
            if not id_info['true']:
                if len(id_info['ids']) > 1:
                    LOG.warning("Gene %s has ambiguous value, please choose one hgnc id in result", gene['hgnc_symbol'])
                gene['hgnc_id'] = ','.join([str(hgnc_id) for hgnc_id in id_info['ids']])

    def get_coding_intervals(self, build='37', genes=None):
        """Return a dictionary with chromosomes as keys and interval trees as values

        Each interval represents a coding region of overlapping genes.

        Args:
            build(str): The genome build
            genes(iterable(scout.models.HgncGene)):

        Returns:
            intervals(dict): A dictionary with chromosomes as keys and overlapping genomic intervals as values
        """
        intervals = {}
        if not genes:
            genes = self.all_genes(build=build)
        LOG.info("Building interval trees...")
        for i,hgnc_obj in enumerate(genes):
            chrom = hgnc_obj['chromosome']
            start = max((hgnc_obj['start'] - 5000), 1)
            end = hgnc_obj['end'] + 5000

            # If this is the first time a chromosome is seen we create a new
            # interval tree with current interval
            if chrom not in intervals:
                intervals[chrom] = intervaltree.IntervalTree()
                intervals[chrom].addi(start, end, i)
                continue

            res = intervals[chrom].search(start, end)

            # If the interval did not overlap any other intervals we insert it and continue
            if not res:
                intervals[chrom].addi(start, end, i)
                continue

            # Loop over the overlapping intervals
            for interval in res:
                # Update the positions to new max and mins
                if interval.begin < start:
                    start = interval.begin

                if interval.end > end:
                    end = interval.end

                # Delete the old interval
                intervals[chrom].remove(interval)

            # Add the new interval consisting och the overlapping ones
            intervals[chrom].addi(start, end, i)

        return intervals

    def load_exons(self, exons, genes=None, build='37'):
        """Create exon objects and insert them into the database

        Args:
            exons(iterable(dict))
        """
        genes = genes or self.ensembl_genes(build)
        for exon in exons:
            exon_obj = build_exon(exon, genes)
            if not exon_obj:
                continue

            res = self.exon_collection.insert_one(exon_obj)

    def exons(self, hgnc_id=None, transcript_id=None,  build=None):
        """Return all exons

        Args:
            hgnc_id(int)
            transcript_id(str)
            build(str)

        Returns:
            exons(iterable(dict))
        """
        query = {}
        if build:
            query['build'] = build
        if hgnc_id:
            query['hgnc_id'] = hgnc_id
        if transcript_id:
            query['transcript_id'] = transcript_id

        return self.exon_collection.find(query)


import logging
import click
import urllib.request

from pprint import pprint as pp

from scout.parse.omim import (parse_genemap2, parse_mim2gene, parse_omim_morbid, parse_mim_titles,
                              get_mim_genes, get_mim_phenotypes)

LOG = logging.getLogger(__name__)

@click.command('omim', short_help='Update omim gene panel')
@click.option('--api-key', help='Specify the api key')
@click.option('--institute',
    help='Specify the owner of the omim panel',
    default='cust002',
    show_default=True,
)
@click.pass_context
def omim(context, api_key, institute):
    """
    Update the automate generated omim gene panel in the database.
    """
    LOG.info("Running scout update omim")
    adapter = context.obj['adapter']

    api_key = api_key or context.obj.get('omim_api_key')
    if not api_key:
        LOG.warning("Please provide a omim api key to load the omim gene panel")
        context.abort()

    institute_obj = adapter.institute(institute)
    if not institute_obj:
        LOG.info("Institute %s could not be found in database", institute)
        LOG.warning("Please specify an existing institute")
        context.abort()

    try:
        adapter.load_omim_panel(api_key, institute=institute)
    except Exception as err:
        LOG.error(err)
        context.abort()