# -*- coding: utf-8 -*-
from pprint import pprint as pp

from scout.server.links import (add_gene_links, add_tx_links, omim)

def gene(store, hgnc_id):
    """Parse information about a gene."""
    res = {'builds': {'37': None, '38': None}, 'symbol': None, 'description': None, 'ensembl_id': None, 'record': None}

    for build in res['builds']:
        record = store.hgnc_gene(hgnc_id, build=build)
        if record:

            record['position'] = "{this[chromosome]}:{this[start]}-{this[end]}".format(this=record)
            res['aliases'] = record['aliases']
            res['hgnc_id'] = record['hgnc_id']
            res['description'] = record['description']
            res['builds'][build] = record
            res['symbol'] = record['hgnc_symbol']
            res['description'] = record['description']
            res['entrez_id'] = record.get('entrez_id')
            res['pli_score'] = record.get('pli_score')

            add_gene_links(record, int(build))

            res['omim_id'] = record.get('omim_id')
            res['incomplete_penetrance'] = record.get('incomplete_penetrance',False)
            res['inheritance_models'] = record.get('inheritance_models',[])
            for transcript in record['transcripts']:
                transcript['position'] = ("{this[chrom]}:{this[start]}-{this[end]}"
                                          .format(this=transcript))
                add_tx_links(transcript, build)

            for phenotype in record.get('phenotypes',[]):
                phenotype['omim_link'] = omim(phenotype.get('mim_number'))

            if not res['record']:
                res['record'] = record

    # If none of the genes where found
    if not any(res.values()):
        raise ValueError

    return res

def genes_to_json(store, query):
    """Fetch matching genes and convert to JSON."""
    gene_query = store.hgnc_genes(query, search=True)
    json_terms = [{'name': "{} | {} ({})".format(gene['hgnc_id'], gene['hgnc_symbol'],
                                                 ', '.join(gene['aliases'])),
                   'id': gene['hgnc_id']} for gene in gene_query]
    return json_terms


import logging

from pprint import pprint as pp

from flask import (abort, Blueprint, current_app, redirect, render_template,
                   request, url_for, send_from_directory, jsonify, flash)
from flask_login import current_user

from scout.server.extensions import store

from .controllers import get_dashboard_info

blueprint = Blueprint('dashboard', __name__, template_folder='templates')

LOG = logging.getLogger(__name__)

@blueprint.route('/dashboard', methods=['GET', 'POST'])
def index():
    """Display the Scout dashboard."""
    accessible_institutes = current_user.institutes
    if not 'admin' in current_user.roles:
        accessible_institutes = current_user.institutes
        if not accessible_institutes:
            flash('Not allowed to see information - please visit the dashboard later!')
            return redirect(url_for('cases.dahboard_general.html'))

    LOG.debug('User accessible institutes: {}'.format(accessible_institutes))
    institutes = [inst for inst in store.institutes(accessible_institutes)]

    # Insert a entry that displays all institutes in the beginning of the array
    institutes.insert(0, {'_id': None, 'display_name': 'All institutes'})

    institute_id = None
    slice_query = None
    panel=1
    if request.method=='POST':
        institute_id = request.form.get('institute')
        slice_query = request.form.get('query')
        panel=request.form.get('pane_id')

    elif request.method=='GET':
        institute_id = request.args.get('institute')
        slice_query = request.args.get('query')

    # User should be restricted to their own institute if:
    #1) Their default institute when the page is first loaded
    #2) if they ask for an institute that they don't belong to
    #3) if they want perform a query on all institutes

    if not institute_id:
        institute_id = accessible_institutes[0]
    elif (not current_user.is_admin) and (slice_query and institute_id == 'None'):
        institute_id = accessible_institutes[0]
    elif (not institute_id in accessible_institutes) and not (institute_id == 'None'):
        institute_id = accessible_institutes[0]

    LOG.info("Fetch all cases with institute: %s", institute_id)

    data = get_dashboard_info(store, institute_id, slice_query)
    data['institutes'] = institutes
    data['choice'] = institute_id
    total_cases = data['total_cases']

    LOG.info("Found %s cases", total_cases)
    if total_cases == 0:
        flash('no cases found for institute {} (with that query) - please visit the dashboard later!'.format(institute_id), 'info')
#        return redirect(url_for('cases.index'))

    return render_template(
        'dashboard/dashboard_general.html', institute=institute_id, query=slice_query, panel=panel, **data)


from __future__ import unicode_literals

from datetime import date
from calendar import day_name
from django.template import Library, TemplateSyntaxError
from django.utils.dates import WEEKDAYS_ABBR, WEEKDAYS

register = Library()

@register.simple_tag
def weekday(year_or_num, month=None, day=None, full=False):
    """Simple tag - returns the weekday of the given (year, month, day) or of given (weekday_number).

    Usage (in template):

    {% weekday 2014 3 3 %}

    Result: Mon

    Return abbreviation by default. To return full name: pass full=True

    {% weekday 2014 3 3 full=True %}

    Result: Monday

    When only number of weekday is given then 0 is considered as "Monday"

    {% weekday 0 full=True %}

    Result: Monday

    """
    if any([month, day]) and not all([month, day]):
        raise TemplateSyntaxError("weekday accepts 1 or 3 arguments plus optional 'full' argument")

    try:
        if all([year_or_num, month, day]):
            weekday_num = date(*map(int, (year_or_num, month, day))).weekday()
        else:
            weekday_num = year_or_num
        if full:
            return WEEKDAYS[weekday_num]
        else:
            return WEEKDAYS_ABBR[weekday_num]
    except Exception:
        return

@register.filter
def weekday_css_class(weekday_num, calendar):
    return calendar.cssclasses[weekday_num]


import logging
import gzip
import urllib.request
from urllib.error import (HTTPError, URLError)

import pybiomart
from scout.constants import CHROMOSOMES

LOG = logging.getLogger(__name__)

HPO_URL = ("http://compbio.charite.de/jenkins/job/hpo.annotations.monthly/"
           "lastStableBuild/artifact/annotation/{0}")

def get_request(url):
    """Return a requests response from url

    Args:
        url(str)

    Returns:
        decoded_data(str): Decoded response
    """
    try:
        LOG.info("Requesting %s", url)
        response = urllib.request.urlopen(url)
        if url.endswith('.gz'):
            LOG.info("Decompress zipped file")
            data = gzip.decompress(response.read())      # a `bytes` object
        else:
            data = response.read()      # a `bytes` object
        decoded_data = data.decode('utf-8')
    except HTTPError as err:
        LOG.warning("Something went wrong, perhaps the api key is not valid?")
        raise err
    except URLError as err:
        LOG.warning("Something went wrong, are you connected to internet?")
        raise err

    if 'Error' in decoded_data:
        raise URLError("Seems like url {} does not exist".format(url))

    return decoded_data

def fetch_resource(url):
    """Fetch a resource and return the resulting lines in a list
    Send file_name to get more clean log messages

    Args:
        url(str)

    Returns:
        lines(list(str))
    """
    try:
        data = get_request(url)
        lines = data.split('\n')
    except Exception as err:
        raise err

    return lines

def fetch_mim_files(api_key, mim2genes=False, mimtitles=False, morbidmap=False, genemap2=False):
    """Fetch the necessary mim files using a api key

    Args:
        api_key(str): A api key necessary to fetch mim data

    Returns:
        mim_files(dict): A dictionary with the neccesary files
    """

    LOG.info("Fetching OMIM files from https://omim.org/")
    mim2genes_url =  'https://omim.org/static/omim/data/mim2gene.txt'
    mimtitles_url= 'https://data.omim.org/downloads/{0}/mimTitles.txt'.format(api_key)
    morbidmap_url = 'https://data.omim.org/downloads/{0}/morbidmap.txt'.format(api_key)
    genemap2_url =  'https://data.omim.org/downloads/{0}/genemap2.txt'.format(api_key)

    mim_files = {}
    mim_urls = {}

    if mim2genes is True:
        mim_urls['mim2genes'] = mim2genes_url
    if mimtitles is True:
        mim_urls['mimtitles'] = mimtitles_url
    if morbidmap is True:
        mim_urls['morbidmap'] = morbidmap_url
    if genemap2 is True:
        mim_urls['genemap2'] = genemap2_url

    for file_name in mim_urls:
        url = mim_urls[file_name]
        mim_files[file_name] = fetch_resource(url)

    return mim_files

def fetch_hpo_terms():
    """Fetch the latest version of the hpo terms in .obo format

    Returns:
        res(list(str)): A list with the lines
    """
    url = "http://purl.obolibrary.org/obo/hp.obo"

    return fetch_resource(url)

def fetch_hpo_to_genes():
    """Fetch the latest version of the map from phenotypes to genes

    Returns:
        res(list(str)): A list with the lines

    """
    file_name = "ALL_SOURCES_ALL_FREQUENCIES_phenotype_to_genes.txt"
    url = HPO_URL.format(file_name)

    return fetch_resource(url)

def fetch_hpo_genes():
    """Fetch the latest version of the map from genes to hpo terms

    Returns:
        res(list(str)): A list with the lines

    """
    file_name = "ALL_SOURCES_ALL_FREQUENCIES_genes_to_phenotype.txt"
    url = HPO_URL.format(file_name)

    return fetch_resource(url)

def fetch_hpo_phenotype_to_terms():
    """Fetch the latest version of the map from phenotype to terms

    Returns:
        res(list(str)): A list with the lines

    """
    file_name = "ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt"
    url = HPO_URL.format(file_name)

    return fetch_resource(url)

def fetch_ensembl_genes(build='37'):
    """Fetch the ensembl genes

    Args:
        build(str): ['37', '38']
    """
    if build == '37':
        url = 'http://grch37.ensembl.org'
    else:
        url = 'http://www.ensembl.org'

    LOG.info("Fetching ensembl genes from %s", url)
    dataset_name = 'hsapiens_gene_ensembl'

    dataset = pybiomart.Dataset(name=dataset_name, host=url)

    attributes = [
        'chromosome_name',
        'start_position',
        'end_position',
        'ensembl_gene_id',
        'hgnc_symbol',
        'hgnc_id',
    ]

    filters = {
        'chromosome_name': CHROMOSOMES,
    }

    result = dataset.query(
        attributes = attributes,
        filters = filters,
        use_attr_names=True,
    )

    return result

def fetch_ensembl_transcripts(build='37', chromosomes=None):
    """Fetch the ensembl genes

    Args:
        build(str): ['37', '38']
        chromosomes(iterable(str))

    Returns:
        result(DataFrame)
    """
    chromosomes = chromosomes or CHROMOSOMES
    LOG.info("Fetching ensembl transcripts build %s ...", build)
    if build == '37':
        url = 'http://grch37.ensembl.org'
    else:
        url = 'http://www.ensembl.org'

    dataset_name = 'hsapiens_gene_ensembl'

    dataset = pybiomart.Dataset(name=dataset_name, host=url)

    attributes = [
        'chromosome_name',
        'ensembl_gene_id',
        'ensembl_transcript_id',
        'transcript_start',
        'transcript_end',
        'refseq_mrna',
		'refseq_mrna_predicted',
		'refseq_ncrna',
    ]

    filters = {
        'chromosome_name': chromosomes,
    }

    result = dataset.query(
        attributes = attributes,
        filters = filters,
        use_attr_names=True,
    )

    return result

def fetch_ensembl_exons(build='37'):
    """Fetch the ensembl genes

    Args:
        build(str): ['37', '38']
    """
    LOG.info("Fetching ensembl exons build %s ...", build)
    if build == '37':
        url = 'http://grch37.ensembl.org'
    else:
        url = 'http://www.ensembl.org'

    dataset_name = 'hsapiens_gene_ensembl'

    dataset = pybiomart.Dataset(name=dataset_name, host=url)

    attributes = [
        'chromosome_name',
        'ensembl_gene_id',
        'ensembl_transcript_id',
        'ensembl_exon_id',
        'exon_chrom_start',
        'exon_chrom_end',
        '5_utr_start',
        '5_utr_end',
        '3_utr_start',
        '3_utr_end',
        'strand',
        'rank'
    ]

    filters = {
        'chromosome_name': CHROMOSOMES,
    }

    result = dataset.query(
        attributes = attributes,
        filters = filters
    )

    return result

def fetch_hgnc():
    """Fetch the hgnc genes file from
        ftp://ftp.ebi.ac.uk/pub/databases/genenames/new/tsv/hgnc_complete_set.txt

    Returns:
        hgnc_gene_lines(list(str))
    """
    file_name = "hgnc_complete_set.txt"
    url = 'ftp://ftp.ebi.ac.uk/pub/databases/genenames/new/tsv/{0}'.format(file_name)
    LOG.info("Fetching HGNC genes")

    hgnc_lines = fetch_resource(url)

    return hgnc_lines

def fetch_exac_constraint():
    """Fetch the file with exac constraint scores

    Returns:
        exac_lines(iterable(str))
    """
    file_name = 'fordist_cleaned_exac_r03_march16_z_pli_rec_null_data.txt'
    url = ('ftp://ftp.broadinstitute.org/pub/ExAC_release/release0.3/functional_gene_constraint'
           '/{0}').format(file_name)

    LOG.info("Fetching ExAC genes")

    try:
        exac_lines = fetch_resource(url)
    except URLError as err:
        LOG.info("Failed to fetch exac constraint scores file from ftp server")
        LOG.info("Try to fetch from google bucket...")
        url = ("https://storage.googleapis.com/gnomad-public/legacy/exacv1_downloads/release0.3.1"
               "/manuscript_data/forweb_cleaned_exac_r03_march16_z_data_pLI.txt.gz")

    exac_lines = fetch_resource(url)

    return exac_lines

def fetch_hpo_files(hpogenes=False, hpoterms=False, phenotype_to_terms=False, hpodisease=False):
    """Fetch the necessary mim files using a api key

    Args:
        api_key(str): A api key necessary to fetch mim data

    Returns:
        mim_files(dict): A dictionary with the neccesary files
    """

    LOG.info("Fetching HPO information from http://compbio.charite.de")
    base_url = ('http://compbio.charite.de/jenkins/job/hpo.annotations.monthly/'
                'lastStableBuild/artifact/annotation/{}')
    hpogenes_url =  base_url.format('ALL_SOURCES_ALL_FREQUENCIES_genes_to_phenotype.txt')
    hpoterms_url= base_url.format('ALL_SOURCES_ALL_FREQUENCIES_phenotype_to_genes.txt')
    hpo_phenotype_to_terms_url = base_url.format('ALL_SOURCES_ALL_FREQUENCIES_diseases_to_genes_to_phenotypes.txt')
    hpodisease_url = base_url.format('diseases_to_genes.txt')

    hpo_files = {}
    hpo_urls = {}

    if hpogenes is True:
        hpo_urls['hpogenes'] = hpogenes_url
    if hpoterms is True:
        hpo_urls['hpoterms'] = hpoterms_url
    if phenotype_to_terms is True:
        hpo_urls['phenotype_to_terms'] = hpo_phenotype_to_terms_url
    if hpodisease is True:
        hpo_urls['hpodisease'] = hpodisease_url

    for file_name in hpo_urls:
        url = hpo_urls[file_name]
        hpo_files[file_name] = request_file(url)

    return hpo_files


import logging
import click
from pprint import pprint as pp

LOG = logging.getLogger(__name__)

@click.command('transcripts', short_help='Display transcripts')
@click.option('-b', '--build', default='37', type=click.Choice(['37', '38']))
@click.option('-i', '--hgnc-id', type=int)
@click.option('--json', is_flag=True)
@click.pass_context
def transcripts(context, build, hgnc_id, json):
    """Show all transcripts in the database"""
    LOG.info("Running scout view transcripts")
    adapter = context.obj['adapter']

    if not json:
        click.echo("Chromosome\tstart\tend\ttranscript_id\thgnc_id\trefseq\tis_primary")
    for tx_obj in adapter.transcripts(build=build, hgnc_id=hgnc_id):
        if json:
            pp(tx_obj)
            continue
        click.echo("{0}\t{1}\t{2}\t{3}\t{4}\t{5}\t{6}".format(
            tx_obj['chrom'],
            tx_obj['start'],
            tx_obj['end'],
            tx_obj['ensembl_transcript_id'],
            tx_obj['hgnc_id'],
            tx_obj.get('refseq_id', ''),
            tx_obj.get('is_primary') or '',
        ))


from __future__ import unicode_literals

import locale

from django.conf import settings

from happenings.models import Event
from .handlers import CountHandler
from .calendars import EventCalendar, MiniEventCalendar
from .common import get_next_and_prev

try:
    CALENDAR_LOCALE = str(getattr(settings, "CALENDAR_LOCALE", ""))
    locale.setlocale(locale.LC_ALL, CALENDAR_LOCALE)
except Exception:
    if CALENDAR_LOCALE == str(''):
        # locale.setlocale(x, '') failes with "Error: unsupported locale setting" on some systems
        CALENDAR_LOCALE = str('en_US.UTF-8')
        locale.setlocale(locale.LC_ALL, CALENDAR_LOCALE)
    else:
        # reraise: maybe wrong locale was set?
        raise

def add_occurrences(events, count):
    """
    Adds an occurrence key to the event object w/ a list of occurrences
    and adds a popover (for use with twitter bootstrap).
    The occurrence is added so that each event can be aware of what
    day(s) it occurs in the month.
    """
    for day in count:
        for item in count[day]:
            for event in events:
                if event.pk == item[1]:
                    try:
                        event.occurrence.append(day)
                    except AttributeError:
                        event.occurrence = []
                        event.occurrence.append(day)

def month_display(year, month, all_month_events,
                  start_day, net, qs, mini=False, request=None, context=None):
    """
    A function that returns an html calendar for the given
    month in the given year, with the number of events for that month
    shown on the generated calendar. Start_day is the day the calendar
    should start on (default is Monday).
    """
    # count the number of times events happen on a given day
    count = CountHandler(year, month, all_month_events).get_count()

    # sort count by start date using all_month_events (which is already sorted)
    for event in all_month_events[::-1]:
        for l in count.values():
            for item in l:
                if item[1] == event.pk:
                    l.insert(0, l.pop(l.index(item)))

    args = (year, month, count, all_month_events, start_day)
    if not mini:
        html_cal = EventCalendar(request=request, context=context, *args).formatmonth(year, month, net=net, qs=qs)
    else:
        html_cal = MiniEventCalendar(request=request, context=context, *args).formatmonth(year, month, net=net, qs=qs)

    nxt, prev = get_next_and_prev(net)
    extra_qs = ('&' + '&'.join(qs)) if qs else ''
    # inject next/prev querystring urls and make them aware of any querystrings
    # already present in the url
    html_cal = html_cal.replace(
        'class="month">\n
',
        'class="month">\n
\
        ←	' % (prev, extra_qs)
    ).replace(
        '%d	' % year,
        '%d	\
        →	' % (year, nxt, extra_qs)
    )

    add_occurrences(all_month_events, count)

    return html_cal

def day_display(year, month, all_month_events, day):
    """
    Returns the events that occur on the given day.
    Works by getting all occurrences for the month, then drilling
    down to only those occurring on the given day.
    """
    # Get a dict with all of the events for the month
    count = CountHandler(year, month, all_month_events).get_count()
    pks = [x[1] for x in count[day]]  # list of pks for events on given day
    # List enables sorting.
    # See the comments in EventMonthView in views.py for more info
    day_events = list(Event.objects.filter(pk__in=pks).order_by(
        'start_date').prefetch_related('cancellations'))
    day_events.sort(key=lambda x: x.l_start_date.hour)
    return day_events


# -*- coding: utf-8 -*-
import logging
import os.path
import urllib.parse

from pprint import pprint as pp

from xlsxwriter import Workbook

from datetime import date
import datetime
from flask import url_for, flash, request
from flask_mail import Message

from scout.constants import (
    CLINSIG_MAP, ACMG_MAP, MANUAL_RANK_OPTIONS, ACMG_OPTIONS, DISMISS_VARIANT_OPTIONS,
    ACMG_COMPLETE_MAP, CALLERS, SPIDEX_HUMAN, VERBS_MAP, MOSAICISM_OPTIONS,
)
from scout.constants.acmg import ACMG_CRITERIA
from scout.constants.variants_export import EXPORT_HEADER, VERIFIED_VARIANTS_HEADER
from scout.export.variant import export_verified_variants
from scout.server.utils import institute_and_case
from scout.server.links import (add_gene_links, ensembl, add_tx_links)
from .forms import CancerFiltersForm
from scout.server.blueprints.genes.controllers import gene

LOG = logging.getLogger(__name__)

class MissingVerificationRecipientError(Exception):
    pass

def variants(store, institute_obj, case_obj, variants_query, page=1, per_page=50):
    """Pre-process list of variants."""
    variant_count = variants_query.count()
    skip_count = per_page * max(page - 1, 0)
    more_variants = True if variant_count > (skip_count + per_page) else False
    variant_res = variants_query.skip(skip_count).limit(per_page)

    genome_build = case_obj.get('genome_build', '37')
    if genome_build not in ['37','38']:
        genome_build = '37'

    variants = []
    for variant_obj in variant_res:
        overlapping_svs = [sv for sv in store.overlapping(variant_obj)]
        variant_obj['overlapping'] = overlapping_svs or None
        variants.append(parse_variant(store, institute_obj, case_obj, variant_obj,
                        update=True, genome_build=genome_build))

    return {
        'variants': variants,
        'more_variants': more_variants,
    }

def sv_variants(store, institute_obj, case_obj, variants_query, page=1, per_page=50):
    """Pre-process list of SV variants."""
    skip_count = (per_page * max(page - 1, 0))
    more_variants = True if variants_query.count() > (skip_count + per_page) else False

    genome_build = case_obj.get('genome_build', '37')
    if genome_build not in ['37','38']:
        genome_build = '37'

    return {
        'variants': (parse_variant(store, institute_obj, case_obj, variant, genome_build=genome_build) for variant in
                     variants_query.skip(skip_count).limit(per_page)),
        'more_variants': more_variants,
    }

def str_variants(store, institute_obj, case_obj, variants_query, page=1, per_page=50):
    """Pre-process list of STR variants."""
    # Nothing unique to STRs on this level. Inheritance?
    return variants(store, institute_obj, case_obj, variants_query, page, per_page)

def str_variant(store, institute_id, case_name, variant_id):
    """Pre-process an STR variant entry for detail page.

    Adds information to display variant

    Args:
        store(scout.adapter.MongoAdapter)
        institute_id(str)
        case_name(str)
        variant_id(str)

    Returns:
        detailed_information(dict): {
            'institute': <institute_obj>,
            'case': <case_obj>,
            'variant': <variant_obj>,
            'overlapping_snvs': <overlapping_snvs>,
            'manual_rank_options': MANUAL_RANK_OPTIONS,
            'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
        }
        """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj =  store.variant(variant_id)

    # fill in information for pilup view
    variant_case(store, case_obj, variant_obj)

    variant_obj['callers'] = callers(variant_obj, category='str')

    # variant_obj['str_ru']
    # variant_obj['str_repid']
    # variant_obj['str_ref']

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    return {
        'institute': institute_obj,
        'case': case_obj,
        'variant': variant_obj,
        'overlapping_snvs': overlapping_snvs,
        'manual_rank_options': MANUAL_RANK_OPTIONS,
        'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
    }

def sv_variant(store, institute_id, case_name, variant_id=None, variant_obj=None, add_case=True,
               get_overlapping=True):
    """Pre-process an SV variant entry for detail page.

    Adds information to display variant

    Args:
        store(scout.adapter.MongoAdapter)
        institute_id(str)
        case_name(str)
        variant_id(str)
        variant_obj(dcit)
        add_case(bool): If information about case files should be added

    Returns:
        detailed_information(dict): {
            'institute': <institute_obj>,
            'case': <case_obj>,
            'variant': <variant_obj>,
            'overlapping_snvs': <overlapping_snvs>,
            'manual_rank_options': MANUAL_RANK_OPTIONS,
            'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
        }
    """
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)

    if not variant_obj:
        variant_obj = store.variant(variant_id)

    if add_case:
        # fill in information for pilup view
        variant_case(store, case_obj, variant_obj)

    # frequencies
    variant_obj['frequencies'] = [
        ('1000G', variant_obj.get('thousand_genomes_frequency')),
        ('1000G (left)', variant_obj.get('thousand_genomes_frequency_left')),
        ('1000G (right)', variant_obj.get('thousand_genomes_frequency_right')),
        ('ClinGen CGH (benign)', variant_obj.get('clingen_cgh_benign')),
        ('ClinGen CGH (pathogenic)', variant_obj.get('clingen_cgh_pathogenic')),
        ('ClinGen NGI', variant_obj.get('clingen_ngi')),
        ('SweGen', variant_obj.get('swegen')),
        ('Decipher', variant_obj.get('decipher')),
    ]

    variant_obj['callers'] = callers(variant_obj, category='sv')

    overlapping_snvs = []
    if get_overlapping:
        overlapping_snvs = (parse_variant(store, institute_obj, case_obj, variant) for variant in
                            store.overlapping(variant_obj))

    # parse_gene function is not called for SVs, but a link to ensembl gene is required
    for gene_obj in variant_obj['genes']:
        if gene_obj.get('common'):
            ensembl_id = gene_obj['common']['ensembl_id']
            try:
                build = int(gene_obj['common'].get('build','37'))
            except Exception:
                build = 37
            gene_obj['ensembl_link'] = ensembl(ensembl_id, build=build)

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    case_clinvars = store.case_to_clinVars(case_obj.get('display_name'))
    if variant_id in case_clinvars:
        variant_obj['clinvar_clinsig'] = case_clinvars.get(variant_id)['clinsig']

    if not 'end_chrom' in variant_obj:
        variant_obj['end_chrom'] = variant_obj['chromosome']

    return {
        'institute': institute_obj,
        'case': case_obj,
        'variant': variant_obj,
        'overlapping_snvs': overlapping_snvs,
        'manual_rank_options': MANUAL_RANK_OPTIONS,
        'dismiss_variant_options': DISMISS_VARIANT_OPTIONS
    }

def parse_variant(store, institute_obj, case_obj, variant_obj, update=False, genome_build='37',
                  get_compounds = True):
    """Parse information about variants.

    - Adds information about compounds
    - Updates the information about compounds if necessary and 'update=True'

    Args:
        store(scout.adapter.MongoAdapter)
        institute_obj(scout.models.Institute)
        case_obj(scout.models.Case)
        variant_obj(scout.models.Variant)
        update(bool): If variant should be updated in database
        genome_build(str)

    """
    has_changed = False
    compounds = variant_obj.get('compounds', [])
    if compounds and get_compounds:
        # Check if we need to add compound information
        # If it is the first time the case is viewed we fill in some compound information
        if 'not_loaded' not in compounds[0]:
            new_compounds = store.update_variant_compounds(variant_obj)
            variant_obj['compounds'] = new_compounds
            has_changed = True

        # sort compounds on combined rank score
        variant_obj['compounds'] = sorted(variant_obj['compounds'],
                                          key=lambda compound: -compound['combined_score'])

    # Update the hgnc symbols if they are incorrect
    variant_genes = variant_obj.get('genes')
    if variant_genes is not None:
        for gene_obj in variant_genes:
            # If there is no hgnc id there is nothin we can do
            if not gene_obj['hgnc_id']:
                continue
            # Else we collect the gene object and check the id
            if gene_obj.get('hgnc_symbol') is None:
                hgnc_gene = store.hgnc_gene(gene_obj['hgnc_id'], build=genome_build)
                if not hgnc_gene:
                    continue
                has_changed = True
                gene_obj['hgnc_symbol'] = hgnc_gene['hgnc_symbol']

    # We update the variant if some information was missing from loading
    # Or if symbold in reference genes have changed
    if update and has_changed:
        variant_obj = store.update_variant(variant_obj)

    variant_obj['comments'] = store.events(institute_obj, case=case_obj,
                                           variant_id=variant_obj['variant_id'], comments=True)

    if variant_genes:
        variant_obj.update(get_predictions(variant_genes))
        if variant_obj.get('category') == 'cancer':
            variant_obj.update(get_variant_info(variant_genes))

    for compound_obj in compounds:
        compound_obj.update(get_predictions(compound_obj.get('genes', [])))

    if isinstance(variant_obj.get('acmg_classification'), int):
        acmg_code = ACMG_MAP[variant_obj['acmg_classification']]
        variant_obj['acmg_classification'] = ACMG_COMPLETE_MAP[acmg_code]

    # convert length for SV variants
    variant_length = variant_obj.get('length')
    variant_obj['length'] = {100000000000: 'inf', -1: 'n.d.'}.get(variant_length, variant_length)
    if not 'end_chrom' in variant_obj:
        variant_obj['end_chrom'] = variant_obj['chromosome']

    return variant_obj

def variant_export_lines(store, case_obj, variants_query):
    """Get variants info to be exported to file, one list (line) per variant.

        Args:
            store(scout.adapter.MongoAdapter)
            case_obj(scout.models.Case)
            variants_query: a list of variant objects, each one is a dictionary

        Returns:
            export_variants: a list of strings. Each string  of the list corresponding to the fields
                             of a variant to be exported to file, separated by comma
    """

    export_variants = []

    for variant in variants_query:
        variant_line = []
        position = variant['position']
        change = variant['reference']+'>'+variant['alternative']
        variant_line.append(variant['rank_score'])
        variant_line.append(variant['chromosome'])
        variant_line.append(position)
        variant_line.append(change)
        variant_line.append('_'.join([str(position), change]))

        # gather gene info:
        gene_list = variant.get('genes') #this is a list of gene objects
        gene_ids = []
        gene_names = []
        hgvs_c = []

        # if variant is in genes
        if len(gene_list) > 0:
            for gene_obj in gene_list:
                hgnc_id = gene_obj['hgnc_id']
                gene_name = gene(store, hgnc_id)['symbol']

                gene_ids.append(hgnc_id)
                gene_names.append(gene_name)

                hgvs_nucleotide = '-'
                # gather HGVS info from gene transcripts
                transcripts_list = gene_obj.get('transcripts')
                for transcript_obj in transcripts_list:
                    if transcript_obj.get('is_canonical') and transcript_obj.get('is_canonical') is True:
                        hgvs_nucleotide = str(transcript_obj.get('coding_sequence_name'))
                hgvs_c.append(hgvs_nucleotide)

            variant_line.append(';'.join( str(x) for x in  gene_ids))
            variant_line.append(';'.join( str(x) for x in  gene_names))
            variant_line.append(';'.join( str(x) for x in  hgvs_c))
        else:
            while i < 4:
                variant_line.append('-') # instead of gene ids
                i = i+1

        variant_gts = variant['samples'] # list of coverage and gt calls for case samples
        for individual in case_obj['individuals']:
            for variant_gt in variant_gts:
                if individual['individual_id'] == variant_gt['sample_id']:
                    # gather coverage info
                    variant_line.append(variant_gt['allele_depths'][0]) # AD reference
                    variant_line.append(variant_gt['allele_depths'][1]) # AD alternate
                    # gather genotype quality info
                    variant_line.append(variant_gt['genotype_quality'])

        variant_line = [str(i) for i in variant_line]
        export_variants.append(",".join(variant_line))

    return export_variants

def variants_export_header(case_obj):
    """Returns a header for the CSV file with the filtered variants to be exported.

        Args:
            case_obj(scout.models.Case)

        Returns:
            header: includes the fields defined in scout.constants.variants_export EXPORT_HEADER
                    + AD_reference, AD_alternate, GT_quality for each sample analysed for a case
    """
    header = []
    header = header + EXPORT_HEADER
    # Add fields specific for case samples
    for individual in case_obj['individuals']:
        display_name = str(individual['display_name'])
        header.append('AD_reference_'+display_name) # Add AD reference field for a sample
        header.append('AD_alternate_'+display_name) # Add AD alternate field for a sample
        header.append('GT_quality_'+display_name) # Add Genotype quality field for a sample
    return header

def get_variant_info(genes):
    """Get variant information"""
    data = {'canonical_transcripts': []}
    for gene_obj in genes:
        if not gene_obj.get('canonical_transcripts'):
            tx = gene_obj['transcripts'][0]
            tx_id = tx['transcript_id']
            exon = tx.get('exon', '-')
            c_seq = tx.get('coding_sequence_name', '-')
        else:
            tx_id = gene_obj['canonical_transcripts']
            exon = gene_obj.get('exon', '-')
            c_seq = gene_obj.get('hgvs_identifier', '-')

        if len(c_seq) > 20:
            c_seq = c_seq[:20] + '...'

        if len(genes) == 1:
            value = ':'.join([tx_id,exon,c_seq])
        else:
            gene_id = gene_obj.get('hgnc_symbol') or str(gene_obj['hgnc_id'])
            value = ':'.join([gene_id, tx_id,exon,c_seq])
        data['canonical_transcripts'].append(value)

    return data

def get_predictions(genes):
    """Get sift predictions from genes."""
    data = {
        'sift_predictions': [],
        'polyphen_predictions': [],
        'region_annotations': [],
        'functional_annotations': []
    }
    for gene_obj in genes:
        for pred_key in data:
            gene_key = pred_key[:-1]
            if len(genes) == 1:
                value = gene_obj.get(gene_key, '-')
            else:
                gene_id = gene_obj.get('hgnc_symbol') or str(gene_obj['hgnc_id'])
                value = ':'.join([gene_id, gene_obj.get(gene_key, '-')])
            data[pred_key].append(value)

    return data

def variant_case(store, case_obj, variant_obj):
    """Pre-process case for the variant view.

    Adds information about files from case obj to variant

    Args:
        store(scout.adapter.MongoAdapter)
        case_obj(scout.models.Case)
        variant_obj(scout.models.Variant)
    """
    case_obj['bam_files'] = []
    case_obj['mt_bams'] = []
    case_obj['bai_files'] = []
    case_obj['mt_bais'] = []
    case_obj['sample_names'] = []
    for individual in case_obj['individuals']:
        bam_path = individual.get('bam_file')
        mt_bam = individual.get('mt_bam')
        case_obj['sample_names'].append(individual.get('display_name'))
        if bam_path and os.path.exists(bam_path):
            case_obj['bam_files'].append(individual['bam_file'])
            case_obj['bai_files'].append(find_bai_file(individual['bam_file']))
        if mt_bam and os.path.exists(mt_bam):
            case_obj['mt_bams'].append(individual['mt_bam'])
            case_obj['mt_bais'].append(find_bai_file(individual['mt_bam']))

        else:
            LOG.debug("%s: no bam file found", individual['individual_id'])

    try:
        genes = variant_obj.get('genes', [])
        if len(genes) == 1:
            hgnc_gene_obj = store.hgnc_gene(variant_obj['genes'][0]['hgnc_id'])
            if hgnc_gene_obj:
                vcf_path = store.get_region_vcf(case_obj, gene_obj=hgnc_gene_obj)
                case_obj['region_vcf_file'] = vcf_path
            else:
                case_obj['region_vcf_file'] = None
        elif len(genes) > 1:
            chrom = variant_obj['genes'][0]['common']['chromosome']
            start = min(gene['common']['start'] for gene in variant_obj['genes'])
            end = max(gene['common']['end'] for gene in variant_obj['genes'])
            # Create a reduced VCF with variants in the region
            vcf_path = store.get_region_vcf(case_obj, chrom=chrom, start=start, end=end)
            case_obj['region_vcf_file'] = vcf_path
    except (SyntaxError, Exception):
        LOG.warning("skip VCF region for alignment view")

def find_bai_file(bam_file):
    """Find out BAI file by extension given the BAM file."""
    bai_file = bam_file.replace('.bam', '.bai')
    if not os.path.exists(bai_file):
        # try the other convention
        bai_file = "{}.bai".format(bam_file)
    return bai_file

def variant(store, institute_obj, case_obj, variant_id=None, variant_obj=None, add_case=True,
            add_other=True, get_overlapping=True):
    """Pre-process a single variant for the detailed variant view.

    Adds information from case and institute that is not present on the variant
    object

    Args:
        store(scout.adapter.MongoAdapter)
        institute_obj(scout.models.Institute)
        case_obj(scout.models.Case)
        variant_id(str)
        variant_obj(dict)
        add_case(bool): If info about files case should be added
        add_other(bool): If information about other causatives should be added
        get_overlapping(bool): If overlapping svs should be collected

    Returns:
        variant_info(dict): {
            'variant': <variant_obj>,
            'causatives': <list(other_causatives)>,
            'events': <list(events)>,
            'overlapping_svs': <list(overlapping svs)>,
            'manual_rank_options': MANUAL_RANK_OPTIONS,
            'dismiss_variant_options': DISMISS_VARIANT_OPTIONS,
            'ACMG_OPTIONS': ACMG_OPTIONS,
            'evaluations': <list(evaluations)>,
        }

    """
    # If the variant is already collected we skip this part
    if not variant_obj:
        default_panels = []
        # Add default panel information to variant
        for panel in case_obj['panels']:
            if not panel.get('is_default'):
                continue
            panel_obj = store.gene_panel(panel['panel_name'], panel.get('version'))
            if not panel:
                LOG.warning("Panel {0} version {1} could not be found".format(
                    panel['panel_name'], panel.get('version')))
                continue
            default_panels.append(panel_obj)

        # NOTE this will query with variant_id == document_id, not the variant_id.
        variant_obj = store.variant(variant_id, gene_panels=default_panels)

    genome_build = case_obj.get('genome_build', '37')
    if genome_build not in ['37','38']:
        genome_build = '37'

    if variant_obj is None:
        return None
    # Add information to case_obj
    if add_case:
        variant_case(store, case_obj, variant_obj)

    # Collect all the events for the variant
    events = list(store.events(institute_obj, case=case_obj, variant_id=variant_obj['variant_id']))
    for event in events:
        event['verb'] = VERBS_MAP[event['verb']]

    other_causatives = []
    # Adds information about other causative variants
    if add_other:
        for other_variant in store.other_causatives(case_obj, variant_obj):
            # This should work with old and new ids
            case_id = other_variant['case_id']
            other_case = store.case(case_id)
            if not other_case:
                continue
            other_variant['case_display_name'] = other_case.get('display_name', case_id)
            other_causatives.append(other_variant)

    variant_obj = parse_variant(store, institute_obj, case_obj, variant_obj, genome_build=genome_build)

    variant_obj['end_position'] = end_position(variant_obj)
    variant_obj['frequency'] = frequency(variant_obj)
    variant_obj['clinsig_human'] = (clinsig_human(variant_obj) if variant_obj.get('clnsig')
                                    else None)
    variant_obj['thousandg_link'] = thousandg_link(variant_obj, genome_build)
    variant_obj['exac_link'] = exac_link(variant_obj)
    variant_obj['gnomad_link'] = gnomad_link(variant_obj)
    variant_obj['swegen_link'] = swegen_link(variant_obj)
    variant_obj['cosmic_link'] = cosmic_link(variant_obj)
    variant_obj['beacon_link'] = beacon_link(variant_obj, genome_build)
    variant_obj['ucsc_link'] = ucsc_link(variant_obj, genome_build)
    variant_obj['alamut_link'] = alamut_link(variant_obj)
    variant_obj['spidex_human'] = spidex_human(variant_obj)
    variant_obj['expected_inheritance'] = expected_inheritance(variant_obj)
    variant_obj['callers'] = callers(variant_obj, category='snv')

    individuals = {individual['individual_id']: individual for individual in
                   case_obj['individuals']}
    for sample_obj in variant_obj['samples']:
        individual = individuals[sample_obj.get('sample_id')]
        if not individual:
            return None
        sample_obj['is_affected'] = True if individual['phenotype'] == 2 else False

    gene_models = set()
    variant_obj['disease_associated_transcripts'] = []

    # Parse the gene models, both from panels and genes
    for gene_obj in variant_obj.get('genes', []):
        # Adds gene level links
        parse_gene(gene_obj, genome_build)
        omim_models = set()
        for disease_term in gene_obj.get('disease_terms', []):
            omim_models.update(disease_term.get('inheritance', []))
        gene_obj['omim_inheritance'] = list(omim_models)

        # Build strings for the disease associated transcripts from gene panel
        for refseq_id in gene_obj.get('disease_associated_transcripts', []):
            hgnc_symbol = (gene_obj['common']['hgnc_symbol'] if gene_obj.get('common') else
                           gene_obj['hgnc_id'])
            transcript_str = "{}:{}".format(hgnc_symbol, refseq_id)
            variant_obj['disease_associated_transcripts'].append(transcript_str)

        gene_models = gene_models | omim_models

    if variant_obj.get('genetic_models'):
        variant_models = set(model.split('_', 1)[0] for model in variant_obj['genetic_models'])
        variant_obj['is_matching_inheritance'] = variant_models & gene_models

    evaluations = []
    for evaluation_obj in store.get_evaluations(variant_obj):
        evaluation(store, evaluation_obj)
        evaluations.append(evaluation_obj)

    case_clinvars = store.case_to_clinVars(case_obj.get('display_name'))
    if variant_id in case_clinvars:
        variant_obj['clinvar_clinsig'] = case_clinvars.get(variant_id)['clinsig']

    svs = []
    if get_overlapping:
        svs = (parse_variant(store, institute_obj, case_obj, variant_obj) for
                            variant_obj in store.overlapping(variant_obj))

    return {
        'variant': variant_obj,
        'causatives': other_causatives,
        'events': events,
        'overlapping_svs': svs,
        'manual_rank_options': MANUAL_RANK_OPTIONS,
        'dismiss_variant_options': DISMISS_VARIANT_OPTIONS,
        'mosaic_variant_options': MOSAICISM_OPTIONS,
        'ACMG_OPTIONS': ACMG_OPTIONS,
        'evaluations': evaluations,
    }

def observations(store, loqusdb, case_obj, variant_obj):
    """Query observations for a variant."""
    composite_id = ("{this[chromosome]}_{this[position]}_{this[reference]}_"
                    "{this[alternative]}".format(this=variant_obj))
    obs_data = loqusdb.get_variant({'_id': composite_id}) or {}
    obs_data['total'] = loqusdb.case_count()

    obs_data['cases'] = []
    institute_id = variant_obj['institute']
    for case_id in obs_data.get('families', []):
        if case_id != variant_obj['case_id'] and case_id.startswith(institute_id):
            other_variant = store.variant(variant_obj['variant_id'], case_id=case_id)
            other_case = store.case(case_id)
            obs_data['cases'].append(dict(case=other_case, variant=other_variant))

    return obs_data

def parse_gene(gene_obj, build=None):
    """Parse variant genes."""
    build = build or 37

    if gene_obj.get('common'):
        add_gene_links(gene_obj, build)
        refseq_transcripts = []
        for tx_obj in gene_obj['transcripts']:
            parse_transcript(gene_obj, tx_obj, build)

            # select refseq transcripts as "primary"
            if not tx_obj.get('refseq_id'):
                continue

            refseq_transcripts.append(tx_obj)

        gene_obj['primary_transcripts'] = (refseq_transcripts if refseq_transcripts else [])

def parse_transcript(gene_obj, tx_obj, build=None):
    """Parse variant gene transcript (VEP)."""
    build = build or 37
    add_tx_links(tx_obj, build)

    if tx_obj.get('refseq_id'):
        gene_name = (gene_obj['common']['hgnc_symbol'] if gene_obj['common'] else
                     gene_obj['hgnc_id'])
        tx_obj['change_str'] = transcript_str(tx_obj, gene_name)

def transcript_str(transcript_obj, gene_name=None):
    """Generate amino acid change as a string."""
    if transcript_obj.get('exon'):
        gene_part, part_count_raw = 'exon', transcript_obj['exon']
    elif transcript_obj.get('intron'):
        gene_part, part_count_raw = 'intron', transcript_obj['intron']
    else:
        # variant between genes
        gene_part, part_count_raw = 'intergenic', '0'

    part_count = part_count_raw.rpartition('/')[0]
    change_str = "{}:{}{}:{}:{}".format(
        transcript_obj.get('refseq_id', ''),
        gene_part,
        part_count,
        transcript_obj.get('coding_sequence_name', 'NA'),
        transcript_obj.get('protein_sequence_name', 'NA'),
    )
    if gene_name:
        change_str = "{}:".format(gene_name) + change_str
    return change_str

def end_position(variant_obj):
    """Calculate end position for a variant."""
    alt_bases = len(variant_obj['alternative'])
    num_bases = max(len(variant_obj['reference']), alt_bases)
    return variant_obj['position'] + (num_bases - 1)

def frequency(variant_obj):
    """Returns a judgement on the overall frequency of the variant.

    Combines multiple metrics into a single call.
    """
    most_common_frequency = max(variant_obj.get('thousand_genomes_frequency') or 0,
                                variant_obj.get('exac_frequency') or 0)
    if most_common_frequency > .05:
        return 'common'
    elif most_common_frequency > .01:
        return 'uncommon'
    else:
        return 'rare'

def clinsig_human(variant_obj):
    """Convert to human readable version of CLINSIG evaluation."""
    for clinsig_obj in variant_obj['clnsig']:
        # The clinsig objects allways have a accession
        if isinstance(clinsig_obj['accession'], int):
            # New version
            link = "https://www.ncbi.nlm.nih.gov/clinvar/variation/{}"
        else:
            # Old version
            link = "https://www.ncbi.nlm.nih.gov/clinvar/{}"

        human_str = 'not provided'
        if clinsig_obj.get('value'):
            try:
                # Old version
                int(clinsig_obj['value'])
                human_str = CLINSIG_MAP.get(clinsig_obj['value'], 'not provided')
            except ValueError:
                # New version
                human_str = clinsig_obj['value']

        clinsig_obj['human'] = human_str
        clinsig_obj['link'] = link.format(clinsig_obj['accession'])

        yield clinsig_obj

def thousandg_link(variant_obj, build=None):
    """Compose link to 1000G page for detailed information."""
    dbsnp_id = variant_obj.get('dbsnp_id')
    build = build or 37

    if not dbsnp_id:
        return None

    if build == 37:
        url_template = ("http://grch37.ensembl.org/Homo_sapiens/Variation/Explore"
                        "?v={};vdb=variation")
    else:
        url_template = ("http://www.ensembl.org/Homo_sapiens/Variation/Explore"
                        "?v={};vdb=variation")

    return url_template.format(dbsnp_id)

def exac_link(variant_obj):
    """Compose link to ExAC website for a variant position."""
    url_template = ("http://exac.broadinstitute.org/variant/"
                    "{this[chromosome]}-{this[position]}-{this[reference]}"
                    "-{this[alternative]}")
    return url_template.format(this=variant_obj)

def gnomad_link(variant_obj):
    """Compose link to gnomAD website."""
    url_template = ("http://gnomad.broadinstitute.org/variant/{this[chromosome]}-"
                    "{this[position]}-{this[reference]}-{this[alternative]}")
    return url_template.format(this=variant_obj)

def swegen_link(variant_obj):
    """Compose link to SweGen Variant Frequency Database."""
    url_template = ("https://swegen-exac.nbis.se/variant/{this[chromosome]}-"
                    "{this[position]}-{this[reference]}-{this[alternative]}")
    return url_template.format(this=variant_obj)

def cosmic_link(variant_obj):
    """Compose link to COSMIC Database.

    Args:
        variant_obj(scout.models.Variant)

    Returns:
        url_template(str): Link to COSMIIC database if cosmic id is present
    """

    cosmic_ids = variant_obj.get('cosmic_ids')

    if not cosmic_ids:
        return None
    else:
        cosmic_id = cosmic_ids[0]
        url_template = ("https://cancer.sanger.ac.uk/cosmic/mutation/overview?id={}")

    return url_template.format(cosmic_id)

def beacon_link(variant_obj, build=None):
    """Compose link to Beacon Network."""
    build = build or 37
    url_template = ("https://beacon-network.org/#/search?pos={this[position]}&"
                    "chrom={this[chromosome]}&allele={this[alternative]}&"
                    "ref={this[reference]}&rs=GRCh37")
    # beacon does not support build 38 at the moment
    # if build == '38':
    #     url_template = ("https://beacon-network.org/#/search?pos={this[position]}&"
    #                     "chrom={this[chromosome]}&allele={this[alternative]}&"
    #                     "ref={this[reference]}&rs=GRCh38")

    return url_template.format(this=variant_obj)

def ucsc_link(variant_obj, build=None):
    """Compose link to UCSC."""
    build = build or 37
    url_template = ("http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg19&"
                        "position=chr{this[chromosome]}:{this[position]}"
                        "-{this[position]}&dgv=pack&knownGene=pack&omimGene=pack")
    if build == 38:
        url_template = ("http://genome.ucsc.edu/cgi-bin/hgTracks?db=hg20&"
                        "position=chr{this[chromosome]}:{this[position]}"
                        "-{this[position]}&dgv=pack&knownGene=pack&omimGene=pack")

    return url_template.format(this=variant_obj)

def alamut_link(variant_obj):
    url_template = ("http://localhost:10000/show?request={this[chromosome]}:"
                    "{this[position]}{this[reference]}>{this[alternative]}")
    return url_template.format(this=variant_obj)

def spidex_human(variant_obj):
    """Translate SPIDEX annotation to human readable string."""
    if variant_obj.get('spidex') is None:
        return 'not_reported'
    elif abs(variant_obj['spidex']) < SPIDEX_HUMAN['low']['pos'][1]:
        return 'low'
    elif abs(variant_obj['spidex']) < SPIDEX_HUMAN['medium']['pos'][1]:
        return 'medium'
    else:
        return 'high'

def expected_inheritance(variant_obj):
    """Gather information from common gene information."""
    manual_models = set()
    for gene in variant_obj.get('genes', []):
        manual_models.update(gene.get('manual_inheritance', []))
    return list(manual_models)

def callers(variant_obj, category='snv'):
    """Return info about callers."""
    calls = set()
    for caller in CALLERS[category]:
        if variant_obj.get(caller['id']):
            calls.add((caller['name'], variant_obj[caller['id']]))

    return list(calls)

def variant_verification(store, mail, institute_obj, case_obj, user_obj, variant_obj, sender, variant_url, order, comment, url_builder=url_for):
    """Sand a verification email and register the verification in the database

        Args:
            store(scout.adapter.MongoAdapter)
            mail(scout.server.extensions.mail): an instance of flask_mail.Mail
            institute_obj(dict): an institute object
            case_obj(dict): a case object
            user_obj(dict): a user object
            variant_obj(dict): a variant object (snv or sv)
            sender(str): current_app.config['MAIL_USERNAME']
            variant_url(str): the complete url to the variant (snv or sv), a link that works from outside scout domain.
            order(str): False == cancel order, True==order verification
            comment(str): sender's entered comment from form
            url_builder(flask.url_for): for testing purposes, otherwise test verification email fails because out of context
    """

    recipients = institute_obj['sanger_recipients']
    if len(recipients) == 0:
        raise MissingSangerRecipientError()

    view_type = None
    email_subject = None
    category = variant_obj.get('category')
    display_name = None
    chromosome = variant_obj['chromosome']
    end_chrom = variant_obj.get('end_chrom', chromosome)
    breakpoint_1 = ':'.join([chromosome, str(variant_obj['position'])])
    breakpoint_2 = ':'.join([end_chrom, str(variant_obj.get('end'))])
    variant_size = variant_obj.get('length')
    panels = ', '.join(variant_obj.get('panels', []))
    hgnc_symbol = ', '.join(variant_obj['hgnc_symbols'])
    email_subj_gene_symbol = None
    if len(variant_obj['hgnc_symbols']) > 3:
        email_subj_gene_symbol = ' '.join([ str(len(variant_obj['hgnc_symbols'])) + 'genes'])
    else:
        email_subj_gene_symbol = hgnc_symbol

    gtcalls = ["<li>{}: {}</li>".format(sample_obj['display_name'],
                                        sample_obj['genotype_call'])
               for sample_obj in variant_obj['samples']]
    tx_changes = []

    if category == 'snv': #SNV
        view_type = 'variants.variant'
        display_name = variant_obj.get('display_name')
        tx_changes = []

        for gene_obj in variant_obj.get('genes', []):
            for tx_obj in gene_obj['transcripts']:
                parse_transcript(gene_obj, tx_obj)
                # select refseq transcripts as "primary"
                if not tx_obj.get('refseq_id'):
                    continue

                for refseq_id in tx_obj.get('refseq_identifiers'):
                    transcript_line = []
                    if "common" in gene_obj:
                        transcript_line.append(gene_obj['common']['hgnc_symbol'])
                    else:
                        transcript_line.append(gene_obj['hgnc_id'])
                    transcript_line.append('-'.join([refseq_id, tx_obj['transcript_id']]))
                    if "exon" in tx_obj:
                        transcript_line.append(''.join([ "exon", tx_obj["exon"]]))
                    elif "intron" in tx_obj:
                        transcript_line.append(''.join([ "intron", tx_obj["intron"]]))
                    else:
                        transcript_line.append('intergenic')
                    if "coding_sequence_name" in tx_obj:
                        transcript_line.append(urllib.parse.unquote(tx_obj['coding_sequence_name']))
                    else:
                        transcript_line.append('')
                    if "protein_sequence_name" in tx_obj:
                        transcript_line.append(urllib.parse.unquote(tx_obj['protein_sequence_name']))
                    else:
                        transcript_line.append('')

                    tx_changes.append("<li>{}</li>".format(':'.join(transcript_line)))

    else: #SV
        view_type = 'variants.sv_variant'
        display_name = '_'.join([breakpoint_1, variant_obj.get('sub_category').upper()])

    # body of the email
    html = verification_email_body(
        case_name = case_obj['display_name'],
        url = variant_url, #this is the complete url to the variant, accessible when clicking on the email link
        display_name = display_name,
        category = category.upper(),
        subcategory = variant_obj.get('sub_category').upper(),
        breakpoint_1 = breakpoint_1,
        breakpoint_2 = breakpoint_2,
        hgnc_symbol = hgnc_symbol,
        panels = panels,
        gtcalls = ''.join(gtcalls),
        tx_changes = ''.join(tx_changes) or 'Not available',
        name = user_obj['name'].encode('utf-8'),
        comment = comment
    )

    # build a local the link to the variant to be included in the events objects (variant and case) created in the event collection.
    local_link = url_builder(view_type, institute_id=institute_obj['_id'],
                           case_name=case_obj['display_name'],
                           variant_id=variant_obj['_id'])

    if order == 'True': # variant verification should be ordered
        # pin variant if it's not already pinned
        if case_obj.get('suspects') is None or variant_obj['_id'] not in case_obj['suspects']:
            store.pin_variant(institute_obj, case_obj, user_obj, local_link, variant_obj)

        email_subject = "SCOUT: validation of {} variant {}, ({})".format( category.upper(), display_name, email_subj_gene_symbol)
        store.order_verification(institute=institute_obj, case=case_obj, user=user_obj, link=local_link, variant=variant_obj)

    else: # variant verification should be cancelled
        email_subject = "SCOUT: validation of {} variant {}, ({}), was CANCELLED!".format(category.upper(), display_name, email_subj_gene_symbol)
        store.cancel_verification(institute=institute_obj, case=case_obj, user=user_obj, link=local_link, variant=variant_obj)

    kwargs = dict(subject=email_subject, html=html, sender=sender, recipients=recipients,
        # cc the sender of the email for confirmation
        cc=[user_obj['email']])

    message = Message(**kwargs)
    # send email using flask_mail
    mail.send(message)

def verification_email_body(case_name, url, display_name, category, subcategory, breakpoint_1, breakpoint_2, hgnc_symbol, panels, gtcalls, tx_changes, name, comment):
    """
        Builds the html code for the variant verification emails (order verification and cancel verification)

        Args:
            case_name(str): case display name
            url(str): the complete url to the variant, accessible when clicking on the email link
            display_name(str): a display name for the variant
            category(str): category of the variant
            subcategory(str): sub-category of the variant
            breakpoint_1(str): breakpoint 1 (format is 'chr:start')
            breakpoint_2(str): breakpoint 2 (format is 'chr:stop')
            hgnc_symbol(str): a gene or a list of genes separated by comma
            panels(str): a gene panel of a list of panels separated by comma
            gtcalls(str): genotyping calls of any sample in the family
            tx_changes(str): amino acid changes caused by the variant, only for snvs otherwise 'Not available'
            name(str): user_obj['name'], uft-8 encoded
            comment(str): sender's comment from form

        Returns:
            html(str): the html body of the variant verification email

    """
    html = """
       <ul>
         <li>
           <strong>Case {case_name}</strong>: <a href="{url}">{display_name}</a>
         </li>
         <li><strong>Variant type</strong>: {category} ({subcategory})
         <li><strong>Breakpoint 1</strong>: {breakpoint_1}</li>
         <li><strong>Breakpoint 2</strong>: {breakpoint_2}</li>
         <li><strong>HGNC symbols</strong>: {hgnc_symbol}</li>
         <li><strong>Gene panels</strong>: {panels}</li>
         <li><strong>GT call</strong></li>
         {gtcalls}
         <li><strong>Amino acid changes</strong></li>
         {tx_changes}
         <li><strong>Comment</strong>: {comment}</li>
         <li><strong>Ordered by</strong>: {name}</li>
       </ul>
    """.format(
        case_name=case_name,
        url=url,
        display_name=display_name,
        category=category,
        subcategory=subcategory,
        breakpoint_1=breakpoint_1,
        breakpoint_2=breakpoint_2,
        hgnc_symbol=hgnc_symbol,
        panels=panels,
        gtcalls=gtcalls,
        tx_changes=tx_changes,
        name=name,
        comment=comment)

    return html

def cancer_variants(store, request_args, institute_id, case_name):
    """Fetch data related to cancer variants for a case."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    form = CancerFiltersForm(request_args)
    variants_query = store.variants(case_obj['_id'], category='cancer', query=form.data).limit(50)
    data = dict(
        institute=institute_obj,
        case=case_obj,
        variants=(parse_variant(store, institute_obj, case_obj, variant, update=True) for
                  variant in variants_query),
        form=form,
        variant_type=request_args.get('variant_type', 'clinical'),
    )
    return data

def clinvar_export(store, institute_id, case_name, variant_id):
    """Gather the required data for creating the clinvar submission form

        Args:
            store(scout.adapter.MongoAdapter)
            institute_id(str): Institute ID
            case_name(str): case ID
            variant_id(str): variant._id

        Returns:
            a dictionary with all the required data (case and variant level) to pre-fill in fields in the clinvar submission form

    """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    pinned = [store.variant(variant_id) or variant_id for variant_id in
                  case_obj.get('suspects', [])]
    variant_obj = store.variant(variant_id)
    return dict(
        today = str(date.today()),
        institute=institute_obj,
        case=case_obj,
        variant=variant_obj,
        pinned_vars=pinned
    )

def get_clinvar_submission(store, institute_id, case_name, variant_id, submission_id):
    """Collects all variants from the clinvar submission collection with a specific submission_id

        Args:
            store(scout.adapter.MongoAdapter)
            institute_id(str): Institute ID
            case_name(str): case ID
            variant_id(str): variant._id
            submission_id(str): clinvar submission id, i.e. SUB76578

        Returns:
            A dictionary with all the data to display the clinvar_update.html template page
    """

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    pinned = [store.variant(variant_id) or variant_id for variant_id in
                  case_obj.get('suspects', [])]
    variant_obj = store.variant(variant_id)
    clinvar_submission_objs = store.clinvars(submission_id=submission_id)
    return dict(
        today = str(date.today()),
        institute=institute_obj,
        case=case_obj,
        variant=variant_obj,
        pinned_vars=pinned,
        clinvars = clinvar_submission_objs
    )

def variant_acmg(store, institute_id, case_name, variant_id):
    """Collect data relevant for rendering ACMG classification form."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    return dict(institute=institute_obj, case=case_obj, variant=variant_obj,
                CRITERIA=ACMG_CRITERIA, ACMG_OPTIONS=ACMG_OPTIONS)

def variant_acmg_post(store, institute_id, case_name, variant_id, user_email, criteria):
    """Calculate an ACMG classification based on a list of criteria."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    user_obj = store.user(user_email)
    variant_link = url_for('variants.variant', institute_id=institute_id,
                           case_name=case_name, variant_id=variant_id)
    classification = store.submit_evaluation(
        institute_obj=institute_obj,
        case_obj=case_obj,
        variant_obj=variant_obj,
        user_obj=user_obj,
        link=variant_link,
        criteria=criteria,
    )
    return classification

def evaluation(store, evaluation_obj):
    """Fetch and fill-in evaluation object."""
    evaluation_obj['institute'] = store.institute(evaluation_obj['institute_id'])
    evaluation_obj['case'] = store.case(evaluation_obj['case_id'])
    evaluation_obj['variant'] = store.variant(evaluation_obj['variant_specific'])
    evaluation_obj['criteria'] = {criterion['term']: criterion for criterion in
                                  evaluation_obj['criteria']}
    evaluation_obj['classification'] = ACMG_COMPLETE_MAP[evaluation_obj['classification']]
    return evaluation_obj

def upload_panel(store, institute_id, case_name, stream):
    """Parse out HGNC symbols from a stream."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    raw_symbols = [line.strip().split('\t')[0] for line in stream if
                   line and not line.startswith('#')]
    # check if supplied gene symbols exist
    hgnc_symbols = []
    for raw_symbol in raw_symbols:
        if store.hgnc_genes(raw_symbol).count() == 0:
            flash("HGNC symbol not found: {}".format(raw_symbol), 'warning')
        else:
            hgnc_symbols.append(raw_symbol)
    return hgnc_symbols

def verified_excel_file(store, institute_list, temp_excel_dir):
    """Collect all verified variants in a list on institutes and save them to file

    Args:
        store(adapter.MongoAdapter)
        institute_list(list): a list of institute ids
        temp_excel_dir(os.Path): folder where the temp excel files are written to

    Returns:
        written_files(int): the number of files written to temp_excel_dir
    """
    document_lines = []
    written_files = 0
    today = datetime.datetime.now().strftime('%Y-%m-%d')
    LOG.info('Creating verified variant document..')

    for cust in institute_list:
        verif_vars = store.verified(institute_id=cust)
        LOG.info('Found {} verified variants for customer {}'.format(len(verif_vars), cust))

        if not verif_vars:
            continue
        unique_callers = set()
        for var_type, var_callers in CALLERS.items():
            for caller in var_callers:
                unique_callers.add(caller.get('id'))
        cust_verified = export_verified_variants(verif_vars, unique_callers)

        document_name = '.'.join([cust, '_verified_variants', today]) + '.xlsx'
        workbook = Workbook(os.path.join(temp_excel_dir,document_name))
        Report_Sheet = workbook.add_worksheet()

        # Write the column header
        row = 0
        for col,field in enumerate(VERIFIED_VARIANTS_HEADER + list(unique_callers)):
            Report_Sheet.write(row,col,field)

        # Write variant lines, after header (start at line 1)
        for row, line in enumerate(cust_verified,1): # each line becomes a row in the document
            for col, field in enumerate(line): # each field in line becomes a cell
                Report_Sheet.write(row,col,field)
        workbook.close()

        if os.path.exists(os.path.join(temp_excel_dir,document_name)):
            written_files += 1

    return written_files


import logging

from scout.models.phenotype_term import HpoTerm

LOG = logging.getLogger(__name__)

def build_hpo_term(hpo_info):
    """Build a hpo_term object

    Check that the information is correct and add the correct hgnc ids to the
    array of genes.

        Args:
            hpo_info(dict)

        Returns:
            hpo_obj(scout.models.HpoTerm): A dictionary with hpo information

    """

    try:
        hpo_id = hpo_info['hpo_id']
    except KeyError:
        raise KeyError("Hpo terms has to have a hpo_id")

    LOG.debug("Building hpo term %s", hpo_id)

    # Add description to HPO term
    try:
        description = hpo_info['description']
    except KeyError:
        raise KeyError("Hpo terms has to have a description")

    hpo_obj = HpoTerm(
        hpo_id = hpo_id,
        description = description
    )

    # Add links to hgnc genes if any
    hgnc_ids = hpo_info.get('genes', set())
    if hgnc_ids:
        hpo_obj['genes'] = list(hgnc_ids)

    return hpo_obj


import logging

LOG = logging.getLogger(__name__)

def export_genes(adapter, build='37'):
    """Export all genes from the database"""
    LOG.info("Exporting all genes to .bed format")

    for gene_obj in adapter.all_genes(build=build):
        yield gene_obj


import logging

from scout.constants import REV_CLINSIG_MAP

logger = logging.getLogger(__name__)

def parse_clnsig(acc, sig, revstat, transcripts):
    """Get the clnsig information

    Args:
        acc(str): The clnsig accession number, raw from vcf
        sig(str): The clnsig significance score, raw from vcf
        revstat(str): The clnsig revstat, raw from vcf
        transcripts(iterable(dict))

    Returns:
        clnsig_accsessions(list): A list with clnsig accessions
    """
    clnsig_accsessions = []

    if acc:
    # New format of clinvar allways have integers as accession numbers
        try:
            acc = int(acc)
        except ValueError:
            pass
        # There are sometimes different separators so we need to check which
        # one to use
        if isinstance(acc, int):
            revstat_groups = []
            if revstat:
                revstat_groups = [rev.lstrip('_') for rev in revstat.split(',')]

            sig_groups = []
            if sig:
                for significance in sig.split('/'):
                    splitted_word = significance.split('_')
                    sig_groups.append(' '.join(splitted_word[:2]))

            for sign_term in sig_groups:
                clnsig_accsessions.append({
                    'value': sign_term,
                    'accession': int(acc),
                    'revstat': ', '.join(revstat_groups),
                })
        else:
            # There are sometimes different separators so we need to check which
            # one to use
            acc_groups = acc.split('|')
            sig_groups = sig.split('|')
            revstat_groups = revstat.split('|')
            for acc_group, sig_group, revstat_group in zip(acc_groups, sig_groups, revstat_groups):
                accessions = acc_group.split(',')
                significances = sig_group.split(',')
                revstats = revstat_group.split(',')
                for accession, significance, revstat in zip(accessions, significances, revstats):
                    clnsig_accsessions.append({
                        'value': int(significance),
                        'accession': accession,
                        'revstat': revstat,
                    })

    elif transcripts:
        clnsig = set()
        for transcript in transcripts:
            for annotation in transcript.get('clinsig', []):
                clnsig.add(annotation)
        for annotation in clnsig:
            clnsig_accsessions.append({'value': annotation})

    return clnsig_accsessions


import logging
import click

LOG = logging.getLogger(__name__)

@click.command('index', short_help='Display all indexes')
@click.option('-n', '--collection-name')
@click.pass_context
def index(context, collection_name):
    """Show all indexes in the database"""
    LOG.info("Running scout view index")
    adapter = context.obj['adapter']

    i = 0
    click.echo("collection\tindex")
    for collection_name in adapter.collections():
        for index in adapter.indexes(collection_name):
            click.echo("{0}\t{1}".format(collection_name, index))
            i += 1

    if i == 0:
        LOG.info("No indexes found")


import logging

from pprint import pprint as pp

import click

LOG = logging.getLogger(__name__)

@click.command('groups', short_help='Update phenotype groups for an institute')
@click.argument('institute-id')
@click.option('-p', '--phenotype-group',
    help="Add one or more phenotype groups to institute",
    multiple = True,
)
@click.option('-a', '--group-abbreviation',
    help="Specify a phenotype group abbreviation",
    multiple = True,
)
@click.option('-f', '--group-file',
    help="CSV file with phenotype groups",
    type=click.File('r'),
)
@click.option('-a', '--add',
    help="If groups should be added instead of replacing existing groups",
    is_flag=True,
)
@click.pass_context
def groups(context, institute_id, phenotype_group, group_abbreviation, group_file, add):
    """
    Update the phenotype for a institute.
    If --add the groups will be added to the default groups. Else the groups will be replaced.
    """
    adapter = context.obj['adapter']
    LOG.info("Running scout update institute")
    if group_file:
        phenotype_group = []
        group_abbreviation = []
        for line in group_file:
            if line.startswith('#'):
                continue
            if len(line) < 7:
                continue
            line = line.rstrip().split('\t')
            phenotype_group.append(line[0])
            if line[1]:
                group_abbreviation.append(line[1])

    if not phenotype_group:
        LOG.info("Please provide some groups")
        return

    if (phenotype_group and group_abbreviation):
        if not len(phenotype_group) == len(group_abbreviation):
            LOG.warning("Specify same number of groups and abbreviations")
            return

    # try:
    adapter.update_institute(
        internal_id=institute_id,
        phenotype_groups=phenotype_group,
        group_abbreviations=group_abbreviation,
        add_groups = add,
        )
    # except Exception as err:
    #     LOG.warning(err)
    #     context.abort()


import logging

from scout.utils.md5 import generate_md5_key

LOG = logging.getLogger(__name__)

def parse_compounds(compound_info, case_id, variant_type):
    """Get a list with compounds objects for this variant.

        Arguments:
            compound_info(str): A Variant dictionary
            case_id (str): unique family id
            variant_type(str): 'research' or 'clinical'

        Returns:
            compounds(list(dict)): A list of compounds
    """
    # We need the case to construct the correct id
    compounds = []
    if compound_info:
        for family_info in compound_info.split(','):
            splitted_entry = family_info.split(':')
            # This is the family id
            if splitted_entry[0] == case_id:
                for compound in splitted_entry[1].split('|'):
                    splitted_compound = compound.split('>')
                    compound_obj = {}
                    compound_name = splitted_compound[0]
                    compound_obj['variant'] = generate_md5_key(compound_name.split('_') +
                                                               [variant_type, case_id])

                    try:
                        compound_score = float(splitted_compound[1])
                    except (TypeError, IndexError):
                        compound_score = 0.0

                    compound_obj['score'] = compound_score
                    compound_obj['display_name'] = compound_name

                    compounds.append(compound_obj)

    return compounds


import logging

import click

from bson.json_util import dumps

from scout.commands.utils import builds_option

from scout.export.gene import export_genes

from .utils import (build_option, json_option)

LOG = logging.getLogger(__name__)

@click.command('genes', short_help='Export genes')
@build_option
@json_option
@click.pass_context
def genes(context, build, json):
    """Export all genes from a build"""
    LOG.info("Running scout export genes")
    adapter = context.obj['adapter']

    result = adapter.all_genes(build=build)
    if json:
        click.echo(dumps(result))
        return

    gene_string = ("{0}\t{1}\t{2}\t{3}\t{4}")
    click.echo("#Chromosom\tStart\tEnd\tHgnc_id\tHgnc_symbol")
    for gene_obj in result:
        click.echo(gene_string.format(
            gene_obj['chromosome'],
            gene_obj['start'],
            gene_obj['end'],
            gene_obj['hgnc_id'],
            gene_obj['hgnc_symbol'],
        ))


import logging

from scout.constants import (REV_PHENOTYPE_MAP, REV_SEX_MAP, ANALYSIS_TYPES)
from scout.exceptions import PedigreeError

log = logging.getLogger(__name__)

def build_individual(ind):
    """Build a Individual object

        Args:
            ind (dict): A dictionary with individual information

        Returns:
            ind_obj (dict): A Individual object

        dict(
            individual_id = str, # required
            display_name = str,
            sex = str,
            phenotype = int,
            father = str, # Individual id of father
            mother = str, # Individual id of mother
            capture_kits = list, # List of names of capture kits
            bam_file = str, # Path to bam file
            vcf2cytosure = str, # Path to CGH file
            analysis_type = str, # choices=ANALYSIS_TYPES
        )

    """

    try:
        ind_obj = dict(
            individual_id=ind['individual_id']
        )
        log.info("Building Individual with id:{0}".format(ind['individual_id']))
    except KeyError as err:
        raise PedigreeError("Individual is missing individual_id")

    ind_obj['display_name'] = ind.get('display_name', ind_obj['individual_id'])

    sex = ind.get('sex', 'unknown')
    # Convert sex to .ped
    try:
        # Check if sex is coded as an integer
        int(sex)
        ind_obj['sex'] = str(sex)
    except ValueError as err:
        try:
            # Sex are numbers in the database
            ind_obj['sex'] = REV_SEX_MAP[sex]
        except KeyError as err:
            raise(PedigreeError("Unknown sex: %s" % sex))

    phenotype = ind.get('phenotype', 'unknown')
    # Make the phenotype integers
    try:
        ped_phenotype = REV_PHENOTYPE_MAP[phenotype]
        if ped_phenotype == -9:
            ped_phenotype = 0
        ind_obj['phenotype'] = ped_phenotype
    except KeyError as err:
        raise(PedigreeError("Unknown phenotype: %s" % phenotype))

    ind_obj['father'] = ind.get('father')
    ind_obj['mother'] = ind.get('mother')
    ind_obj['capture_kits'] = ind.get('capture_kits', [])
    ind_obj['bam_file'] = ind.get('bam_file')
    ind_obj['mt_bam'] = ind.get('mt_bam')
    ind_obj['vcf2cytosure'] = ind.get('vcf2cytosure')
    ind_obj['confirmed_sex'] = ind.get('confirmed_sex')
    ind_obj['confirmed_parent'] = ind.get('confirmed_parent')
    ind_obj['predicted_ancestry'] = ind.get('predicted_ancestry')

    # Check if the analysis type is ok
    # Can be anyone of ('wgs', 'wes', 'mixed', 'unknown')
    analysis_type = ind.get('analysis_type', 'unknown')
    if not analysis_type in ANALYSIS_TYPES:
        raise PedigreeError("Analysis type %s not allowed", analysis_type)
    ind_obj['analysis_type'] = analysis_type

    if 'tmb' in ind:
        ind_obj['tmb'] = ind['tmb']

    if 'msi' in ind:
        ind_obj['msi'] = ind['msi']

    if 'tumor_purity' in ind:
        ind_obj['tumor_purity'] = ind['tumor_purity']

    if 'tumor_type' in ind:
        ind_obj['tumor_type'] = ind['tumor_type']

    return ind_obj


import logging
from pprint import pprint as pp

import click

LOG = logging.getLogger(__name__)

@click.command(short_help='Upload variants to existing case')
@click.argument('case-id')
@click.option('-i', '--institute', help='institute id of related cases')
@click.option('-f', '--force', is_flag=True, help='upload without request')
@click.option('--cancer', is_flag=True, help='Upload clinical cancer variants')
@click.option('--cancer-research', is_flag=True, help='Upload research cancer variants')
@click.option('--sv', is_flag=True, help='Upload clinical structural variants')
@click.option('--sv-research', is_flag=True, help='Upload research structural variants')
@click.option('--snv', is_flag=True, help='Upload clinical SNV variants')
@click.option('--snv-research', is_flag=True, help='Upload research SNV variants')
@click.option('--str-clinical', is_flag=True, help='Upload clinical STR variants')
@click.option('--chrom', help='If region, specify the chromosome')
@click.option('--start', type=int, help='If region, specify the start')
@click.option('--end', type=int, help='If region, specify the end')
@click.option('--hgnc-id', type=int, help='If all variants from a gene, specify the gene id')
@click.option('--hgnc-symbol', help='If all variants from a gene, specify the gene symbol')
@click.option('--rank-treshold', default=5, help='Specify the rank score treshold',
                show_default=True)
@click.pass_context
def variants(context, case_id, institute, force, cancer, cancer_research, sv,
             sv_research, snv, snv_research, str_clinical, chrom, start, end, hgnc_id,
             hgnc_symbol, rank_treshold):
    """Upload variants to a case

        Note that the files has to be linked with the case,
        if they are not use 'scout update case'.
    """
    LOG.info("Running scout load variants")
    adapter = context.obj['adapter']

    if institute:
        case_id = "{0}-{1}".format(institute, case_id)
    else:
        institute = case_id.split('-')[0]
    case_obj = adapter.case(case_id=case_id)
    if case_obj is None:
        LOG.info("No matching case found")
        context.abort()

    files = [
        {'category': 'cancer', 'variant_type': 'clinical', 'upload': cancer},
        {'category': 'cancer', 'variant_type': 'research', 'upload': cancer_research},
        {'category': 'sv', 'variant_type': 'clinical', 'upload': sv},
        {'category': 'sv', 'variant_type': 'research', 'upload': sv_research},
        {'category': 'snv', 'variant_type': 'clinical', 'upload': snv},
        {'category': 'snv', 'variant_type': 'research', 'upload': snv_research},
        {'category': 'str', 'variant_type': 'clinical', 'upload': str_clinical},
    ]

    gene_obj = None
    if (hgnc_id or hgnc_symbol):
        if hgnc_id:
            gene_obj = adapter.hgnc_gene(hgnc_id)
        if hgnc_symbol:
            for res in adapter.gene_by_alias(hgnc_symbol):
                gene_obj = res
        if not gene_obj:
            LOG.warning("The gene could not be found")
            context.abort()

    i = 0
    for file_type in files:
        variant_type = file_type['variant_type']
        category = file_type['category']

        if file_type['upload']:
            i += 1
            if variant_type == 'research':
                if not (force or case_obj['research_requested']):
                    LOG.warn("research not requested, use '--force'")
                    context.abort()

            LOG.info("Delete {0} {1} variants for case {2}".format(
                         variant_type, category, case_id))
            adapter.delete_variants(case_id=case_obj['_id'],
                                    variant_type=variant_type,
                                    category=category)

            LOG.info("Load {0} {1} variants for case {2}".format(
                         variant_type, category, case_id))

            try:
                adapter.load_variants(
                    case_obj=case_obj,
                    variant_type=variant_type,
                    category=category,
                    rank_threshold=rank_treshold,
                    chrom=chrom,
                    start=start,
                    end=end,
                    gene_obj=gene_obj
                )
            except Exception as e:
                LOG.warning(e)
                context.abort()
    if i == 0:
        LOG.info("No files where specified to upload variants from")


from bson import json_util
from flask import Blueprint, Response, abort

from scout.server.extensions import store
from scout.server.utils import institute_and_case

api_bp = Blueprint('api', __name__, url_prefix='/api/v1')

@api_bp.route('/<institute_id>/<case_name>')
def case(institute_id, case_name):
    """Return a variant."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    if case_obj is None:
        return abort(404)
    return Response(json_util.dumps(case_obj), mimetype='application/json')

@api_bp.route('/<institute_id>/<case_name>/<variant_id>')
def variant(institute_id, case_name, variant_id):
    """Display a specific SNV variant."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    return Response(json_util.dumps(variant_obj), mimetype='application/json')


import logging

import click

LOG = logging.getLogger(__name__)

@click.command('collections', short_help='Display all collections')
@click.pass_context
def collections(context):
    """Show all collections in the database"""
    LOG.info("Running scout view collections")

    adapter = context.obj['adapter']

    for collection_name in adapter.collections():
        click.echo(collection_name)


import logging

import click

logger = logging.getLogger(__name__)

from scout.load import load_institute

@click.command('institute', short_help='Load a institute')
@click.option('-i', '--internal-id',
              required=True)
@click.option('-d', '--display-name')
@click.option('-s', '--sanger-recipients', multiple=True)
@click.pass_context
def institute(ctx, internal_id, display_name, sanger_recipients):
    """
    Create a new institute and add it to the database

    """
    adapter = ctx.obj['adapter']

    if not internal_id:
        logger.warning("A institute has to have an internal id")
        ctx.abort()

    if not display_name:
        display_name = internal_id

    if sanger_recipients:
        sanger_recipients = list(sanger_recipients)

    try:
        load_institute(
            adapter=adapter,
            internal_id=internal_id,
            display_name=display_name,
            sanger_recipients=sanger_recipients
        )
    except Exception as e:
        logger.warning(e)
        ctx.abort()


import logging

from pprint import pprint as pp

import click

LOG = logging.getLogger(__name__)

@click.command('institute', short_help='Update institute for a case')
@click.argument('institute-id')
@click.option('-c', '--coverage-cutoff',
    type=int,
    help="Set a new coverage cutoff for a institute",
)
@click.option('-f', '--frequency-cutoff',
    type=float,
    help="Set a new frequency cutoff for a institute",
)
@click.option('-s', '--sanger-recipient',
    help="Specify email adress for a existing user that should be added to the institute",
)
@click.option('-d', '--display-name',
    help="Set a new display name for a insitute",
)
@click.option('-r', '--remove-sanger',
    help="Specify email adress for a existing user that should be removed from sanger recipients",
)
@click.pass_context
def institute(context, institute_id, sanger_recipient, coverage_cutoff, frequency_cutoff,
              display_name, remove_sanger):
    """
    Update an institute
    """
    adapter = context.obj['adapter']
    LOG.info("Running scout update institute")

    try:
        adapter.update_institute(
            internal_id=institute_id,
            sanger_recipient=sanger_recipient,
            coverage_cutoff=coverage_cutoff,
            frequency_cutoff=frequency_cutoff,
            display_name=display_name,
            remove_sanger=remove_sanger,
            )
    except Exception as err:
        LOG.warning(err)
        context.abort()


from codecs import (open, getreader)
import gzip

def get_file_handle(file_path):
    """Return a opened file"""

    if file_path.endswith('.gz'):
        file_handle = getreader('utf-8')(gzip.open(file_path, 'r'), errors='replace')
    else:
        file_handle = open(file_path, 'r', encoding='utf-8')

    return file_handle


from __future__ import unicode_literals

from datetime import date, timedelta
from calendar import monthrange

from django.utils import timezone
from django.utils.six.moves import xrange

get_now = lambda: timezone.localtime(timezone.now())

now = get_now()
# Warning! please do not use 'now' anywhere: it is initialized on first import
# and so it will not change for days in some environments. It's here just for backward
# compatibility

def inc_month(month, year):
    """
    Increment the month and, if neccessary, the year.
    Both month and year should be ints.
    """
    month += 1
    if month > 12:
        month = 1
        year += 1
    return month, year

def _inc_day(year, month, day, net):
    """Increments the day by converting to a datetime.date()."""
    d = date(year, month, day)
    new_d = d + timezone.timedelta(days=net)
    return new_d.year, new_d.month, new_d.day

def get_net_category_tag(req):
    if req.META['QUERY_STRING']:
        net = get_net(req)
        category, tag = get_category_tag(req)
    else:
        net = 0
        category = None
        tag = None
    return net, category, tag

def get_net(req):
    """Get the net of any 'next' and 'prev' querystrings."""
    try:
        nxt, prev = map(
            int, (req.GET.get('cal_next', 0), req.GET.get('cal_prev', 0))
        )
        net = nxt - prev
    except Exception:
        net = 0
    return net

def get_category_tag(req):
    """Get value of any category and/or tag querystrings"""
    return req.GET.get('cal_category', None), req.GET.get('cal_tag', None)

def get_qs(old_qs):
    there = (
        'cal_mini', 'cal_next', 'cal_prev', 'cal_month', 'cal_year'
    )
    return [x for x in old_qs.split('&') if x.split('=')[0] not in there]

# TODO change from d to something more descriptive
def order_events(events, d=False):
    """
    Group events that occur on the same day, then sort them alphabetically
    by title, then sort by day. Returns a list of tuples that looks like
    [(day: [events])], where day is the day of the event(s), and [events]
    is an alphabetically sorted list of the events for the day.
    """
    ordered_events = {}
    for event in events:
        try:
            for occ in event.occurrence:
                try:
                    ordered_events[occ].append(event)
                except Exception:
                    ordered_events[occ] = [event]
        except AttributeError:  # no occurrence for this event
            # This shouldn't happen, since an event w/o an occurrence
            # shouldn't get this far, but if it does, just skip it since
            # it shouldn't be displayed on the calendar anyway.
            pass

    if d:
        # return as a dict without sorting by date
        return ordered_events
    else:
        # return ordered_events as a list tuples sorted by date
        return sorted(ordered_events.items())

def get_next_and_prev(net):
    """Returns what the next and prev querystrings should be."""
    if net == 0:
        nxt = prev = 1
    elif net > 0:
        nxt = net + 1
        prev = -(net - 1)
    else:
        nxt = net + 1
        prev = abs(net) + 1
    return nxt, prev

def _check_year(year, month, error, error_msg):
    """Checks that the year is within 50 years from now."""
    if year not in xrange((now.year - 50), (now.year + 51)):
        year = now.year
        month = now.month
        error = error_msg
    return year, month, error

def clean_year_month_day(year, month, day, net):
    error = False
    error_msg = "The date given was invalid."
    # check that the month is within 1-12
    if month not in xrange(1, 13):
        month = now.month
        error = error_msg
    # check that the day is within range for the month
    if day not in xrange(1, monthrange(year, month)[1] + 1):
        day = 1
        error = error_msg
    # if no error yet, increment the day by net then check the year
    if not error:
        year, month, day = _inc_day(year, month, day, net)
        year, month, error = _check_year(year, month, error, error_msg)
    return year, month, day, error

def clean_year_month(year, month, month_orig):
    """
    If 'month_orig', which is the month given in the url BEFORE any next/prev
    query strings have been applied, is out of range, sets month to the
    current month and returns an error message. Also Returns an error
    message if the year given is +/- 50 years from now.
    If 'month', which is the month given in the url AFTER any next/prev
    query strings have been applied, is out of range, adjusts it to be
    in range (by also adjusting the year).
    """
    error = False
    error_msg = "The date given was invalid."
    if month_orig not in xrange(1, 13) and month_orig is not None:
        month = now.month
        error = error_msg
    # This takes care of 'next' query strings making month > 12
    while month > 12:
        month -= 12
        year += 1
    # This takes care of 'prev' query strings making month < 1
    while month < 1:
        month += 12
        year -= 1
    year, month, error = _check_year(year, month, error, error_msg)
    return year, month, error

def check_weekday(year, month, day, reverse=False):
    """
    Make sure any event day we send back for weekday repeating
    events is not a weekend.
    """
    d = date(year, month, day)
    while d.weekday() in (5, 6):
        if reverse:
            d -= timedelta(days=1)
        else:
            d += timedelta(days=1)
    return d.year, d.month, d.day