from __future__ import unicode_literals

from collections import defaultdict
from datetime import date, timedelta

from django.utils.six.moves import xrange

class Repeater(object):
    def __init__(self, count, year, month, day=None, end_repeat=None,
                 event=None, num=7, count_first=False, end_on=None):
        self.count = count  # defaultdict(list)
        self.year = year
        self.month = month
        self.day = day
        self.end_repeat = end_repeat  # datetime.date()
        self.event = event  # Event object
        self.num = num
        self.count_first = count_first
        self.end_on = end_on
        if end_repeat is None:  # set to far off date to simulate 'forever'
            self.end_repeat = date(2200, 3, 3)

    def count_it(self, day):
        self.count[day].append((self.event.title, self.event.pk))

    def repeat(self, day=None):
        """
        Add 'num' to the day and count that day until we reach end_repeat, or
        until we're outside of the current month, counting the days
        as we go along.
        """
        if day is None:
            day = self.day

        try:
            d = date(self.year, self.month, day)
        except ValueError:  # out of range day
            return self.count

        if self.count_first and d <= self.end_repeat:
            self.count_it(d.day)

        d += timedelta(days=self.num)

        if self.end_on is not None:
            while d.month == self.month and \
                    d <= self.end_repeat and \
                    d.day <= self.end_on:
                self.count_it(d.day)
                d += timedelta(days=self.num)
        else:
            while d.month == self.month and d <= self.end_repeat:
                self.count_it(d.day)
                d += timedelta(days=self.num)

    def repeat_weekdays(self):
        """
        Like self.repeat(), but used to repeat every weekday.
        """
        try:
            d = date(self.year, self.month, self.day)
        except ValueError:  # out of range day
            return self.count

        if self.count_first and \
                d <= self.end_repeat and \
                d.weekday() not in (5, 6):
            self.count_it(d.day)

        d += timedelta(days=1)

        while d.month == self.month and d <= self.end_repeat:
            if d.weekday() not in (5, 6):
                self.count_it(d.day)
            d += timedelta(days=1)

    def repeat_reverse(self, start, end):
        """
        Starts from 'start' day and counts backwards until 'end' day.
        'start' should be >= 'end'. If it's equal to, does nothing.
        If a day falls outside of end_repeat, it won't be counted.
        """
        day = start
        diff = start - end
        try:
            if date(self.year, self.month, day) <= self.end_repeat:
                self.count_it(day)
        # a value error likely means the event runs past the end of the month,
        # like an event that ends on the 31st, but the month doesn't have that
        # many days. Ignore it b/c the dates won't be added to calendar anyway
        except ValueError:
            pass
        for i in xrange(diff):
            day -= 1
            try:
                if date(self.year, self.month, day) <= self.end_repeat:
                    self.count_it(day)
            except ValueError:
                pass

    def repeat_chunk(self, diff):
        for i in xrange(diff):
            self.repeat(self.day + i + 1)

    def repeat_biweekly(self):
        """
        This function is unique b/c it creates an empty defaultdict,
        adds in the event occurrences by creating an instance of Repeater,
        then returns the defaultdict, likely to be merged into the 'main'
        defaultdict (the one holding all event occurrences for this month).
        """
        mycount = defaultdict(list)
        d = self.event.l_start_date
        while d.year != self.year or d.month != self.month:
            d += timedelta(days=14)
        r = self.__class__(
            mycount, self.year, self.month, d.day, self.event.end_repeat,
            self.event, num=self.num, count_first=True
        )
        r.repeat()
        if self.event.is_chunk() and r.count:
            r.day = min(r.count)
            r.repeat_chunk(self.event.start_end_diff)
        return r.count

class YearlyRepeater(Repeater):
    def _repeat_chunk(self):
        self.day = self.event.l_start_date.day
        self.num = 1
        if self.event.end_repeat is not None:
            self.end_repeat = self.event.end_repeat

        if self.event.l_start_date.month == self.month:
            if self.event.starts_ends_same_month():
                self.end_on = self.event.l_end_date.day
            self.repeat()
        elif (self.event.l_end_date.month == self.month
              and not self.event.starts_ends_same_month()):
            self.repeat_reverse(self.event.l_end_date.day, 1)

    def repeat_it(self):
        """
        Events that repeat every year should be shown every year
        on the same date they started e.g. an event that starts on March 23rd
        would appear on March 23rd every year it is scheduled to repeat.
        If the event is a chunk event, hand it over to _repeat_chunk().
        """
        # The start day will be counted if we're in the start year,
        # so only count the day if we're in the same month as
        # l_start_date, but not in the same year.
        if self.event.l_start_date.month == self.month and \
                self.event.l_start_date.year != self.year:
            self.count_it(self.event.l_start_date.day)
        # If we're in the same mo & yr as l_start_date,
        # should already be filled in
        if self.event.is_chunk() and not \
                self.event.starts_same_year_month_as(self.year, self.month):
            self._repeat_chunk()
        return self.count

class MonthlyRepeater(Repeater):
    def _repeat_chunk(self):
        start_day = self.event.l_start_date.day
        last_day_last_mo = date(self.year, self.month, 1) - timedelta(days=1)

        self.day = start_day
        self.num = 1
        if self.event.end_repeat is not None:
            self.end_repeat = self.event.end_repeat

        if not self.event.starts_same_year_month_as(self.year, self.month):
            if not self.event.starts_ends_same_month():
                    self.repeat()  # fill out the end of the month
                    # fill out the beginning of the month, if nec.
                    if start_day <= last_day_last_mo.day:
                        self.repeat_reverse(
                            self.event.l_end_date.day, 1
                        )
            else:
                self.repeat_reverse(
                    self.event.l_end_date.day, start_day + 1
                )

    def repeat_it(self):
        """
        Events that repeat every month should be shown every month
        on the same date they started e.g. an event that starts on the 23rd
        would appear on the 23rd every month it is scheduled to repeat.
        """
        start_day = self.event.l_start_date.day
        if not self.event.starts_same_month_as(self.month):
            self.count_it(start_day)
        elif self.event.starts_same_month_not_year_as(self.month, self.year):
            self.count_it(start_day)

        if self.event.is_chunk():
            self._repeat_chunk()
        return self.count

class DailyRepeater(Repeater):
    """Handles repeating daily and every weekday."""
    def repeat_it(self):
        if self.event.end_repeat is not None:
            self.end_repeat = self.event.end_repeat

        if self.event.starts_same_year_month_as(self.year, self.month):
            # we assume that l_start_date was already counted, so no
            # count_first=True
            self.day = self.event.l_start_date.day
        else:
            # Note count_first=True b/c although the start date isn't this
            # month, the event does begin repeating this month and start_date
            # has not yet been counted.
            self.day = date(self.year, self.month, 1).day
            self.count_first = True

        if self.event.repeats('DAILY'):
            self.num = 1
            self.repeat()
        else:
            self.repeat_weekdays()
        return self.count

class WeeklyRepeater(Repeater):
    def _biweekly_helper(self):
        """Created to take some of the load off of _handle_weekly_repeat_out"""
        self.num = 14
        mycount = self.repeat_biweekly()
        if mycount:
            if self.event.is_chunk() and min(mycount) not in xrange(1, 8):
                mycount = _chunk_fill_out_first_week(
                    self.year, self.month, mycount, self.event,
                    diff=self.event.start_end_diff,
                )
            for k, v in mycount.items():
                for item in v:
                    self.count[k].append(item)

    def _handle_weekly_repeat_out(self):
        """
        Handles repeating an event weekly (or biweekly) if the current
        year and month are outside of its start year and month.
        It takes care of cases 3 and 4 in _handle_weekly_repeat_in() comments.
        """
        start_d = _first_weekday(
            self.event.l_start_date.weekday(), date(self.year, self.month, 1)
        )
        self.day = start_d.day
        self.count_first = True
        if self.event.repeats('BIWEEKLY'):
            self._biweekly_helper()
        elif self.event.repeats('WEEKLY'):
            # Note count_first=True b/c although the start date isn't this
            # month, the event does begin repeating this month and start_date
            # has not yet been counted.
            # Also note we start from start_d.day and not
            # event.l_start_date.day
            self.repeat()
            if self.event.is_chunk():
                diff = self.event.start_end_diff
                self.count = _chunk_fill_out_first_week(
                    self.year, self.month, self.count, self.event, diff
                )
                for i in xrange(diff):
                    # count the chunk days, then repeat them
                    self.day = start_d.day + i + 1
                    self.repeat()

    def _handle_weekly_repeat_in(self):
        """
        Handles repeating both weekly and biweekly events, if the
        current year and month are inside it's l_start_date and l_end_date.
        Four possibilites:
            1. The event starts this month and ends repeating this month.
            2. The event starts this month and doesn't finish repeating
            this month.
            3. The event didn't start this month but ends repeating this month.
            4. The event didn't start this month and doesn't end repeating
            this month.
        """
        self.day = self.event.l_start_date.day
        self.count_first = False
        repeats = {'WEEKLY': 7, 'BIWEEKLY': 14}
        if self.event.starts_same_year_month_as(self.year, self.month):
            # This takes care of 1 and 2 above.
            # Note that 'count' isn't incremented before adding a week (in
            # Repeater.repeat()), b/c it's assumed that l_start_date
            # was already counted.
            for repeat, num in repeats.items():
                self.num = num
                if self.event.repeats(repeat):
                    self.repeat()
                    if self.event.is_chunk():
                        self.repeat_chunk(diff=self.event.start_end_diff)

    def repeat_it(self):
        if self.event.end_repeat is not None:
            self.end_repeat = self.event.end_repeat

        if self.event.starts_same_year_month_as(self.year, self.month):
            self._handle_weekly_repeat_in()
        else:
            self._handle_weekly_repeat_out()
        return self.count

# XXX _first_weekday() and _chunk_fill_out_the_first_week() are currently
# only used in WeeklyRepeat, so maybe move them in there.
def _first_weekday(weekday, d):
    """
    Given a weekday and a date, will increment the date until it's
    weekday matches that of the given weekday, then that date is returned.
    """
    while weekday != d.weekday():
        d += timedelta(days=1)
    return d

def _chunk_fill_out_first_week(year, month, count, event, diff):
    """
    If a repeating chunk event exists in a particular month, but didn't
    start that month, it may be neccessary to fill out the first week.
    Five cases:
        1. event starts repeating on the 1st day of month
        2. event starts repeating past the 1st day of month
        3. event starts repeating before the 1st day of month, and continues
        through it.
        4. event starts repeating before the 1st day of month, and finishes
        repeating before it.
        5. event starts repeating before the 1st day of month, and finishes
        on it.
    """
    first_of_the_month = date(year, month, 1)
    d = _first_weekday(event.l_end_date.weekday(), first_of_the_month)
    d2 = _first_weekday(event.l_start_date.weekday(), first_of_the_month)
    diff_weekdays = d.day - d2.day
    day = first_of_the_month.day
    start = event.l_start_date.weekday()
    first = first_of_the_month.weekday()

    if start == first or diff_weekdays == diff:
        return count
    elif start > first:
        end = event.l_end_date.weekday()
        diff = end - first + 1
    elif start < first:
        diff = d.day
    for i in xrange(diff):
        if event.end_repeat is not None and \
                date(year, month, day) >= event.end_repeat:
            break
        count[day].append((event.title, event.pk))
        day += 1
    return count

class CountHandler(object):
    def __init__(self, year, month, events):
        self.year = year
        self.month = month
        self.events = events
        self.count = defaultdict(list)

    def _handle_single_chunk(self, event):
        """
        This handles either a non-repeating event chunk, or the first
        month of a repeating event chunk.
        """
        if not event.starts_same_month_as(self.month) and not \
                event.repeats('NEVER'):
            # no repeating chunk events if we're not in it's start month
            return

        # add the events into an empty defaultdict. This is better than passing
        # in self.count, which we don't want to make another copy of because it
        # could be very large.
        mycount = defaultdict(list)
        r = Repeater(
            mycount, self.year, self.month, day=event.l_start_date.day,
            end_repeat=event.end_repeat, event=event, count_first=True,
            end_on=event.l_end_date.day, num=1
        )

        if event.starts_same_month_as(self.month):
            if not event.ends_same_month_as(self.month):
                # The chunk event starts this month,
                # but does NOT end this month
                r.end_on = None
        else:
            # event chunks can be maximum of 7 days, so if an event chunk
            # didn't start this month, we know it will end this month.
            r.day = 1
        r.repeat()
        # now we add in the events we generated to self.count
        for k, v in r.count.items():
            self.count[k].extend(v)

    def _handle_month_event(self, event):
        if event.is_chunk():
            self._handle_single_chunk(event)
        elif event.repeats('WEEKDAY'):
            if event.l_start_date.weekday() not in (5, 6):
                self.count[event.l_start_date.day].append(
                    (event.title, event.pk)
                )
        else:
            self.count[event.l_start_date.day].append((event.title, event.pk))

    def get_count(self):
        kwargs = {'year': self.year, 'month': self.month, 'count': self.count}
        for event in self.events:
            kwargs['event'] = event
            if event.starts_ends_yr_mo(self.year, self.month):
                self._handle_month_event(event)
            if event.repeats('WEEKLY') or event.repeats('BIWEEKLY'):
                r = WeeklyRepeater(**kwargs)
            elif event.repeats('MONTHLY'):
                r = MonthlyRepeater(**kwargs)
            elif event.repeats('DAILY') or event.repeats('WEEKDAY'):
                r = DailyRepeater(**kwargs)
            else:
                r = YearlyRepeater(**kwargs)
            self.count = r.repeat_it()
        return self.count


"""scout.load.panel.py

functions to load panels into the database

"""

import logging
import json

from pprint import pprint as pp

from scout.utils.handle import get_file_handle
from scout.parse.panel import (get_panel_info, parse_panel_app_panel, parse_gene_panel)
from scout.utils.requests import get_request

LOG = logging.getLogger(__name__)

def load_panel(panel_path, adapter, date=None, display_name=None, version=None, panel_type=None,
               panel_id=None, institute=None):
    """Load a manually curated gene panel into scout

    Args:
        panel_path(str): path to gene panel file
        adapter(scout.adapter.MongoAdapter)
        date(str): date of gene panel on format 2017-12-24
        display_name(str)
        version(float)
        panel_type(str)
        panel_id(str)
        institute(str)

    """
    panel_lines = get_file_handle(panel_path)

    try:
        # This will parse panel metadata if includeed in panel file
        panel_info = get_panel_info(
            panel_lines=panel_lines,
            panel_id=panel_id,
            institute=institute,
            version=version,
            date=date,
            display_name=display_name
            )
    except Exception as err:
        raise err

    version = None
    if panel_info.get('version'):
        version = float(panel_info['version'])

    panel_id = panel_info['panel_id']
    display_name = panel_info['display_name'] or panel_id
    institute = panel_info['institute']
    date = panel_info['date']

    if not institute:
        raise SyntaxError("A Panel has to belong to a institute")

    #Check if institute exists in database
    if not adapter.institute(institute):
        raise SyntaxError("Institute {0} does not exist in database".format(institute))

    if not panel_id:
        raise SyntaxError("A Panel has to have a panel id")

    if version:
        existing_panel = adapter.gene_panel(panel_id, version)
    else:
        ## Assuming version 1.0
        existing_panel = adapter.gene_panel(panel_id)
        version = 1.0
        LOG.info("Set version to %s", version)

    if existing_panel:
        LOG.info("found existing panel")
        if version == existing_panel['version']:
            LOG.warning("Panel with same version exists in database")
            LOG.info("Reload with updated version")
            raise SyntaxError()
        display_name = display_name or existing_panel['display_name']
        institute = institute or existing_panel['institute']

    parsed_panel = parse_gene_panel(
        path=panel_path,
        institute=institute,
        panel_type=panel_type,
        date=date,
        version=version,
        panel_id=panel_id,
        display_name=display_name,
    )

    try:
        adapter.load_panel(parsed_panel=parsed_panel)
    except Exception as err:
        raise err

def load_panel_app(adapter, panel_id=None, institute='cust000'):
    """Load PanelApp panels into scout database

    If no panel_id load all PanelApp panels

    Args:
        adapter(scout.adapter.MongoAdapter)
        panel_id(str): The panel app panel id
    """
    base_url = 'https://panelapp.genomicsengland.co.uk/WebServices/{0}/'

    hgnc_map = adapter.genes_by_alias()

    if panel_id:
        panel_ids = [panel_id]

    if not panel_id:

        LOG.info("Fetching all panel app panels")
        data = get_request(base_url.format('list_panels'))

        json_lines = json.loads(data)

        panel_ids = [panel_info['Panel_Id'] for panel_info in json_lines['result']]

    for panel_id in panel_ids:
        panel_data = get_request(base_url.format('get_panel') + panel_id)

        parsed_panel = parse_panel_app_panel(
            panel_info = json.loads(panel_data)['result'],
            hgnc_map=hgnc_map,
            institute=institute
        )
        parsed_panel['panel_id'] = panel_id

        if len(parsed_panel['genes']) == 0:
            LOG.warning("Panel {} is missing genes. Skipping.".format(parsed_panel['display_name']))
            continue

        try:
            adapter.load_panel(parsed_panel=parsed_panel)
        except Exception as err:
            raise err


# -*- coding: utf-8 -*-
import logging
import urllib.parse

LOG = logging.getLogger(__name__)

from scout.constants import (CHROMOSOMES, CHROMOSOME_INTEGERS, CALLERS)

def export_variants(adapter, collaborator, document_id=None, case_id=None):
    """Export causative variants for a collaborator

    Args:
        adapter(MongoAdapter)
        collaborator(str)
        document_id(str): Search for a specific variant
        case_id(str): Search causative variants for a case

    Yields:
        variant_obj(scout.Models.Variant): Variants marked as causative ordered by position.
    """

    # Store the variants in a list for sorting
    variants = []
    if document_id:
        yield adapter.variant(document_id)
        return

    variant_ids = adapter.get_causatives(
        institute_id=collaborator,
        case_id=case_id
        )
    ##TODO add check so that same variant is not included more than once
    for document_id in variant_ids:

        variant_obj = adapter.variant(document_id)
        chrom = variant_obj['chromosome']
        # Convert chromosome to integer for sorting
        chrom_int = CHROMOSOME_INTEGERS.get(chrom)
        if not chrom_int:
            LOG.info("Unknown chromosome %s", chrom)
            continue

        # Add chromosome and position to prepare for sorting
        variants.append((chrom_int, variant_obj['position'], variant_obj))

    # Sort varants based on position
    variants.sort(key=lambda x: (x[0], x[1]))

    for variant in variants:
        variant_obj = variant[2]
        yield variant_obj

def export_verified_variants(aggregate_variants, unique_callers):
    """Create the lines for an excel file with verified variants for
        an institute

        Args:
            aggregate_variants(list): a list of variants with aggregates case data
            unique_callers(set): a unique list of available callers

        Returns:
            document_lines(list): list of lines to include in the document
    """
    document_lines = []
    for variant in aggregate_variants:
        # get genotype and allele depth for each sample
        samples = []
        for sample in variant['samples']:
            line = [] # line elements corespond to contants.variants_export.VERIFIED_VARIANTS_HEADER
            line.append(variant['institute'])
            line.append(variant['_id']) # variant database ID
            line.append(variant['category'])
            line.append(variant['variant_type'])
            line.append(variant['display_name'][:30]) # variant display name
            # Build local link to variant:
            case_name = variant['case_obj']['display_name']  # case display name
            local_link = '/'.join([ '', variant['institute'], case_name, variant['_id'] ])
            line.append(local_link)
            line.append(variant.get('validation'))
            line.append(case_name)
            case_individual = next(ind for ind in variant['case_obj']['individuals'] if ind['individual_id'] == sample['sample_id'])
            if case_individual['phenotype'] == 2:
                line.append(' '.join([sample.get('display_name'),'(A)'])) # label sample as affected
            else:
                line.append(sample.get('display_name'))
            line.append(''.join(['chr',variant['chromosome'],':',str(variant['position'])])) # position
            line.append('>'.join([variant.get('reference')[:10],variant.get('alternative')[:10]])) # change
            genes = []
            prot_effect = []
            funct_anno = []
            for gene in variant.get('genes'): # this will be a unique long field in the document
                genes.append(gene.get('hgnc_symbol',''))
                funct_anno.append(gene.get('functional_annotation'))
                for transcript in gene.get('transcripts'):
                    if transcript.get('is_canonical') and transcript.get('protein_sequence_name'):
                        prot_effect.append(urllib.parse.unquote(transcript.get('protein_sequence_name')))
            line.append(','.join(prot_effect))
            line.append(','.join(funct_anno))
            line.append(','.join(genes))
            line.append(variant.get('rank_score'))
            line.append(variant.get('cadd_score'))
            line.append(sample.get('genotype_call'))
            line.append(sample['allele_depths'][0])
            line.append(sample['allele_depths'][1])
            line.append(sample['genotype_quality'])

            # Set callers values. One cell per caller, leave blank if not applicable
            for caller in unique_callers:
                if variant.get(caller):
                    line.append(variant.get(caller))
                else:
                    line.append('-')
            document_lines.append(line)
    return document_lines

def export_mt_variants(variants, sample_id):
    """Export mitochondrial variants for a case to create a MT excel report

    Args:
        variants(list): all MT variants for a case, sorted by position
        sample_id(str) : the id of a sample within the case

    Returns:
        document_lines(list): list of lines to include in the document
    """
    document_lines = []
    for variant in variants:
        line = []
        position = variant.get('position')
        change = '>'.join([variant.get('reference'),variant.get('alternative')])
        line.append(position)
        line.append(change)
        line.append(str(position)+change)
        genes = []
        prot_effect = []
        for gene in variant.get('genes'):
            genes.append(gene.get('hgnc_symbol',''))
            for transcript in gene.get('transcripts'):
                if transcript.get('is_canonical') and transcript.get('protein_sequence_name'):
                    prot_effect.append(urllib.parse.unquote(transcript.get('protein_sequence_name')))
        line.append(','.join(prot_effect))
        line.append(','.join(genes))
        ref_ad = ''
        alt_ad = ''
        for sample in variant['samples']:
            if sample.get('sample_id') == sample_id:
                ref_ad = sample['allele_depths'][0]
                alt_ad = sample['allele_depths'][1]
        line.append(ref_ad)
        line.append(alt_ad)
        document_lines.append(line)
    return document_lines


import logging

import click

LOG = logging.getLogger(__name__)

@click.command('user', short_help='Update a user')
@click.option('--user-id', '-u',
              help="A email adress that identifies the user",
              required=True
              )
@click.option('--update-role', '-r',
              # There will be more roles in the future
              type=click.Choice(['admin', 'mme_submitter']),
              help="Add a role to the user",
              )
@click.option('--remove-admin',
              is_flag=True,
              help="Remove admin rights from user",
              )
@click.option('--add-institute', '-i',
              multiple=True,
              help="Specify the institutes to add",
              )
@click.option('--remove-institute',
              multiple=True,
              help="Specify the institutes to remove",
              )
@click.pass_context
def user(context, user_id, update_role, add_institute, remove_admin, remove_institute):
    """
    Update a user in the database
    """
    adapter = context.obj['adapter']

    user_obj = adapter.user(user_id)

    if not user_obj:
        LOG.warning("User %s could not be found", user_id)
        context.abort()

    existing_roles = set(user_obj.get('roles',[]))
    if update_role:
        if not update_role in user_obj['roles']:
            existing_roles = set(user_obj['roles'])
            existing_roles.add(update_role)
            LOG.info("Adding role %s to user", update_role)
        else:
            LOG.warning("User already have role %s", update_role)

    if remove_admin:
        try:
            existing_roles.remove('admin')
            LOG.info("Removing admin rights from user %s", user_id)
        except KeyError as err:
            LOG.info("User %s does not have admin rights", user_id)

    user_obj['roles'] = list(existing_roles)

    existing_institutes = set(user_obj.get('institutes',[]))
    for institute_id in add_institute:
        institute_obj = adapter.institute(institute_id)
        if not institute_obj:
            LOG.warning("Institute %s could not be found", institute_id)
        else:
            existing_institutes.add(institute_id)
            LOG.info("Adding institute %s to user", institute_id)

    for institute_id in remove_institute:
        try:
            existing_institutes.remove(institute_id)
            LOG.info("Removing institute %s from user", institute_id)
        except KeyError as err:
            LOG.info("User does not have access to institute %s", institute_id)

    user_obj['institutes'] = list(existing_institutes)

    updated_user = adapter.update_user(user_obj)


# -*- coding: utf-8 -*-
import io
import os.path
import shutil
import logging
import datetime
import zipfile
import pathlib

from flask import (Blueprint, request, redirect, abort, flash, current_app, url_for, jsonify, Response,
                    session, send_file)
from werkzeug.datastructures import Headers, MultiDict
from flask_login import current_user

from scout.constants import SEVERE_SO_TERMS
from scout.constants.acmg import ACMG_CRITERIA
from scout.constants import ACMG_MAP
from scout.server.extensions import store, mail, loqusdb
from scout.server.utils import templated, institute_and_case, public_endpoint
from scout.utils.acmg import get_acmg
from scout.parse.clinvar import set_submission_objects
from . import controllers
from .forms import FiltersForm, SvFiltersForm, StrFiltersForm

log = logging.getLogger(__name__)
variants_bp = Blueprint('variants', __name__, static_folder='static', template_folder='templates')

@variants_bp.route('/<institute_id>/<case_name>/variants', methods=['GET','POST'])
@templated('variants/variants.html')
def variants(institute_id, case_name):
    """Display a list of SNV variants."""
    page = int(request.form.get('page', 1))

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_type = request.args.get('variant_type', 'clinical')

    # Update filter settings if Clinical Filter was requested

    default_panels = []
    for panel in case_obj['panels']:
        if panel.get('is_default'):
            default_panels.append(panel['panel_name'])

    request.form.get('gene_panels')
    if bool(request.form.get('clinical_filter')):
        clinical_filter = MultiDict({
            'variant_type': 'clinical',
            'region_annotations': ['exonic','splicing'],
            'functional_annotations': SEVERE_SO_TERMS,
            'clinsig': [4,5],
            'clinsig_confident_always_returned': True,
            'gnomad_frequency': str(institute_obj['frequency_cutoff']),
            'variant_type': 'clinical',
            'gene_panels': default_panels
             })

    if(request.method == "POST"):
        if bool(request.form.get('clinical_filter')):
            form = FiltersForm(clinical_filter)
            form.csrf_token = request.args.get('csrf_token')
        else:
            form = FiltersForm(request.form)
    else:
        form = FiltersForm(request.args)

    # populate available panel choices
    available_panels = case_obj.get('panels', []) + [
        {'panel_name': 'hpo', 'display_name': 'HPO'}]

    panel_choices = [(panel['panel_name'], panel['display_name'])
                     for panel in available_panels]

    form.gene_panels.choices = panel_choices

    # upload gene panel if symbol file exists
    if (request.files):
        file = request.files[form.symbol_file.name]

    if request.files and file and file.filename != '':
        log.debug("Upload file request files: {0}".format(request.files.to_dict()))
        try:
            stream = io.StringIO(file.stream.read().decode('utf-8'), newline=None)
        except UnicodeDecodeError as error:
            flash("Only text files are supported!", 'warning')
            return redirect(request.referrer)

        hgnc_symbols_set = set(form.hgnc_symbols.data)
        log.debug("Symbols prior to upload: {0}".format(hgnc_symbols_set))
        new_hgnc_symbols = controllers.upload_panel(store, institute_id, case_name, stream)
        hgnc_symbols_set.update(new_hgnc_symbols)
        form.hgnc_symbols.data = hgnc_symbols_set
        # reset gene panels
        form.gene_panels.data = ''

    # update status of case if vistited for the first time
    if case_obj['status'] == 'inactive' and not current_user.is_admin:
        flash('You just activated this case!', 'info')
        user_obj = store.user(current_user.email)
        case_link = url_for('cases.case', institute_id=institute_obj['_id'],
                            case_name=case_obj['display_name'])
        store.update_status(institute_obj, case_obj, user_obj, 'active', case_link)

    # check if supplied gene symbols exist
    hgnc_symbols = []
    non_clinical_symbols = []
    not_found_symbols = []
    not_found_ids = []
    if (form.hgnc_symbols.data) and len(form.hgnc_symbols.data) > 0:
        is_clinical = form.data.get('variant_type', 'clinical') == 'clinical'
        clinical_symbols = store.clinical_symbols(case_obj) if is_clinical else None
        for hgnc_symbol in form.hgnc_symbols.data:
            if hgnc_symbol.isdigit():
                hgnc_gene = store.hgnc_gene(int(hgnc_symbol))
                if hgnc_gene is None:
                    not_found_ids.append(hgnc_symbol)
                else:
                    hgnc_symbols.append(hgnc_gene['hgnc_symbol'])
            elif store.hgnc_genes(hgnc_symbol).count() == 0:
                  not_found_symbols.append(hgnc_symbol)
            elif is_clinical and (hgnc_symbol not in clinical_symbols):
                 non_clinical_symbols.append(hgnc_symbol)
            else:
                hgnc_symbols.append(hgnc_symbol)

    if (not_found_ids):
        flash("HGNC id not found: {}".format(", ".join(not_found_ids)), 'warning')
    if (not_found_symbols):
        flash("HGNC symbol not found: {}".format(", ".join(not_found_symbols)), 'warning')
    if (non_clinical_symbols):
        flash("Gene not included in clinical list: {}".format(", ".join(non_clinical_symbols)), 'warning')
    form.hgnc_symbols.data = hgnc_symbols

    # handle HPO gene list separately
    if form.data['gene_panels'] == ['hpo']:
        hpo_symbols = list(set(term_obj['hgnc_symbol'] for term_obj in
                               case_obj['dynamic_gene_list']))
        form.hgnc_symbols.data = hpo_symbols

    variants_query = store.variants(case_obj['_id'], query=form.data)
    data = {}

    if request.form.get('export'):
        document_header = controllers.variants_export_header(case_obj)
        export_lines = []
        if form.data['chrom'] == 'MT':
            # Return all MT variants
            export_lines = controllers.variant_export_lines(store, case_obj, variants_query)
        else:
            # Return max 500 variants
            export_lines = controllers.variant_export_lines(store, case_obj, variants_query.limit(500))

        def generate(header, lines):
            yield header + '\n'
            for line in lines:
                yield line + '\n'

        headers = Headers()
        headers.add('Content-Disposition','attachment', filename=str(case_obj['display_name'])+'-filtered_variants.csv')

        # return a csv with the exported variants
        return Response(generate(",".join(document_header), export_lines), mimetype='text/csv',
                        headers=headers)

    data = controllers.variants(store, institute_obj, case_obj, variants_query, page)

    return dict(institute=institute_obj, case=case_obj, form=form,
                    severe_so_terms=SEVERE_SO_TERMS, page=page, **data)

@variants_bp.route('/<institute_id>/<case_name>/<variant_id>')
@templated('variants/variant.html')
def variant(institute_id, case_name, variant_id):
    """Display a specific SNV variant."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    log.debug("Variants view requesting data for variant {}".format(variant_id))
    data = controllers.variant(store, institute_obj, case_obj, variant_id=variant_id)
    if data is None:
        log.warning("An error occurred: variants view requesting data for variant {}".format(variant_id))
        flash('An error occurred while retrieving variant object', 'danger')
        return redirect(request.referrer)

    if current_app.config.get('LOQUSDB_SETTINGS'):
        data['observations'] = controllers.observations(store, loqusdb,
            case_obj, data['variant'])
    data['cancer'] = request.args.get('cancer') == 'yes'
    return dict(institute=institute_obj, case=case_obj, **data)

@variants_bp.route('/<institute_id>/<case_name>/str/variants')
@templated('variants/str-variants.html')
def str_variants(institute_id, case_name):
    """Display a list of STR variants."""
    page = int(request.args.get('page', 1))
    variant_type = request.args.get('variant_type', 'clinical')

    form = StrFiltersForm(request.args)

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)

    query = form.data
    query['variant_type'] = variant_type

    variants_query = store.variants(case_obj['_id'], category='str',
        query=query)
    data = controllers.str_variants(store, institute_obj, case_obj,
        variants_query, page)
    return dict(institute=institute_obj, case=case_obj,
        variant_type = variant_type, form=form, page=page, **data)

@variants_bp.route('/<institute_id>/<case_name>/sv/variants',
                   methods=['GET','POST'])
@templated('variants/sv-variants.html')
def sv_variants(institute_id, case_name):
    """Display a list of structural variants."""
    page = int(request.form.get('page', 1))

    variant_type = request.args.get('variant_type', 'clinical')

    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)

    form = SvFiltersForm(request.form)

    default_panels = []
    for panel in case_obj['panels']:
        if panel['is_default']:
            default_panels.append(panel['panel_name'])

    request.form.get('gene_panels')
    if bool(request.form.get('clinical_filter')):
        clinical_filter = MultiDict({
            'variant_type': 'clinical',
            'region_annotations': ['exonic','splicing'],
            'functional_annotations': SEVERE_SO_TERMS,
            'thousand_genomes_frequency': str(institute_obj['frequency_cutoff']),
            'variant_type': 'clinical',
            'clingen_ngi': 10,
            'swegen': 10,
            'size': 100,
            'gene_panels': default_panels
             })

    if(request.method == "POST"):
        if bool(request.form.get('clinical_filter')):
            form = SvFiltersForm(clinical_filter)
            form.csrf_token = request.args.get('csrf_token')
        else:
            form = SvFiltersForm(request.form)
    else:
        form = SvFiltersForm(request.args)

    available_panels = case_obj.get('panels', []) + [
        {'panel_name': 'hpo', 'display_name': 'HPO'}]

    panel_choices = [(panel['panel_name'], panel['display_name'])
                     for panel in available_panels]
    form.gene_panels.choices = panel_choices

    if form.data['gene_panels'] == ['hpo']:
        hpo_symbols = list(set(term_obj['hgnc_symbol'] for term_obj in
                               case_obj['dynamic_gene_list']))
        form.hgnc_symbols.data = hpo_symbols

    # update status of case if vistited for the first time
    if case_obj['status'] == 'inactive' and not current_user.is_admin:
        flash('You just activated this case!', 'info')
        user_obj = store.user(current_user.email)
        case_link = url_for('cases.case', institute_id=institute_obj['_id'],
                            case_name=case_obj['display_name'])
        store.update_status(institute_obj, case_obj, user_obj, 'active', case_link)

    variants_query = store.variants(case_obj['_id'], category='sv',
                                    query=form.data)
    data = {}
    # if variants should be exported
    if request.form.get('export'):
        document_header = controllers.variants_export_header(case_obj)
        export_lines = []
        # Return max 500 variants
        export_lines = controllers.variant_export_lines(store, case_obj, variants_query.limit(500))

        def generate(header, lines):
            yield header + '\n'
            for line in lines:
                yield line + '\n'

        headers = Headers()
        headers.add('Content-Disposition','attachment', filename=str(case_obj['display_name'])+'-filtered_sv-variants.csv')
        return Response(generate(",".join(document_header), export_lines), mimetype='text/csv', headers=headers) # return a csv with the exported variants

    else:
        data = controllers.sv_variants(store, institute_obj, case_obj,
                                       variants_query, page)

    return dict(institute=institute_obj, case=case_obj, variant_type=variant_type,
                form=form, severe_so_terms=SEVERE_SO_TERMS, page=page, **data)

@variants_bp.route('/<institute_id>/<case_name>/sv/variants/<variant_id>')
@templated('variants/sv-variant.html')
def sv_variant(institute_id, case_name, variant_id):
    """Display a specific structural variant."""
    data = controllers.sv_variant(store, institute_id, case_name, variant_id)
    return data

@variants_bp.route('/<institute_id>/<case_name>/str/variants/<variant_id>')
@templated('variants/str-variant.html')
def str_variant(institute_id, case_name, variant_id):
    """Display a specific STR variant."""
    data = controllers.str_variant(store, institute_id, case_name, variant_id)
    return data

@variants_bp.route('/<institute_id>/<case_name>/<variant_id>/update', methods=['POST'])
def variant_update(institute_id, case_name, variant_id):
    """Update user-defined information about a variant: manual rank & ACMG."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    user_obj = store.user(current_user.email)
    link = request.referrer

    manual_rank = request.form.get('manual_rank')
    if manual_rank:
        new_manual_rank = int(manual_rank) if manual_rank != '-1' else None
        store.update_manual_rank(institute_obj, case_obj, user_obj, link, variant_obj,
                                 new_manual_rank)
        if new_manual_rank:
            flash("updated variant tag: {}".format(new_manual_rank), 'info')
        else:
            flash("reset variant tag: {}".format(variant_obj.get('manual_rank', 'NA')), 'info')
    elif request.form.get('acmg_classification'):
        new_acmg = request.form['acmg_classification']
        acmg_classification = variant_obj.get('acmg_classification')
        if isinstance(acmg_classification, int) and (new_acmg == ACMG_MAP[acmg_classification]):
            new_acmg = None
        store.update_acmg(institute_obj, case_obj, user_obj, link, variant_obj, new_acmg)
        flash("updated ACMG classification: {}".format(new_acmg), 'info')

    new_dismiss = request.form.getlist('dismiss_variant')
    if request.form.getlist('dismiss_variant'):
        store.update_dismiss_variant(institute_obj, case_obj, user_obj, link, variant_obj,
                                     new_dismiss)
        if new_dismiss:
            flash("Dismissed variant: {}".format(new_dismiss), 'info')

    if variant_obj.get('dismiss_variant') and not new_dismiss:
        if 'dismiss' in request.form:
            store.update_dismiss_variant(institute_obj, case_obj, user_obj, link, variant_obj,
                                     new_dismiss)
            flash("Reset variant dismissal: {}".format(variant_obj.get('dismiss_variant')), 'info')
        else:
            log.debug("DO NOT reset variant dismissal: {}".format(variant_obj.get('dismiss_variant')), 'info')

    mosaic_tags = request.form.getlist('mosaic_tags')
    if mosaic_tags:
        store.update_mosaic_tags(institute_obj, case_obj, user_obj, link, variant_obj,
                                     mosaic_tags)
        if new_dismiss:
            flash("Added mosaic tags: {}".format(mosaic_tags), 'info')

    if variant_obj.get('mosaic_tags') and not mosaic_tags:
        if 'mosaic' in request.form:
            store.update_mosaic_tags(institute_obj, case_obj, user_obj, link, variant_obj,
                                     mosaic_tags)
            flash("Reset mosaic tags: {}".format(variant_obj.get('mosaic_tags')), 'info')

    return redirect(request.referrer)

@variants_bp.route('/<institute_id>/<case_name>/<variant_id>/<variant_category>/<order>', methods=['POST'])
def verify(institute_id, case_name, variant_id, variant_category, order):
    """Start procedure to validate variant using other techniques."""
    institute_obj, case_obj = institute_and_case(store, institute_id, case_name)
    variant_obj = store.variant(variant_id)
    user_obj = store.user(current_user.email)

    comment = request.form.get('verification_comment')

    try:
        controllers.variant_verification(store=store, mail=mail, institute_obj=institute_obj, case_obj=case_obj, user_obj=user_obj, comment=comment,
                           variant_obj=variant_obj, sender=current_app.config['MAIL_USERNAME'], variant_url=request.referrer, order=order, url_builder=url_for)
    except controllers.MissingVerificationRecipientError:
        flash('No verification recipients added to institute.', 'danger')

    return redirect(request.referrer)

@variants_bp.route('/<institute_id>/<case_name>/<variant_id>/clinvar', methods=['POST', 'GET'])
@templated('variants/clinvar.html')
def clinvar(institute_id, case_name, variant_id):
    """Build a clinVar submission form for a variant."""
    data = controllers.clinvar_export(store, institute_id, case_name, variant_id)
    if request.method == 'GET':
        return data
    else: #POST
        form_dict = request.form.to_dict()
        submission_objects = set_submission_objects(form_dict) # A tuple of submission objects (variants and casedata objects)

        # Add submission data to an open clinvar submission object,
        # or create a new if no open submission is found in database
        open_submission = store.get_open_clinvar_submission(current_user.email, institute_id)
        updated_submission = store.add_to_submission(open_submission['_id'], submission_objects)

        # Redirect to clinvar submissions handling page, and pass it the updated_submission_object
        return redirect(url_for('cases.clinvar_submissions', institute_id=institute_id))

@variants_bp.route('/<institute_id>/<case_name>/cancer/variants')
@templated('variants/cancer-variants.html')
def cancer_variants(institute_id, case_name):
    """Show cancer variants overview."""
    data = controllers.cancer_variants(store, request.args, institute_id, case_name)
    return data

@variants_bp.route('/<institute_id>/<case_name>/<variant_id>/acmg', methods=['GET', 'POST'])
@templated('variants/acmg.html')
def variant_acmg(institute_id, case_name, variant_id):
    """ACMG classification form."""
    if request.method == 'GET':
        data = controllers.variant_acmg(store, institute_id, case_name, variant_id)
        return data
    else:
        criteria = []
        criteria_terms = request.form.getlist('criteria')
        for term in criteria_terms:
            criteria.append(dict(
                term=term,
                comment=request.form.get("comment-{}".format(term)),
                links=[request.form.get("link-{}".format(term))],
            ))
        acmg = controllers.variant_acmg_post(store, institute_id, case_name, variant_id,
                                             current_user.email, criteria)
        flash("classified as: {}".format(acmg), 'info')
        return redirect(url_for('.variant', institute_id=institute_id, case_name=case_name,
                                variant_id=variant_id))

@variants_bp.route('/evaluations/<evaluation_id>', methods=['GET', 'POST'])
@templated('variants/acmg.html')
def evaluation(evaluation_id):
    """Show or delete an ACMG evaluation."""
    evaluation_obj = store.get_evaluation(evaluation_id)
    controllers.evaluation(store, evaluation_obj)
    if request.method == 'POST':
        link = url_for('.variant', institute_id=evaluation_obj['institute']['_id'],
                       case_name=evaluation_obj['case']['display_name'],
                       variant_id=evaluation_obj['variant_specific'])
        store.delete_evaluation(evaluation_obj)
        return redirect(link)
    return dict(evaluation=evaluation_obj, institute=evaluation_obj['institute'],
                case=evaluation_obj['case'], variant=evaluation_obj['variant'],
                CRITERIA=ACMG_CRITERIA)

@variants_bp.route('/api/v1/acmg')
@public_endpoint
def acmg():
    """Calculate an ACMG classification from submitted criteria."""
    criteria = request.args.getlist('criterion')
    classification = get_acmg(criteria)
    return jsonify(dict(classification=classification))

@variants_bp.route('/<institute_id>/<case_name>/upload', methods=['POST'])
def upload_panel(institute_id, case_name):
    """Parse gene panel file and fill in HGNC symbols for filter."""
    file = form.symbol_file.data

    if file.filename == '':
        flash('No selected file', 'warning')
        return redirect(request.referrer)

    try:
        stream = io.StringIO(file.stream.read().decode('utf-8'), newline=None)
    except UnicodeDecodeError as error:
        flash("Only text files are supported!", 'warning')
        return redirect(request.referrer)

    category = request.args.get('category')

    if(category == 'sv'):
        form = SvFiltersForm(request.args)
    else:
        form = FiltersForm(request.args)

    hgnc_symbols = set(form.hgnc_symbols.data)
    new_hgnc_symbols = controllers.upload_panel(store, institute_id, case_name, stream)
    hgnc_symbols.update(new_hgnc_symbols)
    form.hgnc_symbols.data = ','.join(hgnc_symbols)
    # reset gene panels
    form.gene_panels.data = ''
    # HTTP redirect code 307 asks that the browser preserves the method of request (POST).
    if(category == 'sv'):
        return redirect(url_for('.sv_variants', institute_id=institute_id, case_name=case_name,
                            **form.data), code=307)
    else:
        return redirect(url_for('.variants', institute_id=institute_id, case_name=case_name,
                            **form.data), code=307)

@variants_bp.route('/verified', methods=['GET'])
def download_verified():
    """Download all verified variants for user's cases"""
    user_obj = store.user(current_user.email)
    user_institutes = user_obj.get('institutes')
    temp_excel_dir = os.path.join(variants_bp.static_folder, 'verified_folder')
    os.makedirs(temp_excel_dir, exist_ok=True)

    written_files = controllers.verified_excel_file(store, user_institutes, temp_excel_dir)
    if written_files:
        today = datetime.datetime.now().strftime('%Y-%m-%d')
        # zip the files on the fly and serve the archive to the user
        data = io.BytesIO()
        with zipfile.ZipFile(data, mode='w') as z:
            for f_name in pathlib.Path(temp_excel_dir).iterdir():
                zipfile.ZipFile
                z.write(f_name, os.path.basename(f_name))
        data.seek(0)

        # remove temp folder with excel files in it
        shutil.rmtree(temp_excel_dir)

        return send_file(
            data,
            mimetype='application/zip',
            as_attachment=True,
            attachment_filename='_'.join(['scout', 'verified_variants', today])+'.zip'
        )
    else:
        flash("No verified variants could be exported for user's institutes", 'warning')
        return redirect(request.referrer)


import sys
import logging

from pprint import pprint as pp

from pandas import DataFrame

from scout.parse.hgnc import parse_hgnc_genes
from scout.parse.ensembl import (parse_ensembl_transcripts, parse_ensembl_exons,
                                 parse_ensembl_genes, parse_ensembl_gene_request)
from scout.parse.exac import parse_exac_genes
from scout.parse.hpo import get_incomplete_penetrance_genes
from scout.parse.omim import get_mim_genes

LOG = logging.getLogger(__name__)

def genes_by_alias(hgnc_genes):
    """Return a dictionary with hgnc symbols as keys

    Value of the dictionaries are information about the hgnc ids for a symbol.
    If the symbol is primary for a gene then 'true_id' will exist.
    A list of hgnc ids that the symbol points to is in ids.

    Args:
        hgnc_genes(dict): a dictionary with hgnc_id as key and gene info as value

    Returns:
        alias_genes(dict):
            {
                'hgnc_symbol':{
                    'true_id': int,
                    'ids': list(int)
                    }
            }
    """
    alias_genes = {}
    for hgnc_id in hgnc_genes:
        gene = hgnc_genes[hgnc_id]
        # This is the primary symbol:
        hgnc_symbol = gene['hgnc_symbol']

        for alias in gene['previous_symbols']:
            true_id = None
            if alias == hgnc_symbol:
                true_id = hgnc_id
            if alias in alias_genes:
                alias_genes[alias.upper()]['ids'].add(hgnc_id)
                if true_id:
                    alias_genes[alias.upper()]['true_id'] = hgnc_id
            else:
                alias_genes[alias.upper()] = {
                    'true': true_id,
                    'ids': set([hgnc_id])
                }

    return alias_genes

def add_ensembl_info(genes, ensembl_lines):
    """Add the coordinates from ensembl

    Args:
        genes(dict): Dictionary with all genes
        ensembl_lines(iteable): Iteable with raw ensembl info
    """

    LOG.info("Adding ensembl coordinates")
    # Parse and add the ensembl gene info
    if isinstance(ensembl_lines, DataFrame):
        ensembl_genes = parse_ensembl_gene_request(ensembl_lines)
    else:
        ensembl_genes = parse_ensembl_genes(ensembl_lines)

    for ensembl_gene in ensembl_genes:
        gene_obj = genes.get(ensembl_gene['hgnc_id'])
        if not gene_obj:
            continue
        gene_obj['chromosome'] = ensembl_gene['chrom']
        gene_obj['start'] = ensembl_gene['gene_start']
        gene_obj['end'] = ensembl_gene['gene_end']
        # ensembl ids can differ between builds. There is one stated in HGNC
        # that is true for build 38. So we add information from ensembl
        gene_obj['ensembl_gene_id'] = ensembl_gene['ensembl_gene_id']

def add_exac_info(genes, alias_genes, exac_lines):
    """Add information from the exac genes

    Currently we only add the pLi score on gene level

    The exac resource only use HGNC symbol to identify genes so we need
    our alias mapping.

    Args:
        genes(dict): Dictionary with all genes
        alias_genes(dict): Genes mapped to all aliases
        ensembl_lines(iteable): Iteable with raw ensembl info

    """
    LOG.info("Add exac pli scores")
    for exac_gene in parse_exac_genes(exac_lines):
        hgnc_symbol = exac_gene['hgnc_symbol'].upper()
        pli_score = exac_gene['pli_score']

        for hgnc_id in get_correct_ids(hgnc_symbol, alias_genes):
            genes[hgnc_id]['pli_score'] = pli_score

def add_omim_info(genes, alias_genes, genemap_lines, mim2gene_lines):
    """Add omim information

    We collect information on what phenotypes that are associated with a gene,
    what inheritance models that are associated and the correct omim id.

    Args:
        genes(dict): Dictionary with all genes
        alias_genes(dict): Genes mapped to all aliases
        genemap_lines(iterable): Iterable with raw omim info
        mim2gene_lines(iterable): Iterable with raw omim info

    """
    LOG.info("Add omim info")
    omim_genes = get_mim_genes(genemap_lines, mim2gene_lines)

    for hgnc_symbol in omim_genes:
        omim_info = omim_genes[hgnc_symbol]
        inheritance = omim_info.get('inheritance', set())

        for hgnc_id in get_correct_ids(hgnc_symbol, alias_genes):
            gene_info = genes[hgnc_id]

            # Update the omim id to the one found in omim
            gene_info['omim_id'] = omim_info['mim_number']

            gene_info['inheritance_models'] = list(inheritance)
            gene_info['phenotypes'] = omim_info.get('phenotypes', [])

def add_incomplete_penetrance(genes, alias_genes, hpo_lines):
    """Add information of incomplete penetrance"""
    LOG.info("Add incomplete penetrance info")
    for hgnc_symbol in get_incomplete_penetrance_genes(hpo_lines):
        for hgnc_id in get_correct_ids(hgnc_symbol, alias_genes):
            genes[hgnc_id]['incomplete_penetrance'] = True

def get_correct_ids(hgnc_symbol, alias_genes):
    """Try to get the correct gene based on hgnc_symbol

    The HGNC symbol is unfortunately not a persistent gene identifier.
    Many of the resources that are used by Scout only provides the hgnc symbol to
    identify a gene. We need a way to guess what gene is pointed at.

    Args:
        hgnc_symbol(str): The symbol used by a resource
        alias_genes(dict): A dictionary with all the alias symbols (including the current symbol)
                           for all genes

    Returns:
        hgnc_ids(iterable(int)): Hopefully only one but a symbol could map to several ids
    """
    hgnc_ids = set()
    hgnc_symbol = hgnc_symbol.upper()
    if hgnc_symbol in alias_genes:
        hgnc_id_info = alias_genes[hgnc_symbol]
        if hgnc_id_info['true']:
            return set([hgnc_id_info['true']])
        else:
            return set(hgnc_id_info['ids'])
    return hgnc_ids

def link_genes(ensembl_lines, hgnc_lines, exac_lines, mim2gene_lines,
               genemap_lines, hpo_lines):
    """Gather information from different sources and return a gene dict

    Extract information collected from a number of sources and combine them
    into a gene dict with HGNC symbols as keys.

    hgnc_id works as the primary symbol and it is from this source we gather
    as much information as possible (hgnc_complete_set.txt)

    Coordinates are gathered from ensemble and the entries are linked from hgnc
    to ensembl via ENSGID.

    From exac the gene intolerance scores are collected, genes are linked to hgnc
    via hgnc symbol. This is a unstable symbol since they often change.

        Args:
            ensembl_lines(iterable(str)): Strings with ensembl gene information
            hgnc_lines(iterable(str)): Strings with hgnc gene information
            exac_lines(iterable(str)): Strings with exac PLi score info
            mim2gene_lines(iterable(str))
            genemap_lines(iterable(str))
            hpo_lines(iterable(str)): Strings with hpo gene information

        Yields:
            gene(dict): A dictionary with gene information
    """
    genes = {}
    LOG.info("Linking genes")
    # HGNC genes are the main source, these define the gene dataset to use
    # Try to use as much information as possible from hgnc
    for hgnc_gene in parse_hgnc_genes(hgnc_lines):
        hgnc_id = hgnc_gene['hgnc_id']
        genes[hgnc_id] = hgnc_gene

    add_ensembl_info(genes, ensembl_lines)

    symbol_to_id = genes_by_alias(genes)

    add_exac_info(genes, symbol_to_id, exac_lines)

    add_omim_info(genes, symbol_to_id, genemap_lines, mim2gene_lines)

    add_incomplete_penetrance(genes, symbol_to_id, hpo_lines)

    return genes


# -*- coding: UTF-8 -*-
import logging
import requests
import json
import datetime;
from werkzeug.datastructures import Headers

LOG = logging.getLogger(__name__)

def matchmaker_request(url, token, method, content_type=None, accept=None, data=None):
    """Send a request to MatchMaker and return its response

    Args:
        url(str): url to send request to
        token(str): MME server authorization token
        method(str): 'GET', 'POST' or 'DELETE'
        content_type(str): MME request Content-Type
        accept(str): accepted response
        data(dict): eventual data to send in request

    Returns:
        json_response(dict): server response
    """
    headers = Headers()
    headers = { 'X-Auth-Token': token}
    if content_type:
        headers['Content-Type'] = content_type
    if accept:
        headers['Accept'] = accept

    #sending data anyway so response will not be cached
    req_data = data or {'timestamp' : datetime.datetime.now().timestamp()}
    json_response = None
    try:
        LOG.info('Sending {} request to MME url {}. Data sent: {}'.format(
            method, url, req_data))
        resp = requests.request(
            method = method,
            url = url,
            headers = headers,
            data = json.dumps(req_data)
        )
        json_response = resp.json()
        LOG.info('MME server response was:{}'.format(json_response))

        if isinstance(json_response, str):
            json_response = {
                'message' : json_response,
            }
        elif isinstance(json_response, list): #asking for connected nodes
            return json_response
        json_response['status_code'] = resp.status_code
    except Exception as err:
        LOG.info('An error occurred while sending HTTP request to server ({})'.format(err))
        json_response = {
            'message' : str(err)
        }
    return json_response

def mme_nodes(mme_base_url, token):
    """Return the available MatchMaker nodes

    Args:
        mme_base_url(str): base URL of MME service
        token(str): MME server authorization token

    Returns:
        nodes(list): a list of node disctionaries
    """
    nodes = []
    if not mme_base_url or not token:
        return nodes
    url = ''.join([mme_base_url, '/nodes'])
    nodes = matchmaker_request(url=url, token=token, method='GET')
    LOG.info('Matchmaker has the following connected nodes:{}'.format(nodes))
    return nodes


from scout.constants import (CYTOBANDS, BND_ALT_PATTERN, CHR_PATTERN)

def get_cytoband_coordinates(chrom, pos):
    """Get the cytoband coordinate for a position

    Args:
        chrom(str)
        pos(int)

    Returns:
        coordinate(str)
    """
    coordinate = ""

    if chrom in CYTOBANDS:
        for interval in CYTOBANDS[chrom][pos]:
            coordinate = interval.data

    return coordinate

def get_sub_category(alt_len, ref_len, category, svtype=None):
    """Get the subcategory for a VCF variant

    The sub categories are:
        'snv', 'indel', 'del', 'ins', 'dup', 'bnd', 'inv'

    Args:
        alt_len(int)
        ref_len(int)
        category(str)
        svtype(str)

    Returns:
        subcategory(str)
    """
    subcategory = ''

    if category in ('snv', 'indel', 'cancer'):
        if ref_len == alt_len:
            subcategory = 'snv'
        else:
            subcategory = 'indel'
    elif category == 'sv':
        subcategory = svtype

    return subcategory

def get_length(alt_len, ref_len, category, pos, end, svtype=None, svlen=None):
    """Return the length of a variant

    Args:
        alt_len(int)
        ref_len(int)
        category(str)
        svtype(str)
        svlen(int)
    """
    # -1 would indicate uncertain length
    length = -1
    if category in ('snv', 'indel', 'cancer'):
        if ref_len == alt_len:
            length = alt_len
        else:
            length = abs(ref_len - alt_len)

    elif category == 'sv':
        if svtype == 'bnd':
            length = int(10e10)
        else:
            if svlen:
                length = abs(int(svlen))
            # Some software does not give a length but they give END
            elif end:
                if end != pos:
                    length = end - pos
    return length

def get_end(pos, alt, category, snvend=None, svend=None, svlen=None):
    """Return the end coordinate for a variant

    Args:
        pos(int)
        alt(str)
        category(str)
        snvend(str)
        svend(int)
        svlen(int)

    Returns:
        end(int)
    """
    # If nothing is known we set end to be same as start
    end = pos
    # If variant is snv or indel we know that cyvcf2 can handle end pos
    if category in ('snv', 'indel', 'cancer'):
        end = snvend

    # With SVs we have to be a bit more careful
    elif category == 'sv':
        # The END field from INFO usually works fine
        end = svend

        # For some cases like insertions the callers set end to same as pos
        # In those cases we can hope that there is a svlen...
        if svend == pos:
            if svlen:
                end = pos + svlen
        # If variant is 'BND' they have ':' in alt field
        # Information about other end is in the alt field
        if ':' in alt:
            match = BND_ALT_PATTERN.match(alt)
            if match:
                end = int(match.group(2))

    return end

def parse_coordinates(variant, category):
    """Find out the coordinates for a variant

    Args:
        variant(cyvcf2.Variant)

    Returns:
        coordinates(dict): A dictionary on the form:
        {
            'position':<int>,
            'end':<int>,
            'end_chrom':<str>,
            'length':<int>,
            'sub_category':<str>,
            'mate_id':<str>,
            'cytoband_start':<str>,
            'cytoband_end':<str>,
        }
    """
    ref = variant.REF

    if variant.ALT:
        alt = variant.ALT[0]
    if category=="str" and not variant.ALT:
        alt = '.'

    chrom_match = CHR_PATTERN.match(variant.CHROM)
    chrom = chrom_match.group(2)

    svtype = variant.INFO.get('SVTYPE')
    if svtype:
        svtype = svtype.lower()

    mate_id = variant.INFO.get('MATEID')

    svlen = variant.INFO.get('SVLEN')

    svend = variant.INFO.get('END')
    snvend = int(variant.end)

    position = int(variant.POS)

    ref_len = len(ref)
    alt_len = len(alt)

    sub_category = get_sub_category(alt_len, ref_len, category, svtype)
    end = get_end(position, alt, category, snvend, svend)

    length = get_length(alt_len, ref_len, category, position, end, svtype, svlen)
    end_chrom = chrom

    if sub_category == 'bnd':
        if ':' in alt:
            match = BND_ALT_PATTERN.match(alt)
            # BND will often be translocations between different chromosomes
            if match:
                other_chrom = match.group(1)
                match = CHR_PATTERN.match(other_chrom)
                end_chrom = match.group(2)

    cytoband_start = get_cytoband_coordinates(chrom, position)
    cytoband_end = get_cytoband_coordinates(end_chrom, end)

    coordinates = {
        'position': position,
        'end': end,
        'length': length,
        'sub_category': sub_category,
        'mate_id': mate_id,
        'cytoband_start': cytoband_start,
        'cytoband_end': cytoband_end,
        'end_chrom': end_chrom,
    }

    return coordinates


import logging
import intervaltree

def parse_cytoband(lines):
    """Parse iterable with cytoband coordinates

    Args:
        lines(iterable): Strings on format "chr1\t2300000\t5400000\tp36.32\tgpos25"

    Returns:
        cytobands(dict): Dictionary with chromosome names as keys and
                         interval trees as values
    """
    cytobands = {}
    for line in lines:
        line = line.rstrip()
        splitted_line = line.split('\t')
        chrom = splitted_line[0].lstrip('chr')
        start = int(splitted_line[1])
        stop = int(splitted_line[2])
        name = splitted_line[3]
        if chrom in cytobands:
            # Add interval to existing tree
            cytobands[chrom][start:stop] = name
        else:
            # Create a new interval tree
            new_tree = intervaltree.IntervalTree()
            # create the interval
            new_tree[start:stop] = name
            # Add the interval tree
            cytobands[chrom] = new_tree

    return cytobands

import click
from scout.utils.handle import get_file_handle
from scout.resources import cytobands_path

@click.command()
@click.option('--infile', default=cytobands_path)
def cli(infile):
    """docstring for cli"""
    lines = get_file_handle(infile)
    cytobands = parse_cytoband(lines)

    print("Check some coordinates:")

    print("checking chrom 1 pos 2")
    intervals = cytobands['1'][2]
    for interval in intervals:
        print(interval)
        print(interval.begin)
        print(interval.end)
        print(interval.data)
        # print(interval.__dict__)
    print(cytobands['1'][2])

    print("checking chrom 8 pos 101677777")
    print(cytobands['8'][101677777])

    print("checking chrom X pos 4200000 - 6000000")
    print(cytobands['X'][4200000:6000000])

if __name__ == '__main__':
    cli()


import logging

from pprint import pprint as pp

from scout.exceptions import IntegrityError

LOG = logging.getLogger(__name__)

def update_panel(adapter, panel_name, panel_version, new_version=None, new_date=None):
    """Update a gene panel in the database

    We need to update the actual gene panel and then all cases that refers to the panel.

    Args:
        adapter(scout.adapter.MongoAdapter)
        panel_name(str): Unique name for a gene panel
        panel_version(float)
        new_version(float)
        new_date(datetime.datetime)

    Returns:
        updated_panel(scout.models.GenePanel): The updated gene panel object
    """
    panel_obj = adapter.gene_panel(panel_name, panel_version)

    if not panel_obj:
        raise IntegrityError("Panel %s version %s does not exist" % (panel_name, panel_version))

    updated_panel = adapter.update_panel(panel_obj, new_version, new_date)

    panel_id = updated_panel['_id']

    # We need to alter the embedded panels in all affected cases
    update = {'$set': {}}
    if new_version:
        update['$set']['panels.$.version'] = updated_panel['version']
    if new_date:
        update['$set']['panels.$.updated_at'] = updated_panel['date']

    LOG.info('Updating affected cases with {0}'.format(update))

    query = {'panels': { '$elemMatch': {'panel_name': panel_name}}}
    adapter.case_collection.update_many(query, update)

    return updated_panel


# -*- coding: utf-8 -*-
import sys
import logging

from pprint import pprint as pp

import click
import coloredlogs
import yaml

# Adapter stuff
from scout.adapter.mongo import MongoAdapter
from scout.adapter.client import get_connection
from pymongo.errors import (ConnectionFailure, OperationFailure)

# General, logging
from scout import __version__

# Commands
from scout.commands.load import load as load_command
from scout.commands.export import export
from scout.commands.wipe_database import wipe
from scout.commands.setup import setup as setup_command
from scout.commands.convert import convert
from scout.commands.query import query as query_command
from scout.commands.view import view as view_command
from scout.commands.delete import delete
from scout.commands.serve import serve
from scout.commands.update import update as update_command
from scout.commands.index_command import index as index_command

from scout.adapter.utils import check_connection

try:
    from scoutconfig import *
except ImportError:
    pass

LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
LOG = logging.getLogger(__name__)

@click.group()
@click.option('--loglevel', default='INFO', type=click.Choice(LOG_LEVELS),
              help="Set the level of log output.", show_default=True)
@click.option('-db', '--mongodb', help='Name of mongo database [scout]')
@click.option('-u', '--username')
@click.option('-p', '--password')
@click.option('-a', '--authdb', help='database to use for authentication')
@click.option('-port', '--port', help="Specify on what port to listen for the mongod")
@click.option('-h', '--host', help="Specify the host for the mongo database.")
@click.option('-c', '--config', type=click.Path(exists=True),
              help="Specify the path to a config file with database info.")
@click.option('--demo', is_flag=True, help="If the demo database should be used")
@click.version_option(__version__)
@click.pass_context
def cli(context, mongodb, username, password, authdb, host, port, loglevel, config, demo):
    """scout: manage interactions with a scout instance."""
    # log_format = "%(message)s" if sys.stdout.isatty() else None
    log_format = None
    coloredlogs.install(level=loglevel, fmt=log_format)
    LOG.info("Running scout version %s", __version__)
    LOG.debug("Debug logging enabled.")

    mongo_config = {}
    cli_config = {}
    if config:
        LOG.debug("Use config file %s", config)
        with open(config, 'r') as in_handle:
            cli_config = yaml.load(in_handle)

    mongo_config['mongodb'] = (mongodb or cli_config.get('mongodb') or 'scout')
    if demo:
        mongo_config['mongodb'] = 'scout-demo'

    mongo_config['host'] = (host or cli_config.get('host') or 'localhost')
    mongo_config['port'] = (port or cli_config.get('port') or 27017)
    mongo_config['username'] = username or cli_config.get('username')
    mongo_config['password'] = password or cli_config.get('password')
    mongo_config['authdb'] = authdb or cli_config.get('authdb') or mongo_config['mongodb']
    mongo_config['omim_api_key'] = cli_config.get('omim_api_key')

    if context.invoked_subcommand in ('setup', 'serve'):
        mongo_config['adapter'] = None
    else:
        LOG.info("Setting database name to %s", mongo_config['mongodb'])
        LOG.debug("Setting host to %s", mongo_config['host'])
        LOG.debug("Setting port to %s", mongo_config['port'])

        try:
            client = get_connection(**mongo_config)
        except ConnectionFailure:
            context.abort()

        database = client[mongo_config['mongodb']]

        LOG.info("Setting up a mongo adapter")
        mongo_config['client'] = client
        adapter = MongoAdapter(database)
        mongo_config['adapter'] = adapter

        LOG.info("Check if authenticated...")
        try:
            for ins_obj in adapter.institutes():
                pass
        except OperationFailure as err:
            LOG.info("User not authenticated")
            context.abort()

    context.obj = mongo_config

cli.add_command(load_command)
cli.add_command(wipe)
cli.add_command(setup_command)
cli.add_command(export)
cli.add_command(convert)
cli.add_command(query_command)
cli.add_command(view_command)
cli.add_command(delete)
cli.add_command(serve)
cli.add_command(update_command)
cli.add_command(index_command)


import logging
from pprint import pprint as pp
logger = logging.getLogger(__name__)

def parse_exac_line(line, header):
    """Parse an exac formated line

        Args:
            line(list): A list with exac gene info
            header(list): A list with the header info

        Returns:
            exac_info(dict): A dictionary with the relevant info
    """
    exac_gene = {}
    splitted_line = line.rstrip().split('\t')
    exac_gene = dict(zip(header, splitted_line))
    exac_gene['hgnc_symbol'] = exac_gene['gene']
    exac_gene['pli_score'] = float(exac_gene['pLI'])
    exac_gene['raw'] = line

    return exac_gene

def parse_exac_genes(lines):
    """Parse lines with exac formated genes

        This is designed to take a dump with genes from exac.
        This is downloaded from:
            ftp.broadinstitute.org/pub/ExAC_release//release0.3/functional_gene_constraint/
            fordist_cleaned_exac_r03_march16_z_pli_rec_null_data.txt

        Args:
            lines(iterable(str)): An iterable with ExAC formated genes
        Yields:
            exac_gene(dict): A dictionary with the relevant information
    """
    header = []
    logger.info("Parsing exac genes...")
    for index, line in enumerate(lines):
        if index == 0:
            header = line.rstrip().split('\t')
        elif len(line) > 0:
            exac_gene = parse_exac_line(line, header)

            yield exac_gene


# -*- coding: utf-8 -*-
import logging
import datetime

from flask import abort, Blueprint, request, redirect, url_for, flash, render_template
from flask_weasyprint import HTML, render_pdf
from flask_login import current_user

from scout.server.extensions import store
from scout.server.utils import templated, user_institutes
from .forms import PanelGeneForm
from . import controllers

log = logging.getLogger(__name__)
panels_bp = Blueprint('panels', __name__, template_folder='templates')

@panels_bp.route('/panels', methods=['GET', 'POST'])
@templated('panels/panels.html')
def panels():
    """Show all panels for a case."""
    if request.method == 'POST':
        # update an existing panel
        csv_file = request.files['csv_file']
        content = csv_file.stream.read()
        lines = None
        try:
            if b'\n' in content:
                lines = content.decode('utf-8', 'ignore').split('\n')
            else:
                lines = content.decode('windows-1252').split('\r')
        except Exception as err:
            flash('Something went wrong while parsing the panel CSV file! ({})'.format(err), 'danger')
            return redirect(request.referrer)

        new_panel_name = request.form.get('new_panel_name')
        if new_panel_name: #create a new panel
            new_panel_id = controllers.new_panel(
                store=store,
                institute_id=request.form['institute'],
                panel_name=new_panel_name,
                display_name=request.form['display_name'],
                csv_lines=lines,
            )
            if new_panel_id is None:
                flash('Something went wrong and the panel list was not updated!','warning')
                return redirect(request.referrer)
            else:
                flash("new gene panel added, {}!".format(new_panel_name),'success')
            return redirect(url_for('panels.panel', panel_id=new_panel_id))

        else: # modify an existing panel
            update_option = request.form['modify_option']
            panel_obj= controllers.update_panel(
                                   store=store,
                                   panel_name=request.form['panel_name'],
                                   csv_lines=lines,
                                   option=update_option
             )
            if panel_obj is None:
                return abort(404, "gene panel not found: {}".format(request.form['panel_name']))
            else:
                return redirect(url_for('panels.panel', panel_id=panel_obj['_id']))

    institutes = list(user_institutes(store, current_user))
    panel_names = [name
                   for institute in institutes
                   for name in
                   store.gene_panels(institute_id=institute['_id']).distinct('panel_name')]

    panel_versions = {}
    for name in panel_names:
        panel_versions[name]=store.gene_panels(panel_id=name)

    panel_groups = []
    for institute_obj in institutes:
         institute_panels = store.latest_panels(institute_obj['_id'])
         panel_groups.append((institute_obj, institute_panels))

    return dict(panel_groups=panel_groups, panel_names=panel_names,
                panel_versions=panel_versions, institutes=institutes)

@panels_bp.route('/panels/<panel_id>', methods=['GET', 'POST'])
@templated('panels/panel.html')
def panel(panel_id):
    """Display (and add pending updates to) a specific gene panel."""
    panel_obj = store.gene_panel(panel_id) or store.panel(panel_id)
    if request.method == 'POST':
        raw_hgnc_id = request.form['hgnc_id']
        if '|' in raw_hgnc_id:
            raw_hgnc_id = raw_hgnc_id.split(' | ', 1)[0]
        hgnc_id = 0
        try:
            hgnc_id = int(raw_hgnc_id)
        except:
            flash("Provided HGNC is not valid : '{}'". format(raw_hgnc_id), 'danger')
            return redirect(request.referrer)
        action = request.form['action']
        gene_obj = store.hgnc_gene(hgnc_id)
        if gene_obj is None:
            flash("HGNC id not found: {}".format(hgnc_id), 'warning')
            return redirect(request.referrer)

        if action == 'add':
            panel_gene = controllers.existing_gene(store, panel_obj, hgnc_id)
            if panel_gene:
                flash("gene already in panel: {}".format(panel_gene['symbol']),
                      'warning')
            else:
                # ask user to fill-in more information about the gene
                return redirect(url_for('.gene_edit', panel_id=panel_id,
                                        hgnc_id=hgnc_id))
        elif action == 'delete':
            log.debug("marking gene to be deleted: %s", hgnc_id)
            panel_obj = store.add_pending(panel_obj, gene_obj, action='delete')

    data = controllers.panel(store, panel_obj)
    if request.args.get('case_id'):
        data['case'] = store.case(request.args['case_id'])
    if request.args.get('institute_id'):
        data['institute'] = store.institute(request.args['institute_id'])
    return data

@panels_bp.route('/panels/<panel_id>/update', methods=['POST'])
def panel_update(panel_id):
    """Update panel to a new version."""
    panel_obj = store.panel(panel_id)
    update_version = request.form.get('version', None)
    new_panel_id = store.apply_pending(panel_obj, update_version)
    return redirect(url_for('panels.panel', panel_id=new_panel_id))

@panels_bp.route('/panels/export-panel/<panel_id>', methods=['GET', 'POST'])
def panel_export(panel_id):
    """Export panel to PDF file"""
    panel_obj = store.panel(panel_id)
    data = controllers.panel_export(store, panel_obj)
    data['report_created_at'] = datetime.datetime.now().strftime("%Y-%m-%d")
    html_report = render_template('panels/panel_pdf_simple.html', **data)
    return render_pdf(HTML(string=html_report), download_filename=data['panel']['panel_name']+'_'+str(data['panel']['version'])+'_'+datetime.datetime.now().strftime("%Y-%m-%d")+'_scout.pdf')

@panels_bp.route('/panels/<panel_id>/update/<int:hgnc_id>', methods=['GET', 'POST'])
@templated('panels/gene-edit.html')
def gene_edit(panel_id, hgnc_id):
    """Edit additional information about a panel gene."""
    panel_obj = store.panel(panel_id)
    hgnc_gene = store.hgnc_gene(hgnc_id)
    panel_gene = controllers.existing_gene(store, panel_obj, hgnc_id)

    form = PanelGeneForm()
    transcript_choices = []
    for transcript in hgnc_gene['transcripts']:
        if transcript.get('refseq_id'):
            refseq_id = transcript.get('refseq_id')
            transcript_choices.append((refseq_id, refseq_id))
    form.disease_associated_transcripts.choices = transcript_choices

    if form.validate_on_submit():
        action = 'edit' if panel_gene else 'add'
        info_data = form.data.copy()
        if 'csrf_token' in info_data:
            del info_data['csrf_token']
        store.add_pending(panel_obj, hgnc_gene, action=action, info=info_data)
        return redirect(url_for('.panel', panel_id=panel_id))

    if panel_gene:
        for field_key in ['disease_associated_transcripts', 'reduced_penetrance',
                          'mosaicism', 'inheritance_models', 'database_entry_version', 'comment']:
            form_field = getattr(form, field_key)
            if not form_field.data:
                panel_value = panel_gene.get(field_key)
                if panel_value is not None:
                    form_field.process_data(panel_value)
    return dict(panel=panel_obj, form=form, gene=hgnc_gene, panel_gene=panel_gene)


import logging
import click

from scout.load.report import load_delivery_report

LOG = logging.getLogger(__name__)

@click.command('delivery-report')
@click.argument('case_id')
@click.argument('report_path', type=click.Path(exists=True))
@click.option('-update', '--update', is_flag=True, help='update delivery report for a sample')
@click.pass_context
def delivery_report(context, case_id, report_path,
                    update):
    """Add delivery report to an existing case."""

    adapter = context.obj['adapter']

    try:
        load_delivery_report(adapter=adapter, case_id=case_id,
                             report_path=report_path, update=update)
        LOG.info("saved report to case!")
    except Exception as e:
        LOG.error(e)
        context.abort()


from scout.utils.convert import (make_bool, convert_number)

def parse_peddy_ped(lines):
    """Parse a peddy.ped file

    Args:
        lines(iterable(str))

    Returns:
        peddy_ped(list(dict))
    """
    peddy_ped = []
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if i == 0:
            # Header line
            header = line.lstrip('#').split('\t')
        else:
            ind_info = dict(zip(header, line.split('\t')))

            # PC1/PC2/PC3/PC4: the first 4 values after this sample was
            # projected onto the thousand genomes principle components.
            ind_info['PC1'] = convert_number(ind_info['PC1'])
            ind_info['PC2'] = convert_number(ind_info['PC2'])
            ind_info['PC3'] = convert_number(ind_info['PC3'])
            # ancestry-prediction one of AFR AMR EAS EUR SAS UNKNOWN

            ind_info['het_call_rate'] = convert_number(ind_info['het_call_rate'])

            # idr_baf: inter-decile range (90th percentile - 10th percentile)
            # of b-allele frequency. We make a distribution of all sites of
            # alts / (ref + alts) and then report the difference between the
            # 90th and the 10th percentile.
            # Large values indicated likely sample contamination.
            ind_info['het_idr_baf'] = convert_number(ind_info['het_idr_baf'])

            ind_info['het_mean_depth'] = convert_number(ind_info['het_mean_depth'])

            peddy_ped.append(ind_info)
    return peddy_ped

def parse_peddy_ped_check(lines):
    """Parse a .ped_check.csv file

    Args:
        lines(iterable(str))

    Returns:
        ped_check(list(dict))
    """
    ped_check = []
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if i == 0:
            # Header line
            header = line.lstrip('#').split(',')
        else:
            pair_info = dict(zip(header, line.split(',')))

            # the number of sites at which sample_a was heterozygous
            pair_info['hets_a'] = convert_number(pair_info['hets_a'])

            # the number of sites at which sample_b was heterozygous
            pair_info['hets_b'] = convert_number(pair_info['hets_b'])

            # the number of sites at which the 2 samples shared no alleles
            # (should approach 0 for parent-child pairs).
            pair_info['ibs0'] = convert_number(pair_info['ibs0'])

            # the number of sites and which the 2 samples where both
            # hom-ref, both het, or both hom-alt.
            pair_info['ibs2'] = convert_number(pair_info['ibs2'])

            # the number of sites that was used to predict the relatedness.
            pair_info['n'] = convert_number(pair_info['n'])

            # the relatedness reported in the ped file.
            pair_info['rel'] = convert_number(pair_info['rel'])

            # the relatedness reported in the ped file.
            pair_info['pedigree_relatedness'] = convert_number(pair_info['pedigree_relatedness'])

            # difference between the preceding 2 colummns.
            pair_info['rel_difference'] = convert_number(pair_info['rel_difference'])

            # the number of sites at which both samples were hets.
            pair_info['shared_hets'] = convert_number(pair_info['shared_hets'])

            # boolean indicating that this pair is a parent-child pair
            # according to the ped file.
            pair_info['pedigree_parents'] = make_bool(pair_info.get('pedigree_parents'))

            # boolean indicating that this pair is expected to be a parent-child
            # pair according to the ibs0 (< 0.012) calculated from the genotypes.
            pair_info['predicted_parents'] = make_bool(pair_info.get('predicted_parents'))

            # boolean indicating that the preceding 2 columns do not match
            pair_info['parent_error'] = make_bool(pair_info.get('parent_error'))

            #  boolean indicating that rel > 0.75 and ibs0 < 0.012
            pair_info['sample_duplication_error'] = make_bool(pair_info.get('sample_duplication_error'))

            ped_check.append(pair_info)

    return ped_check

def parse_peddy_sex_check(lines):
    """Parse a .ped_check.csv file

    Args:
        lines(iterable(str))

    Returns:
        sex_check(list(dict))
    """
    sex_check = []
    header = []
    for i,line in enumerate(lines):
        line = line.rstrip()
        if i == 0:
            # Header line
            header = line.lstrip('#').split(',')
        else:
            ind_info = dict(zip(header, line.split(',')))

            # boolean indicating wether there is a mismatch between X
            # genotypes and ped sex.
            ind_info['error'] = make_bool(ind_info.get('error'))

            # number of homozygous-alternate calls
            ind_info['hom_alt_count'] = convert_number(ind_info['hom_alt_count'])
            #number of homozygous-reference calls
            ind_info['hom_ref_count'] = convert_number(ind_info['hom_ref_count'])
            # number of heterozygote calls
            ind_info['het_count'] = convert_number(ind_info['het_count'])

            # ratio of het_count / hom_alt_count. Low for males, high for females
            ind_info['het_ratio'] = convert_number(ind_info['het_ratio'])

            sex_check.append(ind_info)

    return sex_check


# -*- coding: utf-8 -*-

def hpo_terms(store, query = None, limit = None):
    """Retrieves a list of HPO terms from scout database

    Args:
        store (obj): an adapter to the scout database
        query (str): the term to search in the database
        limit (str): the number of desired results

    Returns:
        hpo_phenotypes (dict): the complete list of HPO objects stored in scout

    """
    hpo_phenotypes = {}
    if limit:
        limit=int(limit)

    hpo_phenotypes['phenotypes'] = list(store.hpo_terms(text=query, limit=limit))
    return hpo_phenotypes


import logging
import click

LOG = logging.getLogger(__name__)

@click.command('whitelist', short_help='Display whitelist')
@click.pass_context
def whitelist(context):
    """Show all objects in the whitelist collection"""
    LOG.info("Running scout view users")
    adapter = context.obj['adapter']

    ## TODO add a User interface to the adapter
    for whitelist_obj in adapter.whitelist_collection.find():
        click.echo(whitelist_obj['_id'])


import logging

from datetime import datetime

from scout.exceptions import (PedigreeError, ConfigError, IntegrityError)

from . import build_individual

log = logging.getLogger(__name__)

def build_phenotype(phenotype_id, adapter):
    """Build a small phenotype object

        Build a dictionary with phenotype_id and description

    Args:
        phenotype_id (str): The phenotype id
        adapter (scout.adapter.MongoAdapter)

    Returns:
        phenotype_obj (dict):

        dict(
            phenotype_id = str,
            feature = str, # description of phenotype
            )
    """
    phenotype_obj = {}
    phenotype = adapter.hpo_term(phenotype_id)
    if phenotype:
        phenotype_obj['phenotype_id'] = phenotype['hpo_id']
        phenotype_obj['feature'] = phenotype['description']
    return phenotype

def build_case(case_data, adapter):
    """Build a case object that is to be inserted to the database

    Args:
        case_data (dict): A dictionary with the relevant case information
        adapter (scout.adapter.MongoAdapter)

    Returns:
        case_obj (dict): A case object

    dict(
        case_id = str, # required=True, unique
        display_name = str, # If not display name use case_id
        owner = str, # required

        # These are the names of all the collaborators that are allowed to view the
        # case, including the owner
        collaborators = list, # List of institute_ids
        assignee = str, # _id of a user
        individuals = list, # list of dictionaries with individuals
        created_at = datetime,
        updated_at = datetime,
        suspects = list, # List of variants referred by there _id
        causatives = list, # List of variants referred by there _id

        synopsis = str, # The synopsis is a text blob
        status = str, # default='inactive', choices=STATUS
        is_research = bool, # default=False
        research_requested = bool, # default=False
        rerun_requested = bool, # default=False

        analysis_date = datetime,
        analyses = list, # list of dict

        # default_panels specifies which panels that should be shown when
        # the case is opened
        panels = list, # list of dictionaries with panel information

        dynamic_gene_list = list, # List of genes

        genome_build = str, # This should be 37 or 38
        genome_version = float, # What version of the build

        rank_model_version = str,
        rank_score_threshold = int, # default=8

        phenotype_terms = list, # List of dictionaries with phenotype information
        phenotype_groups = list, # List of dictionaries with phenotype information

        madeline_info = str, # madeline info is a full xml file

        multiqc = str, # path to dir with multiqc information

        vcf_files = dict, # A dictionary with vcf files

        diagnosis_phenotypes = list, # List of references to diseases
        diagnosis_genes = list, # List of references to genes

        has_svvariants = bool, # default=False

        is_migrated = bool # default=False

    )
    """
    log.info("build case with id: {0}".format(case_data['case_id']))
    case_obj = {
        '_id': case_data['case_id'],
        'display_name': case_data.get('display_name', case_data['case_id']),
    }

    # Check if institute exists in database
    try:
        institute_id = case_data['owner']
    except KeyError as err:
        raise ConfigError("Case has to have a institute")
    institute_obj = adapter.institute(institute_id)
    if not institute_obj:
        raise IntegrityError("Institute %s not found in database" % institute_id)
    case_obj['owner'] = case_data['owner']

    # Owner allways has to be part of collaborators
    collaborators = set(case_data.get('collaborators', []))
    collaborators.add(case_data['owner'])
    case_obj['collaborators'] = list(collaborators)

    if case_data.get('assignee'):
        case_obj['assignees'] = [case_data['assignee']]

    # Individuals
    ind_objs = []
    try:
        for individual in case_data.get('individuals', []):
            ind_objs.append(build_individual(individual))
    except Exception as error:
        ## TODO add some action here
        raise error
    # sort the samples to put the affected individual first
    sorted_inds = sorted(ind_objs, key=lambda ind: -ind['phenotype'])
    case_obj['individuals'] = sorted_inds

    now = datetime.now()
    case_obj['created_at'] = now
    case_obj['updated_at'] = now

    if case_data.get('suspects'):
        case_obj['suspects'] = case_data['suspects']
    if case_data.get('causatives'):
        case_obj['causatives'] = case_data['causatives']

    case_obj['synopsis'] = case_data.get('synopsis', '')

    case_obj['status'] = 'inactive'
    case_obj['is_research'] = False
    case_obj['research_requested'] = False
    case_obj['rerun_requested'] = False

    analysis_date = case_data.get('analysis_date')
    if analysis_date:
        case_obj['analysis_date'] = analysis_date

    # We store some metadata and references about gene panels in 'panels'
    case_panels = case_data.get('gene_panels', [])
    default_panels = case_data.get('default_panels', [])
    panels = []

    for panel_name in case_panels:
        panel_obj = adapter.gene_panel(panel_name)
        if not panel_obj:
            raise IntegrityError("Panel %s does not exist in database" % panel_name)
        panel = {
            'panel_id': panel_obj['_id'],
            'panel_name': panel_obj['panel_name'],
            'display_name': panel_obj['display_name'],
            'version': panel_obj['version'],
            'updated_at': panel_obj['date'],
            'nr_genes': len(panel_obj['genes'])
        }
        if panel_name in default_panels:
            panel['is_default'] = True
        else:
            panel['is_default'] = False
        panels.append(panel)

    case_obj['panels'] = panels

    case_obj['dynamic_gene_list'] = {}

    # Meta data
    genome_build = case_data.get('genome_build', '37')
    if not genome_build in ['37', '38']:
        pass
        ##TODO raise exception if invalid genome build was used

    case_obj['genome_build'] = genome_build
    case_obj['genome_version'] = case_data.get('genome_version')

    if case_data.get('rank_model_version'):
        case_obj['rank_model_version'] = str(case_data['rank_model_version'])

    if case_data.get('sv_rank_model_version'):
        case_obj['sv_rank_model_version'] = str(case_data['sv_rank_model_version'])

    if case_data.get('rank_score_threshold'):
        case_obj['rank_score_threshold'] = float(case_data['rank_score_threshold'])

    # phenotype information
    phenotypes = []
    for phenotype in case_data.get('phenotype_terms', []):
        phenotype_obj = build_phenotype(phenotype, adapter)
        if phenotype_obj:
            phenotypes.append(phenotype_obj)
    if phenotypes:
        case_obj['phenotype_terms'] = phenotypes

    # phenotype groups
    phenotype_groups = []
    for phenotype in case_data.get('phenotype_groups', []):
        phenotype_obj = build_phenotype(phenotype, adapter)
        if phenotype_obj:
            phenotype_groups.append(phenotype_obj)
    if phenotype_groups:
        case_obj['phenotype_groups'] = phenotype_groups

    # Files
    case_obj['madeline_info'] = case_data.get('madeline_info')
    if 'multiqc' in case_data:
        case_obj['multiqc'] = case_data.get('multiqc')
    case_obj['vcf_files'] = case_data.get('vcf_files', {})
    case_obj['delivery_report'] = case_data.get('delivery_report')

    case_obj['has_svvariants'] = False
    if (case_obj['vcf_files'].get('vcf_sv') or case_obj['vcf_files'].get('vcf_sv_research')):
        case_obj['has_svvariants'] = True

    case_obj['has_strvariants'] = False
    if (case_obj['vcf_files'].get('vcf_str')):
        case_obj['has_strvariants'] = True

    case_obj['is_migrated'] = False

    case_obj['track'] = case_data.get('track', 'rare')

    return case_obj