import os
import re

from toil.lib.docker import dockerCall

def gatk_select_variants(job, mode, vcf_id, ref_fasta, ref_fai, ref_dict):
    """
    Isolates a particular variant type from a VCF file using GATK SelectVariants

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: variant type (i.e. SNP or INDEL)
    :param str vcf_id: FileStoreID for input VCF file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :return: FileStoreID for filtered VCF
    :rtype: str
    """
    job.fileStore.logToMaster('Running GATK SelectVariants to select %ss' % mode)

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf_id}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'SelectVariants',
               '-R', 'genome.fa',
               '-V', 'input.vcf',
               '-o', 'output.vcf',
               '-selectType', mode]

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.vcf'))

def gatk_variant_filtration(job, vcf_id, filter_name, filter_expression, ref_fasta, ref_fai, ref_dict):
    """
    Filters VCF file using GATK VariantFiltration. Fixes extra pair of quotation marks in VCF header that
    may interfere with other VCF tools.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str vcf_id: FileStoreID for input VCF file
    :param str filter_name: Name of filter for VCF header
    :param str filter_expression: JEXL filter expression
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :return: FileStoreID for filtered VCF file
    :rtype: str
    """
    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf_id}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'VariantFiltration',
               '-R', 'genome.fa',
               '-V', 'input.vcf',
               '--filterName', filter_name,   # Documents filter name in header
               '--filterExpression', filter_expression,
               '-o', 'filtered_variants.vcf']

    job.fileStore.logToMaster('Running GATK VariantFiltration using {name}: '
                              '{expression}'.format(name=filter_name, expression=filter_expression))

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    # Remove extra quotation marks around filter expression.
    malformed_header = os.path.join(work_dir, 'filtered_variants.vcf')
    fixed_header = os.path.join(work_dir, 'fixed_header.vcf')
    filter_regex = re.escape('"%s"' % filter_expression)
    with open(malformed_header, 'r') as f, open(fixed_header, 'w') as g:
        for line in f:
            g.write(re.sub(filter_regex, filter_expression, line))

    return job.fileStore.writeGlobalFile(fixed_header)

def gatk_variant_recalibrator(job,
                              mode,
                              vcf,
                              ref_fasta, ref_fai, ref_dict,
                              annotations,
                              hapmap=None, omni=None, phase=None, dbsnp=None, mills=None,
                              max_gaussians=4,
                              unsafe_mode=False):
    """
    Runs either SNP or INDEL variant quality score recalibration using GATK VariantRecalibrator. Because the VQSR method
    models SNPs and INDELs differently, VQSR must be run separately for these variant types.

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: Determines variant recalibration mode (SNP or INDEL)
    :param str vcf: FileStoreID for input VCF file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param list[str] annotations: List of GATK variant annotations to filter on
    :param str hapmap: FileStoreID for HapMap resource file, required for SNP VQSR
    :param str omni: FileStoreID for Omni resource file, required for SNP VQSR
    :param str phase: FileStoreID for 1000G resource file, required for SNP VQSR
    :param str dbsnp: FilesStoreID for dbSNP resource file, required for SNP and INDEL VQSR
    :param str mills: FileStoreID for Mills resource file, required for INDEL VQSR
    :param int max_gaussians: Number of Gaussians used during training, default is 4
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for the variant recalibration table, tranche file, and plots file
    :rtype: tuple
    """
    mode = mode.upper()

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf}

    # Refer to GATK documentation for description of recommended parameters:
    # https://software.broadinstitute.org/gatk/documentation/article?id=1259
    # https://software.broadinstitute.org/gatk/documentation/article?id=2805

    # This base command includes parameters for both INDEL and SNP VQSR.
    command = ['-T', 'VariantRecalibrator',
               '-R', 'genome.fa',
               '-input', 'input.vcf',
               '-tranche', '100.0',
               '-tranche', '99.9',
               '-tranche', '99.0',
               '-tranche', '90.0',
               '--maxGaussians', str(max_gaussians),
               '-recalFile', 'output.recal',
               '-tranchesFile', 'output.tranches',
               '-rscriptFile', 'output.plots.R']

    # Parameters and resource files for SNP VQSR.
    if mode == 'SNP':
        command.extend(
            ['-resource:hapmap,known=false,training=true,truth=true,prior=15.0', 'hapmap.vcf',
             '-resource:omni,known=false,training=true,truth=true,prior=12.0', 'omni.vcf',
             '-resource:dbsnp,known=true,training=false,truth=false,prior=2.0', 'dbsnp.vcf',
             '-resource:1000G,known=false,training=true,truth=false,prior=10.0', '1000G.vcf',
             '-mode', 'SNP'])

        inputs['hapmap.vcf'] = hapmap
        inputs['omni.vcf'] = omni
        inputs['dbsnp.vcf'] = dbsnp
        inputs['1000G.vcf'] = phase

    # Parameters and resource files for INDEL VQSR
    elif mode == 'INDEL':
        command.extend(
            ['-resource:mills,known=false,training=true,truth=true,prior=12.0', 'mills.vcf',
             '-resource:dbsnp,known=true,training=false,truth=false,prior=2.0', 'dbsnp.vcf',
             '-mode', 'INDEL'])

        inputs['mills.vcf'] = mills
        inputs['dbsnp.vcf'] = dbsnp

    else:
        raise ValueError('Variant filter modes can be SNP or INDEL, got %s' % mode)

    for annotation in annotations:
        command.extend(['-an', annotation])

    if unsafe_mode:
        command.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    # Delay reading in files until function is configured
    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    job.fileStore.logToMaster('Running GATK VariantRecalibrator on {mode}s using the following annotations:\n'
                              '{annotations}'.format(mode=mode, annotations='\n'.join(annotations)))

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    recal_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.recal'))
    tranches_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.tranches'))
    plots_id = job.fileStore.writeGlobalFile(os.path.join(work_dir, 'output.plots.R'))
    return recal_id, tranches_id, plots_id

def gatk_apply_variant_recalibration(job,
                                     mode,
                                     vcf,
                                     recal_table, tranches,
                                     ref_fasta, ref_fai, ref_dict,
                                     ts_filter_level=99.0,
                                     unsafe_mode=False):
    """
    Applies variant quality score recalibration to VCF file using GATK ApplyRecalibration

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str mode: Determines variant recalibration mode (SNP or INDEL)
    :param str vcf: FileStoreID for input VCF file
    :param str recal_table: FileStoreID for recalibration table file
    :param str tranches: FileStoreID for tranches file
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param float ts_filter_level: Sensitivity expressed as a percentage, default is 99.0
    :param bool unsafe_mode: If True, runs gatk UNSAFE mode: "-U ALLOW_SEQ_DICT_INCOMPATIBILITY"
    :return: FileStoreID for recalibrated VCF file
    :rtype: str
    """
    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict,
              'input.vcf': vcf,
              'recal': recal_table,
              'tranches': tranches}

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    mode = mode.upper()

    # GATK recommended parameters:
    # https://software.broadinstitute.org/gatk/documentation/article?id=2805
    command = ['-T', 'ApplyRecalibration',
               '-mode', mode,
               '-R', 'genome.fa',
               '-input', 'input.vcf',
               '-o', 'vqsr.vcf',
               '-ts_filter_level', str(ts_filter_level),
               '-recalFile', 'recal',
               '-tranchesFile', 'tranches']

    if unsafe_mode:
        command.extend(['-U', 'ALLOW_SEQ_DICT_INCOMPATIBILITY'])

    job.fileStore.logToMaster('Running GATK ApplyRecalibration on {mode}s '
                              'with a sensitivity of {sensitivity}%'.format(mode=mode,
                                                                            sensitivity=ts_filter_level))
    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'vqsr.vcf'))

def gatk_combine_variants(job, vcfs, ref_fasta, ref_fai, ref_dict, merge_option='UNIQUIFY'):
    """
    Merges VCF files using GATK CombineVariants

    :param JobFunctionWrappingJob job: Toil Job instance
    :param dict vcfs: Dictionary of VCF FileStoreIDs {sample identifier: FileStoreID}
    :param str ref_fasta: FileStoreID for reference genome fasta
    :param str ref_fai: FileStoreID for reference genome index file
    :param str ref_dict: FileStoreID for reference genome sequence dictionary file
    :param str merge_option: Value for --genotypemergeoption flag (Default: 'UNIQUIFY')
                            'UNIQUIFY': Multiple variants at a single site are merged into a
                                        single variant record.
                            'UNSORTED': Used to merge VCFs from the same sample
    :return: FileStoreID for merged VCF file
    :rtype: str
    """
    job.fileStore.logToMaster('Running GATK CombineVariants')

    inputs = {'genome.fa': ref_fasta,
              'genome.fa.fai': ref_fai,
              'genome.dict': ref_dict}
    inputs.update(vcfs)

    work_dir = job.fileStore.getLocalTempDir()
    for name, file_store_id in inputs.iteritems():
        job.fileStore.readGlobalFile(file_store_id, os.path.join(work_dir, name))

    command = ['-T', 'CombineVariants',
               '-R', '/data/genome.fa',
               '-o', '/data/merged.vcf',
               '--genotypemergeoption', merge_option]

    for uuid, vcf_id in vcfs.iteritems():
        command.extend(['--variant', os.path.join('/data', uuid)])

    docker_parameters = ['--rm', 'log-driver', 'none',
                         '-e', 'JAVA_OPTS=-Djava.io.tmpdir=/data/ -Xmx{}'.format(job.memory)]
    dockerCall(job=job, workDir=work_dir,
               parameters=command,
               tool='quay.io/ucsc_cgl/gatk:3.5--dba6dae49156168a909c43330350c6161dc7ecc2',
               dockerParameters=docker_parameters)

    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'merged.vcf'))


import os
import subprocess

from toil_lib import require

def bam_quickcheck(bam_path):
    """
    Perform a quick check on a BAM via `samtools quickcheck`.
    This will detect obvious BAM errors such as truncation.

    :param str bam_path: path to BAM file to checked

    :rtype: boolean
    :return: True if the BAM is valid, False is BAM is invalid or something related to the call went wrong
    """
    directory, bam_name = os.path.split(bam_path)
    exit_code = subprocess.call(['docker', 'run', '-v', directory + ':/data',
                                 'quay.io/ucsc_cgl/samtools:1.3--256539928ea162949d8a65ca5c79a72ef557ce7c',
                                 'quickcheck', '-vv', '/data/' + bam_name])
    if exit_code != 0:
        return False
    return True

def require_bam_quickcheck(bam_path):
    require(bam_quickcheck(bam_path), "The BAM at '%s' is invalid or something else is wrong!" % bam_path)


"""
home.collect.handlers
=====================

Handlers are defined to handle events that Home collects. They will either
just record the event, or store it in a particular way or take action based on
the event.
"""

from logging import getLogger

from home import db
from home.exceptions import HandlerConfigError
from home.ts.models import Device, Series, DataPoint

def importer(dotted_path):
    module, object_ = dotted_path.rsplit('.', 1)
    try:
        mod = __import__(module, fromlist=[object_, ])
    except ImportError as e:
        raise ImportError("Failed to load {0}".format(dotted_path)) from e
    return getattr(mod, object_)

def load_handlers(handler_mapping):
    """
    Given a dictionary mapping which looks like the following, import the
    objects based on the dotted path and yield the packet type and handler as
    pairs.

    If the special string '*' is passed, don't process that, pass it on as it
    is a wildcard.

    If an non-string object is given for either packet or handler (key or
    value) assume these are the objects to use and yield them.

    ::
        {
        'rfxcom.protocol.Status': 'home.collect.logging_handler',
        'rfxcom.protocol.Elec': 'home.collect.elec_handler',
        'rfxcom.protocol.TempHumidity': 'home.collect.temp_humidity_handler',
        '*': 'home.collect.logging_handler'
        }
    """

    handlers = {}

    for packet_type, handler in handler_mapping.items():

        if packet_type == '*':
            Packet = packet_type
        elif isinstance(packet_type, str):
            Packet = importer(packet_type)
        else:
            Packet = packet_type

        if isinstance(handler, str):
            Handler = importer(handler)
        else:
            Handler = handler

        if Packet in handlers:
            raise HandlerConfigError(
                "Handler already provided for packet %s" % Packet)

        handlers[Packet] = Handler

    return handlers

class BaseHandler:

    def __init__(self):

        self.log = getLogger('home.collect.%s' % self.__class__.__name__)

    def format_packet(self, pkt):

        return " ".join("0x{0:02x}".format(x) for x in pkt)

class LoggingHandler(BaseHandler):

    def __call__(self, packet):

        self.log.warning(
            "Ignoring packet: {0} ({1})".format(
                packet, self.format_packet(packet.raw)))

class RecordingHandler(BaseHandler):

    def __init__(self, mapping):
        super().__init__()
        self.mapping = mapping

    def __call__(self, packet):

        device = Device.get_or_create(packet.data['packet_type'],
                                      packet.data['sub_type'],
                                      packet.data['id'])

        for series_name, value_name in self.mapping.items():

            series = Series.get_or_create(name=series_name)
            db.session.commit()
            try:
                val = packet.data[value_name]
            except KeyError:
                self.log.error("Failed to find %s in packet. Key list: %r" % (
                    value_name, sorted(packet.data.keys())))
                return

            id_ = packet.data.get('id')
            self.log.info("ID=%s, %s=%s, Device ID=%s, Series ID=%s" % (
                id_, series_name, val, device.id, series.id))
            DataPoint.record(series, device, val)

        db.session.commit()


# -*- coding: utf-8 -*-

import json

import os

__all__ = [
    'CONFIG_DIRECTORY',
    'CONFIG_PATH',
    'BASE_URL',
]

CONFIG_DIRECTORY = os.path.join(os.path.expanduser('~'), '.config', 'ols_client')

if not os.path.exists(CONFIG_DIRECTORY):
    os.makedirs(CONFIG_DIRECTORY)

CONFIG_PATH = os.path.join(CONFIG_DIRECTORY, 'config.json')

def write_config(configuration):
    """Helper to write the JSON configuration to a file"""
    with open(CONFIG_PATH, 'w') as f:
        json.dump(configuration, f, indent=2, sort_keys=True)

def get_config():
    """Gets the configuration for this project from the default JSON file, or writes one if it doesn't exist

    :rtype: dict
    """
    if not os.path.exists(CONFIG_PATH):
        write_config({})

    with open(CONFIG_PATH) as f:
        return json.load(f)

config = get_config()
config.setdefault('BASE_URL', 'http://www.ebi.ac.uk/ols')

BASE_URL = config['BASE_URL']


# -*- coding: utf-8 -*-

import logging
import time

import requests

from ols_client.constants import BASE_URL

__all__ = [
    'OlsClient'
]

log = logging.getLogger(__name__)
log.setLevel(logging.INFO)

HIERARCHICAL_CHILDREN = 'hierarchicalChildren'

api_ontology = '/api/ontologies/{ontology}'
api_terms = '/api/ontologies/{ontology}/terms'
api_term = '/api/ontologies/{ontology}/terms/{iri}'
api_properties = '/api/ontologies/{ontology}/properties/{iri}'
api_indivduals = '/api/ontologies/{ontology}/individuals/{iri}'
api_suggest = '/api/suggest'
api_search = '/api/search'
api_descendants = '/api/ontologies/{ontology}/terms/{iri}/hierarchicalDescendants'

def _iterate_response_terms(response):
    """Iterates over the terms in the given response"""
    for term in response['_embedded']['terms']:
        yield term

def _help_iterate_labels(term_iterator):
    for term in term_iterator:
        yield term['label']

class OlsClient:
    """Wraps the functions to query the Ontology Lookup Service such that alternative base URL's can be used."""

    def __init__(self, ols_base=None):
        """
        :param ols_base: An optional, custom URL for the OLS RESTful API.
        """
        self.base = (ols_base if ols_base is not None else BASE_URL).rstrip('/')

        self.ontology_terms_fmt = self.base + api_terms
        self.ontology_term_fmt = self.base + api_term
        self.ontology_metadata_fmt = self.base + api_ontology
        self.ontology_suggest = self.base + api_suggest
        self.ontology_search = self.base + api_search
        self.ontology_term_descendants_fmt = self.base + api_descendants

    def get_ontology(self, ontology):
        """Gets the metadata for a given ontology

        :param str ontology: The name of the ontology
        :return: The dictionary representing the JSON from the OLS
        :rtype: dict
        """
        url = self.ontology_metadata_fmt.format(ontology=ontology)
        response = requests.get(url)
        return response.json()

    def get_term(self, ontology, iri):
        """Gets the data for a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :rtype: dict
        """
        url = self.ontology_term_fmt.format(ontology, iri)
        response = requests.get(url)

        return response.json()

    def search(self, name, query_fields=None):
        """Searches the OLS with the given term

        :param str name:
        :param list[str] query_fields: Fields to query
        :return: dict
        """
        params = {'q': name}
        if query_fields is not None:
            params['queryFields'] = '{{{}}}'.format(','.join(query_fields))
        response = requests.get(self.ontology_search, params=params)

        return response.json()

    def suggest(self, name, ontology=None):
        """Suggest terms from an optional list of ontologies

        :param str name:
        :param list[str] ontology:
        :rtype: dict

        .. seealso:: https://www.ebi.ac.uk/ols/docs/api#_suggest_term
        """
        params = {'q': name}
        if ontology:
            params['ontology'] = ','.join(ontology)
        response = requests.get(self.ontology_suggest, params=params)

        return response.json()

    @staticmethod
    def _iter_terms_helper(url, size=None, sleep=None):
        """Iterates over all terms, lazily with paging

        :param str url: The url to query
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to none.
        :rtype: iter[dict]
        """
        if size is None:
            size = 500
        elif size > 500:
            raise ValueError('Maximum size is 500. Given: {}'.format(size))

        t = time.time()
        response = requests.get(url, params={'size': size}).json()
        links = response['_links']

        for response_term in _iterate_response_terms(response):
            yield response_term

        t = time.time() - t

        log.info(
            'Page %s/%s done in %.2f seconds',
            response['page']['number'] + 1,
            response['page']['totalPages'],
            t
        )

        log.info('Estimated time until done: %.2f minutes', t * response['page']['totalPages'] / 60)

        while 'next' in links:
            if sleep:
                time.sleep(sleep)

            t = time.time()
            response = requests.get(links['next']['href'], params={'size': size}).json()
            links = response['_links']

            for response_term in _iterate_response_terms(response):
                yield response_term

            log.info(
                'Page %s/%s done in %.2f seconds',
                response['page']['number'],
                response['page']['totalPages'],
                time.time() - t
            )

    def iter_terms(self, ontology, size=None, sleep=None):
        """Iterates over all terms, lazily with paging

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[dict]
        """
        url = self.ontology_terms_fmt.format(ontology=ontology)
        for term in self._iter_terms_helper(url, size=size, sleep=sleep):
            yield term

    def iter_descendants(self, ontology, iri, size=None, sleep=None):
        """Iterates over the descendants of a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[dict]
        """
        url = self.ontology_term_descendants_fmt.format(ontology=ontology, iri=iri)
        log.info('getting %s', url)
        for term in self._iter_terms_helper(url, size=size, sleep=sleep):
            yield term

    def iter_descendants_labels(self, ontology, iri, size=None, sleep=None):
        """Iterates over the labels for the descendants of a given term

        :param str ontology: The name of the ontology
        :param str iri: The IRI of a term
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[str]
        """
        for label in _help_iterate_labels(self.iter_descendants(ontology, iri, size=size, sleep=sleep)):
            yield label

    def iter_labels(self, ontology, size=None, sleep=None):
        """Iterates over the labels of terms in the ontology. Automatically wraps the pager returned by the OLS.

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[str]
        """
        for label in _help_iterate_labels(self.iter_terms(ontology=ontology, size=size, sleep=sleep)):
            yield label

    def iter_hierarchy(self, ontology, size=None, sleep=None):
        """Iterates over parent-child relations

        :param str ontology: The name of the ontology
        :param int size: The size of each page. Defaults to 500, which is the maximum allowed by the EBI.
        :param int sleep: The amount of time to sleep between pages. Defaults to 0 seconds.
        :rtype: iter[tuple[str,str]]
        """
        for term in self.iter_terms(ontology=ontology, size=size, sleep=sleep):
            try:
                hierarchy_children_link = term['_links'][HIERARCHICAL_CHILDREN]['href']
            except KeyError:  # there's no children for this one
                continue

            response = requests.get(hierarchy_children_link).json()

            for child_term in response['_embedded']['terms']:
                yield term['label'], child_term['label']  # TODO handle different relation types

    def get_description(self, ontology):
        """Gets the description of a given ontology

        :param str ontology: The name of the ontology
        :rtype: str
        """
        response = self.get_ontology(ontology)

        return response['config'].get('description')


import os

from toil.lib.docker import dockerCall

from toil_lib.files import tarball_files

def run_fastqc(job, r1_id, r2_id):
    """
    Run Fastqc on the input reads

    :param JobFunctionWrappingJob job: passed automatically by Toil
    :param str r1_id: FileStoreID of fastq read 1
    :param str r2_id: FileStoreID of fastq read 2
    :return: FileStoreID of fastQC output (tarball)
    :rtype: str
    """
    work_dir = job.fileStore.getLocalTempDir()
    job.fileStore.readGlobalFile(r1_id, os.path.join(work_dir, 'R1.fastq'))
    parameters = ['/data/R1.fastq']
    output_names = ['R1_fastqc.html', 'R1_fastqc.zip']
    if r2_id:
        job.fileStore.readGlobalFile(r2_id, os.path.join(work_dir, 'R2.fastq'))
        parameters.extend(['-t', '2', '/data/R2.fastq'])
        output_names.extend(['R2_fastqc.html', 'R2_fastqc.zip'])
    dockerCall(job=job, tool='quay.io/ucsc_cgl/fastqc:0.11.5--be13567d00cd4c586edf8ae47d991815c8c72a49',
               workDir=work_dir, parameters=parameters)
    output_files = [os.path.join(work_dir, x) for x in output_names]
    tarball_files(tar_name='fastqc.tar.gz', file_paths=output_files, output_dir=work_dir)
    return job.fileStore.writeGlobalFile(os.path.join(work_dir, 'fastqc.tar.gz'))


from .._stream import Stream, query_maker

def get_stream(cdb, stream):
    if isinstance(stream, Stream):
        return stream.path
    elif stream.count("/") == 0:
        return cdb.path + "/" + stream
    elif stream.count("/") == 2:
        return stream
    else:
        raise Exception("Stream '%s' invalid" % (stream, ))

class Merge(object):
    """Merge represents a query which allows to merge multiple streams into one
    when reading, with all the streams merged together by increasing timestamp.
    The merge query is used as a constructor-type object::

        m = Merge(cdb)
        m.addStream("mystream1",t1=time.time()-10)
        m.addStream("mystream2",t1=time.time()-10)
        result = m.run()
    """

    def __init__(self, cdb):
        """Given a ConnectorDB object, begins the construction of a Merge query"""
        self.cdb = cdb

        self.query = []

    def addStream(self, stream, t1=None, t2=None, limit=None, i1=None, i2=None, transform=None):
        """Adds the given stream to the query construction. The function supports both stream
        names and Stream objects."""
        params = query_maker(t1, t2, limit, i1, i2, transform)

        params["stream"] = get_stream(self.cdb, stream)

        # Now add the stream to the query parameters
        self.query.append(params)

    def run(self):
        """Runs the merge query, and returns the result"""
        return self.cdb.db.query("merge", self.query)


"""
home.__init__
=============

A simple module that defines how the flask app is created.

"""
__version__ = '0.3.4'

from os import environ

from flask import Flask
from flask.ext.admin import Admin
from flask.ext.login import LoginManager
from flask.ext.migrate import Migrate
from flask.ext.sqlalchemy import SQLAlchemy

from home.ts.redis import RedisSeries

db = SQLAlchemy()
redis_series = RedisSeries()
login_manager = LoginManager()

def create_app(config=None):
    """ This needs some tidying up. To avoid circular imports we import
    everything here but it makes this method a bit more gross.
    """

    # Initialise the app
    from home.config import TEMPLATE_FOLDER, STATIC_FOLDER
    app = Flask(__name__, static_folder=STATIC_FOLDER,
                template_folder=TEMPLATE_FOLDER)

    app.config['SECRET_KEY'] = 'ssh, its a secret.'

    # Load the default config, the specified config file and then any
    # overwrites that are manually passed in.
    app.config.from_object('home.config')

    if 'HOME_SETTINGS' in environ:
        app.config.from_envvar('HOME_SETTINGS')

    app.config.from_object(config)

    # Register the web front end and the API.
    from home.dash.web import web
    from home.dash.api import api
    app.register_blueprint(web)
    app.register_blueprint(api, url_prefix='/api')

    login_manager.init_app(app)
    login_manager.login_view = 'Dashboard Web.login'

    from home.dash.models import User

    @login_manager.user_loader
    def load_user(user_id):
        return User.query.get(int(user_id))

    # Initialise the migrations app, we want to store all migrations within
    # the project directory for easier packaging.
    Migrate(app, db, directory=app.config['MIGRATE_DIRECTORY'])

    admin = Admin(app)

    from home.dash.admin import setup_admin
    setup_admin(admin)

    # Wire up the database to the app so it gets the config.
    db.init_app(app)

    return app


# Copyright (C) 2016 Regents of the University of California
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import logging
import multiprocessing
import os
import subprocess
import time

from toil.job import Job
from toil.lib.docker import STOP, dockerCheckOutput

_log = logging.getLogger(__name__)
_SPARK_MASTER_PORT = "7077"

def spawn_spark_cluster(job,
                        numWorkers,
                        cores=None,
                        memory=None,
                        disk=None,
                        overrideLeaderIP=None):
    '''
    :param numWorkers: The number of worker nodes to have in the cluster. \
    Must be greater than or equal to 1.
    :param cores: Optional parameter to set the number of cores per node. \
    If not provided, we use the number of cores on the node that launches \
    the service.
    :param memory: Optional parameter to set the memory requested per node.
    :param disk: Optional parameter to set the disk requested per node.
    :type leaderMemory: int or string convertable by bd2k.util.humanize.human2bytes to an int
    :type numWorkers: int
    :type cores: int
    :type memory: int or string convertable by bd2k.util.humanize.human2bytes to an int
    :type disk: int or string convertable by bd2k.util.humanize.human2bytes to an int
    '''

    if numWorkers < 1:
        raise ValueError("Must have more than one worker. %d given." % numWorkers)

    leaderService = SparkService(cores=cores,
                                 memory=memory,
                                 disk=disk,
                                 overrideLeaderIP=overrideLeaderIP)
    leaderIP = job.addService(leaderService)
    for i in range(numWorkers):
        job.addService(WorkerService(leaderIP,
                                     cores=cores,
                                     disk=disk,
                                     memory=memory),
                       parentService=leaderService)

    return leaderIP

def _checkContainerStatus(sparkContainerID,
                          hdfsContainerID,
                          sparkNoun='leader',
                          hdfsNoun='namenode'):

    containers = subprocess.check_output(["docker", "ps", "-q"])

    # docker ps emits shortened versions of the hash
    # these shortened hashes are 12 characters long
    shortSpark = sparkContainerID[0:11]
    shortHdfs = hdfsContainerID[0:11]

    if ((sparkContainerID not in containers and
         shortSpark not in containers) or
        (hdfsContainerID not in containers and
         shortHdfs not in containers)):
        raise RuntimeError('Lost both Spark %s and HDFS %s.' % (sparkNoun, hdfsNoun))
    elif sparkContainerID not in containers and shortSpark not in containers:
        raise RuntimeError('Lost Spark %s. %r' % sparkNoun)
    elif hdfsContainerID not in containers and shortHdfs not in containers:
        raise RuntimeError('Lost HDFS %s. %r' % hdfsNoun)
    else:
        return True

class SparkService(Job.Service):
    """
    A Service job that spins up a Spark cluster that child jobs can then attach
    to. If the job that spawns this job is run with `checkpoint = True`, then
    this service will robustly restart the Spark cluster upon the loss of any
    nodes in the cluster.
    """

    def __init__(self,
                 memory=None,
                 disk=None,
                 cores=None,
                 overrideLeaderIP=None):
        """
        :param memory: The amount of memory to be requested for the Spark leader. Optional.
        :param disk: The amount of disk to be requested for the Spark leader. Optional.
        :param cores: Optional parameter to set the number of cores per node. \
        If not provided, we use the number of cores on the node that launches \
        the service.
        :type memory: int or string convertable by bd2k.util.humanize.human2bytes to an int
        :type disk: int or string convertable by bd2k.util.humanize.human2bytes to an int
        :type cores: int
        """

        if cores is None:
            cores = multiprocessing.cpu_count()

        self.hostname = overrideLeaderIP

        Job.Service.__init__(self, memory=memory, cores=cores, disk=disk)

    def start(self, job):
        """
        Start spark and hdfs master containers

        :param job: The underlying job.
        """

        if self.hostname is None:
            self.hostname = subprocess.check_output(["hostname", "-f",])[:-1]

        _log.info("Started Spark master container.")
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-master:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e", "SPARK_MASTER_IP=" + self.hostname,
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.hostname])[:-1]
        _log.info("Started HDFS Datanode.")
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                           defer=STOP,
                                           workDir=os.getcwd(),
                                           tool="quay.io/ucsc_cgl/apache-hadoop-master:2.6.2",
                                           dockerParameters=["--net=host",
                                                              "-d"],
                                           parameters=[self.hostname])[:-1]

        return self.hostname

    def stop(self, job):
        """
        Stop and remove spark and hdfs master containers

        :param job: The underlying job.
        """

        subprocess.call(["docker", "exec", self.sparkContainerID, "rm", "-r", "/ephemeral/spark"])
        subprocess.call(["docker", "stop", self.sparkContainerID])
        subprocess.call(["docker", "rm", self.sparkContainerID])
        _log.info("Stopped Spark master.")

        subprocess.call(["docker", "stop", self.hdfsContainerID])
        subprocess.call(["docker", "rm", self.hdfsContainerID])
        _log.info("Stopped HDFS namenode.")

        return

    def check(self):
        """
        Checks to see if Spark master and HDFS namenode are still running.
        """

        status = _checkContainerStatus(self.sparkContainerID, self.hdfsContainerID)

        return status

class WorkerService(Job.Service):
    """
    Service Job that implements the worker nodes in a Spark/HDFS cluster.
    Should not be called outside of `SparkService`.
    """

    def __init__(self, masterIP, memory=None, cores=None, disk=None):
        """
        :param memory: The memory requirement for each node in the cluster. Optional.
        :param disk: The disk requirement for each node in the cluster. Optional.
        :param cores: Optional parameter to set the number of cores per node. \
        If not provided, we use the number of cores on the node that launches \
        the service.
        :type memory: int or string convertable by bd2k.util.humanize.human2bytes to an int
        :type disk: int or string convertable by bd2k.util.humanize.human2bytes to an int
        :type cores: int
        """

        self.masterIP = masterIP

        if cores is None:
            cores = multiprocessing.cpu_count()

        Job.Service.__init__(self, memory=memory, cores=cores, disk=disk)

    def start(self, job):
        """
        Start spark and hdfs worker containers

        :param job: The underlying job.
        """

        # start spark and our datanode
        self.sparkContainerID = dockerCheckOutput(job=job,
                                                  defer=STOP,
                                                  workDir=os.getcwd(),
                                                  tool="quay.io/ucsc_cgl/apache-spark-worker:1.5.2",
                                                  dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw",
                                                                    "-e",
                                                                    "\"SPARK_MASTER_IP=" + self.masterIP + ":" + _SPARK_MASTER_PORT + "\"",
                                                                    "-e", "SPARK_LOCAL_DIRS=/ephemeral/spark/local",
                                                                    "-e", "SPARK_WORKER_DIR=/ephemeral/spark/work"],
                                                  parameters=[self.masterIP + ":" + _SPARK_MASTER_PORT])[:-1]
        self.__start_datanode(job)

        # fake do/while to check if HDFS is up
        hdfs_down = True
        retries = 0
        while hdfs_down and (retries < 5):

            _log.info("Sleeping 30 seconds before checking HDFS startup.")
            time.sleep(30)
            clusterID = ""
            try:
                clusterID = subprocess.check_output(["docker",
                                                     "exec",
                                                     self.hdfsContainerID,
                                                     "grep",
                                                     "clusterID",
                                                     "-R",
                                                     "/opt/apache-hadoop/logs"])
            except:
                # grep returns a non-zero exit code if the pattern is not found
                # we expect to not find the pattern, so a non-zero code is OK
                pass

            if "Incompatible" in clusterID:
                _log.warning("Hadoop Datanode failed to start with: %s", clusterID)
                _log.warning("Retrying container startup, retry #%d.", retries)
                retries += 1

                _log.warning("Removing ephemeral hdfs directory.")
                subprocess.check_call(["docker",
                                       "exec",
                                       self.hdfsContainerID,
                                       "rm",
                                       "-rf",
                                       "/ephemeral/hdfs"])

                _log.warning("Killing container %s.", self.hdfsContainerID)
                subprocess.check_call(["docker",
                                       "kill",
                                       self.hdfsContainerID])

                # todo: this is copied code. clean up!
                _log.info("Restarting datanode.")
                self.__start_datanode(job)

            else:
                _log.info("HDFS datanode started up OK!")
                hdfs_down = False

        if retries >= 5:
            raise RuntimeError("Failed %d times trying to start HDFS datanode." % retries)

        return

    def __start_datanode(self, job):
        """
        Launches the Hadoop datanode.

        :param job: The underlying job.
        """
        self.hdfsContainerID = dockerCheckOutput(job=job,
                                                 defer=STOP,
                                                 workDir=os.getcwd(),
                                                 tool="quay.io/ucsc_cgl/apache-hadoop-worker:2.6.2",
                                                 dockerParameters=["--net=host",
                                                                    "-d",
                                                                    "-v", "/mnt/ephemeral/:/ephemeral/:rw"],
                                                 parameters=[self.masterIP])[:-1]

    def stop(self, fileStore):
        """
        Stop spark and hdfs worker containers

        :param job: The underlying job.
        """

        subprocess.call(["docker", "exec", self.sparkContainerID, "rm", "-r", "/ephemeral/spark"])
        subprocess.call(["docker", "stop", self.sparkContainerID])
        subprocess.call(["docker", "rm", self.sparkContainerID])
        _log.info("Stopped Spark worker.")

        subprocess.call(["docker", "exec", self.hdfsContainerID, "rm", "-r", "/ephemeral/hdfs"])
        subprocess.call(["docker", "stop", self.hdfsContainerID])
        subprocess.call(["docker", "rm", self.hdfsContainerID])
        _log.info("Stopped HDFS datanode.")

        return

    def check(self):
        """
        Checks to see if Spark worker and HDFS datanode are still running.
        """

        status = _checkContainerStatus(self.sparkContainerID,
                                       self.hdfsContainerID,
                                       sparkNoun='worker',
                                       hdfsNoun='datanode')

        return status