#!/usr/bin/env python
"""Entry-point for suite of sub-commands.

The only executable module in the project; this module
 * validates command line args
 * manages use of temp directories (to keep output clean and atomic)
 * dipatches to sub-commands as appropriate
 * attempts to deal with usage and run-time errors

Jacquard first writes results to temp dir and only copies results on successful
    completion.

Then architecture of Jacquard modules can be divided into:
 * commands : These transform files or directories (e.g. translate.py) and are
     indirectly executed through the jacquard module. Each command must
     implement an execute method that does the heavy lifting along with some
     simpler methods that expedite command validation.
 * transforms : These transform VcfRecords (e.g. mutect.py). They typically have
     a collection of tag classes; where each tag holds the metaheader and
     code to transform a single VcfRecord. Note that a caller could
     manipulate any aspect of a VcfRecord, but (by strong convention) typically
     only adds information, for example add a sample-format tag, add an info
     field, or add a filter field.
 * utils : Common functionality (e.g. command_validator, vcf, logger, etc.)
"""
##   Copyright 2014 Bioinformatics Core, University of Michigan
##
##   Licensed under the Apache License, Version 2.0 (the "License");
##   you may not use this file except in compliance with the License.
##   You may obtain a copy of the License at
##
##       http://www.apache.org/licenses/LICENSE-2.0
##
##   Unless required by applicable law or agreed to in writing, software
##   distributed under the License is distributed on an "AS IS" BASIS,
##   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
##   See the License for the specific language governing permissions and
##   limitations under the License.
from __future__ import print_function, absolute_import, division

import argparse
from datetime import datetime
import distutils.dir_util
import os
import re
import shutil
import signal
import sys
import traceback

from jacquard import __version__
import jacquard.utils.command_validator as command_validator
import jacquard.expand as expand
import jacquard.utils.logger as logger
import jacquard.merge as merge
import jacquard.summarize as summarize
import jacquard.translate as translate
import jacquard.utils.utils as utils
from jacquard.variant_caller_transforms import variant_caller_factory

_SUBCOMMANDS = [translate,
                merge,
                summarize,
                expand]

class _JacquardArgumentParser(argparse.ArgumentParser):
    """Argument parser that raises UsageError instead of exiting."""
    #pylint: disable=too-few-public-methods

    @staticmethod
    def _remessage_invalid_subparser(message):
        pattern = r"argument subparser_name: invalid choice: '(.*?)' \((.*)\)"
        match = re.match(pattern, message)
        if match:
            message = ("'{}' is not a Jacquard command;"
                       " {}.").format(match.group(1), match.group(2))
        return message

    def error(self, message):
        '''Suppress default exit behavior'''
        message = self._remessage_invalid_subparser(message)
        raise utils.UsageError(message)

def _cleanup(temp_working_dir):
    if temp_working_dir and os.path.exists(temp_working_dir):
        logger.debug("Cleaning up tmp directory")
        shutil.rmtree(temp_working_dir)

def _move_tmp_contents_to_original(args):
    temp_dir = args.temp_working_dir
    dest_dir = os.path.dirname(args.temp_working_dir)

    logger.debug("Moving files from tmp directory [{}] to output "
                 "directory [{}]",
                 temp_dir,
                 dest_dir)
    distutils.dir_util.copy_tree(temp_dir, dest_dir)
    logger.debug("Output saved to [{}]", os.path.abspath(args.original_output))
    logger.info("Output saved to [{}]", args.original_output)
    _cleanup(temp_dir)

def _parse_command_line_args(modules, arguments):
    parser = _JacquardArgumentParser(
        usage="jacquard",
        formatter_class=argparse.RawTextHelpFormatter,
        # pylint: disable=line-too-long
        description='''type 'jacquard <subcommand> -h' for help on a specific command''',
        epilog="See https://github.com/umich-brcf-bioinf/Jacquard for more info.")

    parser.add_argument("-V",
                        "--version",
                        action='version',
                        version=_version_text())
    subparsers = parser.add_subparsers(title="subcommands",
                                       dest="subparser_name")

    module_dispatch = {}
    for module in modules:
        module.add_subparser(subparsers)
        short_name = module.__name__.split('.')[-1]
        module_dispatch[short_name] = module

    args = parser.parse_args(arguments)
    return module_dispatch[args.subparser_name], args

def _set_interrupt_handler(target=signal.signal):
    def _handler(dummy_signum, dummy_frame):
        msg = "WARNING: Jacquard was interrupted before completing."
        try:
            logger.debug(msg)
        finally:
            print(msg, file=sys.stderr)
            exit(1)
    target(signal.SIGINT, _handler)
    target(signal.SIGTERM, _handler)

def _version_text():
    callers = variant_caller_factory.SUPPORTED_CALLER_VERSIONS.items()
    caller_versions = [key + " " + value for key, value in callers]
    caller_version_string = "\n\t".join(caller_versions)
    return "Jacquard v{0}\nSupported variant callers:\n\t{1}".\
        format(__version__, caller_version_string)

def _get_execution_context(command):
    cwd = os.path.dirname(os.getcwd())
    now = datetime.now().strftime('%Y-%m-%d %H:%M:%S')

    return [('##jacquard=<timestamp="{}",command="{}",cwd="{}",'
             'source="Jacquard",version="{}">').format(now,
                                                       command,
                                                       cwd,
                                                       __version__)]

def _dispatch(modules, arguments):
    try:
        command, args = _parse_command_line_args(modules, arguments)
        execution_context = _get_execution_context(arguments)

        logger.initialize_logger(args)
        logger.debug("Jacquard run begins")
        logger.debug("cwd|{}", os.getcwd())
        logger.debug("command|{}", " ".join(arguments))

        command_validator.preflight(command, args)

        logger.info("Jacquard begins (v{})", __version__)
        logger.info("Saving log to [{}]",
                    os.path.basename(logger.log_filename))
        logger.debug("Writing output to tmp directory [{}]",
                     args.temp_working_dir)

        command.execute(args, execution_context)

        _move_tmp_contents_to_original(args)

        if logger.WARNING_OCCURRED:
            logger.info("Done. (See warnings above)")
        else:
            logger.info("Done")

    except utils.UsageError as usage_error:
        message = "Jacquard usage problem: {}".format(str(usage_error))
        logger.debug(message)
        print(message, file=sys.stderr)
        try:
            print("See 'jacquard {} --help'.".format(args.subparser_name),
                  file=sys.stderr)
        except NameError: #could not determine the command
            print("See 'jacquard --help'.", file=sys.stderr)
        sys.exit(1)
    except Exception as exception: #pylint: disable=broad-except
        logger.error(str(exception))
        logger.error("Jacquard encountered an unanticipated problem. "
                     "Please review log file and contact your sysadmin "
                     "or Jacquard support for assistance.")
        logger.debug(traceback.format_exc())
        sys.exit(1)
    finally:
        try:
            _cleanup(args.temp_working_dir)
        except Exception: #pylint: disable=broad-except
            pass #we tried

def main():
    _set_interrupt_handler()
    _dispatch(_SUBCOMMANDS, sys.argv[1:])

if __name__ == '__main__':
    main()


"""Interprets MuTect VCF files adding Jacquard standard information.

MuTect VCFs are assumed to have a ".vcf" extension and have a valid
"##MuTect=..." metaheader.
"""
from __future__ import print_function, absolute_import, division

import argparse
import re

import jacquard.utils.utils as utils
import jacquard.variant_caller_transforms.common_tags as common_tags
import jacquard.utils.vcf as vcf

JQ_MUTECT_TAG = "JQ_MT_"
MUTECT_ABBREVIATION = "MT"
VERSION = "v1.1 - v4.0"

class _GenotypeTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    @classmethod
    def _standardize_gt(cls, value):
        if value == "0":
            value = "0/0"
        return value

    def __init__(self):
        super(self.__class__,
              self).__init__(MUTECT_ABBREVIATION,
                             common_tags.GENOTYPE_TAG,
                             "Jacquard genotype (based on GT)")

    def add_tag_values(self, vcf_record):
        if "GT" in vcf_record.format_tags:
            sample_values = {}
            for samp in vcf_record.sample_tag_values:
                genotype = vcf_record.sample_tag_values[samp]["GT"]
                sample_values[samp] = _GenotypeTag._standardize_gt(genotype)
            vcf_record.add_sample_tag_value(self.tag_id, sample_values)

class _AlleleFreqTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(MUTECT_ABBREVIATION,
                             common_tags.ALLELE_FREQ_TAG,
                             ('Jacquard allele frequency for MuTect: '
                              'Decimal allele frequency rounded to 4 digits '
                              '(based on FA)'))

    def add_tag_values(self, vcf_record):
        if "FA" in vcf_record.format_tags:
            sample_values = {}
            for sample in vcf_record.sample_tag_values:
                freq = vcf_record.sample_tag_values[sample]["FA"].split(",")
                sample_values[sample] = self._standardize_af(freq)
            vcf_record.add_sample_tag_value(self.tag_id, sample_values)

    @staticmethod
    def _standardize_af(value):
        new_values = []
        for val in value:
            new_values.append(utils.round_digits(val))
        return ",".join(new_values)

class _DepthTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(MUTECT_ABBREVIATION,
                             common_tags.DEPTH_TAG,
                             'Jacquard depth for MuTect (based on DP)')

    def add_tag_values(self, vcf_record):
        if "DP" in vcf_record.format_tags:
            sample_values = {}
            for samp in vcf_record.sample_tag_values:
                sample_values[samp] = vcf_record.sample_tag_values[samp]["DP"]
            vcf_record.add_sample_tag_value(self.tag_id, sample_values)

class _SomaticTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    _DESCRIPTION = (\
'''Jacquard somatic status for MuTect: 0=non-somatic,1=somatic (based on SS
 FORMAT tag)''').replace("\n","")
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(MUTECT_ABBREVIATION,
                             common_tags.SOMATIC_TAG,
                             self._DESCRIPTION)

    def add_tag_values(self, vcf_record):
        sample_values = {}
        if "SS" in vcf_record.format_tags:
            for sample in vcf_record.sample_tag_values:
                somatic_status = vcf_record.sample_tag_values[sample]["SS"]
                sample_values[sample] = self._somatic_status(somatic_status)
        else:
            for sample in vcf_record.sample_tag_values:
                sample_values[sample] = "0"
        vcf_record.add_sample_tag_value(self.tag_id, sample_values)

    @staticmethod
    def _somatic_status(ss_value):
        if ss_value == "2":
            return "1"
        else:
            return "0"

class _Mutect1Parser(object):
    _MUTECT1_METAHEADER_REGEX = re.compile('^##MuTect=')
    _MUTECT1_METAHEADER_DICT = re.compile('^##MuTect="(.*)"')
    _MUTECT1_DICT_KEY_NORMAL = 'normal_sample_name'
    _MUTECT1_DICT_KEY_TUMOR = 'tumor_sample_name'

    @staticmethod
    def is_mutect_metaheader(metaheader):
        return _Mutect1Parser._MUTECT1_METAHEADER_REGEX.search(metaheader)

    @staticmethod
    def build_mutect_dict(metaheaders, normal_key, tumor_key):
        def get_mutect_header(metaheaders):
            for metaheader in metaheaders:
                match = _Mutect1Parser._MUTECT1_METAHEADER_DICT.search(metaheader)
                if match:
                    return match.group(1)
            return None

        mutect_dict = {}
        header = get_mutect_header(metaheaders)
        for token in header.split():
            try:
                key, value = token.split("=")
                if key == _Mutect1Parser._MUTECT1_DICT_KEY_NORMAL:
                    mutect_dict[normal_key] = value
                elif key == _Mutect1Parser._MUTECT1_DICT_KEY_TUMOR:
                    mutect_dict[tumor_key] = value
            except ValueError:
                pass
        return mutect_dict

class _Mutect2Parser(object):
    _MUTECT2_METAHEADER_REGEX = re.compile('^##GATKCommandLine.*?=<.*ID=Mu[tT]ect2')
    _MUTECT2_METAHEADER_SAMPLE_REGEX = re.compile('^##SAMPLE=<ID=(.*?),SampleName=(.*?),.*')
    _MUTECT2_METAHEADER_COMMAND_REGEX = re.compile('^##GATKCommandLine.*?=<.*ID=Mu[tT]ect2.*CommandLine.*?="(.*?)"')

    @staticmethod
    def is_mutect_metaheader(metaheader):
        return _Mutect2Parser._MUTECT2_METAHEADER_REGEX.search(metaheader)

    @staticmethod
    def _mutect_dict_from_sample_metalines(metaheaders, normal_key, tumor_key):
        mutect_keys = {'NORMAL': normal_key, 'TUMOR': tumor_key}
        mutect_dict = {}
        for metaheader in metaheaders:
            match = _Mutect2Parser._MUTECT2_METAHEADER_SAMPLE_REGEX.search(metaheader)
            if match and match.group(1) in mutect_keys:
                key = mutect_keys[match.group(1)]
                mutect_dict[key] = match.group(2)
        return mutect_dict

    @staticmethod
    def _mutect_dict_from_command_line(metaheaders, normal_key, tumor_key):
        def get_mutect_header(metaheaders):
            for metaheader in metaheaders:
                match = _Mutect2Parser._MUTECT2_METAHEADER_COMMAND_REGEX.search(metaheader)
                if match:
                    return match.group(1)
            return None

        command_line_args = get_mutect_header(metaheaders).split()
        header_parser = argparse.ArgumentParser(add_help=False)
        header_parser.add_argument("--normal-sample")
        header_parser.add_argument("--tumor-sample")
        args, _ = header_parser.parse_known_args(command_line_args)
        mutect_dict = {}
        if args.normal_sample:
            mutect_dict[normal_key] = args.normal_sample
        if args.tumor_sample:
            mutect_dict[tumor_key] = args.tumor_sample
        return mutect_dict

    @staticmethod
    def build_mutect_dict(metaheaders, normal_key, tumor_key):
        mutect_dict = _Mutect2Parser._mutect_dict_from_sample_metalines(metaheaders, normal_key, tumor_key)
        if not mutect_dict:
            mutect_dict = _Mutect2Parser._mutect_dict_from_command_line(metaheaders, normal_key, tumor_key)
        return mutect_dict

def _get_mutect_parser(metaheaders):
    for metaheader in metaheaders:
        if _Mutect1Parser.is_mutect_metaheader(metaheader):
            return _Mutect1Parser
        elif _Mutect2Parser.is_mutect_metaheader(metaheader):
            return _Mutect2Parser
    return None

class Mutect(object):
    #pylint: disable=too-few-public-methods
    """Recognize and transform MuTect VCFs to standard Jacquard format.

    MuTect VCFs are blessedly compliant and straightforward to translate, with
    the following exception. The incoming header has the sample name values
    (derived from the input alignments). To play well with other callers like
    Strelka and VarScan, the sample headers are replaced with Normal and Tumor.
    """
    _NORMAL_SAMPLE_KEY = "normal_sample_name"
    _TUMOR_SAMPLE_KEY = "tumor_sample_name"

    def __init__(self):
        self.name = "MuTect"
        self.abbr = "MT"

    @staticmethod
    def _is_mutect_vcf(file_reader):
        if not file_reader.file_name.lower().endswith(".vcf"):
            return False
        vcf_reader = vcf.VcfReader(file_reader)
        return _get_mutect_parser(vcf_reader.metaheaders) != None

    def claim(self, file_readers):
        """Recognizes and claims MuTect VCFs form the set of all input VCFs.

        Each defined caller has a chance to evaluate and claim all the incoming
        files as something that it can process.

        Args:
            file_readers: the collection of currently unclaimed files

        Returns:
            A tuple of unclaimed readers and MuTectVcfReaders.
        """
        unclaimed_readers = []
        vcf_readers = []
        for file_reader in file_readers:
            if self._is_mutect_vcf(file_reader):
                vcf_reader = vcf.VcfReader(file_reader)
                vcf_readers.append(_MutectVcfReader(vcf_reader))
            else:
                unclaimed_readers.append(file_reader)
        return (unclaimed_readers, vcf_readers)

    @staticmethod
    def _build_mutect_dict(metaheaders):
        mutect_parser = _get_mutect_parser(metaheaders)
        return mutect_parser.build_mutect_dict(metaheaders,
                                               Mutect._NORMAL_SAMPLE_KEY,
                                               Mutect._TUMOR_SAMPLE_KEY)

    def _get_new_column_header(self, vcf_reader):
        """Returns a standardized column header.

        MuTect sample headers include the name of input alignment, which is
        nice, but doesn't match up with the sample names reported in Strelka
        or VarScan. To fix this, we replace with NORMAL and TUMOR using the
        MuTect metadata command line to replace them correctly."""
        mutect_dict = self._build_mutect_dict(vcf_reader.metaheaders)

        new_header_list = []
        required_keys = set([self._NORMAL_SAMPLE_KEY, self._TUMOR_SAMPLE_KEY])
        mutect_keys = set(mutect_dict.keys())

        if not required_keys.issubset(mutect_keys):
            raise utils.JQException("Unable to determine normal "
                                    "and tumor sample ordering "
                                    "based on MuTect metaheader.")

        for field_name in vcf_reader.column_header.split("\t"):
            if field_name == mutect_dict[self._NORMAL_SAMPLE_KEY]:
                field_name = "NORMAL"
            elif field_name == mutect_dict[self._TUMOR_SAMPLE_KEY]:
                field_name = "TUMOR"
            new_header_list.append(field_name)

        return "\t".join(new_header_list)

class _MutectVcfReader(object):
    """Adapter that presents a MuTect VCF as a VcfReader.

    This follows the VcfReader interface, delegating calls to the base
    VcfReader, adjusting metaheaders, column header, and individual
    variants as appropriate.

    See VcfReader for more info.
    """
    def __init__(self, vcf_reader):
        self._vcf_reader = vcf_reader
        self._caller = Mutect()
        self.tags = [common_tags.ReportedTag(MUTECT_ABBREVIATION),
                     common_tags.PassedTag(MUTECT_ABBREVIATION),
                     _AlleleFreqTag(),
                     _DepthTag(),
                     _SomaticTag(),
                     _GenotypeTag()]

    def _get_new_metaheaders(self):
        return [tag.metaheader for tag in self.tags]

    @property
    def caller_name(self):
        return self._caller.name

    @property
    def file_name(self):
        return self._vcf_reader.file_name

    def open(self):
        return self._vcf_reader.open()

    def close(self):
        return self._vcf_reader.close()

    @staticmethod
    def expected_file_format():
        return [""]

    @property
    def metaheaders(self):
        new_metaheaders = list(self._vcf_reader.metaheaders)
        new_metaheaders.extend(self._get_new_metaheaders())
        caller_metaheader = "##jacquard.translate.caller={0}".\
                format(self._caller.name)
        new_metaheaders.append(caller_metaheader)

        return new_metaheaders

    @property
    def column_header(self):
        return self._caller._get_new_column_header(self._vcf_reader)

    def tagged_vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield vcf_record

    def vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield self._add_tags(vcf_record)

    def _add_tags(self, vcf_record):
        for tag in self.tags:
            tag.add_tag_values(vcf_record)
        return vcf_record


from __future__ import print_function
from setuptools import setup
from setuptools import find_packages
import sys
import os
import platform
import jacquard

_REQUIRED_PYTHON_VERSION = (2, 7)

def check_python_version():
    if sys.version_info < _REQUIRED_PYTHON_VERSION:
        msg_format = '''
Problem: Python v{0}.{1} or above is required but you are using v{2}.
Please install a supported version of Python and try again.\
'''
        message = msg_format.format(_REQUIRED_PYTHON_VERSION[0],
                                    _REQUIRED_PYTHON_VERSION[1],
                                    platform.python_version())
        print(message, file=sys.stderr)
        sys.exit(1)

def read(*paths):
    """Build a file path from *paths* and return the contents."""
    with open(os.path.join(*paths), 'r') as filename:
        return filename.read()

check_python_version()

setup(name='jacquard',
      version=jacquard.__version__,
      description=('Command-line tools to expedite analysis of '
                   'Variant Call Format (VCF) files.'),
      long_description=(read('README.rst') + '\n\n' +
                        read('INSTALL.rst') + '\n\n' +
                        read('CHANGELOG.rst') + '\n\n' +
                        read('AUTHORS.rst')),
      url='https://github.com/umich-brcf-bioinf/Jacquard',
      author='University of Michigan Bioinformatics Core',
      author_email='bfx-jacquard@umich.edu',
      license='Apache',
      packages=find_packages(exclude=['test*']),
      classifiers=['Development Status :: 4 - Beta',
                   'Environment :: Console',
                   'Intended Audience :: Science/Research',
                   'License :: OSI Approved :: Apache Software License',
                   'Operating System :: OS Independent',
                   'Programming Language :: Python :: 2.7',
                   'Programming Language :: Python :: 3',
                   'Topic :: Scientific/Engineering :: Bio-Informatics'],
      keywords='VCF bioinformatic exome-seq DNA-seq variant-call-format',
      install_requires=['natsort'],
      entry_points={'console_scripts': ['jacquard=jacquard.jacquard:main']},
      test_suite='nose.collector',
      tests_require=['nose', 'testfixtures', 'numpy'],
      zip_safe=False)


"""Interprets VarScan2 VCF files adding Jacquard standard information.

* VarScan VCFs are assumed to have a ".vcf" extension and have a
    "##source=VarScan2" metaheader.
* VarScan produces a separate file for SNPs and indels. Jacquard can process
    either or both.
* Jacquard requires the VarScan VCF outputs. The VarScan workflow has optional
    extra steps to:
    a) partition the results into Germline, LOH, and Somatic files
    b) filter to a subset of high-confidence variants
    If these files are provided in the input, Jacquard will use them to
    flag variants not found in the high-confidence files so that they can be
    optionally filtered out.
* If provided, high-confidence files should follow the naming convention of
    patientName.*.fpfilter.pass
    The high-confidence file suffix can be supplied as a command line arg.

See tag definitions for more info.
"""
from __future__ import print_function, absolute_import, division

from collections import defaultdict, OrderedDict
import os
import re

import jacquard.utils.logger as logger
import jacquard.utils.utils as utils
import jacquard.variant_caller_transforms.common_tags as common_tags
import jacquard.utils.vcf as vcf

_VARSCAN_SOMATIC_HEADER = ("#CHROM|POS|ID|REF|ALT|QUAL|FILTER|INFO|FORMAT|"
                           "NORMAL|TUMOR").replace("|", "\t")
JQ_VARSCAN_TAG = "JQ_VS_"
VARSCAN_ABBREVIATION = "VS"
VERSION = "v2.3 - v2.4"

def _varscan_hc_fileheader(line):
    return line.startswith("chrom\tposition")

class _GenotypeTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(VARSCAN_ABBREVIATION,
                             common_tags.GENOTYPE_TAG,
                             'Jacquard genotype (based on GT)')

    def add_tag_values(self, vcf_record):
        sample_values = {}
        if "GT" in vcf_record.format_tags:
            for sample in vcf_record.sample_tag_values:
                genotype = vcf_record.sample_tag_values[sample]["GT"]
                sample_values[sample] = genotype
            vcf_record.add_sample_tag_value(self.tag_id,
                                            sample_values)

class _AlleleFreqTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    @classmethod
    def _standardize_af(cls, value):
        new_values = []
        for val in value:
            new_val = str(round(float(val.strip("%")) / 100, 6))
            new_values.append(utils.round_digits(new_val))
        return ",".join(new_values)

    def __init__(self):
        super(self.__class__,
              self).__init__(VARSCAN_ABBREVIATION,
                             common_tags.ALLELE_FREQ_TAG,
                             ('Jacquard allele frequency for VarScan: Decimal '
                              'allele frequency rounded to 4 digits (based on '
                              'FREQ)'))

    def add_tag_values(self, vcf_record):
        sample_values = {}
        if "FREQ" in vcf_record.format_tags:
            for sample in vcf_record.sample_tag_values:
                freq = vcf_record.sample_tag_values[sample]["FREQ"].split(",")
                sample_values[sample] = _AlleleFreqTag._standardize_af(freq)
            vcf_record.add_sample_tag_value(self.tag_id, sample_values)

class _DepthTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods
    def __init__(self):
        super(self.__class__,
              self).__init__(VARSCAN_ABBREVIATION,
                             common_tags.DEPTH_TAG,
                             'Jacquard depth for VarScan (based on DP)')

    def add_tag_values(self, vcf_record):
        if "DP" in vcf_record.format_tags:
            sample_values = {}
            for sample in vcf_record.sample_tag_values:
                depth = vcf_record.sample_tag_values[sample]["DP"]
                sample_values[sample] = depth
            vcf_record.add_sample_tag_value(self.tag_id, sample_values)

##TODO (cgates): Make this robust to sample order changes
class _SomaticTag(common_tags.AbstractJacquardTag):
    #pylint: disable=too-few-public-methods

    @staticmethod
    def _somatic_status(sample_index):
        if sample_index == 0:  # it's NORMAL
            return "0"
        else:  # it's TUMOR
            return "1"

    def __init__(self):
        super(self.__class__,
              self).__init__(VARSCAN_ABBREVIATION,
                             common_tags.SOMATIC_TAG,
                             ('Jacquard somatic status for VarScan: '
                              '0=non-somatic,1=somatic (based on SOMATIC info '
                              'tag and if sample is TUMOR)'))

    def add_tag_values(self, vcf_record):
        info_array = vcf_record.info.split(";")
        sample_values = {}

        if "SS=2" in info_array and vcf_record.filter == "PASS":
            for i, sample in enumerate(vcf_record.sample_tag_values):
                sample_values[sample] = _SomaticTag._somatic_status(i)
        else:
            for sample in vcf_record.sample_tag_values:
                sample_values[sample] = "0"

        vcf_record.add_sample_tag_value(self.tag_id, sample_values)

#TODO: (cgates/jebene): All tags should have _TAG_ID as implemented below
class _HCTag(object):
    #pylint: disable=too-few-public-methods
    _FILTERS_TO_REPLACE = set(["", ".", "pass"])
    _TAG_ID = "{}LOW_CONFIDENCE".format(JQ_VARSCAN_TAG)

    def __init__(self, file_reader):
        #pylint: disable=line-too-long
        self.metaheader = ('##FILTER=<ID={},'
                           'Number=1,'
                           'Type=Flag,'
                           'Description="Jacquard high-confidence somatic flag for VarScan. Based on intersection with filtered VarScan variants">')\
                           .format(self._TAG_ID)
        self._hc_loci = self._parse_file_reader(file_reader)

    @staticmethod
    def _parse_file_reader(file_reader):
        hc_loci = set()
        file_reader.open()
        for line in file_reader.read_lines():
            if not _varscan_hc_fileheader(line):
                split_line = line.split("\t")
                hc_loci.add((split_line[0], split_line[1]))
        file_reader.close()

        return hc_loci

    def add_tag_values(self, vcf_record):
        if (vcf_record.chrom, vcf_record.pos) not in self._hc_loci:
            vcf_record.add_or_replace_filter(self._TAG_ID)
        return vcf_record

class Varscan(object):
    #pylint: disable=too-few-public-methods
    """Recognize and transform VarScan VCFs to standard Jacquard format."""

    _DEFAULT_REGEX = "Somatic.hc.fpfilter.pass"

    def __init__(self, args=None):
        self.name = "VarScan"
        self.abbr = "VS"
        self.meta_header = "##jacquard.normalize_varscan.sources={0},{1}\n"

        self.hc_file_pattern = self._get_hc_file_pattern(args)

    @staticmethod
    def _get_hc_file_pattern(args):
        msg = ("The specified regex [{}] could not be compiled. "
               "Review inputs and try again")
        if args and args.varscan_hc_filter_file_regex:
            try:
                compiled_regex = re.compile(args.varscan_hc_filter_file_regex)
            except:
                raise utils.UsageError(msg, args.varscan_hc_filter_file_regex)
        else:
            try:
                compiled_regex = re.compile(Varscan._DEFAULT_REGEX)
            except:
                raise utils.UsageError(msg, Varscan._DEFAULT_REGEX)

        return compiled_regex

    @staticmethod
    def _validate_filter_file(file_reader):
        header_is_valid = False
        file_is_empty = True
        file_reader.open()
        for line in file_reader.read_lines():
            file_is_empty = False
            if _varscan_hc_fileheader(line):
                header_is_valid = True
                break
        file_reader.close()

        if file_is_empty or header_is_valid:
            return file_reader
        return None

    @staticmethod
    def _is_varscan_vcf(file_reader):
        if file_reader.file_name.endswith(".vcf"):
            vcf_reader = vcf.VcfReader(file_reader)
            return "##source=VarScan2" in vcf_reader.metaheaders
        return False

    def _is_varscan_hc_filename(self, file_reader):
        return self.hc_file_pattern.search(file_reader.file_name)

    @staticmethod
    def _raise_invalid_filter_exception(invalid_filter_files):
        if invalid_filter_files[5:]:
            omitted_files = ("...({} file(s) omitted)")\
                            .format(len(invalid_filter_files[5:]))
        else:
            omitted_files = ""
        first_five_fnames = [i.file_name for i in invalid_filter_files[:5]]
        raise utils.JQException("The [{}] input files [{}{}] match "
                                "high-confidence file names, but the "
                                "file header is invalid or missing. "
                                "Review inputs and try again.",
                                len(invalid_filter_files),
                                first_five_fnames,
                                omitted_files)

    @staticmethod
    def _raise_error_for_unpaired_files(unpaired_files,
                                        found_orphan,
                                        missing_counterpart):
        singleton_error_msg = ("The VarScan {0} [{{}}] has no "
                               "matching {1}.").format(found_orphan,
                                                       missing_counterpart)
        multiple_error_msg = ("[{{1}}] VarScan {0}s [{{}}] did "
                              "not have a matching {1}. See log for more "
                              "details. Review inputs and try "
                              "again.").format(found_orphan,
                                               missing_counterpart)
        total_unpaired_files = len(unpaired_files)
        msg = None
        if total_unpaired_files == 1:
            msg = singleton_error_msg.format(unpaired_files[0])
        else:
            cutoff = 5
            files = unpaired_files[0:min(cutoff, total_unpaired_files)]
            unpaired_hc_files_list = ", ".join(files)
            if total_unpaired_files > cutoff:
                omitted = total_unpaired_files - cutoff
                unpaired_hc_files_list += ", ...({} file(s) omitted)".\
                        format(omitted)
            msg = multiple_error_msg.format(total_unpaired_files,
                                             unpaired_hc_files_list)
        raise utils.UsageError(msg)

    def _find_varscan_files(self, file_readers):
        unclaimed_set = set()
        prefix_file_readers = OrderedDict()
        filter_files = set()

        for file_reader in file_readers:
            if self._is_varscan_vcf(file_reader):
                prefix, _ = os.path.splitext(file_reader.file_name)
                prefix_file_readers[prefix] = file_reader
            elif self._is_varscan_hc_filename(file_reader):
                filter_files.add(file_reader)
            else:
                unclaimed_set.add(file_reader)

        if not filter_files:
            if not self.hc_file_pattern.match(Varscan._DEFAULT_REGEX):
                msg = ("The VarScan high-confidence filename regex [{}] "
                       "didn't match any files in the input directory. "
                       "The beginning of the high-confidence filename must "
                       "exactly match a VCF filename up to the .vcf extension. "
                       "Review inputs/command options and try again.")
                raise utils.UsageError(msg, self.hc_file_pattern.pattern)

        return prefix_file_readers, filter_files, unclaimed_set

    @staticmethod
    def _split_prefix_by_patient(prefix_vcf_readers):
        prefix_by_patients = defaultdict(list)
        snp_indel = ["snp", "indel"]
        for prefix in prefix_vcf_readers:
            patient_names = [i for i in prefix.split(".") if i not in snp_indel]
            patient_name = ".".join(patient_names)
            prefix_by_patients[patient_name].append(prefix)

        return prefix_by_patients

    @staticmethod
    def _validate_vcf_readers(prefix_by_patients):
        number_of_files = set()
        for file_names in prefix_by_patients.values():
            if len(file_names) == 1:
                for file_name in file_names:
                    if re.search("snp", file_name):
                        msg = "VarScan VCF [{}] has no indel file."
                        logger.error(msg, file_name)
                    elif re.search("indel", file_name):
                        msg = "VarScan VCF [{}] has no snp file."
                        logger.error(msg, file_name)
            number_of_files.add(len(file_names))

        if len(number_of_files) > 1:
            msg = ("Some Varscan VCFs were missing either a snp or indel file. "
                   "Review inputs/command options and try again.")
            raise utils.JQException(msg)

    @staticmethod
    def _remove_paired_filters(vcf_dict, filter_dict):
        for filter_file in vcf_dict.values():
            try:
                filter_dict.pop(filter_file)
            except KeyError:
                pass
        return filter_dict

    @staticmethod
    def _dictionaries_to_tuples(vcf_dict, filter_dict):
        flipped_filter = OrderedDict((y, x) for x, y in filter_dict.items())
        pairs = vcf_dict.copy()
        pairs.update(flipped_filter)
        return [(k, v) for k, v in pairs.items()]

    def _pair_files(self, prefixes, filter_files):
        vcf_dict = OrderedDict()
        filter_dict = OrderedDict()
        invalid_filter_files = []

        for prefix, file_reader in prefixes.items():
            vcf_dict[file_reader] = None
            for filter_reader in filter_files:
                filter_dict[filter_reader] = None

                if filter_reader.file_name.startswith(prefix):
                    if self._validate_filter_file(filter_reader):
                        vcf_dict[file_reader] = filter_reader
                    else:
                        invalid_filter_files.append(filter_reader)

        if invalid_filter_files:
            self._raise_invalid_filter_exception(invalid_filter_files)
        filter_dict = self._remove_paired_filters(vcf_dict, filter_dict)

        return self._dictionaries_to_tuples(vcf_dict, filter_dict)

    def _validate_vcf_hc_pairs(self, vcf_hc_pairs):
        hc_files = [i[1] for i in vcf_hc_pairs if i[1]]
        if not hc_files:
            return

        unpaired_vcf_files = []
        unpaired_hc_files = []

        for vcf_file_reader, hc_file_reader in vcf_hc_pairs:
            if vcf_file_reader and not hc_file_reader:
                unpaired_vcf_files.append(vcf_file_reader.file_name)
            if hc_file_reader and not vcf_file_reader:
                unpaired_hc_files.append(hc_file_reader.file_name)

        if unpaired_hc_files:
            self._raise_error_for_unpaired_files(unpaired_hc_files,
                                                "high-confidence file",
                                                "VCF file")
        elif unpaired_vcf_files:
            self._raise_error_for_unpaired_files(unpaired_vcf_files,
                                                "VCF file",
                                                "high-confidence file")

    @staticmethod
    def _create_vcf_readers(pair_tuples):
        vcf_readers = []
        for vcf_file_reader, hc_file_reader in pair_tuples:
            if vcf_file_reader and hc_file_reader:
                vcf_reader = _VarscanVcfReader(vcf.VcfReader(vcf_file_reader),
                                               hc_file_reader)
            elif vcf_file_reader and not hc_file_reader:
                vcf_reader = _VarscanVcfReader(vcf.VcfReader(vcf_file_reader))
            vcf_readers.append(vcf_reader)

        return vcf_readers

    def claim(self, file_readers):
        """Recognizes and claims VarScan VCFs form the set of all input VCFs.

        Each defined caller has a chance to evaluate and claim all the incoming
        files as something that it can process. Since VarScan can claim
        high-confidence files as well, this process is significantly more
        complex than for other callers.

        Args:
            file_readers: the collection of currently unclaimed files

        Returns:
            A tuple of unclaimed readers and VarScanVcfReaders.
        """

        (prefix_to_readers,
         filter_files,
         unclaimed_set) = self._find_varscan_files(file_readers)

        prefix_by_patients = self._split_prefix_by_patient(prefix_to_readers)
        self._validate_vcf_readers(prefix_by_patients)
        vcf_hc_pairs = self._pair_files(prefix_to_readers, filter_files)
        self._validate_vcf_hc_pairs(vcf_hc_pairs)
        vcf_readers = self._create_vcf_readers(vcf_hc_pairs)

        return list(unclaimed_set), vcf_readers

class _VarscanVcfReader(object):
    """Adapter that presents a VarScan VCF as a VcfReader.

    This follows the VcfReader interface, delegating calls to the base
    VcfReader, adjusting metaheaders and individual
    variants as appropriate.

    See VcfReader for more info.
    """
    def __init__(self, vcf_reader, som_hc_file_reader=None):
        self._vcf_reader = vcf_reader
        self._som_hc_file_reader = som_hc_file_reader
        self._caller = Varscan()
        self.tags = [common_tags.ReportedTag(VARSCAN_ABBREVIATION),
                     common_tags.PassedTag(VARSCAN_ABBREVIATION),
                     _AlleleFreqTag(),
                     _DepthTag(),
                     _SomaticTag(),
                     _GenotypeTag()]

        if som_hc_file_reader:
            self.tags.insert(0, _HCTag(som_hc_file_reader))

    @property
    def file_name(self):
        return self._vcf_reader.file_name

    def _get_new_metaheaders(self):
        return [tag.metaheader for tag in self.tags]

    @property
    def caller_name(self):
        return self._caller.name

    def open(self):
        return self._vcf_reader.open()

    def close(self):
        return self._vcf_reader.close()

    @staticmethod
    def expected_file_format():
        return ["snp", "indel"]

    @property
    def metaheaders(self):
        new_metaheaders = list(self._vcf_reader.metaheaders)
        new_metaheaders.extend(self._get_new_metaheaders())
        caller_metaheader = "##jacquard.translate.caller={0}".\
                format(self._caller.name)
        new_metaheaders.append(caller_metaheader)

        return new_metaheaders

    @property
    def column_header(self):
        return self._vcf_reader.column_header

    def tagged_vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield vcf_record

    def vcf_records(self):
        for vcf_record in self._vcf_reader.vcf_records():
            yield self._add_tags(vcf_record)

    def _add_tags(self, vcf_record):
        for tag in self.tags:
            tag.add_tag_values(vcf_record)
        return vcf_record


"""Classes to summarize zscore data for a sample-variant.

A collection of individual Tag classes hold the metaheader and logic to
transform Jacquard-standardized VcfRecords.

These transforms combine info from a single sample-variant tag with aggregate
info about all sample-variants in the VCF. For example, the zscore of a
sample-variant depth is:
    (my_depth - average of all depths) / stddev of all depths
For this reason, zscore values require the full VCF as one of the inputs.
"""
from __future__ import print_function, absolute_import, division

import math

import jacquard.utils.utils as utils
import jacquard.variant_caller_transforms.common_tags as common_tags

_JQ_SUMMARY_TAG = "JQ_SUMMARY_"
SUMMARY_TAG = "SUMMARY"

SUMMARY_ALLELE_FREQ_ZSCORE = common_tags.TagType("AF_ZSCORE", "Float", "1")
SUMMARY_DEPTH_ZSCORE = common_tags.TagType("DP_ZSCORE", "Float", "1")

#TODO: (cgates): Define tag ids as public, class-level constants so dependent
# tags can reference them directly
class _AlleleFreqZScoreTag(common_tags.AbstractJacquardTag):
    #TODO: (jebene) change the way allelefreqzscore understands the tags
    #it's dependent on (_range_tag should not be defined here)
    _RANGE_TAG = "{0}AF_RANGE".format(_JQ_SUMMARY_TAG)

    def __init__(self, vcf_reader):
        super(self.__class__,
              self).__init__(SUMMARY_TAG,
                             SUMMARY_ALLELE_FREQ_ZSCORE,
                             ('Concordance of reported allele frequencies '
                              'across callers: [(this AF range - mean AF range)'
                              '/standard dev(all AF ranges)]. Values with null '
                              'or missing AF range will be assigned zscore of '
                              '\'.\'; for multi-valued ranges, zscore is of '
                              'largest range.'))
        self.tag = _ZScoreTag(self.tag_id,
                              self.metaheader,
                              self._RANGE_TAG,
                              vcf_reader)
    @property
    def metaheaders(self):
        return self.tag.metaheaders

    def add_tag_values(self, vcf_record):
        self.tag.add_tag_values(vcf_record)

class _DepthZScoreTag(common_tags.AbstractJacquardTag):
    #TODO: (jebene) change the way depthzscore understands the tags
    #it's dependent on (_range_tag should not be defined here)
    _RANGE_TAG = "{0}DP_RANGE".format(_JQ_SUMMARY_TAG)

    def __init__(self, vcf_reader):
        super(self.__class__,
              self).__init__(SUMMARY_TAG,
                             SUMMARY_DEPTH_ZSCORE,
                             ('Concordance of reported depth across callers: '
                              '[(this DP range - mean DP range)/standard '
                              'dev(all DP ranges)]. Values with null or '
                              'missing DP range will be assigned zscore '
                              'of \'.\'.'))
        self.tag = _ZScoreTag(self.tag_id,
                              self.metaheader,
                              self._RANGE_TAG,
                              vcf_reader)

    @property
    def metaheaders(self):
        return self.tag.metaheaders

    def add_tag_values(self, vcf_record):
        self.tag.add_tag_values(vcf_record)

class _ZScoreTag(object):
    '''Utility tag to add zscore for dependent tag (e.g. depth or allele freq)

    Given a dependent tag and a vcf reader, calculates mean and stdev on
    construction and then adds zscores for each value.
    '''
    _EXECUTION_FORMAT = "##jacquard.summarize.{0}.{1}_{2}={3}"
    _MAX_PRECISION = 13

    def __init__(self,
                 tag_id,
                 metaheader,
                 dependent_tag_id,
                 vcf_reader):

        self._tag_id = tag_id
        self._dependent_tag_id = dependent_tag_id
        self._mean, self._stdev = self._init_population_stats(vcf_reader,
                                                              dependent_tag_id)
        self._metaheaders = self._init_metaheaders(tag_id,
                                                   metaheader,
                                                   dependent_tag_id,
                                                   self._mean,
                                                   self._stdev)

    def _init_metaheaders(self,
                          tag_id,
                          metaheader,
                          dependent_tag_id,
                          mean,
                          stdev):
        #pylint: disable=too-many-arguments
        metaheaders = []
        metaheaders.append(self._EXECUTION_FORMAT.format(tag_id,
                                                         dependent_tag_id,
                                                         "mean",
                                                         repr(mean)))
        metaheaders.append(self._EXECUTION_FORMAT.format(tag_id,
                                                         dependent_tag_id,
                                                         "stdev",
                                                         repr(stdev)))
        metaheaders.append(metaheader)
        return tuple(metaheaders)

    @property
    def metaheaders(self):
        return self._metaheaders

    def _ok_to_add_tag_values(self, vcf_record):
        return self._stdev and self._dependent_tag_id in vcf_record.format_tags

    @staticmethod
    def _zscore_as_str(zscore):
        if zscore == ".":
            return zscore
        else:
            return utils.round_digits(repr(zscore))

    def add_tag_values(self, vcf_record):
        if not self._ok_to_add_tag_values(vcf_record):
            return

        sample_values = {}
        for sample_name in vcf_record.sample_tag_values:
            zscore = "."
            tag_values = vcf_record.sample_tag_values[sample_name]
            value = self._get_dependent_value(tag_values,
                                              self._dependent_tag_id)

            if  value is not None:
                zscore = (value - self._mean) / self._stdev
            sample_values[sample_name] = self._zscore_as_str(zscore)

        vcf_record.add_sample_tag_value(self._tag_id,
                                        sample_values)

    @staticmethod
    def _get_dependent_value(tag_values, dependent_tag_id):
        '''Extract (float) value of dependent tag or None if absent.'''
        try:
            values = tag_values[dependent_tag_id].split(",")
            return max([float(value) for value in values])
        except KeyError:
            return None
        except ValueError:
            return None

    def _init_population_stats(self, vcf_reader, dependent_tag_id):
        '''Derive mean and stdev.

        Adapted from online variance algorithm from Knuth, The Art of Computer
        Programming, volume 2

        Returns: mean and stdev when len(values) > 1, otherwise (None, None)
            Values rounded to _MAX_PRECISION to ameliorate discrepancies between
            python versions.'''
        #pylint: disable=invalid-name
        n = 0
        mean = 0
        M2 = 0
        try:
            vcf_reader.open()
            for vcf_record in vcf_reader.vcf_records():
                for tag_values in vcf_record.sample_tag_values.values():
                    value = self._get_dependent_value(tag_values,
                                                      dependent_tag_id)
                    if value is not None:
                        n += 1
                        delta = value - mean
                        mean += delta / n
                        M2 += delta * (value - mean)
        finally:
            vcf_reader.close()

        mean = round(mean, self._MAX_PRECISION)

        stdev = 0
        if n == 0:
            mean = None
            stdev = None
        elif n >= 2:
            variance = M2/n
            stdev = round(math.sqrt(variance), self._MAX_PRECISION)

        return mean, stdev

class ZScoreCaller(object):
    """Provides metaheaders for VcfReader; adds summary tags to VcfRecords."""
    def __init__(self, vcf_reader):
        self._tags = [_AlleleFreqZScoreTag(vcf_reader),
                      _DepthZScoreTag(vcf_reader)]
        self._metaheaders = self._init_metaheaders(self._tags)

    @staticmethod
    def _init_metaheaders(tags):
        metaheaders = []
        for tag in tags:
            metaheaders.extend(tag.metaheaders)
        return tuple(metaheaders)

    @property
    def metaheaders(self):
        return self._metaheaders

    def add_tags(self, vcf_record):
        for tag in self._tags:
            tag.add_tag_values(vcf_record)
        return vcf_record.text()


"""Delegates to third-party variant callers.

To keep things simple, fair, and encapsulated route any requests to variant
callers through this module; likewise, don't directly reference individual
variant callers outside of this module.
"""
from __future__ import print_function, absolute_import, division

import jacquard.variant_caller_transforms.mutect as mutect
import jacquard.variant_caller_transforms.strelka as strelka
import jacquard.variant_caller_transforms.varscan as varscan

SUPPORTED_CALLER_VERSIONS = {"VarScan": varscan.VERSION,
                             "MuTect": mutect.VERSION,
                             "Strelka": strelka.VERSION}

class VariantCallerFactory(object):
    #pylint: disable=too-few-public-methods
    def __init__(self, args=None):
        self._callers = [varscan.Varscan(args),
                         strelka.Strelka(),
                         mutect.Mutect()]

    def claim(self, unclaimed_file_readers):
        """Allows each caller to claim incoming files as they are recognized.

        Args:
            unclaimed_file_readers: Usually, all files in the input dir.

        Returns:
            A tuple of unclaimed file readers and claimed VcfReaders. The
            presence of any unclaimed file readers could indicate stray files
            in the input dir.
        """
        claimed_vcf_readers = []
        for caller in self._callers:
            (unclaimed_file_readers,
             translated_vcf_readers) = caller.claim(unclaimed_file_readers)
            claimed_vcf_readers.extend(translated_vcf_readers)

        return unclaimed_file_readers, claimed_vcf_readers


"""Merges a set of VCF files into a single VCF file.

* Merge assumes the incoming VCFs are aligned to the same set of contigs.
* Merge assumes incoming VCFs names follow this pattern:
    patientIdentifier.*.vcf
    Specifically, the first element of the VCF file name should be the patient
    name; for example you mihght have this:
        patientA.mutect.vcf, patientA.strelka.vcf, patientA.varscan.vcf,
        patientB.mutect.vcf, patientB.strelka.vcf, patientB.varscan.vcf,
        etc.
* Merge assumes the sample names (i.e. the VCF sample column headers, typically
    TUMOR and NORMAL) are consistent across the input VCFs. (The preceding
    Jacquard command "translate" ensures this is true.)
* Each incoming VCF record is joined with other VCF records that share the same
    "coordinate", where coordinate is the (chrom, pos, ref, and alt).
* Calls for the same patient/sample are joined into a single column. Each output
    column is the patient name (prefix of the file name) plus the sample name
    from the column header.
* By default, the merged file will have as many records as the distinct set of
    (chrom, pos, ref, alt) across all input files.
    Running Merge with "--include_variants" or "--include_loci" will reduce the
    number of records in the merged file to include only those which conform
    with the desired flag.
* Each variant record will have the minimal set of incoming format tags for
    that variant (i.e. the list of format tags is specific to each record).
* Incoming QUAL and INFO fields are ignored.
* By default, merge only includes "Jacquard" FORMAT tags, but other tags can be
    included through and optional a command line arg.
"""
from __future__ import print_function, absolute_import, division

from collections import defaultdict, OrderedDict
import glob
import os
import re

import natsort
import textwrap

import jacquard.utils.logger as logger
import jacquard.utils.utils as utils
from jacquard.utils.vcf import FileWriter
import jacquard.utils.vcf as vcf

_DEFAULT_INCLUDED_FORMAT_TAGS = ["JQ_.*"]
_MULT_ALT_TAG = "JQ_MULT_ALT_LOCUS"
_JQ_SOMATIC_TAG = "HC_SOM"

_MULT_ALT_HEADER = ('##INFO=<ID={},Number=0,Type=Flag,'
                    'Description="More than one alt allele was seen at this '
                    'locus.">').format(_MULT_ALT_TAG)
_FILE_FORMAT = ["##fileformat=VCFv4.1"]
_FILE_OUTPUT_SUFFIX = "merged"

class _BufferedReader(object):
    """A look-ahead reader that expedites merging.

    Accepts an iterator and returns element (advancing current element) if
    requested element equals next element in collection and None otherwise.
    Never returns StopIteration so cannot be used as iteration control-flow.

    This behavior is useful only because VcfRecord equality is based on their
    coordinate (chrom, pos, ref, alt), so by iterating over a list of
    coordinates, you can either park on your next record or return the
    current record and advance the reader to the next.

    Each incoming VcfReader is wrapped in a a BufferedReader and readers are
    advanced to the next reader when their current coordinate is requested.
    This approach avoids reading all VCFs into memory, but does require a file
    handle for each VCF you are merging and also requires that VcfReaders are
    sorted identically.

    Stylistic note:
    This class must capture the state of the incoming iterator and provide
    modified behavior based on data in that iterator. A small class works ok,
    but suspect there may be a more pythonic way to curry iterator in a
    partial function. Uncertain if that would be clearer/simpler. [cgates]
    """
    #pylint: disable=too-few-public-methods
    def __init__(self, iterator):
        self._iterator = iterator
        try:
            self._current_element = next(self._iterator)
        except StopIteration:
            self._current_element = self._iterator

    def next_if_equals(self, requested_element):
        result = None
        if requested_element == self._current_element:
            result = self._current_element
            self._current_element = self._get_next()
        return result

    def _get_next(self):
        try:
            return next(self._iterator)
        except StopIteration:
            return None

class MergeVcfReader(vcf.VcfReader):
    def __init__(self, file_reader):
        super(self.__class__,self).__init__(file_reader)
        self.format_tags = {}

    def modify_metaheader(self, original_metaheader, transformed_tag):
        updated_metaheader = re.sub(r'(^##FORMAT=.*?[<,]ID=)([^,>]*)',
                                    r'\g<1>%s' % transformed_tag,
                                    original_metaheader)

        self.metaheaders.append(updated_metaheader)
        if original_metaheader in self.metaheaders:
            self.metaheaders.remove(original_metaheader)

    def store_format_tags(self, original_tag, new_tag):
        self.format_tags[original_tag] = new_tag

    @staticmethod
    def modify_format_tag(vcf_record, format_tags):
        for tags in list(vcf_record.sample_tag_values.values()):
            for original_tag, new_tag in list(format_tags.items()):
                if new_tag not in tags and original_tag in tags:
                    tags[new_tag] = tags[original_tag]
                    del tags[original_tag]

        return vcf_record

    def vcf_records(self, format_tags=None, qualified=False):
        """Generates parsed VcfRecord objects.

        Typically called in a for loop to process each vcf record in a
        VcfReader. VcfReader must be opened in advanced and closed when
        complete. Skips all headers.

        Args:
            qualified: When True, sample names are prefixed with file name

        Returns:
            Parsed VcfRecord

        Raises:
            StopIteration: when reader is exhausted.
            TypeError: if reader is closed.
        """
        if qualified:
            sample_names = self.qualified_sample_names
        else:
            sample_names = self.sample_names

        for line in self._file_reader.read_lines():
            if line.startswith("#"):
                continue
            vcf_record = vcf.VcfRecord.parse_record(line, sample_names)
            if format_tags:
                vcf_record = self.modify_format_tag(vcf_record, format_tags)
            yield vcf_record

class _Filter(object):
    """Interprets command-line args to initialize variant and locus filters
    """
    #pylint: disable=too-few-public-methods
    def __init__(self, args):
        self._cell_filters = {"all" : _Filter._include_cell_if_all,
                              "valid": _Filter._include_cell_if_valid,
                              "passed" : _Filter._include_cell_if_passed,
                              "somatic" : _Filter._include_cell_if_somatic}
        self._row_filters = {"all": _Filter._include_row_if_all,
                             "all_passed": _Filter._include_row_if_all_passed,
                             "at_least_one_passed": _Filter.\
                                                    _include_row_if_any_passed,
                             "all_somatic": _Filter._include_row_if_all_somatic,
                             "at_least_one_somatic": _Filter.\
                                                    _include_row_if_any_somatic}
        self._args = args
#         if args.include_all and (args.include_cells or args.include_rows):
#             msg = ('Unable to process command-line arguments. Neither '
#                    '--include_cells nor --include_rows can be specified if '
#                    '--include_all is specified.')
#             raise utils.UsageError(msg)

        if args.include_all:
            cell_filter_strategy = self._cell_filters["all"]
            row_filter_strategy = self._row_filters["all"]
        else:
            cell_filter_strategy = self._cell_filters[args.include_cells]
            row_filter_strategy = self._row_filters[args.include_rows]

        self._cell_filter_strategy = cell_filter_strategy
        self._row_filter_strategy = row_filter_strategy
        self.row_count = 0
        self.rows_excluded = 0
        self.cell_count = 0
        self.cells_excluded = 0

        self.excluded_breakdown = defaultdict(int)

    def include_cell(self, vcf_record):
        self.cell_count += 1
        include = self._cell_filter_strategy(self, vcf_record)
        if not include:
            self.cells_excluded += 1
        return include

    def include_row(self, vcf_records):
        self.row_count += 1
        include = self._row_filter_strategy(vcf_records)
        if not include:
            self.rows_excluded += 1
        return include

    def log_statistics(self):
        cell_msg = "{}% ({}) cells were excluded because (--include_cells={})"
        logger.info(cell_msg,
                    int(round(100 * self.cells_excluded / self.cell_count, 0)),
                    self.cells_excluded,
                    self._args.include_cells)

        row_msg = "{}% ({}) rows were excluded because (--include_rows={})"
        logger.info(row_msg,
                    int(round(100 * self.rows_excluded / self.row_count, 0)),
                    self.rows_excluded,
                    self._args.include_rows)

        for key, count in self.excluded_breakdown.items():
            msg = "{} cells were excluded with [{}]"
            logger.debug(msg, count, key)

    #TODO: (cgates/jebene): Should this be part of VcfRecord?
    @staticmethod
    def _is_somatic(record):
        for sample in record.sample_tag_values:
            for tag in record.sample_tag_values[sample]:
                is_somatic = record.sample_tag_values[sample][tag] == "1"
                if re.search(_JQ_SOMATIC_TAG, tag) and is_somatic:
                    return True
        return False

    def _include_cell_if_all(self, dummy):
        #pylint: disable=no-self-use
        return True

    def _include_cell_if_somatic(self, record):
        include = _Filter._is_somatic(record)
        if not include:
            self.excluded_breakdown["not somatic"] += 1
        return include

    def _include_cell_if_valid(self, record):
        include ="JQ_EXCLUDE" not in record.filter
        if not include:
            self.excluded_breakdown[record.filter] += 1
        return include

    def _include_cell_if_passed(self, record):
        include = record.filter == "PASS"
        if not include:
            self.excluded_breakdown[record.filter] += 1
        return include

    @staticmethod
    def _include_row_if_all(dummy):
        return True

    @staticmethod
    def _include_row_if_all_passed(records):
        for record in records:
            if record.filter != "PASS":
                return False
        return True

    @staticmethod
    def _include_row_if_any_passed(records):
        for record in records:
            if record.filter == "PASS":
                return True
        return False

    @staticmethod
    def _include_row_if_all_somatic(records):
        for record in records:
            if not _Filter._is_somatic(record):
                return False
        return True

    @staticmethod
    def _include_row_if_any_somatic(records):
        for record in records:
            if _Filter._is_somatic(record):
                return True
        return False

def _get_caller_name(merge_vcf_reader):
    for metaheader in merge_vcf_reader.metaheaders:
        tag = re.match("##jacquard.translate.caller=(.*)", metaheader)
        if tag:
            return tag.group(1)
    return False

def _alter_description(metaheader, caller_name):
    return re.sub(r'^##FORMAT=.*?[<,]Description="([,>]*)',
                  r'\g<0>[%s]: ' % caller_name,
                  metaheader)

def _get_format_tags(merge_vcf_readers):
    format_tags = defaultdict(list)
    caller_names = defaultdict(list)
    for merge_vcf_reader in merge_vcf_readers:
        caller_name = _get_caller_name(merge_vcf_reader)
        for tag, metahdr in list(merge_vcf_reader.format_metaheaders.items()):
            format_tags[tag].append(metahdr)
            caller_names[tag].append(caller_name)

    for tag, callers in caller_names.items():
        if len(set(callers)) != 1:
            for i, caller in enumerate(callers):
                format_tags[tag][i] = _alter_description(format_tags[tag][i],
                                                         caller)
        else:
            format_tags[tag] = [format_tags[tag][0]]

    return format_tags

def _disambiguate_format_tags(merge_vcf_readers, format_tags):
    for tag, metaheaders in list(format_tags.items()):
        for merge_vcf_reader in merge_vcf_readers:
            for i, metahdr in enumerate(list(metaheaders)):
                vcf_metaheaders = merge_vcf_reader.format_metaheaders.values()
                if metahdr in list(vcf_metaheaders):
                    if len(metaheaders) > 1:
                        new_tag = "JX{}_{}".format(i+1, tag)
                        format_tags[new_tag] = metaheaders

                        merge_vcf_reader.modify_metaheader(metahdr, new_tag)
                        merge_vcf_reader.store_format_tags(tag, new_tag)
                    else:
                        merge_vcf_reader.store_format_tags(tag, tag)

    return format_tags

def _build_format_tags(format_tag_regex, vcf_readers):
    retained_tags = set()
    regexes_used = set()

    for vcf_reader in vcf_readers:
        for tag_regex in format_tag_regex:
            for original_tag, new_tag in list(vcf_reader.format_tags.items()):
                if re.match(tag_regex + "$", original_tag):
                    retained_tags.add(new_tag)
                    regexes_used.add(tag_regex)

    if len(retained_tags) == 0:
        msg = ("The specified format tag regex [{}] would exclude all format "
               "tags. Review inputs/usage and try again")
        raise utils.JQException(msg, format_tag_regex)

    unused_regexes = set(format_tag_regex).difference(regexes_used)
    if unused_regexes:
        for unused_regex in unused_regexes:
            msg = ("In the specified list of regexes {}, the regex [{}] does "
                   "not match any format tags; this expression may be "
                   "irrelevant.")
            logger.warning(msg, format_tag_regex, unused_regex)
    return sorted(list(retained_tags))

def _compile_metaheaders(incoming_headers,
                         vcf_readers,
                         all_sample_names,
                         contigs_to_keep,
                         format_tags_to_keep,
                         info_tags_to_keep):
    #pylint: disable=too-many-arguments
    ordered_metaheaders = list(incoming_headers)
    all_info_metaheaders = {}
    all_format_metaheaders = {}
    all_contig_metaheaders = {}

    for vcf_reader in vcf_readers:
        all_info_metaheaders.update(vcf_reader.info_metaheaders)
        all_format_metaheaders.update(vcf_reader.format_metaheaders)
        all_contig_metaheaders.update(vcf_reader.contig_metaheaders)

    all_info_metaheaders[_MULT_ALT_TAG] = _MULT_ALT_HEADER

    if all_contig_metaheaders:
        for contig_id in contigs_to_keep:
            ordered_metaheaders.append(all_contig_metaheaders[contig_id])
    for tag in info_tags_to_keep:
        ordered_metaheaders.append(all_info_metaheaders[tag])
    for tag in format_tags_to_keep:
        ordered_metaheaders.append(all_format_metaheaders[tag])

    sorted_metaheaders = utils.sort_metaheaders(ordered_metaheaders)

    column_header = vcf_readers[0].column_header.split("\t")[0:9]
    column_header.extend(all_sample_names)
    sorted_metaheaders.append("\t".join(column_header))

    return sorted_metaheaders

def _write_metaheaders(file_writer, all_headers):
    file_writer.write("\n".join(all_headers) + "\n")

def _create_vcf_readers(file_readers):
    merge_vcf_readers = []
    for file_reader in file_readers:
#         vcf_reader = vcf.VcfReader(file_reader)
        merge_vcf_reader = MergeVcfReader(file_reader)
        merge_vcf_readers.append(merge_vcf_reader)

    return merge_vcf_readers

def _create_buffered_readers(vcf_readers):
    buffered_readers = []
    for vcf_reader in vcf_readers:
        vcf_reader.open()
        records = vcf_reader.vcf_records(vcf_reader.format_tags, qualified=True)
        buffered_readers.append(_BufferedReader(records))

    return buffered_readers

def _build_coordinates(vcf_readers):
    coordinate_set = set()
    mult_alts = defaultdict(set)

    for i, vcf_reader in enumerate(vcf_readers):
        logger.info("Reading [{}] ({}/{})",
                    os.path.basename(vcf_reader.file_name),
                    i + 1,
                    len(vcf_readers))
        try:
            vcf_reader.open()

            for vcf_record in vcf_reader.vcf_records():
                coordinate_set.add(vcf_record.get_empty_record())
                ref_alt = vcf_record.ref, vcf_record.alt
                locus = vcf_record.chrom, vcf_record.pos
                mult_alts[locus].add(ref_alt)
        finally:
            vcf_reader.close()

    for vcf_record in coordinate_set:
        ref_alts_for_this_locus = mult_alts[vcf_record.chrom, vcf_record.pos]
        inferred_mult_alt = len(ref_alts_for_this_locus) > 1
        explicit_mult_alt = "," in next(iter(ref_alts_for_this_locus))[1]

        if inferred_mult_alt or explicit_mult_alt:
            vcf_record.add_info_field(_MULT_ALT_TAG)

    return sorted(list(coordinate_set))

def _write_headers(reader, file_writer):
    headers = reader.metaheaders
    headers.append(reader.column_header)

    file_writer.write("\n".join(headers) + "\n")

def _sort_vcf(reader, sorted_dir):
    vcf_records = []
    reader.open()
    for vcf_record in reader.vcf_records():
        vcf_records.append(vcf_record)

    reader.close()
    vcf_records.sort()
    writer = FileWriter(os.path.join(sorted_dir,
                                     reader.file_name))
    writer.open()
    writer.write("\n".join(reader.metaheaders) + "\n")
    writer.write(reader.column_header + "\n")
    for vcf_record in vcf_records:
        writer.write(vcf_record.text())

    writer.close()
    reader = MergeVcfReader(vcf.FileReader(writer.output_filepath))
    return reader

def _get_unsorted_readers(vcf_readers):
    unsorted_readers = []
    for i, reader in enumerate(vcf_readers):
        logger.info("Checking sort order of [{}] ({}/{})",
                    reader.file_name,
                    i+1,
                    len(vcf_readers)
                    )
        previous_record = None
        reader.open()
        for vcf_record in reader.vcf_records():
            if previous_record and vcf_record < previous_record:
                logger.debug("VCF file:chrom:pos [{}:{}:{}] is out of order"
                             .format(reader.file_name,
                                     vcf_record.chrom,
                                     vcf_record.pos))
                unsorted_readers.append(reader)
                break
            else:
                previous_record = vcf_record
        reader.close()
    return unsorted_readers

def _sort_readers(vcf_readers, output_path):
    unsorted_readers = _get_unsorted_readers(vcf_readers)
    sorted_readers = []
    unsorted_count = 0
    if unsorted_readers:
        sorted_dir = os.path.join(os.path.dirname(output_path), "tmp")
        if not os.path.isdir(sorted_dir):
            os.makedirs(sorted_dir)

    for reader in vcf_readers:
        if reader in unsorted_readers:
            unsorted_count += 1
            logger.info("Sorting vcf [{}] ({}/{})",
                        reader.file_name,
                        unsorted_count,
                        len(unsorted_readers))
            reader = _sort_vcf(reader, sorted_dir)
        sorted_readers.append(reader)
    return sorted_readers

def _build_merged_record(coordinate,
                         vcf_records,
                         all_sample_names,
                         tags_to_keep):

    all_tags = set()
    sparse_matrix = {}

    for record in vcf_records:
        for sample, tags in record.sample_tag_values.items():
            if sample not in sparse_matrix:
                sparse_matrix[sample] = {}
            for tag, value in list(tags.items()):
                if tag in tags_to_keep:
                    all_tags.add(tag)
                    sparse_matrix[sample][tag] = value

    full_matrix = OrderedDict()
    for sample in all_sample_names:
        full_matrix[sample] = OrderedDict()
        for tag in sorted(all_tags):
            try:
                full_matrix[sample][tag] = sparse_matrix[sample][tag]
            except KeyError:
                full_matrix[sample][tag] = "."

    merged_record = vcf.VcfRecord(coordinate.chrom,
                                  coordinate.pos,
                                  coordinate.ref,
                                  coordinate.alt,
                                  coordinate.vcf_id,
                                  coordinate.qual,
                                  coordinate.filter,
                                  coordinate.info,
                                  sample_tag_values=full_matrix)

    return merged_record

def _pull_matching_records(filter_strategy, coordinate, buffered_readers):
    vcf_records = []
    for buffered_reader in buffered_readers:
        record = buffered_reader.next_if_equals(coordinate)
        if record and filter_strategy.include_cell(record):
            vcf_records.append(record)
    if filter_strategy.include_row(vcf_records):
        return vcf_records
    else:
        return []

def _build_sample_list(vcf_readers):
    all_sample_names = set()
    patient_to_file = defaultdict(list)

    for vcf_reader in vcf_readers:
        for sample_name in vcf_reader.qualified_sample_names:
            all_sample_names.add(sample_name)
            patient_to_file[sample_name].append(vcf_reader.file_name)
    sorted_sample_names = natsort.natsorted(all_sample_names)

    sorted_patient_to_file = OrderedDict()
    for sample_name in sorted_sample_names:
        sorted_patient_to_file[sample_name] = patient_to_file[sample_name]

    merge_metaheaders = _build_merge_metaheaders(sorted_patient_to_file)

    return sorted_sample_names, merge_metaheaders

def _build_info_tags(coordinates):
    all_info_tags = set()
    for record in coordinates:
        all_info_tags.update(record.info_dict.keys())
    ordered_tags = sorted(all_info_tags)

    return ordered_tags

def _build_contigs(coordinates):
    all_contigs = set()
    for record in coordinates:
        all_contigs.add(record.chrom)
    ordered_contigs = natsort.natsorted(all_contigs)

    return ordered_contigs

def _build_merge_metaheaders(patient_to_file):
    metaheaders = []
    metaheader_fmt = "##jacquard.merge.sample=<Column={0},Name={1},Source={2}>"
    for i, patient in enumerate(patient_to_file):
        filenames_string = "|".join(patient_to_file[patient])
        metaheader = metaheader_fmt.format(i + 1, patient, filenames_string)
        metaheaders.append(metaheader)

    return metaheaders

def _build_writers_to_readers(vcf_readers, output_path):
    writers_to_readers = {}
    basename, extension = os.path.splitext(output_path)
    new_filename = basename + ".merged" + extension

    file_writer = vcf.FileWriter(new_filename)
    writers_to_readers[file_writer] = vcf_readers

    return writers_to_readers

def _get_readers_per_patient(file_readers):
    readers_per_patient = defaultdict(list)
    for file_reader in file_readers:
        file_reader.open()
        patient = file_reader.file_name.split(".")[0]
        for line in file_reader.read_lines():
            caller_meta_header = "##jacquard.translate.caller"
            if line.startswith(caller_meta_header):
                readers_per_patient[patient].append(line.split("=")[1])
        file_reader.close()

    return OrderedDict(sorted(readers_per_patient.items()))

def _validate_consistent_samples(file_readers):
    readers_per_patient = _get_readers_per_patient(file_readers)

    all_callers = set()
    for callers in readers_per_patient.values():
        all_callers.update(callers)
    warning = 0
    for patient, callers in readers_per_patient.items():
        missing_callers = set(all_callers).difference(set(callers))
        if missing_callers:
            warning = 1
            msg = "Sample [{}] is missing VCF(s): {}"
            logger.warning(msg,
                           patient,
                           sorted(list(missing_callers)))
    if warning:
        msg = "Some samples appear to be missing VCF(s)"
        logger.warning(msg)

def _validate_consistent_input(vcf_readers, include_all):
    if not include_all:
        untranslated_readers = []
        translated = False
        untranslated = False

        for vcf_reader in vcf_readers:
            if '##jacquard.translate.caller' in vcf_reader.metaheaders:
                translated = True
            else:
                untranslated = True
                untranslated_readers.append(vcf_reader)

        if translated and untranslated:
            msg = ("Some input VCFs [{}] were not translated by Jacquard. "
                    "Review input and/or use --included_all flag")\
                   .format(untranslated_readers)
            raise utils.UsageError(msg)

def _merge_records(vcf_readers,
                   coordinates,
                   filter_strategy,
                   all_sample_names,
                   format_tags_to_keep,
                   file_writer):
    #pylint: disable=too-many-arguments
    row_count = 0
    next_breakpoint = 0
    buffered_readers = _create_buffered_readers(vcf_readers)

    for coordinate in coordinates:
        vcf_records = _pull_matching_records(filter_strategy,
                                             coordinate,
                                             buffered_readers)

        if vcf_records:
            merged_record = _build_merged_record(coordinate,
                                                 vcf_records,
                                                 all_sample_names,
                                                 format_tags_to_keep)
            file_writer.write(merged_record.text())

        row_count += 1
        progress = 100 * row_count / len(coordinates)
        if progress > next_breakpoint:
            logger.info("Merging: {} rows processed ({}%)",
                        row_count,
                        next_breakpoint)
            next_breakpoint = 10 * int(progress/10) + 10

    logger.info("Merge complete: {} rows processed (100%)", row_count)
    filter_strategy.log_statistics()

def _get_format_tag_regex(args):
    if args.tags and args.include_all:
        msg =("Unable to process command-line arguments. --include_format_tags cannot "
              "be specified if --include_all is specified.")
        raise utils.UsageError(msg)

    if args.include_all:
        format_tag_regex = ['.*']
    elif args.tags:
        format_tag_regex = args.tags.split(",")
    else:
        format_tag_regex = _DEFAULT_INCLUDED_FORMAT_TAGS

    return format_tag_regex

def add_subparser(subparser):
    #pylint: disable=line-too-long

    parser = subparser.add_parser("merge",
                                  formatter_class=utils._JacquardHelpFormatter,
                                  usage=[" [--include_format_tags=JQ_.*]\n"
                                         "\t\t\t\t\t[--include_cells=valid]\n"
                                         "\t\t\t\t\t[--include_rows=at_least_one_somatic]"],
                                  help="Accepts a directory of VCFs and returns a single merged VCF file.")
    parser.add_argument("input", help="Directory containing VCF files. Other file types ignored")
    parser.add_argument("output", help="VCF file")
    parser.add_argument("--include_all", action="store_true", help=textwrap.fill("Useful when merging untranslated VCFs which have already been filtered to passing variants. Equivalent to --include_format_tags='.*' --include_cells=all --include_rows=all"))
    parser.add_argument("--include_cells",
                        choices=["all", "valid", "passed", "somatic"],
                        default="valid",
                        help=("all: Include all variants\n"
                              "valid: Only include valid variants (invalid variants become '.')\n"
                              "passed: Only include variants which passed their respective filter (failed variants become '.')\n"
                              "somatic: Only include somatic variants (non-somatic variants become '.')"),
                        metavar="")
    parser.add_argument("--include_rows",
                        choices=["all", "at_least_one_passed", "all_passed", "at_least_one_somatic", "all_somatic"],
                        default="at_least_one_somatic",
                        help=("all: Include all loci\n"
                              "at_least_one_passed: Include loci where at least one variant passed\n"
                              "at_least_one_somatic: Include loci where at least one variant was somatic\n"
                              "all_passed: Include loci where all variants passed\n"
                              "all_somatic: Include loci where all variants were somatic\n"),
                        metavar="")
    parser.add_argument("--include_format_tags", dest='tags', help=textwrap.fill("Comma-separated user-defined list of regular expressions for format tags to be included in output"), metavar="")
    parser.add_argument("--force", action='store_true', help="Overwrite contents of output directory")
    parser.add_argument("--log_file", help="Log file destination", metavar="")
    parser.add_argument("-v", "--verbose", action='store_true')

def _predict_output(args):
    desired_output_files = set([os.path.basename(args.output)])

    return desired_output_files

def report_prediction(args):
    return _predict_output(args)

def get_required_input_output_types():
    return ("directory", "file")

#TODO (cgates): Validate should actually validate
def validate_args(dummy):
    pass

def execute(args, execution_context):
    input_path = os.path.abspath(args.input)
    output_path = os.path.abspath(args.output)
    filter_strategy = _Filter(args)
    format_tag_regex = _get_format_tag_regex(args)

    input_files = sorted(glob.glob(os.path.join(input_path, "*.vcf")))
    file_readers = [vcf.FileReader(i) for i in input_files]
    _validate_consistent_samples(file_readers)

    try:
        file_writer = vcf.FileWriter(output_path)
        file_writer.open()

#TODO: jebene: _build_format_tags, _built_sample_list, _build_info_tags,
#and _build_contigs behave differently. It seems like we could make the
#signatures of these methods more similar or even combine some methods to
#reduce excess iterations over the coordinates/vcf_readers
        merge_vcf_readers = _create_vcf_readers(file_readers)
        _validate_consistent_input(merge_vcf_readers, args.include_all)
        merge_vcf_readers = _sort_readers(merge_vcf_readers, output_path)
        format_tags = _get_format_tags(merge_vcf_readers)
        _disambiguate_format_tags(merge_vcf_readers, format_tags)
        format_tags_to_keep = _build_format_tags(format_tag_regex,
                                                 merge_vcf_readers)
        (all_sample_names,
         merge_metaheaders) = _build_sample_list(merge_vcf_readers)

        coordinates = _build_coordinates(merge_vcf_readers)
        info_tags_to_keep = _build_info_tags(coordinates)
        contigs_to_keep = _build_contigs(coordinates)
        incoming_headers = _FILE_FORMAT + execution_context + merge_metaheaders
        headers = _compile_metaheaders(incoming_headers,
                                       merge_vcf_readers,
                                       all_sample_names,
                                       contigs_to_keep,
                                       format_tags_to_keep,
                                       info_tags_to_keep)

        _write_metaheaders(file_writer, headers)

        _merge_records(merge_vcf_readers,
                       coordinates,
                       filter_strategy,
                       all_sample_names,
                       format_tags_to_keep,
                       file_writer)
    finally:
        for vcf_reader in merge_vcf_readers:
            vcf_reader.close()
        file_writer.close()