class FileEntry(MyEntry):
    """
    Creates a labeled Entry field for a file or directory.

    *text* is the Label/error box text.
    *directory* is ``True`` if selecting a directory (instead of a file).
    *extensions* is a list of valid file endings

    """

    def __init__(
        self, text, cfg, key, optional=False, directory=False, extensions=None
    ):
        MyEntry.__init__(self, text, cfg, key, optional)
        self.choose = None
        self.clear = None

        self.directory = directory
        if extensions is not None:
            self.extensions = [x.lower() for x in extensions]
        else:
            self.extensions = None

    def body(self, master, row, columns=DEFAULT_COLUMNS, **kwargs):
        """
        Place the required elements using the grid layout method.

        Returns the number of rows taken by this element.
        """
        label = ttk.Label(master, text=self.text)
        label.grid(row=row, column=0, columnspan=1, sticky="e")
        self.entry = ttk.Entry(master, textvariable=self.value)
        self.entry.grid(row=row, column=1, columnspan=columns - 1, sticky="ew")
        if self.directory:
            self.choose = ttk.Button(
                master,
                text="Choose...",
                command=lambda: self.value.set(tkFileDialog.askdirectory()),
            )
        else:
            self.choose = ttk.Button(
                master,
                text="Choose...",
                command=lambda: self.value.set(tkFileDialog.askopenfilename()),
            )
        self.choose.grid(row=row + 1, column=1, sticky="w")
        if self.optional:
            self.clear = ttk.Button(
                master, text="Clear", command=lambda: self.value.set("")
            )
            self.clear.grid(row=row + 1, column=2, sticky="e")
        return 2

    def validate(self):
        if not self.enabled:
            return True
        elif len(self.value.get()) == 0:
            if not self.optional:
                tkMessageBox.showwarning("", "{} not specified.".format(self.text))
                return False
            else:
                return True
        else:
            if os.path.exists(self.value.get()):
                if self.extensions is not None:
                    if any(
                        self.value.get().lower().endswith(x) for x in self.extensions
                    ):
                        return True
                    else:
                        tkMessageBox.showwarning(
                            "", "Invalid file extension " "for {}.".format(self.text)
                        )
                        return False
                else:  # no extension restriction
                    return True
            else:
                tkMessageBox.showwarning(
                    "", "{} file does not exist." "".format(self.text)
                )
                return False

    def enable(self):
        self.enabled = True
        self.entry.state(["!disabled"])
        self.choose.state(["!disabled"])
        if self.optional:
            self.clear.state(["!disabled"])

    def disable(self):
        self.enabled = False
        self.entry.state(["disabled"])
        self.choose.state(["disabled"])
        if self.optional:
            self.clear.state(["disabled"]) class IntegerEntry(MyEntry):
    """
    Creates a labeled Entry field for an integer.

    *text* is the Label/error box text.
    """

    def __init__(self, text, cfg, key, optional=False, minvalue=0):
        MyEntry.__init__(self, text, cfg, key, optional)
        self.minvalue = minvalue

    def body(self, master, row, columns=DEFAULT_COLUMNS, width=4, left=False, **kwargs):
        """
        Add the labeled entry to the Frame *master* using grid at *row*.

        *width* controls the width of the Entry.
        *left* is ``True`` if the Entry is to the left of the Label.
        *columns* is the number of columns in *master*.

        Returns the number of rows taken by this element.
        """
        if left:
            entry_column = 0
            entry_sticky = "e"
            entry_width = 1
            label_column = 1
            label_sticky = "w"
            label_width = columns - 1
        else:
            entry_column = 1
            entry_sticky = "w"
            entry_width = columns - 1
            label_column = 0
            label_sticky = "e"
            label_width = 1

        label = ttk.Label(master, text=self.text)
        label.grid(
            row=row, column=label_column, columnspan=label_width, sticky=label_sticky
        )
        self.entry = ttk.Entry(master, textvariable=self.value, width=width)
        self.entry.grid(
            row=row, column=entry_column, columnspan=entry_width, sticky=entry_sticky
        )
        return 1

    def validate(self):
        """
        Returns ``True`` if the value entered validates; else ``False``.

        If *self.optional* is ``True``, the field can be empty.
        Checks the *self.minvalue* that was passed on creation.
        """
        if not self.enabled:
            return True
        else:
            try:
                intvalue = int(self.value.get())
            except ValueError:
                if len(self.value.get()) == 0:
                    if not self.optional:
                        tkMessageBox.showwarning(
                            "", "{} not specified." "".format(self.text)
                        )
                        return False
                    else:
                        return True
                else:
                    tkMessageBox.showwarning(
                        "", "{} is not an integer." "".format(self.text)
                    )
                    return False
            else:
                if intvalue < self.minvalue:
                    tkMessageBox.showwarning(
                        "",
                        "{} lower than minimum value "
                        "({}).".format(self.text, self.minvalue),
                    )
                    return False
                else:
                    return True

    def apply(self):
        if self.enabled and len(self.value.get()) > 0:
            self.cfg[self.key] = int(self.value.get())
        else:
            self.cfg[self.key] = None class Checkbox(object):
    def __init__(self, text, cfg, key):
        self.checkbox = None
        self.enabled = True

        self.value = tk.BooleanVar()
        self.text = text
        self.cfg = cfg
        self.key = key
        try:
            if self.cfg[self.key] not in (True, False):
                self.value.set(False)
            else:
                self.value.set(self.cfg[self.key])
        except KeyError:
            self.value.set(False)  # default to False

    def body(self, master, row, columns=DEFAULT_COLUMNS, **kwargs):
        """
        Place the required elements using the grid layout method.

        Returns the number of rows taken by this element.
        """
        self.checkbox = ttk.Checkbutton(master, text=self.text, variable=self.value)
        self.checkbox.grid(row=row, column=0, columnspan=columns, sticky="w")
        return 1

    def validate(self):
        return True

    def apply(self):
        if self.enabled:
            self.cfg[self.key] = self.value.get()
        else:
            self.cfg[self.key] = None

    def enable(self):
        self.enabled = True
        self.checkbox.state(["!disabled"])

    def disable(self):
        self.enabled = False
        self.checkbox.state(["disabled"]) class StringEntry(MyEntry):
    """
    Creates a labeled Entry field for a string.

    *text* is the Label/error box text.
    """

    def __init__(self, text, cfg, key, optional=False):
        MyEntry.__init__(self, text, cfg, key, optional)

    def body(self, master, row, columns=DEFAULT_COLUMNS, **kwargs):
        """
        Place the required elements using the grid layout method.

        Returns the number of rows taken by this element.
        """
        label = ttk.Label(master, text=self.text)
        label.grid(row=row, column=0, columnspan=1, sticky="e")
        self.entry = ttk.Entry(master, textvariable=self.value)
        self.entry.grid(row=row, column=1, columnspan=columns - 1, sticky="ew")
        return 1 class SectionLabel(object):
    def __init__(self, text):
        self.text = text

    def body(self, master, row, columns=DEFAULT_COLUMNS, **kwargs):
        label = ttk.Label(master, text=self.text)
        label.grid(row=row, column=0, columnspan=columns, sticky="w")
        return 1

    def validate(self):
        return True

    def apply(self):
        return None

    def enable(self):
        pass

    def disable(self):
        pass DEFAULT_COLUMNS = 3 class BasicSeqLib(VariantSeqLib):
    """
    Class for count data from sequencing libraries with a single read for
    each variant. Creating a :py:class:`BasicSeqLib` requires a valid
    *config* object, usually from a ``.json`` configuration file.
    """

    treeview_class_name = "Basic SeqLib"

    def __init__(self):
        VariantSeqLib.__init__(self)
        self.reads = None
        self.revcomp_reads = None
        self.trim_start = None
        self.trim_length = None
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from
        a ``.json`` file.
        """
        VariantSeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )

        # if counts are specified, copy them later
        # else handle the FASTQ config options and check the files
        if self.counts_file is None:
            self.configure_fastq(cfg)
            try:
                if split_fastq_path(self.reads) is None:
                    raise IOError(
                        "FASTQ file error: unrecognized extension "
                        "[{}]".format(self.name)
                    )
            except IOError as fqerr:
                raise IOError("FASTQ file error [{}]: {}".format(self.name, fqerr))

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for
        dumping to a config file.
        """
        cfg = VariantSeqLib.serialize(self)

        cfg["fastq"] = self.serialize_fastq()

        return cfg

    def configure_fastq(self, cfg):
        """
        Set up the object's FASTQ_ file handling and filtering options.
        """
        try:
            self.reads = cfg["fastq"]["reads"]

            if "reverse" in cfg["fastq"]:
                self.revcomp_reads = cfg["fastq"]["reverse"]
            else:
                self.revcomp_reads = False

            if "start" in cfg["fastq"]:
                self.trim_start = cfg["fastq"]["start"]
            else:
                self.trim_start = 1

            if "length" in cfg["fastq"]:
                self.trim_length = cfg["fastq"]["length"]
            else:
                self.trim_length = sys.maxsize

            self.filters = cfg["fastq"]["filters"]
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]"
                "".format(key=key, name=self.name)
            )

    def serialize_fastq(self):
        """
        Serialize this object's FASTQ_ file handling and filtering options.
        """
        fastq = {"filters": self.serialize_filters()}
        fastq["reads"] = self.reads

        if self.revcomp_reads:
            fastq["reverse"] = True
        else:
            fastq["reverse"] = False

        if self.trim_start > 1:
            fastq["start"] = self.trim_start

        if self.trim_length < sys.maxsize:
            fastq["length"] = self.trim_length

        return fastq

    def counts_from_reads(self):
        """
        Reads the forward or reverse FASTQ_ file (reverse reads are
        reverse-complemented), performs quality-based filtering, and counts
        the variants.
        """
        df_dict = dict()

        self.logger.info("Counting variants")
        max_mut_variants = 0
        for fq in read_fastq(self.reads):
            fq.trim_length(self.trim_length, start=self.trim_start)
            if self.revcomp_reads:
                fq.revcomp()

            if self.read_quality_filter(fq):
                mutations = self.count_variant(fq.sequence)
                if mutations is None:  # too many mutations
                    max_mut_variants += 1
                    if self.report_filtered:
                        self.report_filtered_variant(fq.sequence, 1)
                else:
                    try:
                        df_dict[mutations] += 1
                    except KeyError:
                        df_dict[mutations] = 1

        self.save_counts("variants", df_dict, raw=True)
        del df_dict

        if self.aligner is not None:
            self.logger.info("Aligned {} variants".format(self.aligner.calls))
            self.aligner_cache = None
        self.logger.info(
            "Removed {} total variants with excess mutations"
            "".format(max_mut_variants)
        )
        self.save_filter_stats()

    def calculate(self):
        """
        Counts variants from counts file or FASTQ.
        """
        if not self.check_store("/main/variants/counts"):
            if not self.check_store("/raw/variants/counts"):
                if self.counts_file is not None:
                    self.counts_from_file(self.counts_file)
                else:
                    self.counts_from_reads()
            self.save_filtered_counts("variants", "count >= self.variant_min_count")
        self.count_synonymous() class BcvSeqLib(VariantSeqLib, BarcodeSeqLib):
    """
    Class for counting variant data from barcoded sequencing libraries. 
    Creating a :py:class:`BcvSeqLib` requires a valid *config* 
    object with an ``'barcodes'`` entry and information about the wild type 
    sequence.

    The ``barcode_map`` keyword argument can be used to pass an existing 
    :py:class:`~seqlib.barcodemap.BarcodeMap`. Ensuring this is the 
    right :py:class:`~seqlib.barcodemap.BarcodeMap` is the responsibility 
    of the caller.
    """

    treeview_class_name = "Barcoded Variant SeqLib"

    def __init__(self):
        VariantSeqLib.__init__(self)
        BarcodeSeqLib.__init__(self)
        self.barcode_map = None
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg, barcode_map=None):
        """
        Set up the object using the config object *cfg*, usually derived from 
        a ``.json`` file.
        """
        VariantSeqLib.configure(self, cfg)
        BarcodeSeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )
        try:
            if barcode_map is not None:
                if barcode_map.filename == cfg["barcodes"]["map file"]:
                    self.barcode_map = barcode_map
                else:
                    raise ValueError(
                        "Attempted to assign non-matching barcode map [{}]".format(
                            self.name
                        )
                    )
            else:
                self.barcode_map = BarcodeMap(
                    cfg["barcodes"]["map file"], is_variant=True
                )
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]".format(
                    key=key, name=self.name
                )
            )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for dumping to a config file.
        """
        cfg = VariantSeqLib.serialize(self)
        cfg.update(BarcodeSeqLib.serialize(self))

        if self.barcode_map is not None:  # required for creating new objects in GUI
            cfg["barcodes"]["map file"] = self.barcode_map.filename

        return cfg

    def calculate(self):
        """
        Counts the barcodes using :py:meth:`BarcodeSeqLib.count` and combines them into 
        variant counts using the :py:class:`BarcodeMap`.
        """
        if not self.check_store("/main/variants/counts"):
            BarcodeSeqLib.calculate(self)  # count the barcodes
            df_dict = dict()
            barcode_variants = dict()

            self.logger.info("Converting barcodes to variants")
            # store mapped barcodes
            self.save_filtered_counts(
                "barcodes",
                "index in self.barcode_map.keys() & count >= self.barcode_min_count",
            )

            # count variants associated with the barcodes
            max_mut_barcodes = 0
            max_mut_variants = 0
            for bc, count in self.store["/main/barcodes/counts"].iterrows():
                count = count["count"]
                variant = self.barcode_map[bc]
                mutations = self.count_variant(variant)
                if mutations is None:  # variant has too many mutations
                    max_mut_barcodes += 1
                    max_mut_variants += count
                    if self.report_filtered:
                        self.report_filtered_variant(variant, count)
                else:
                    try:
                        df_dict[mutations] += count
                    except KeyError:
                        df_dict[mutations] = count
                    barcode_variants[bc] = mutations

            # save counts, filtering based on the min count
            self.save_counts(
                "variants",
                {k: v for k, v in df_dict.iteritems() if v >= self.variant_min_count},
                raw=False,
            )
            del df_dict

            # write the active subset of the BarcodeMap to the store
            barcodes = barcode_variants.keys()
            barcode_variants = pd.DataFrame(
                {"value": [barcode_variants[bc] for bc in barcodes]}, index=barcodes
            )
            del barcodes
            barcode_variants.sort_values("value", inplace=True)
            self.store.put(
                "/raw/barcodemap",
                barcode_variants,
                data_columns=barcode_variants.columns,
                format="table",
            )
            del barcode_variants

            if self.aligner is not None:
                self.logger.info("Aligned {} variants".format(self.aligner.calls))
                self.aligner_cache = None
            # self.report_filter_stats()
            self.logger.info(
                "Removed {} unique barcodes ({} total variants) "
                "with excess mutations".format(max_mut_barcodes, max_mut_variants)
            )
            self.save_filter_stats()

        self.count_synonymous()

    def make_plots(self):
        """
        Make plots for :py:class:`~seqlib.seqlib.BcvSeqLib` objects.

        Creates plot of the number of barcodes mapping to each variant.
        """
        if self.plots_requested:
            SeqLib.make_plots(self)
            # open the PDF file
            pdf = PdfPages(os.path.join(self.plot_dir, "barcodes_per_variant.pdf"))
            barcodemap_plot(self, pdf)
            pdf.close() class BcidSeqLib(BarcodeSeqLib):
    """
    Class for counting data from barcoded sequencing libraries with non-variant
    identifiers. 
    Creating a :py:class:`BcidSeqLib` requires a valid *config* 
    object with an ``'barcodes'`` entry and information.

    The ``barcode_map`` keyword argument can be used to pass an existing 
    :py:class:`~seqlib.barcodemap.BarcodeMap`. Ensuring this is the 
    right :py:class:`~seqlib.barcodemap.BarcodeMap` is the responsibility 
    of the caller.
    """

    treeview_class_name = "Barcoded ID SeqLib"

    def __init__(self):
        BarcodeSeqLib.__init__(self)
        self.barcode_map = None
        self.identifier_min_count = None
        self.add_label("identifiers")
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg, barcode_map=None):
        """
        Set up the object using the config object *cfg*, usually derived from 
        a ``.json`` file.
        """
        BarcodeSeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )
        try:
            if "min count" in cfg["identifiers"]:
                self.identifier_min_count = int(cfg["identifiers"]["min count"])
            else:
                self.identifier_min_count = 0

            if barcode_map is not None:
                if barcode_map.filename == cfg["barcodes"]["map file"]:
                    self.barcode_map = barcode_map
                else:
                    raise ValueError(
                        "Attempted to assign non-matching barcode map [{}]".format(
                            self.name
                        )
                    )
            else:
                self.barcode_map = BarcodeMap(
                    cfg["barcodes"]["map file"], is_variant=False
                )
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]".format(
                    key=key, name=self.name
                )
            )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for dumping to a config file.
        """
        cfg = BarcodeSeqLib.serialize(self)

        cfg["identifiers"] = dict()
        if self.identifier_min_count > 0:
            cfg["identifiers"]["min count"] = self.identifier_min_count

        if self.barcode_map is not None:  # required for creating new objects in GUI
            cfg["barcodes"]["map file"] = self.barcode_map.filename

        return cfg

    def calculate(self):
        """
        Counts the barcodes using :py:meth:`BarcodeSeqLib.count` and combines them into 
        identifier counts using the :py:class:`BarcodeMap`.
        """
        if not self.check_store("/main/identifiers/counts"):
            BarcodeSeqLib.calculate(self)  # count the barcodes
            df_dict = dict()
            barcode_identifiers = dict()

            self.logger.info("Converting barcodes to identifiers")
            # store mapped barcodes
            self.save_filtered_counts(
                "barcodes",
                "index in self.barcode_map.keys() & count >= self.barcode_min_count",
            )

            # count identifiers associated with the barcodes
            for bc, count in self.store["/main/barcodes/counts"].iterrows():
                count = count["count"]
                identifier = self.barcode_map[bc]
                try:
                    df_dict[identifier] += count
                except KeyError:
                    df_dict[identifier] = count
                barcode_identifiers[bc] = identifier

            # save counts, filtering based on the min count
            self.save_counts(
                "identifiers",
                {
                    k: v
                    for k, v in df_dict.iteritems()
                    if v >= self.identifier_min_count
                },
                raw=False,
            )
            del df_dict

            # write the active subset of the BarcodeMap to the store
            barcodes = barcode_identifiers.keys()
            barcode_identifiers = pd.DataFrame(
                {"value": [barcode_identifiers[bc] for bc in barcodes]}, index=barcodes
            )
            del barcodes
            barcode_identifiers.sort_values("value", inplace=True)
            self.store.put(
                "/raw/barcodemap",
                barcode_identifiers,
                data_columns=barcode_identifiers.columns,
                format="table",
            )
            del barcode_identifiers

            # self.report_filter_stats()
            self.save_filter_stats()

    def make_plots(self):
        """
        Make plots for :py:class:`~seqlib.seqlib.BcidSeqLib` objects.

        Creates plot of the number of barcodes mapping to each identifier.
        """
        if self.plots_requested:
            SeqLib.make_plots(self)
            # open the PDF file
            pdf = PdfPages(os.path.join(self.plot_dir, "barcodes_per_identifier.pdf"))
            barcodemap_plot(self, pdf)
            pdf.close() class BarcodeSeqLib(SeqLib):
    """
    Class for count data from barcoded sequencing libraries. Designed for
    barcode-only scoring or as a parent class for
    :py:class:`~seqlib.barcodevariant.BcvSeqLib` and
    :py:class:`~seqlib.barcodeid.BcidSeqLib`.
    """

    treeview_class_name = "Barcode SeqLib"

    def __init__(self):
        # Init step handled by VariantSeqLib's init for Barcode-variant
        if type(self).__name__ != "BcvSeqLib":
            SeqLib.__init__(self)
        self.reads = None
        self.revcomp_reads = None
        self.trim_start = None
        self.trim_length = None
        self.barcode_min_count = None
        self.add_label("barcodes")
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from
        a ``.json`` file.
        """
        SeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )

        # handle non-FASTQ config options
        try:
            if "min count" in cfg["barcodes"]:
                self.barcode_min_count = int(cfg["barcodes"]["min count"])
            else:
                self.barcode_min_count = 0
        except KeyError as key:
            raise KeyError("Missing required config value {}".format(key), self.name)

        # if counts are specified, copy them later
        # else handle the FASTQ config options and check the files
        if self.counts_file is None:
            self.configure_fastq(cfg)
            try:
                if split_fastq_path(self.reads) is None:
                    raise ValueError(
                        "FASTQ file error: unrecognized file " "extension", self.name
                    )
            except IOError as fqerr:
                raise IOError("FASTQ file error: {}".format(fqerr), self.name)

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for
        dumping to a config file.
        """
        cfg = SeqLib.serialize(self)

        cfg["barcodes"] = dict()
        if self.barcode_min_count > 0:
            cfg["barcodes"]["min count"] = self.barcode_min_count

        cfg["fastq"] = self.serialize_fastq()

        return cfg

    def configure_fastq(self, cfg):
        """
        Set up the object's FASTQ_ file handling and filtering options.
        """
        try:
            self.reads = cfg["fastq"]["reads"]
            self.revcomp_reads = cfg["fastq"]["reverse"]

            if "start" in cfg["fastq"]:
                self.trim_start = cfg["fastq"]["start"]
            else:
                self.trim_start = 1

            if "length" in cfg["fastq"]:
                self.trim_length = cfg["fastq"]["length"]
            else:
                self.trim_length = sys.maxsize

            self.filters = cfg["fastq"]["filters"]
        except KeyError as key:
            raise KeyError("Missing required config value {}".format(key), self.name)

    def serialize_fastq(self):
        """
        Serialize this object's FASTQ_ file handling and filtering options.
        """
        fastq = {
            "reads": self.reads,
            "reverse": self.revcomp_reads,
            "filters": self.serialize_filters(),
        }
        if self.trim_start > 1:
            fastq["start"] = self.trim_start

        if self.trim_length < sys.maxsize:
            fastq["length"] = self.trim_length

        return fastq

    def counts_from_reads(self):
        """
        Reads the forward or reverse FASTQ_ file (reverse reads are
        reverse-complemented), performs quality-based filtering, and counts
        the barcodes.

        Barcode counts after read-level filtering are stored under
        ``"/raw/barcodes/counts"``.
        """
        df_dict = dict()

        filter_flags = dict()
        for key in self.filters:
            filter_flags[key] = False

        # count all the barcodes
        self.logger.info("Counting barcodes")
        for fqr in read_fastq(self.reads):
            fqr.trim_length(self.trim_length, start=self.trim_start)
            if self.revcomp_reads:
                fqr.revcomp()

            if self.read_quality_filter(fqr):  # passed filtering
                try:
                    df_dict[fqr.sequence.upper()] += 1
                except KeyError:
                    df_dict[fqr.sequence.upper()] = 1

        self.save_counts("barcodes", df_dict, raw=True)
        del df_dict

    def calculate(self):
        """
        Counts the barcodes from the FASTQ file or from the provided counts
        file depending on the config.

        Barcodes that pass the minimum count
        filtering are stored under ``"/main/barcodes/counts"``.

        If ``"/main/barcodes/counts"`` already exists, those will be used
        instead of re-counting.
        """
        if self.check_store("/main/barcodes/counts"):
            return

        # no raw counts present
        if not self.check_store("/raw/barcodes/counts"):
            if self.counts_file is not None:
                self.counts_from_file(self.counts_file)
            else:
                self.counts_from_reads()

        if len(self.labels) == 1:  # only barcodes
            self.save_filtered_counts("barcodes", "count >= self.barcode_min_count")
            self.save_filter_stats() class IdOnlySeqLib(SeqLib):
    """
    Class for counting data with non-variant identifiers and no associated
    FASTQ_ data.
    """

    treeview_class_name = "ID-only SeqLib"

    def __init__(self):
        SeqLib.__init__(self)
        self.identifier_min_count = None
        self.add_label("identifiers")
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from
        a ``.json`` file.
        """
        SeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )
        try:
            if "min count" in cfg["identifiers"]:
                self.identifier_min_count = int(cfg["identifiers"]["min count"])
            else:
                self.identifier_min_count = 0
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]"
                "".format(key=key, name=self.name)
            )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for
        dumping to a config file.
        """
        cfg = SeqLib.serialize(self)

        cfg["identifiers"] = dict()
        if self.identifier_min_count > 0:
            cfg["identifiers"]["min count"] = self.identifier_min_count

        return cfg

    def calculate(self):
        """
        Get the identifier counts from the counts file.
        """
        if not self.check_store("/main/identifiers/counts"):
            if self.counts_file is not None:
                self.counts_from_file(self.counts_file)
            else:
                raise ValueError("Missing counts file [{}]".format(self.name))
            self.save_filtered_counts(
                "identifiers", "count >= self.identifier_min_count"
            ) class OverlapSeqLib(VariantSeqLib):
    """
    Class for count data from sequencing libraries with overlapping paired-end 
    reads for each variant. Creating a 
    :py:class:`~seqlib.overlap.OverlapSeqLib` requires a valid *config* object 
    with an ``'overlap'`` entry.

    The ``"fastq"`` config entry must contain two read files, with the keys 
    ``"forward"`` and ``"reverse"``. Information about how to combine these 
    reads is in the ``"overlap"`` config entry.

    The ``"overlap"`` config entry contains the following keys:

    * ``"forward start"`` --- position in the forward read where the \
        overlapping region begins
    * ``"reverse start"`` --- position in the reverse read where the \
        overlapping region begins (before being reverse-complemented)
    * ``"length"`` --- number of bases in the overlapping region
    * ``"max mismatches"`` --- maximum number of mismatches tolerated in the \
        overlapping region before discarding the read
    * ``"overlap only"`` --- whether to trim the merged read to contain only \
        the overlapping region (optional, defaults to ``False``)

    Here is a schematic of the case in the above JSON example::

        forward ---> 1   
                     CGACGCAAGGA
                       |||||||||
                       ACTCCTTGCGTCG
                                   1 <--- reverse

    Note that the merged sequence is identical to the wild type sequence given 
    in the JSON file.
    """

    treeview_class_name = "Overlap SeqLib"

    def __init__(self):
        VariantSeqLib.__init__(self)
        self.forward = None
        self.reverse = None
        self.fwd_start = None
        self.rev_start = None
        self.overlap_length = None
        self.trim = None
        self.max_overlap_mismatches = None
        self.merge_mismatches = None
        self.default_filters.update({"merge failure": True})
        self.default_filters.update({"remove unresolvable": False})
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from 
        a ``.json`` file.
        """
        VariantSeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )

        # if counts are specified, copy them later
        # else handle the FASTQ config options and check the files
        if self.counts_file is None:
            self.configure_fastq(cfg)
            try:
                self.fwd_start = int(cfg["overlap"]["forward start"])
                self.rev_start = int(cfg["overlap"]["reverse start"])
                self.overlap_length = int(cfg["overlap"]["length"])
                self.trim = cfg["overlap"]["trim"]
                self.max_overlap_mismatches = int(cfg["overlap"]["max mismatches"])

                forward_error = False
                reverse_error = False
                if split_fastq_path(self.forward) is None:
                    forward_error = True
                if split_fastq_path(self.reverse) is None:
                    reverse_error = True
                if forward_error and reverse_error:
                    raise IOError(
                        "FASTQ file error: unrecognized extension (forward and reverse) [{}]".format(
                            self.name
                        )
                    )
                elif forward_error:
                    raise IOError(
                        "FASTQ file error: unrecognized extension (forward) [{}]".format(
                            self.name
                        )
                    )
                elif reverse_error:
                    raise IOError(
                        "FASTQ file error: unrecognized extension (reverse) [{}]".format(
                            self.name
                        )
                    )
            except IOError as fqerr:
                raise IOError("FASTQ file error [{}]: {}".format(self.name, fqerr))
            except KeyError as key:
                raise KeyError(
                    "Missing required config value {key} [{name}]".format(
                        key=key, name=self.name
                    )
                )
            except ValueError as value:
                raise ValueError(
                    "Invalid parameter value {value} [{name}]".format(
                        value=value, name=self.name
                    )
                )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for dumping to a config file.
        """
        cfg = VariantSeqLib.serialize(self)

        cfg["fastq"] = self.serialize_fastq()
        cfg["overlap"] = {
            "forward start": self.fwd_start,
            "reverse start": self.rev_start,
            "length": self.overlap_length,
            "trim": self.trim,
            "max mismatches": self.max_overlap_mismatches,
        }

        return cfg

    def configure_fastq(self, cfg):
        """
        Set up the object's FASTQ_ file handling and filtering options.
        """
        try:
            self.forward = cfg["fastq"]["forward reads"]
            self.reverse = cfg["fastq"]["reverse reads"]

            if "merge failure" in cfg["fastq"]["filters"]:
                raise ValueError(
                    "'merge failure' is not user-configurable [{}]".format(self.name)
                )
            self.filters = cfg["fastq"]["filters"]
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]".format(
                    key=key, name=self.name
                )
            )

    def serialize_fastq(self):
        """
        Serialize this object's FASTQ_ file handling and filtering options.
        """
        fastq = {
            "forward reads": self.forward,
            "reverse reads": self.reverse,
            "filters": self.serialize_filters(),
        }

        return fastq

    def merge_reads(self, fwd, rev):
        """
        Combines the *fwd* and *rev* :py:class:`~fqread.FQRead` objects into a 
        single :py:class:`~fqread.FQRead` with the same header information as 
        *fwd*. Mismatches are resolved by taking the highest quality base. If 
        discrepant bases have the same quality value, this position is 
        unresolvable and an ``'X'`` is inserted. Quality values in the 
        resulting :py:class:`~fqread.FQRead` are the maximum quality for the 
        given base at that position. Returns ``None`` if the maximum number of 
        mismatches in the overlap region is exceded.
        """
        rev.revcomp()

        # print(fwd.sequence, "-" * (self.rev_start - 1), sep="")
        # print("-" * (self.fwd_start - 1), rev.sequence, sep="")
        rev_extra_start = len(rev) - self.rev_start + 1
        fwd_end = self.fwd_start + self.overlap_length - 1
        merge = FQRead(
            header=fwd.header + "|" + rev.header,
            sequence="A",
            header2=fwd.header2 + "|" + rev.header2,
            quality="#",
            qbase=fwd.qbase,
        )
        merge.sequence = list(fwd.sequence[:fwd_end] + rev.sequence[rev_extra_start:])
        merge.quality = fwd.quality[:fwd_end] + rev.quality[rev_extra_start:]

        mismatches = 0
        first = True
        for i in xrange(self.overlap_length):
            a = self.fwd_start - 1 + i
            b = len(rev) - self.rev_start - self.overlap_length + i + 1
            try:
                if fwd.sequence[a] == rev.sequence[b]:
                    # take the highest quality value
                    if rev.quality[b] > fwd.quality[a]:
                        merge.quality[a] = rev.quality[b]
                else:
                    if fwd.quality[a] == rev.quality[b]:
                        merge.sequence[a] = "X"  # unresolvable
                        self.merge_mismatches.iloc[i]["unresolved"] += 1
                    elif rev.quality[b] > fwd.quality[a]:
                        merge.sequence[a] = rev.sequence[b]
                        merge.quality[a] = rev.quality[b]
                        self.merge_mismatches.iloc[i]["resolved"] += 1
                    else:
                        # overlap region already same as fwd
                        self.merge_mismatches.iloc[i]["resolved"] += 1
                    mismatches += 1
                    if first:
                        self.merge_mismatches.iloc[i]["first"] += 1
                        first = False
            except IndexError:
                raise IndexError(
                    "Failed to calculate overlap (a={a}, len(a)={lena}, b={b}, len(b)={lenb}) [{name}]".format(
                        a=a,
                        b=b,
                        lena=len(fwd.sequence),
                        lenb=len(rev.sequence),
                        name=self.name,
                    )
                )

        if mismatches > self.max_overlap_mismatches:
            return None  # merge failed

        merge.sequence = "".join(merge.sequence)
        if self.trim:
            merge.trim_length(self.overlap_length, self.fwd_start)
        return merge

    def counts_from_reads(self):
        df_dict = dict()

        self.merge_mismatches = pd.DataFrame(
            data=0,
            index=[
                x + self.fwd_start + self.wt.dna_offset
                for x in xrange(0, self.overlap_length)
            ],
            columns=["resolved", "unresolved", "first"],
        )

        self.logger.info("Counting variants")
        max_mut_variants = 0
        for fwd, rev in read_fastq_multi([self.forward, self.reverse]):
            # filter on chastity before merge
            chaste = True
            if self.filters["chastity"]:
                if not fwd.is_chaste():
                    chaste = False
                    if self.report_filtered:
                        self.report_filtered_read(fwd, "chastity")
                if not rev.is_chaste():
                    chaste = False
                    if self.report_filtered:
                        self.report_filtered_read(rev, "chastity")
                if not chaste:
                    self.filter_stats["chastity"] += 1
                    self.filter_stats["total"] += 1
                    continue

            merge = self.merge_reads(fwd, rev)
            if merge is None:  # merge failed
                self.filter_stats["merge failure"] += 1
                self.filter_stats["total"] += 1
                if self.report_filtered:
                    self.report_filtered_read(fwd, filter_flags)
                    self.report_filtered_read(rev, filter_flags)
            else:
                if self.read_quality_filter(merge):
                    mutations = self.count_variant(merge.sequence)
                    if mutations is None:  # merge read has too many mutations
                        max_mut_variants += 1
                        if self.report_filtered:
                            self.report_filtered_variant(merge.sequence, 1)
                    else:
                        try:
                            df_dict[mutations] += 1
                        except KeyError:
                            df_dict[mutations] = 1

        self.store.put(
            "/raw/overlap_mismatches",
            self.merge_mismatches,
            format="table",
            data_columns=self.merge_mismatches.columns,
        )
        self.merge_mismatches = None
        self.save_counts("variants", df_dict, raw=True)
        del df_dict

        if self.aligner is not None:
            self.logger.info("Aligned {} variants".format(self.aligner.calls))
            self.aligner_cache = None
        self.logger.info(
            "Removed {} total variants with excess mutations"
            "".format(max_mut_variants)
        )
        self.save_filter_stats()

    def calculate(self):
        """
        Reads the forward and reverse reads, merges them, performs 
        quality-based filtering, and counts the variants.
        """
        if not self.check_store("/main/variants/counts"):
            if not self.check_store("/raw/variants/counts"):
                if self.counts_file is not None:
                    self.counts_from_file(self.counts_file)
                else:  # count everything
                    self.counts_from_reads()
            self.save_filtered_counts("variants", "count >= self.variant_min_count")

        self.count_synonymous()

    def make_plots(self):
        """
        Make plots for :py:class:`~seqlib.seqlib.OverlapSeqLib` objects.

        Creates plots of the location of merged read mismatches.
        """
        if self.plots_requested:
            SeqLib.make_plots(self)
            pdf = PdfPages(os.path.join(self.plot_dir, "overlap_mismatches.pdf"))
            overlap_merge_plot(self, pdf)
            pdf.close() class SeqLib(StoreManager):
    """
    Abstract class for handling count data from a single sequencing library.
    """

    # Note: the following block is referenced by line number above
    # When adding new messages, update the documentation line numbers also!
    filter_messages = OrderedDict(
        [
            ("min quality", "single-base quality"),
            ("avg quality", "average quality"),
            ("max N", "excess N bases"),
            ("chastity", "not chaste"),
            ("remove unresolvable", "unresolvable mismatch"),
            ("merge failure", "unable to merge reads"),
            ("total", "total"),
        ]
    )

    store_suffix = "lib"

    def __init__(self):
        StoreManager.__init__(self)
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))
        self.timepoint = None
        self.counts_file = None
        self.report_filtered = None
        self._filters = dict()
        self.filter_stats = dict()
        self.default_filters = dict()
        self.default_filters.update({"min quality": 0})
        self.default_filters.update({"max N": sys.maxsize})
        self.default_filters.update({"avg quality": 0})
        self.default_filters.update({"chastity": False})

    @property
    def filters(self):
        return self._filters

    @filters.setter
    def filters(self, config_filters):
        """
        Set up the filters dictionary using the options selected in
        *config_filters*, filling in missing entries with defaults.
        """
        self._filters.clear()
        self._filters.update(self.default_filters)

        unused = list()
        for key in config_filters:
            if key in self._filters:
                if config_filters[key] is not None:
                    self._filters[key] = config_filters[key]
            else:
                unused.append(key)
        if len(unused) > 0:
            self.logger.warning(
                "Unused filter parameters ({})" "".format(", ".join(unused))
            )

        self.filter_stats.clear()
        for key in self._filters:
            self.filter_stats[key] = 0
        self.filter_stats["total"] = 0

    def serialize_filters(self):
        """
        Return a dictionary of filtering options that have non-default values.
        """
        cfg = dict()
        for key in self.filters.keys():
            if self.filters[key] != self.default_filters[key]:
                cfg[key] = self.filters[key]
        return cfg

    def _children(self):
        """
        These objects have no children. Returns ``None``.
        """
        return None

    def add_child(self, child):
        """
        No children, raises an AttributeError.
        """
        raise AttributeError("SeqLib objects do not support adding children")

    def remove_child_id(self, tree_id):
        """
        No children, raises an AttributeError.
        """
        raise AttributeError("SeqLib objects do not support removing children")

    def validate(self):
        """
        Validates paramaters for a configured SeqLib. Currently does nothing.
        """
        pass

    def has_wt_sequence(self):
        """
        Returns whether or not the object has a wild type sequence. Returns
        ``False`` unless overloaded by a derived class (such as
        :py:class:`~seqlib.seqlib.VariantSeqLib`).
        """
        return False

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from
        a ``.json`` file.
        """
        StoreManager.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )
        try:
            self.timepoint = int(cfg["timepoint"])
            if "report filtered reads" in cfg:
                self.report_filtered = cfg["report filtered reads"]
            else:
                self.report_filtered = False
            if "counts file" in cfg:
                self.counts_file = cfg["counts file"]
            else:
                self.counts_file = None
        except KeyError as key:
            raise KeyError(
                "Missing required config value {key}" "".format(key=key), self.name
            )
        except ValueError as value:
            raise ValueError(
                "Invalid parameter value {value}" "".format(value=value), self.name
            )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for
        dumping to a config file.
        """
        cfg = StoreManager.serialize(self)
        cfg["timepoint"] = self.timepoint
        cfg["report filtered reads"] = self.report_filtered
        if self.counts_file is not None:
            cfg["counts file"] = self.counts_file

        return cfg

    def calculate(self):
        """
        Pure virtual method that defines how the data are counted.
        """
        raise NotImplementedError("must be implemented by subclass")

    def report_filtered_read(self, fq, filter_flags):
        """
        Write the :py:class:`~fqread.FQRead` object *fq* to the ``DEBUG``
        log. The dictionary *filter_flags* contains ``True``
        values for each filtering option that applies to *fq*. Keys in
        *filter_flags* are converted to messages using the
        ``SeqLib.filter_messages`` dictionary.
        """
        self.logger.debug(
            "Filtered read ({messages})\n{read!s}".format(
                messages=", ".join(
                    SeqLib.filter_messages[x] for x in filter_flags if filter_flags[x]
                ),
                name=self.name,
                read=fq,
            )
        )

    def save_counts(self, label, df_dict, raw):
        """
        Convert the counts in the dictionary *df_dict* into a DataFrame object
        and save it to the data store.

        If *raw* is ``True``, the counts are stored under
        ``"/raw/label/counts"``; else ``"/main/label/counts"``.
        """
        if len(df_dict.keys()) == 0:
            raise ValueError("Failed to count {} [{}]".format(label, self.name))
        df = pd.DataFrame.from_dict(df_dict, orient="index", dtype=np.int32)
        df.columns = ["count"]
        df.sort_values("count", ascending=False, inplace=True)
        self.logger.info(
            "Counted {n} {label} ({u} unique)".format(
                n=df["count"].sum(), u=len(df.index), label=label
            )
        )
        if raw:
            key = "/raw/{}/counts".format(label)
        else:
            key = "/main/{}/counts".format(label)
        self.store.put(key, df, format="table", data_columns=df.columns)
        del df

    def save_filtered_counts(self, label, query):
        """
        Filter the counts in ``"/raw/label/counts"`` using the *query* string
        and store the result in ``"/main/label/counts"``

        For more information on building query strings, see
        http://pandas.pydata.org/pandas-docs/stable/io.html#querying-a-table
        """
        self.logger.info("Converting raw {} counts to main counts".format(label))
        raw_table = "/raw/{}/counts".format(label)
        main_table = "/main/{}/counts".format(label)
        self.map_table(source=raw_table, destination=main_table, source_query=query)
        self.logger.info(
            "Counted {n} {label} ({u} unique) after query".format(
                n=self.store[main_table]["count"].sum(),
                u=len(self.store[main_table].index),
                label=label,
            )
        )

    def report_filter_stats(self):
        """
        Create report file for the number of filtered reads.

        The report file is located in the output directory, named
        ``SeqLibName.filter.txt``.
        It contains the number of reads filtered for each category, plus the
        total number filtered.

        .. note:: Reads are checked for all quality-based criteria before \
        filtering.
        """
        with open(
            os.path.join(self.output_dir, fix_filename(self.name) + ".filter.txt"), "w"
        ) as handle:
            for key in sorted(
                self.filter_stats, key=self.filter_stats.__getitem__, reverse=True
            ):
                if key != "total" and self.filter_stats[key] > 0:
                    print(
                        SeqLib.filter_messages[key],
                        self.filter_stats[key],
                        sep="\t",
                        file=handle,
                    )
            print("total", self.filter_stats["total"], sep="\t", file=handle)
        self.logger.info("Wrote filtering statistics")

    def save_filter_stats(self):
        """
        Save a DataFrame containing the number of filtered reads under
        ``'/raw/filter'``.

        This DataFrame contains the same information as ``report_filter_stats``
        """
        df = pd.DataFrame(index=SeqLib.filter_messages.values(), columns=["count"])
        for key in self.filter_stats.keys():
            if self.filter_stats[key] > 0 or key == "total":
                df.loc[SeqLib.filter_messages[key], "count"] = self.filter_stats[key]
        df.dropna(inplace=True)
        self.store.put(
            "/raw/filter", df.astype(int), format="table", data_columns=df.columns
        )

    def read_quality_filter(self, fq):
        """
        Check the quality of the FQRead object *fq*.

        Checks ``'chastity'``, ``'min quality'``, ``'avg quality'``,
        ``'max N'``, and ``'remove unresolvable'``.
        Counts failed reads for later output and reports the filtered read if
        desired. 
        Returns ``True`` if the read passes all filters, else ``False``.
        """
        filter_flags = dict()
        for key in self.filters:
            filter_flags[key] = False

        if self.filters["chastity"]:
            if not fq.is_chaste():
                self.filter_stats["chastity"] += 1
                filter_flags["chastity"] = True

        if self.filters["min quality"] > 0:
            if fq.min_quality() < self.filters["min quality"]:
                self.filter_stats["min quality"] += 1
                filter_flags["min quality"] = True

        if self.filters["avg quality"] > 0:
            if fq.mean_quality() < self.filters["avg quality"]:
                self.filter_stats["avg quality"] += 1
                filter_flags["avg quality"] = True

        if self.filters["max N"] >= 0:
            if fq.sequence.upper().count("N") > self.filters["max N"]:
                self.filter_stats["max N"] += 1
                filter_flags["max N"] = True

        if "remove unresolvable" in self.filters:  # OverlapSeqLib only
            if self.filters["remove unresolvable"]:
                if "X" in fq.sequence:
                    self.filter_stats["remove unresolvable"] += 1
                    filter_flags["remove unresolvable"] = True

        # update total and report if failed
        if any(filter_flags.values()):
            self.filter_stats["total"] += 1
            if self.report_filtered:
                self.report_filtered_read(fq, filter_flags)
            return False
        else:
            return True

    def make_plots(self):
        """
        Make plots that are shared by all :py:class:`~seqlib.seqlib.SeqLib`
        objects.

        Creates counts histograms for all labels.
        """
        if self.plots_requested:
            self.logger.info("Creating plots")

            pdf = PdfPages(os.path.join(self.plot_dir, "counts.pdf"))
            for label in self.labels:
                counts_plot(self, label, pdf, log=True)
                counts_plot(self, label, pdf, log=False)
            pdf.close()

    def write_tsv(self):
        """
        Write each table from the store to its own tab-separated file.

        Files are written to a ``tsv`` directory in the default output
        location.
        File names are the HDF5 key with ``'_'`` substituted for ``'/'``.
        """
        if self.tsv_requested:
            self.logger.info("Generating tab-separated output files")
            for k in self.store.keys():
                self.write_table_tsv(k)

    def counts_from_file_h5(self, fname):
        """
        If an HDF store containing raw counts has been specified, open the
        store, copy those counts into this store, and close the counts store.

        Copies all tables in the ``'/raw'`` group along with their metadata.
        """
        store = pd.HDFStore(fname)
        self.logger.info(
            "Using existing HDF5 data store '{}' for raw data" "".format(fname)
        )
        # this could probably be much more efficient, but the PyTables docs
        # don't explain copying subsets of files adequately
        raw_keys = [key for key in store.keys() if key.startswith("/raw/")]
        if len(raw_keys) == 0:
            raise ValueError(
                "No raw counts found in '{}' [{}]" "".format(fname, self.name)
            )
        else:
            for k in raw_keys:
                # copy the data table
                raw = store[k]
                self.store.put(k, raw, format="table", data_columns=raw.columns)
                # copy the metadata
                self.set_metadata(k, self.get_metadata(k, store=store), update=False)
                self.logger.info("Copied raw data '{}'".format(k))
        store.close()

    def counts_from_file_tsv(self, fname):
        """
        If a counts file in tsv format has been specified, read the counts into
        a new dataframe and save as raw counts.
        """
        df = pd.read_table(fname, sep="\t", header=0, index_col=0)
        if df.columns != ["count"]:
            raise ValueError(
                "Invalid column names for counts file [{}]" "".format(self.name)
            )
        if len(df) == 0:
            raise ValueError("Empty counts file [{}]".format(self.name))
        label = None
        for elem in ELEMENT_LABELS:
            if elem in self.labels:
                label = elem
                break
        if label is None:
            raise ValueError("No valid element labels [{}]".format(self.name))
        key = "/raw/{}/counts".format(label)
        self.store.put(key, df, format="table", data_columns=df.columns, dtype=np.int32)

    def counts_from_file(self, fname):
        """Get raw counts from a counts file instead of FASTQ_ file.

        The ``'/raw/<element>/counts'`` table will be populated using the given
        input file. The input file should be a two-column file readable by
        ``pandas`` as a series or two-column dataframe or an Enrich2 HDF5 file.

        If the input file is a two-column file, the index will be checked using
        the SeqLib's ``validate_index()`` method.

        If the input file is an HDF5 file, the entire set of ``'/raw'`` tables
        will be copied over, with the metadata intact.
        """
        if not os.path.exists(fname):
            raise IOError("Counts file '{}' not found [{}]" "".format(fname, self.name))
        elif os.path.splitext(fname)[-1].lower() in (".h5"):
            self.counts_from_file_h5(self.counts_file)
        elif os.path.splitext(fname)[-1].lower() in (".txt", ".tsv", ".csv"):
            self.counts_from_file_tsv(self.counts_file)
        else:
            raise ValueError(
                "Unrecognized counts file extension for '{}' "
                "[{}]".format(fname, self.name)
            ) class VariantSeqLib(SeqLib):
    """
    Abstract :py:class:`~seqlib.seqlib.SeqLib` class for for Enrich libraries
    containing variants. Implements core functionality for assessing variants,
    either coding or noncoding. Subclasess must evaluate the variant DNA
    sequences that are being counted.
    """

    def __init__(self):
        SeqLib.__init__(self)
        self.wt = WildTypeSequence(self.name)
        self.aligner = None
        self.aligner_cache = None
        self.variant_min_count = None
        self.max_mutations = None
        # 'synonymous' label may be added in configure() if wt is coding
        self.add_label("variants")
        self.logger = logging.getLogger("{}.{}".format(__name__, self.__class__))

    def configure(self, cfg):
        """
        Set up the object using the config object *cfg*, usually derived from
        a ``.json`` file.
        """
        SeqLib.configure(self, cfg)
        self.logger = logging.getLogger(
            "{}.{} - {}".format(__name__, self.__class__.__name__, self.name)
        )

        self.wt.configure(cfg["variants"]["wild type"])
        if self.is_coding():
            self.add_label("synonymous")
        try:
            if "use aligner" in cfg["variants"]:
                if cfg["variants"]["use aligner"]:
                    self.aligner = Aligner()
                    self.aligner_cache = dict()
                else:
                    self.aligner = None
                    self.aligner_cache = None

            if "min count" in cfg["variants"]:
                self.variant_min_count = int(cfg["variants"]["min count"])
            else:
                self.variant_min_count = 0

            if "max mutations" in cfg["variants"]:
                self.max_mutations = int(cfg["variants"]["max mutations"])
            else:
                self.max_mutations = DEFAULT_MAX_MUTATIONS

        except KeyError as key:
            raise KeyError(
                "Missing required config value {key} [{name}]"
                "".format(key=key, name=self.name)
            )

    def serialize(self):
        """
        Format this object (and its children) as a config object suitable for
        dumping to a config file.
        """
        cfg = SeqLib.serialize(self)

        cfg["variants"] = dict()
        cfg["variants"]["wild type"] = self.wt.serialize()
        cfg["variants"]["use aligner"] = self.aligner is not None
        if self.max_mutations != DEFAULT_MAX_MUTATIONS:
            cfg["variants"]["max mutations"] = self.max_mutations
        if self.variant_min_count > 0:
            cfg["variants"]["min count"] = self.variant_min_count

        return cfg

    def is_coding(self):
        """
        Return ``True`` if the variants are protein-coding, else ``False``.
        """
        return self.wt.is_coding()

    def has_wt_sequence(self):
        """
        Returns ``True``, because :py:class:`~seqlib.seqlib.VariantSeqLib`
        objects have a wild type sequence. Raises a ValueError if the wild type
        sequence is not set properly.
        """
        if self.wt is not None:
            return True
        else:
            raise ValueError("Wild type not set properly [{}]".format(self.name))

    def align_variant(self, variant_dna):
        """
        Use the local :py:class:`~seqlib.aligner.Aligner` instance to align the
        *variant_dna* to the wild type sequence. Returns a list of HGVS_
        variant strings.

        Aligned variants are stored in a local dictionary to avoid recomputing
        alignments. This dictionary should be cleared after all variants are
        counted, to save memory.

        .. warning:: Using the :py:class:`~seqlib.aligner.Aligner` \
        dramatically increases runtime.
        """
        if variant_dna in self.aligner_cache.keys():
            return self.aligner_cache[variant_dna]

        mutations = list()
        traceback = self.aligner.align(self.wt.dna_seq, variant_dna)
        for x, y, cat, length in traceback:
            if cat == "match":
                continue
            elif cat == "mismatch":
                mut = "{pre}>{post}".format(pre=self.wt.dna_seq[x], post=variant_dna[y])
            elif cat == "insertion":
                if y > length:
                    dup = variant_dna[y : y + length]
                    if dup == variant_dna[y - length : y]:
                        mut = "dup{seq}".format(seq=dup)
                    else:
                        mut = "_{pos}ins{seq}".format(pos=x + 2, seq=dup)
                else:
                    mut = "_{pos}ins{seq}".format(
                        pos=x + 2, seq=variant_dna[y : y + length]
                    )
            elif cat == "deletion":
                mut = "_{pos}del".format(pos=x + length)
            else:
                raise ValueError("Unable to resolve mutation [{}]".format(self.name))
            mutations.append((x, mut))

        self.aligner_cache[variant_dna] = mutations
        return mutations

    def count_variant(self, variant_dna, include_indels=True):
        """
        Identifies mutations and counts the *variant_dna* sequence.
        The algorithm attempts to call variants by comparing base-by-base.
        If the *variant_dna* and wild type DNA are different lengths, or if
        there are an excess of mismatches (indicating a possible indel), local
        alignment is performed using :py:meth:`align_variant` if this option
        has been selected in the configuration.

        Each variant is stored as a tab-delimited string of mutations in HGVS_
        format. Returns a list of HGVS_ variant strings. Returns an empty list
        if the variant is wild type. Returns None if the variant was discarded
        due to excess mismatches.
        """
        if not re.match("^[ACGTNXacgtnx]+$", variant_dna):
            raise ValueError(
                "Variant DNA sequence contains unexpected "
                "characters [{}]".format(self.name)
            )

        variant_dna = variant_dna.upper()

        if len(variant_dna) != len(self.wt.dna_seq):
            if self.aligner is not None:
                mutations = self.align_variant(variant_dna)
            else:
                return None
        else:
            mutations = list()
            for i in xrange(len(variant_dna)):
                if variant_dna[i] != self.wt.dna_seq[i]:
                    mutations.append(
                        (
                            i,
                            "{pre}>{post}".format(
                                pre=self.wt.dna_seq[i], post=variant_dna[i]
                            ),
                        )
                    )
                    if len(mutations) > self.max_mutations:
                        if self.aligner is not None:
                            mutations = self.align_variant(variant_dna)
                            if len(mutations) > self.max_mutations:
                                # too many mutations post-alignment
                                return None
                            else:
                                # stop looping over this variant
                                break
                        else:
                            # too many mutations and not using aligner
                            return None

        mutation_strings = list()
        if self.is_coding():
            variant_protein = ""
            for i in xrange(0, len(variant_dna), 3):
                try:
                    variant_protein += CODON_TABLE[variant_dna[i : i + 3]]
                except KeyError:  # garbage codon due to indel, X, or N
                    variant_protein += "?"

            for pos, change in mutations:
                ref_dna_pos = pos + self.wt.dna_offset + 1
                ref_pro_pos = pos / 3 + self.wt.protein_offset + 1
                mut = "c.{pos}{change}".format(pos=ref_dna_pos, change=change)
                if has_indel(change):
                    mut += " (p.{pre}{pos}fs)".format(
                        pre=AA_CODES[self.wt.protein_seq[pos / 3]], pos=ref_pro_pos
                    )
                elif variant_protein[pos / 3] == self.wt.protein_seq[pos / 3]:
                    mut += " (p.=)"
                else:
                    mut += " (p.{pre}{pos}{post})".format(
                        pre=AA_CODES[self.wt.protein_seq[pos / 3]],
                        pos=ref_pro_pos,
                        post=AA_CODES[variant_protein[pos / 3]],
                    )
                mutation_strings.append(mut)
        else:
            for pos, change in mutations:
                ref_dna_pos = pos + self.wt.dna_offset + 1
                mut = "n.{pos}{change}".format(pos=ref_dna_pos, change=change)
                mutation_strings.append(mut)

        if len(mutation_strings) > 0:
            variant_string = ", ".join(mutation_strings)
        else:
            variant_string = WILD_TYPE_VARIANT
        return variant_string

    def count_synonymous(self):
        """
        Combine counts for synonymous variants (defined as variants that differ
        at the nucleotide level but not at the amino acid level) and store them 
        under the ``synonymous`` label.

        This method should be called only after ``variants`` have been counted.

        .. note:: The total number of ``synonymous`` variants may be greater \
        than the total number of ``variants`` after filtering. This is \
        because ``variants`` are combined into ``synonymous`` entries at the \
        :py:class:`~seqlib.seqlib.SeqLib` level before count-based filtering, \
        allowing filtered ``variants`` to contribute counts to their \
        ``synonymous`` entry.
        """
        if not self.is_coding():
            self.logger.warning("Cannot count synonymous mutations in noncoding data")
            return

        if self.check_store("/main/synonymous/counts"):
            return

        self.logger.info("Counting synonymous variants")

        df_dict = dict()

        for variant, count in self.store["/main/variants/counts"].iterrows():
            if variant == WILD_TYPE_VARIANT:
                df_dict[variant] = count["count"]
            else:
                variant = protein_variant(variant)
                if len(variant) == 0:
                    variant = SYNONYMOUS_VARIANT
                try:
                    df_dict[variant] += count["count"]
                except KeyError:
                    df_dict[variant] = count["count"]

        self.save_counts("synonymous", df_dict, raw=False)
        del df_dict

    def report_filtered_variant(self, variant, count):
        """
        Outputs a summary of the filtered variant to *handle*. Internal filter
        names are converted to messages using the ``SeqLib.filter_messages``
        dictionary. Related to :py:meth:`SeqLib.report_filtered`.
        """
        self.logger.debug(
            "Filtered variant (quantity={n}) (excess mutations)"
            "\n{read!s}".format(n=count, read=variant)
        )
import Tkinter as tk
import ttk
import tkSimpleDialog
import tkMessageBox
from sys import maxsize
from collections import OrderedDict
from .dialog_elements import (
    FileEntry,
    IntegerEntry,
    Checkbox,
    StringEntry,
    SectionLabel,
    DEFAULT_COLUMNS,
)
from ..basic import BasicSeqLib
from ..barcodevariant import BcvSeqLib
from ..barcodeid import BcidSeqLib
from ..barcode import BarcodeSeqLib
from ..idonly import IdOnlySeqLib
from ..overlap import OverlapSeqLib
from ..seqlib import SeqLib
from ..variant import VariantSeqLib
        ("main", "counts"),
        ("fastq", "trimming", "filters"),
        ("variants",),
    ],
    "BarcodeSeqLib": [
        ("main", "counts"),
        ("fastq", "trimming", "filters"),
        ("barcodes",),
    ],
    "IdOnlySeqLib": [("main", "counts"), ("identifiers",)],
    "Selection": [("main",)],
    "Condition": [("main",)],
    "Experiment": [("main",)],
}


class CountsToggle(object):
    def __init__(self, frame_dict):
        self.mode = tk.StringVar()
        self.frame_dict = frame_dict
        self.rb_fastq = None
        self.rb_coutns = None

    def body(self, master, row, columns=DEFAULT_COLUMNS, **kwargs):
        self.rb_fastq = ttk.Radiobutton(
            master,
            text="FASTQ File Mode",
            variable=self.mode,
            value="FASTQ",
            command=self.fastq_mode,
        )
        self.rb_fastq.grid(row=row, column=0, columnspan=columns, sticky="ew")
        self.rb_counts = ttk.Radiobutton(
            master,
            text="Count File Mode",
            variable=self.mode,
            value="Counts",
            command=self.counts_mode,
        )
        self.rb_counts.grid(row=row + 1, column=0, columnspan=columns, sticky="ew")
        return 2

    def counts_mode(self):
        for k in ["filters", "fastq", "overlap", "trimming"]:
            if k in self.frame_dict:
                for x in self.frame_dict[k]:
                    try:
                        x.disable()
                    except AttributeError:
                        print(x, k)
        for k in ["counts"]:
            if k in self.frame_dict:
                for x in self.frame_dict[k]:
                    try:
                        x.enable()
                    except AttributeError:
                        print(x, k)

    def fastq_mode(self):
        for k in ["counts"]:
            if k in self.frame_dict:
                for x in self.frame_dict[k]:
                    try:
                        x.disable()
                    except AttributeError:
                        print(x, k)
        for k in ["filters", "fastq", "overlap", "trimming"]:
            if k in self.frame_dict:
                for x in self.frame_dict[k]:
                    try:
                        x.enable()
                    except AttributeError:
                        print(x, k)

    def validate(self):
        return True

    def apply(self):
        return None

    def enable(self):
        pass

    def disable(self):
        pass


class EditDialog(tkSimpleDialog.Dialog):
    """
    Dialog box for editing elements. Also used to set properties on newly-created elements.

    *parent_window* is the Tk window that owns this child window
    *tree* is the object containing the config tree and associated Treeview
    *new* is ``True`` if we are creating a new child of the focused item or ``False`` if we are editing the focused item
    """

    def __init__(self, parent_window, tree, element, title="Configure Object"):
        self.tree = tree
        self.element = element
        self.element_cfg = None
        self.frame_dict = OrderedDict()
        self.toggle = None

        # create the editable version of the config object
        self.element_cfg = self.element.serialize()

        # dialog options common to all elements
        self.frame_dict["main"] = list()
        self.name_entry = StringEntry("Name", self.element_cfg, "name", optional=False)
        self.frame_dict["main"].append(self.name_entry)
        if "output directory" in self.element_cfg:
            self.frame_dict["main"].append(
                FileEntry(
                    "Output Directory",
                    self.element_cfg,
                    "output directory",
                    optional=self.element != self.tree.root_element,
                    directory=True,
                )
            )

        if isinstance(self.element, SeqLib):