Runs step2 (duplicated snps).

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``tfile``).

    This function calls the :py:mod:`pyGenClean.DupSNPs.duplicated_snps`
    module. The required file type for this module is ``tfile``, hence the need
    to use the :py:func:`check_input_files` to check if the file input file
    type is the good one, or to create it if needed.

    .. note::
        This function creates a ``map`` file, needed for the
        :py:mod:`pyGenClean.DupSNPs.duplicated_snps` module.

def run_duplicated_snps(in_prefix, in_type, out_prefix, base_dir, options):
    """Runs step2 (duplicated snps).

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``tfile``).

    This function calls the :py:mod:`pyGenClean.DupSNPs.duplicated_snps`
    module. The required file type for this module is ``tfile``, hence the need
    to use the :py:func:`check_input_files` to check if the file input file
    type is the good one, or to create it if needed.

    .. note::
        This function creates a ``map`` file, needed for the
        :py:mod:`pyGenClean.DupSNPs.duplicated_snps` module.

    """
    # Creating the output directory
    os.mkdir(out_prefix)

    # We know we need a tfile
    required_type = "tfile"
    check_input_files(in_prefix, in_type, required_type)

    # This step require a map file (we now have a tfile)
    if not os.path.isfile(in_prefix + ".map"):
        outputFile = None
        try:
            outputFile = open(in_prefix + ".map", "w")
        except IOError:
            msg = "{}: can't write file".format(in_prefix + ".map")
            raise ProgramError(msg)
        try:
            with open(in_prefix + ".tped", 'r') as inputFile:
                for line in inputFile:
                    row = createRowFromPlinkSpacedOutput(line)
                    print >>outputFile, "\t".join(row[:4])
        except IOError:
            msg = "{}: no such file".format(in_prefix + ".tped")
            raise ProgramError(msg)
        outputFile.close()

    # We need to inject the name of the input file and the name of the output
    # prefix
    script_prefix = os.path.join(out_prefix, "dup_snps")
    options += ["--{}".format(required_type), in_prefix,
                "--out", script_prefix]

    # We run the script
    try:
        duplicated_snps.main(options)
    except duplicated_snps.ProgramError as e:
        msg = "duplicated_snps: {}".format(e)
        raise ProgramError(msg)

    # Reading the number of duplicated markers
    duplicated_count = defaultdict(int)
    if os.path.isfile(script_prefix + ".duplicated_snps.tped"):
        with open(script_prefix + ".duplicated_snps.tped", "r") as i_file:
            duplicated_count = Counter(
                (i[0], i[3]) for i in [
                    tuple(createRowFromPlinkSpacedOutput(line)[:4])
                    for line in i_file
                ]
            )

    # Counting the number of zeroed out genotypes per duplicated markers
    zeroed_out = defaultdict(int)
    if os.path.isfile(script_prefix + ".zeroed_out"):
        with open(script_prefix + ".zeroed_out", "r") as i_file:
            zeroed_out = Counter([
                tuple(line.rstrip("\r\n").split("\t")[:2])
                for line in i_file.read().splitlines()[1:]
            ])
    nb_zeroed_out = sum(zeroed_out.values())

    # Checking the not good enough markers
    not_good_enough = set()
    if os.path.isfile(script_prefix + ".not_good_enough"):
        with open(script_prefix + ".not_good_enough", "r") as i_file:
            not_good_enough = {
                line.rstrip("\r\n").split("\t")[0]
                for line in i_file.read().splitlines()[1:]
            }

    # Checking which markers were chosen
    chosen_markers = set()
    if os.path.isfile(script_prefix + ".chosen_snps.info"):
        with open(script_prefix + ".chosen_snps.info", "r") as i_file:
            chosen_markers = set(i_file.read().splitlines())

    # Finding if some 'not_good_enough' samples were chosen
    not_good_still = chosen_markers & not_good_enough

    # Adding the 'not chosen markers' to the list of excluded markers
    removed_markers = set()
    o_filename = os.path.join(base_dir, "excluded_markers.txt")
    if os.path.isfile(script_prefix + ".removed_duplicates"):
        with open(script_prefix + ".removed_duplicates", "r") as i_file:
            removed_markers = set(i_file.read().splitlines())
            with open(o_filename, "a") as o_file:
                for marker_id in removed_markers:
                    print >>o_file, marker_id + "\t" + "removed duplicate"

    # Writing the summary results
    total_remaining = 0
    with open(os.path.join(base_dir, "results_summary.txt"), "a") as o_file:
        print >>o_file, "# {}".format(script_prefix)
        rep_counter = Counter(duplicated_count.values()).most_common()
        if rep_counter:
            print >>o_file, "Number of replicated markers"
        else:
            print >>o_file, "Number of replicated markers\t0"
        total_nb_removed_rep = 0
        for rep_type, rep_count in rep_counter:
            nb_removed_rep = (rep_count * rep_type) - rep_count
            print >>o_file, "  - x{}\t{:,d}\t-{:,d}".format(
                rep_type,
                rep_count,
                nb_removed_rep,
            )
            total_nb_removed_rep += nb_removed_rep
        total_remaining = total_nb_removed_rep - len(removed_markers)
        print >>o_file, (
            "Number of replicated markers kept\t{nb:,d}\t+{nb:,d}".format(
                nb=total_remaining,
            )
        )
        print >>o_file, ("Poorly chosen replicated markers\t"
                         "{nb:,d}".format(nb=len(not_good_still)))
        print >>o_file, ("Final number of excluded markers\t"
                         "{nb:,d}".format(nb=len(removed_markers)))
        print >>o_file, "---"

    # We create a LaTeX summary
    latex_file = os.path.join(script_prefix + ".summary.tex")
    try:
        with open(latex_file, "w") as o_file:
            print >>o_file, latex_template.subsection(
                duplicated_snps.pretty_name
            )

            text = (
                "A total of {:,d} duplicated marker{} {} found.".format(
                    len(duplicated_count),
                    "s" if len(duplicated_count) > 1 else "",
                    "were" if len(duplicated_count) > 1 else "was",
                )
            )
            print >>o_file, latex_template.wrap_lines(text)

            if len(duplicated_count) > 0:
                text = (
                    "While merging duplicates, a total of {:,d} genotype{} {} "
                    "zeroed out. A total of {:,d} marker{} {} found to be not "
                    "good enough for duplicate completion.".format(
                        nb_zeroed_out,
                        "s" if nb_zeroed_out > 1 else "",
                        "were" if nb_zeroed_out > 1 else "was",
                        len(not_good_enough),
                        "s" if len(not_good_enough) > 1 else "",
                        "were" if len(not_good_enough) > 1 else "was",
                    )
                )
                print >>o_file, latex_template.wrap_lines(text)

                text = (
                    "A total of {:,d} marker{} {} excluded while creating the "
                    "final dataset.".format(
                        len(removed_markers),
                        "s" if len(removed_markers) > 1 else "",
                        "were" if len(removed_markers) > 1 else "was",
                    )
                )
                print >>o_file, latex_template.wrap_lines(text)

                if total_remaining > 0:
                    text = latex_template.textbf(
                        "In total, {:,d} maker{} {} not merged for different "
                        "reasons (low completion rate, discordant allele, "
                        "discordant MAF, etc) and {} still present in the "
                        "dataset.".format(
                            total_remaining,
                            "s" if total_remaining > 1 else "",
                            "were" if total_remaining > 1 else "was",
                            "are" if total_remaining > 1 else "is",
                        )
                    )
                    print >>o_file, latex_template.wrap_lines(text)

                if len(not_good_still) > 0:
                    start = "A total of"
                    end = " and {} still present in the final dataset.".format(
                        "are" if len(not_good_still) > 1 else "is",
                    )
                    if total_remaining > 0:
                        start = "Out of these,"
                        end = "."
                    text = latex_template.textbf(
                        start + " {:,d} marker{} {} not good enough for "
                        "completion, but {} still selected as the best "
                        "duplicate{}".format(
                            len(not_good_still),
                            "s" if len(not_good_still) > 1 else "",
                            "were" if len(not_good_still) > 1 else "was",
                            "were" if len(not_good_still) > 1 else "was",
                            end,
                        )
                    )
                    print >>o_file, latex_template.wrap_lines(text)

    except IOError:
        msg = "{}: cannot write LaTeX summary".format(latex_file)
        raise ProgramError(msg)

    # We know this step does produce a new data set (tfile), so we return it
    return _StepResult(
        next_file=os.path.join(out_prefix, "dup_snps.final"),
        next_file_type="tfile",
        latex_summary=latex_file,
        description=duplicated_snps.desc,
        long_description=duplicated_snps.long_desc,
        graph_path=None,
    )