Runs step1 (duplicated samples).

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``tfile``).

    This function calls the :py:mod:`pyGenClean.DupSamples.duplicated_samples`
    module. The required file type for this module is ``tfile``, hence the need
    to use the :py:func:`check_input_files` to check if the file input file
    type is the good one, or to create it if needed.

def run_duplicated_samples(in_prefix, in_type, out_prefix, base_dir, options):
    """Runs step1 (duplicated samples).

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``tfile``).

    This function calls the :py:mod:`pyGenClean.DupSamples.duplicated_samples`
    module. The required file type for this module is ``tfile``, hence the need
    to use the :py:func:`check_input_files` to check if the file input file
    type is the good one, or to create it if needed.

    """
    # Creating the output directory
    os.mkdir(out_prefix)

    # We know we need tfile
    required_type = "tfile"
    check_input_files(in_prefix, in_type, required_type)

    # We need to inject the name of the input file and the name of the output
    # prefix
    script_prefix = os.path.join(out_prefix, "dup_samples")
    options += ["--{}".format(required_type), in_prefix,
                "--out", script_prefix]

    # We run the script
    try:
        duplicated_samples.main(options)
    except duplicated_samples.ProgramError as e:
        msg = "duplicated_samples: {}".format(e)
        raise ProgramError(msg)

    # Reading the number of duplicated samples
    duplicated_count = defaultdict(int)
    if os.path.isfile(script_prefix + ".duplicated_samples.tfam"):
        with open(script_prefix + ".duplicated_samples.tfam", "r") as i_file:
            duplicated_count = Counter([
                tuple(createRowFromPlinkSpacedOutput(line)[:2])
                for line in i_file
            ])

    # Counting the number of zeroed out genotypes per duplicated sample
    zeroed_out = defaultdict(int)
    if os.path.isfile(script_prefix + ".zeroed_out"):
        with open(script_prefix + ".zeroed_out", "r") as i_file:
            zeroed_out = Counter([
                tuple(line.rstrip("\r\n").split("\t")[:2])
                for line in i_file.read().splitlines()[1:]
            ])
    nb_zeroed_out = sum(zeroed_out.values())

    # Checking the not good enough samples
    not_good_enough = set()
    if os.path.isfile(script_prefix + ".not_good_enough"):
        with open(script_prefix + ".not_good_enough", "r") as i_file:
            not_good_enough = {
                tuple(line.rstrip("\r\n").split("\t")[:4])
                for line in i_file.read().splitlines()[1:]
            }

    # Checking which samples were chosen
    chosen_sample = set()
    if os.path.isfile(script_prefix + ".chosen_samples.info"):
        with open(script_prefix + ".chosen_samples.info", "r") as i_file:
            chosen_sample = {
                tuple(line.rstrip("\r\n").split("\t"))
                for line in i_file.read().splitlines()[1:]
            }

    # Finding if some 'not_good_enough' samples were chosen
    not_good_still = {s[2:] for s in chosen_sample & not_good_enough}

    # We create a LaTeX summary
    latex_file = os.path.join(script_prefix + ".summary.tex")
    try:
        with open(latex_file, "w") as o_file:
            print >>o_file, latex_template.subsection(
                duplicated_samples.pretty_name
            )
            text = (
                "A total of {:,d} duplicated sample{} {} found.".format(
                    len(duplicated_count),
                    "s" if len(duplicated_count) > 1 else "",
                    "were" if len(duplicated_count) > 1 else "was",
                )
            )
            print >>o_file, latex_template.wrap_lines(text)

            if len(duplicated_count) > 0:
                text = (
                    "While merging duplicates, a total of {:,d} genotype{} {} "
                    "zeroed out. A total of {:,d} sample{} {} found to be not "
                    "good enough for duplicate completion.".format(
                        nb_zeroed_out,
                        "s" if nb_zeroed_out > 1 else "",
                        "were" if nb_zeroed_out > 1 else "was",
                        len(not_good_enough),
                        "s" if len(not_good_enough) > 1 else "",
                        "were" if len(not_good_enough) > 1 else "was",
                    )
                )
                print >>o_file, latex_template.wrap_lines(text)

                table_label = re.sub(
                    r"[/\\]",
                    "_",
                    script_prefix,
                ) + "_dup_samples"
                text = (
                    r"Table~\ref{" + table_label + "} summarizes the number "
                    "of each duplicated sample with some characteristics."
                )
                print >>o_file, latex_template.wrap_lines(text)

                if len(not_good_still) > 0:
                    text = latex_template.textbf(
                        "There {} {:,d} sample{} that {} not good due to low "
                        "completion or concordance, but {} still selected as "
                        "the best duplicate (see Table~{}).".format(
                            "were" if len(not_good_still) > 1 else "was",
                            len(not_good_still),
                            "s" if len(not_good_still) > 1 else "",
                            "were" if len(not_good_still) > 1 else "was",
                            "were" if len(not_good_still) > 1 else "was",
                            r"~\ref{" + table_label + "}",
                        )
                    )
                    print >>o_file, latex_template.wrap_lines(text)

                # Getting the template
                longtable_template = latex_template.jinja2_env.get_template(
                    "longtable_template.tex",
                )

                # The table caption
                table_caption = (
                    "Summary of the {:,d} duplicated sample{}. The number of "
                    "duplicates and the total number of zeroed out genotypes "
                    "are shown.".format(
                        len(duplicated_count),
                        "s" if len(duplicated_count) > 1 else "",
                    )
                )

                if len(not_good_still) > 0:
                    table_caption += (
                        " A total of {:,d} sample{} (highlighted) {} not good "
                        "enough for completion, but {} chosen as the best "
                        "duplicate, and {} still in the final "
                        "dataset).".format(
                            len(not_good_still),
                            "s" if len(not_good_still) > 1 else "",
                            "were" if len(not_good_still) > 1 else "was",
                            "were" if len(not_good_still) > 1 else "was",
                            "are" if len(not_good_still) > 1 else "is",
                        )
                    )

                duplicated_samples_list = duplicated_count.most_common()
                print >>o_file, longtable_template.render(
                    table_caption=table_caption,
                    table_label=table_label,
                    nb_col=4,
                    col_alignments="llrr",
                    text_size="scriptsize",
                    header_data=[("FID", 1), ("IID", 1), ("Nb Duplicate", 1),
                                 ("Nb Zeroed", 1)],
                    tabular_data=[
                        [latex_template.sanitize_tex(fid),
                         latex_template.sanitize_tex(iid),
                         "{:,d}".format(nb),
                         "{:,d}".format(zeroed_out[(fid, iid)])]
                        for (fid, iid), nb in duplicated_samples_list
                    ],
                    highlighted=[
                        (fid, iid) in not_good_still
                        for fid, iid in [i[0] for i in duplicated_samples_list]
                    ],
                )

    except IOError:
        msg = "{}: cannot write LaTeX summary".format(latex_file)
        raise ProgramError(msg)

    # Writing the summary results
    with open(os.path.join(base_dir, "results_summary.txt"), "a") as o_file:
        print >>o_file, "# {}".format(script_prefix)
        counter = Counter(duplicated_count.values()).most_common()
        if counter:
            print >>o_file, "Number of replicated samples"
        else:
            print >>o_file, "Number of replicated samples\t0"
        for rep_type, rep_count in counter:
            print >>o_file, "  - x{}\t{:,d}\t\t-{:,d}".format(
                rep_type,
                rep_count,
                (rep_count * rep_type) - rep_count,
            )
        print >>o_file, ("Poorly chosen replicated "
                         "samples\t{:,d}".format(len(not_good_still)))
        print >>o_file, "---"

    # We know this step does produce a new data set (tfile), so we return it
    return _StepResult(
        next_file=os.path.join(out_prefix, "dup_samples.final"),
        next_file_type="tfile",
        latex_summary=latex_file,
        description=duplicated_samples.desc,
        long_description=duplicated_samples.long_desc,
        graph_path=None,
    )