Subsets the data.

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``bfile``).

    This function calls the
    :py:mod:`pyGenClean.pyGenClean.PlinkUtils.subset_data` module. The required
    file type for this module is ``bfile``, hence the need to use the
    :py:func:`check_input_files` to check if the file input file type is the
    good one, or to create it if needed.

    .. note::
        The output file type is the same as the input file type.

def run_subset_data(in_prefix, in_type, out_prefix, base_dir, options):
    """Subsets the data.

    :param in_prefix: the prefix of the input files.
    :param in_type: the type of the input files.
    :param out_prefix: the output prefix.
    :param base_dir: the output directory.
    :param options: the options needed.

    :type in_prefix: str
    :type in_type: str
    :type out_prefix: str
    :type base_dir: str
    :type options: list

    :returns: a tuple containing the prefix of the output files (the input
              prefix for the next script) and the type of the output files
              (``bfile``).

    This function calls the
    :py:mod:`pyGenClean.pyGenClean.PlinkUtils.subset_data` module. The required
    file type for this module is ``bfile``, hence the need to use the
    :py:func:`check_input_files` to check if the file input file type is the
    good one, or to create it if needed.

    .. note::
        The output file type is the same as the input file type.

    """
    # Creating the output directory
    os.mkdir(out_prefix)

    # The prefix of the script
    script_prefix = os.path.join(out_prefix, "subset")

    # The extension of the marker and sample file
    markers_ext = None
    samples_ext = None

    # Looking at what we have
    required_type = None
    if in_type == "bfile":
        required_type = "bfile"
        markers_ext = ".bim"
        samples_ext = ".fam"

    elif in_type == "tfile":
        required_type = "tfile"
        markers_ext = ".tped"
        samples_ext = ".tfam"

    else:
        required_type = "file"
        markers_ext = ".map"
        samples_ext = ".ped"

    if "--is-bfile" in set(options):
        required_type = "bfile"

    # Checking the input file
    check_input_files(in_prefix, in_type, required_type)

    # What is going to be done
    is_extract = "--extract" in options
    is_exclude = "--exclude" in options
    is_keep = "--keep" in options
    is_remove = "--remove" in options

    # Checking if we have a reason for markers
    m_subset_reason = None
    if "--reason-marker" in options:
        m_subset_reason = latex_template.sanitize_tex(
            options.pop(options.index("--reason-marker")+1),
        )
        options.pop(options.index("--reason-marker"))

        if (not is_extract) and (not is_exclude):
            # There was a reason for marker exclusion, but no exclusion will be
            # performed, so we skip in the report
            m_subset_reason = None

    # Checking if we have a reason for samples
    s_subset_reason = None
    if "--reason-sample" in options:
        s_subset_reason = latex_template.sanitize_tex(
            options.pop(options.index("--reason-sample")+1),
        )
        options.pop(options.index("--reason-sample"))

        if (not is_keep) and (not is_remove):
            # There was a reason for sample exclusion, but no exclusion will be
            # performed, so we skip in the report
            s_subset_reason = None

    # We need to inject the name of the input file and the name of the output
    # prefix
    options += ["--ifile", in_prefix,
                "--out", os.path.join(out_prefix, "subset")]

    if required_type == "bfile":
        options.append("--is-bfile")
    elif required_type == "tfile":
        options.append("--is-tfile")
    else:
        options.append("--is-file")

    # We run the script
    try:
        subset_data.main(options)
    except subset_data.ProgramError as e:
        msg = "subset_data: {}".format(e)
        raise ProgramError(msg)

    # The files before the subset
    samples_before_fn = in_prefix
    markers_before_fn = in_prefix

    # The files after the subset
    samples_after_fn = script_prefix
    markers_after_fn = script_prefix

    # The name of the subset file for markers and samples
    sample_subset_fn = None
    marker_subset_fn = None

    # The samples and markers that were removed
    removed_samples = {}
    removed_markers = {}

    # The set of samples and markers before subset
    samples_before = None
    markers_before = None

    # The set of remaining samples and markers after subset
    samples_after = None
    markers_after = None

    # Markers were extracted
    nb_extract = None
    if is_extract:
        # Counting the number of markers that were extracted
        marker_subset_fn = options[options.index("--extract") + 1]
        with open(marker_subset_fn, "r") as i_file:
            nb_extract = sum(1 for line in i_file)

    # Markers were excluded
    nb_exclude = None
    if is_exclude:
        marker_subset_fn = options[options.index("--exclude") + 1]
        with open(marker_subset_fn, "r") as i_file:
            nb_exclude = sum(1 for line in i_file)

    # Checking the difference between both (for markers)
    if is_extract or is_exclude:
        markers_before_fn += markers_ext
        markers_after_fn += ".bim" if required_type == "bfile" else markers_ext

        markers_before = None
        with open(markers_before_fn, "r") as i_file:
            markers_before = {
                createRowFromPlinkSpacedOutput(line)[1] for line in i_file
            }

        with open(markers_after_fn, "r") as i_file:
            markers_after = {
                createRowFromPlinkSpacedOutput(line)[1] for line in i_file
            }

        reason = os.path.relpath(marker_subset_fn, base_dir)
        if m_subset_reason is not None:
            reason = m_subset_reason
        removed_markers = {
            name: "subset {}".format(reason)
            for name in markers_before - markers_after
        }

    # Samples were kept
    nb_keep = None
    if is_keep:
        sample_subset_fn = options[options.index("--keep") + 1]
        with open(sample_subset_fn, "r") as i_file:
            nb_keep = sum(1 for line in i_file)

    # Samples were removed
    nb_remove = None
    if is_remove:
        sample_subset_fn = options[options.index("--remove") + 1]
        with open(sample_subset_fn, "r") as i_file:
            nb_remove = sum(1 for line in i_file)

    if is_keep or is_remove:
        samples_before_fn += samples_ext
        samples_after_fn += ".fam" if required_type == "bfile" else samples_ext

        samples_before = None
        with open(samples_before_fn, "r") as i_file:
            samples_before = {
                tuple(createRowFromPlinkSpacedOutput(line)[:2])
                for line in i_file
            }

        with open(samples_after_fn, "r") as i_file:
            samples_after = {
                tuple(createRowFromPlinkSpacedOutput(line)[:2])
                for line in i_file
            }

        reason = os.path.relpath(sample_subset_fn, base_dir)
        if s_subset_reason is not None:
            reason = s_subset_reason
        removed_samples = {
            "\t".join(name): "subset {}".format(reason)
            for name in samples_before - samples_after
        }

    # Reading the log file to gather what is left
    log_file = None
    with open(script_prefix + ".log", "r") as i_file:
        log_file = i_file.read()

    # Checking the number of markers at the beginning
    nb_marker_start = re.search(
        r"(\d+) (\(of \d+\) )?markers to be included "
        "from \[ {}".format(in_prefix),
        log_file,
    )
    if nb_marker_start:
        nb_marker_start = int(nb_marker_start.group(1))

    # Checking the number of markers at the end
    nb_marker_end = re.search(
        r"Before frequency and genotyping pruning, there are (\d+) SNPs",
        log_file,
    )
    if nb_marker_end:
        nb_marker_end = int(nb_marker_end.group(1))

    # Checking the number of samples
    if (markers_after is not None) and (nb_marker_end != len(markers_after)):
        raise ProgramError("Something went wrong with Plink's subset (numbers "
                           "are different from data and log file)")
    if nb_marker_start - nb_marker_end != len(removed_markers):
        raise ProgramError("Something went wrong with Plink's subset (numbers "
                           "are different from data and log file)")

    # Checking the number of samples at the beginning
    nb_sample_start = re.search(
        r"(\d+) individuals read from \[ {}".format(in_prefix),
        log_file,
    )
    if nb_sample_start:
        nb_sample_start = int(nb_sample_start.group(1))

    # Checking the number of samples at the end
    nb_sample_end = re.search(
        r"(\d+) founders and (\d+) non-founders found",
        log_file,
    )
    if nb_sample_end:
        nb_sample_end = int(nb_sample_end.group(1)) + \
                        int(nb_sample_end.group(2))

    # Checking the number of samples
    if (samples_after is not None) and (nb_sample_end != len(samples_after)):
        raise ProgramError("Something went wrong with Plink's subset (numbers "
                           "are different from data and log file)")
    if nb_sample_start - nb_sample_end != len(removed_samples):
        raise ProgramError("Something went wrong with Plink's subset (numbers "
                           "are different from data and log file)")

    # Creating the reasons
    reasons = []
    if m_subset_reason is not None:
        reasons.append(m_subset_reason)
    if s_subset_reason is not None:
        reasons.append(s_subset_reason)

    # We write a LaTeX summary
    latex_file = os.path.join(script_prefix + ".summary.tex")
    try:
        with open(latex_file, "w") as o_file:
            section_name = subset_data.pretty_name
            if len(reasons) > 0:
                section_name += " ({})".format(", ".join(reasons))
            print >>o_file, latex_template.subsection(section_name)
            text = ""
            if is_extract:
                text += (
                    "The file for marker extraction contained {:,d} marker{}. "
                    "Out of a total of {:,d} marker{}, {:,d} "
                    "remained ({:,d} excluded).".format(
                        nb_extract,
                        "s" if nb_extract > 1 else "",
                        nb_marker_start,
                        "s" if nb_marker_start > 1 else "",
                        nb_marker_end,
                        nb_marker_start - nb_marker_end,
                    )
                )
            if is_exclude:
                text += (
                    "The file for marker exclusion contained {:,d} marker{}. "
                    "Out of a total of {:,d} marker{}, {:,d} "
                    "remained ({:,d} excluded). ".format(
                        nb_exclude,
                        "s" if nb_exclude > 1 else "",
                        nb_marker_start,
                        "s" if nb_marker_start > 1 else "",
                        nb_marker_end,
                        nb_marker_start - nb_marker_end,
                    )
                )
            if is_keep:
                text += (
                    "The file containing samples to keep contained {:,d} "
                    "sample{}. Out of a total of {:,d} sample{}, {:,d} "
                    "remained ({:,d} removed). ".format(
                        nb_keep,
                        "s" if nb_keep > 1 else "",
                        nb_sample_start,
                        "s" if nb_sample_start > 1 else "",
                        nb_sample_end,
                        nb_sample_start - nb_sample_end,
                    )
                )
            if is_remove:
                text += (
                    "The file containing samples to remove contained {:,d} "
                    "sample{}. Out of a total of {:,d} sample{}, {:,d} "
                    "remained ({:,d} removed). ".format(
                        nb_remove,
                        "s" if nb_remove > 1 else "",
                        nb_sample_start,
                        "s" if nb_sample_start > 1 else "",
                        nb_sample_end,
                        nb_sample_start - nb_sample_end,
                    )
                )
            print >>o_file, latex_template.wrap_lines(text)

    except IOError:
        msg = "{}: cannot write LaTeX summary".format(latex_file)
        raise ProgramError(msg)

    # Writing the excluded samples
    o_filename = os.path.join(base_dir, "excluded_samples.txt")
    with open(o_filename, "a") as o_file:
        for value in removed_samples.items():
            print >>o_file, "\t".join(value)

    # Writing the excluded markers to file
    o_filename = os.path.join(base_dir, "excluded_markers.txt")
    with open(o_filename, "a") as o_file:
        for value in removed_markers.items():
            print >>o_file, "\t".join(value)

    # Writing the summary results
    with open(os.path.join(base_dir, "results_summary.txt"), "a") as o_file:
        print >>o_file, "# {}".format(script_prefix)
        if nb_marker_end != nb_marker_start:
            reason = "_file_path:" + marker_subset_fn
            if m_subset_reason is not None:
                reason = m_subset_reason

            print >>o_file, "Number of markers excluded"
            print >>o_file, ("  - {reason}\t{nb:,d}\t-{nb:,d}".format(
                reason=reason,
                nb=nb_marker_start - nb_marker_end,
            ))
            print >>o_file, "---"
        if nb_sample_end != nb_sample_start:
            reason = "_file_path:" + sample_subset_fn
            if s_subset_reason is not None:
                reason = s_subset_reason

            print >>o_file, "Number of samples removed"
            print >>o_file, ("  - {reason}\t{nb:,d}\t\t-{nb:,d}".format(
                reason=reason,
                nb=nb_sample_start - nb_sample_end,
            ))
            print >>o_file, "---"

    # Modifying the description
    description = subset_data.desc
    if len(reasons) > 0:
        description += " ({})".format(", ".join(reasons))

    # We know this step does produce a new data set (bfile), so we return it
    return _StepResult(
        next_file=os.path.join(out_prefix, "subset"),
        next_file_type=required_type,
        latex_summary=latex_file,
        description=description,
        long_description=subset_data.long_desc,
        graph_path=None,
    )