import os
import sys
import json
import jinja2
import shutil
import logging
import requests

from collections import defaultdict
from os.path import dirname, join, abspath, split, splitext, exists, basename

logger = logging.getLogger("main.{}".format(__name__))

try:
    import generator.process as pc
    import generator.error_handling as eh
    from __init__ import __version__
    from generator import header_skeleton as hs
    from generator import footer_skeleton as fs
    from generator.process_details import colored_print
    from generator.pipeline_parser import guess_process
except ImportError:
    import flowcraft.generator.process as pc
    import flowcraft.generator.error_handling as eh
    from flowcraft import __version__
    from flowcraft.generator import header_skeleton as hs
    from flowcraft.generator import footer_skeleton as fs
    from flowcraft.generator.process_details import colored_print
    from flowcraft.generator.pipeline_parser import guess_process

class NextflowGenerator:

    def __init__(self, process_connections, nextflow_file, process_map,
                 pipeline_name="flowcraft", ignore_dependencies=False,
                 auto_dependency=True, merge_params=True, export_params=False):

        self.processes = []

        self.process_map = process_map
        """
        dict: Maps the nextflow template name to the corresponding Process
        class of the component.
        """

        # Create the processes attribute with the first special init process.
        # This process will handle the forks of the raw input channels and
        # secondary inputs
        self.processes = [pc.Init(template="init")]
        """
        list: Stores the process interfaces in the specified order
        """

        self._fork_tree = defaultdict(list)
        """
        dict: A dictionary with the fork tree of the pipeline, which consists
        on the the paths of each lane. For instance, a single fork with two
        sinks is represented as: {1: [2,3]}. Subsequent forks are then added
        sequentially: {1:[2,3], 2:[3,4,5]}. This allows the path upstream
        of a process in a given lane to be traversed until the start of the
        pipeline.
        """

        self.lanes = 0
        """
        int: Stores the number of lanes in the pipelines
        """

        self.export_parameters = export_params
        """
        bool: Determines whether the build mode is only for the export of
        parameters in JSON format. Setting to True will disabled some checks,
        such as component dependency requirements
        """

        # When the export_params option is used, disable the auto dependency
        # feature automatically
        auto_deps = auto_dependency if not self.export_parameters else False

        # Builds the connections in the processes, which parses the
        # process_connections dictionary into the self.processes attribute
        # list.
        self._build_connections(process_connections, ignore_dependencies,
                                auto_deps)

        self.nf_file = nextflow_file
        """
        str: Path to file where the pipeline will be generated
        """

        self.pipeline_name = pipeline_name
        """
        str: Name of the pipeline, for customization and help purposes.
        """

        self.template = ""
        """
        str: String that will harbour the pipeline code
        """

        self.secondary_channels = {}
        """
        dict: Stores secondary channel links
        """

        self.main_raw_inputs = {}
        """
        list: Stores the main raw inputs from the user parameters into the
        first process(es).
        """

        self.merge_params = merge_params
        """
        bool: Determines whether the params of the pipeline should be merged
        (i.e., the same param name in multiple components is merged into one)
        or if they should be unique and specific to each component.
        """

        self.extra_inputs = {}
        """
        """

        self.status_channels = []
        """
        list: Stores the status channels from each process
        """

        self.skip_class = [pc.Compiler]
        """
        list: Stores the Process classes that should be skipped when iterating
        over the :attr:`~NextflowGenerator.processes` list.
        """

        self.resources = ""
        """
        str: Stores the resource directives string for each nextflow process.
        See :func:`NextflowGenerator._get_resources_string`.
        """

        self.containers = ""
        """
        str: Stores the container directives string for each nextflow process.
        See :func:`NextflowGenerator._get_container_string`.
        """

        self.params = ""
        """
        str: Stores the params directives string for the nextflow pipeline.
        See :func:`NextflowGenerator._get_params_string`
        """

        self.manifest = ""
        """
        str: Stores de manifest directives string for the nextflow pipeline.
        See :func:`NextflowGenerator._get_manifest_string`
        """

        self.user_config = ""
        """
        str: Stores the user configuration file placeholder. This is an
        empty configuration file that is only added the first time to a
        project directory. If the file already exists, it will not overwrite
        it.
        """

        self.compilers = {
            "patlas_consensus": {
                "cls": pc.PatlasConsensus,
                "template": "patlas_consensus"
            }
        }
        """
        dict: Maps the information about each available compiler process in
        flowcraft. The key of each entry is the name/signature of the
        compiler process. The value is a json/dict object that contains two
        key:pair values:
            - ``cls``: The reference to the compiler class object.
            - ``template``: The nextflow template file of the process.
        """

    @staticmethod
    def _parse_process_name(name_str):
        """Parses the process string and returns the process name and its
        directives

        Process strings my contain directive information with the following
        syntax::

            proc_name={'directive':'val'}

        This method parses this string and returns the process name as a
        string and the directives information as a dictionary.

        Parameters
        ----------
        name_str : str
            Raw string with process name and, potentially, directive
            information

        Returns
        -------
        str
            Process name
        dict or None
            Process directives
        """

        directives = None

        fields = name_str.split("=")
        process_name = fields[0]

        if len(fields) == 2:
            _directives = fields[1].replace("'", '"')
            try:
                directives = json.loads(_directives)
            except json.decoder.JSONDecodeError:
                raise eh.ProcessError(
                    "Could not parse directives for process '{}'. The raw"
                    " string is: {}\n"
                    "Possible causes include:\n"
                    "\t1. Spaces inside directives\n"
                    "\t2. Missing '=' symbol before directives\n"
                    "\t3. Missing quotes (' or \") around directives\n"
                    "A valid example: process_name={{'cpus':'2'}}".format(
                        process_name, name_str))

        return process_name, directives

    def _build_connections(self, process_list, ignore_dependencies,
                           auto_dependency):
        """Parses the process connections dictionaries into a process list

        This method is called upon instantiation of the NextflowGenerator
        class. Essentially, it sets the main input/output channel names of the
        processes so that they can be linked correctly.

        If a connection between two consecutive process is not possible due
        to a mismatch in the input/output types, it exits with an error.

        Returns
        -------

        """

        logger.debug("=============================")
        logger.debug("Building pipeline connections")
        logger.debug("=============================")

        logger.debug("Processing connections: {}".format(process_list))

        for p, con in enumerate(process_list):

            logger.debug("Processing connection '{}': {}".format(p, con))

            # Get lanes
            in_lane = con["input"]["lane"]
            out_lane = con["output"]["lane"]
            logger.debug("[{}] Input lane: {}".format(p, in_lane))
            logger.debug("[{}] Output lane: {}".format(p, out_lane))

            # Update the total number of lines of the pipeline
            if out_lane > self.lanes:
                self.lanes = out_lane

            # Get process names and directives for the output process
            p_in_name, p_out_name, out_directives = self._get_process_names(
                con, p)

            # Check if process is available or correctly named
            if p_out_name not in self.process_map:
                logger.error(colored_print(
                    "\nThe process '{}' is not available."
                        .format(p_out_name), "red_bold"))
                guess_process(p_out_name, self.process_map)
                sys.exit(1)

            # Instance output process
            out_process = self.process_map[p_out_name](template=p_out_name)

            # Update directives, if provided
            if out_directives:
                out_process.update_attributes(out_directives)

            # Set suffix strings for main input/output channels. Suffixes are
            # based on the lane and the arbitrary and unique process id
            # e.g.: 'process_1_1'
            input_suf = "{}_{}".format(in_lane, p)
            output_suf = "{}_{}".format(out_lane, p)
            logger.debug("[{}] Setting main channels with input suffix '{}'"
                         " and output suffix '{}'".format(
                            p, input_suf, output_suf))
            out_process.set_main_channel_names(input_suf, output_suf, out_lane)

            # Instance input process, if it exists. In case of init, the
            # output process forks from the raw input user data
            if p_in_name != "__init__":
                # Create instance of input process
                in_process = self.process_map[p_in_name](template=p_in_name)
                # Test if two processes can be connected by input/output types
                logger.debug("[{}] Testing connection between input and "
                             "output processes".format(p))
                self._test_connection(in_process, out_process)
                out_process.parent_lane = in_lane
            else:
                # When the input process is __init__, set the parent_lane
                # to None. This will tell the engine that this process
                # will receive the main input from the raw user input.
                out_process.parent_lane = None
            logger.debug("[{}] Parent lane: {}".format(
                p, out_process.parent_lane))

            # If the current connection is a fork, add it to the fork tree
            if in_lane != out_lane:
                logger.debug("[{}] Connection is a fork. Adding lanes to "
                             "fork list".format(p))
                self._fork_tree[in_lane].append(out_lane)
                # Update main output fork of parent process
                try:
                    parent_process = [
                        x for x in self.processes if x.lane == in_lane and
                        x.template == p_in_name
                    ][0]
                    logger.debug(
                        "[{}] Updating main forks of parent fork '{}' with"
                        " '{}'".format(p, parent_process,
                                       out_process.input_channel))
                    parent_process.update_main_forks(out_process.input_channel)
                except IndexError:
                    pass
            else:
                # Get parent process, naive version
                parent_process = self.processes[-1]

                # Check if the last process' lane matches the lane of the
                # current output process. If not, get the last process
                # in the same lane
                if parent_process.lane and parent_process.lane != out_lane:
                    parent_process = [x for x in self.processes[::-1]
                                      if x.lane == out_lane][0]

                if parent_process.output_channel:
                    logger.debug(
                        "[{}] Updating input channel of output process"
                        " with '{}'".format(
                            p, parent_process.output_channel))
                    out_process.input_channel = parent_process.output_channel

            # Check for process dependencies
            if out_process.dependencies and not ignore_dependencies:
                logger.debug("[{}] Dependencies found for process '{}': "
                             "{}".format(p, p_out_name,
                                         out_process.dependencies))
                parent_lanes = self._get_fork_tree(out_lane)
                for dep in out_process.dependencies:
                    if not self._search_tree_backwards(dep, parent_lanes):
                        if auto_dependency:
                            self._add_dependency(
                                out_process, dep, in_lane, out_lane, p)
                        elif not self.export_parameters:
                            logger.error(colored_print(
                                "\nThe following dependency of the process"
                                " '{}' is missing: {}".format(p_out_name, dep),
                                "red_bold"))
                            sys.exit(1)

            self.processes.append(out_process)

        logger.debug("Completed connections: {}".format(self.processes))
        logger.debug("Fork tree: {}".format(self._fork_tree))

    def _get_process_names(self, con, pid):
        """Returns the input/output process names and output process directives

        Parameters
        ----------
        con : dict
            Dictionary with the connection information between two processes.

        Returns
        -------
        input_name : str
            Name of the input process
        output_name : str
            Name of the output process
        output_directives : dict
            Parsed directives from the output process
        """

        try:
            _p_in_name = con["input"]["process"]
            p_in_name, _ = self._parse_process_name(_p_in_name)
            logger.debug("[{}] Input channel: {}".format(pid, p_in_name))
            _p_out_name = con["output"]["process"]
            p_out_name, out_directives = self._parse_process_name(
                _p_out_name)
            logger.debug("[{}] Output channel: {}".format(pid, p_out_name))
        # Exception is triggered when the process name/directives cannot
        # be processes.
        except eh.ProcessError as ex:
            logger.error(colored_print(ex.value, "red_bold"))
            sys.exit(1)

        return p_in_name, p_out_name, out_directives

    def _add_dependency(self, p, template, inlane, outlane, pid):
        """Automatically Adds a dependency of a process.

        This method adds a template to the process list attribute as a
        dependency. It will adapt the input lane, output lane and process
        id of the process that depends on it.

        Parameters
        ----------
        p : Process
            Process class that contains the dependency.
        template : str
            Template name of the dependency.
        inlane : int
            Input lane.
        outlane : int
            Output lane.
        pid : int
            Process ID.
        """

        dependency_proc = self.process_map[template](template=template)

        if dependency_proc.input_type != p.input_type:
            logger.error("Cannot automatically add dependency with different"
                         " input type. Input type of process '{}' is '{}."
                         " Input type of dependency '{}' is '{}'".format(
                            p.template, p.input_type, template,
                            dependency_proc.input_type))

        input_suf = "{}_{}_dep".format(inlane, pid)
        output_suf = "{}_{}_dep".format(outlane, pid)
        dependency_proc.set_main_channel_names(input_suf, output_suf, outlane)

        # To insert the dependency process before the current process, we'll
        # need to move the input channel name of the later to the former, and
        # set a new connection between the dependency and the process.
        dependency_proc.input_channel = p.input_channel
        p.input_channel = dependency_proc.output_channel

        # If the current process was the first in the pipeline, change the
        # lanes so that the dependency becomes the first process
        if not p.parent_lane:
            p.parent_lane = outlane
            dependency_proc.parent_lane = None
        else:
            dependency_proc.parent_lane = inlane
            p.parent_lane = outlane

        self.processes.append(dependency_proc)

    def _search_tree_backwards(self, template, parent_lanes):
        """Searches the process tree backwards in search of a provided process

        The search takes into consideration the provided parent lanes and
        searches only those

        Parameters
        ----------
        template : str
            Name of the process template attribute being searched
        parent_lanes : list
            List of integers with the parent lanes to be searched

        Returns
        -------
        bool
            Returns True when the template is found. Otherwise returns False.
        """

        for p in self.processes[::-1]:

            # Ignore process in different lanes
            if p.lane not in parent_lanes:
                continue

            # template found
            if p.template == template:
                return True

        return False

    @staticmethod
    def _test_connection(parent_process, child_process):
        """Tests if two processes can be connected by input/output type

        Parameters
        ----------
        parent_process : flowcraft.Process.Process
            Process that will be sending output.
        child_process : flowcraft.Process.Process
            Process that will receive output.

        """

        # If any of the processes has an ignore type attribute set to True,
        # don't perform the check
        if parent_process.ignore_type or child_process.ignore_type:
            return

        if parent_process.output_type != child_process.input_type:
            logger.error(
                "The output of the '{}' process ({}) cannot link with the "
                "input of the '{}' process ({}). Please check the order of "
                "the processes".format(parent_process.template,
                                       parent_process.output_type,
                                       child_process.template,
                                       child_process.input_type))
            sys.exit(1)

    def _build_header(self):
        """Adds the header template to the master template string
        """

        logger.debug("===============")
        logger.debug("Building header")
        logger.debug("===============")
        self.template += hs.header

    def _build_footer(self):
        """Adds the footer template to the master template string"""

        logger.debug("===============")
        logger.debug("Building header")
        logger.debug("===============")
        self.template += fs.footer

    def _update_raw_input(self, p, sink_channel=None, input_type=None):
        """Given a process, this method updates the
        :attr:`~Process.main_raw_inputs` attribute with the corresponding
        raw input channel of that process. The input channel and input type
        can be overridden if the `input_channel` and `input_type` arguments
        are provided.

        Parameters
        ----------
        p : flowcraft.Process.Process
            Process instance whose raw input will be modified
        sink_channel: str
            Sets the channel where the raw input will fork into. It overrides
            the process's `input_channel` attribute.
        input_type: str
            Sets the type of the raw input. It overrides the process's
            `input_type` attribute.
        """

        process_input = input_type if input_type else p.input_type
        process_channel = sink_channel if sink_channel else p.input_channel

        logger.debug("[{}] Setting raw input channel "
                     "with input type '{}'".format(p.template, process_input))
        # Get the dictionary with the raw forking information for the
        # provided input
        raw_in = p.get_user_channel(process_channel, process_input)
        logger.debug("[{}] Fetched process raw user: {}".format(p.template,
                                                                raw_in))

        if process_input in self.main_raw_inputs:
            self.main_raw_inputs[process_input]["raw_forks"].append(
                raw_in["input_channel"])
        else:
            self.main_raw_inputs[process_input] = {
                "channel": raw_in["channel"],
                "channel_str": "{}\n{} = {}".format(
                    raw_in["checks"].format(raw_in["params"]),
                    raw_in["channel"],
                    raw_in["channel_str"].format(raw_in["params"])),
                "raw_forks": [raw_in["input_channel"]]
            }
        logger.debug("[{}] Updated main raw inputs: {}".format(
            p.template, self.main_raw_inputs))

    def _update_extra_inputs(self, p):
        """Given a process, this method updates the
        :attr:`~Process.extra_inputs` attribute with the corresponding extra
        inputs of that process

        Parameters
        ----------
        p : flowcraft.Process.Process
        """

        if p.extra_input:
            logger.debug("[{}] Found extra input: {}".format(
                p.template, p.extra_input))

            if p.extra_input == "default":
                # Check if the default type is now present in the main raw
                # inputs. If so, issue an error. The default param can only
                # be used when not present in the main raw inputs
                if p.input_type in self.main_raw_inputs:
                    logger.error(colored_print(
                        "\nThe default input param '{}' of the process '{}'"
                        " is already specified as a main input parameter of"
                        " the pipeline. Please choose a different extra_input"
                        " name.".format(p.input_type, p.template), "red_bold"))
                    sys.exit(1)
                param = p.input_type
            else:
                param = p.extra_input

            dest_channel = "EXTRA_{}_{}".format(p.template, p.pid)

            if param not in self.extra_inputs:
                self.extra_inputs[param] = {
                    "input_type": p.input_type,
                    "channels": [dest_channel]
                }
            else:
                if self.extra_inputs[param]["input_type"] != p.input_type:
                    logger.error(colored_print(
                        "\nThe extra_input parameter '{}' for process"
                        " '{}' was already defined with a different "
                        "input type '{}'. Please choose a different "
                        "extra_input name.".format(
                            p.input_type, p.template,
                            self.extra_inputs[param]["input_type"]),
                        "red_bold"))
                    sys.exit(1)
                self.extra_inputs[param]["channels"].append(dest_channel)

            logger.debug("[{}] Added extra channel '{}' linked to param: '{}' "
                         "".format(p.template, param,
                                   self.extra_inputs[param]))
            p.update_main_input(
                "{}.mix({})".format(p.input_channel, dest_channel)
            )

    def _get_fork_tree(self, lane):
        """

        Parameters
        ----------
        p

        Returns
        -------
        """

        parent_lanes = [lane]

        while True:
            original_lane = lane
            for fork_in, fork_out in self._fork_tree.items():
                if lane in fork_out:
                    lane = fork_in
                    parent_lanes.append(fork_in)
            if lane == original_lane:
                break

        return parent_lanes

    def _set_implicit_link(self, p, link):
        """

        Parameters
        ----------
        p
        link

        Returns
        -------

        """

        output_type = link["link"].lstrip("_")
        parent_forks = self._get_fork_tree(p.lane)
        fork_sink = "{}_{}".format(link["alias"], p.pid)

        for proc in self.processes[::-1]:
            if proc.lane not in parent_forks:
                continue
            if proc.output_type == output_type:
                proc.update_main_forks(fork_sink)
                logger.debug("[{}] Found special implicit link '{}' with "
                             "output type '{}'. Linked '{}' with process "
                             "{}".format(
                                     p.template, link["link"], output_type,
                                     link["alias"], proc))
                return

        self._update_raw_input(p, fork_sink, output_type)

    def _update_secondary_channels(self, p):
        """Given a process, this method updates the
        :attr:`~Process.secondary_channels` attribute with the corresponding
        secondary inputs of that channel.

        The rationale of the secondary channels is the following:

            - Start storing any secondary emitting channels, by checking the
              `link_start` list attribute of each process. If there are
              channel names in the link start, it adds to the secondary
              channels dictionary.
            - Check for secondary receiving channels, by checking the
              `link_end` list attribute. If the link name starts with a
              `__` signature, it will created an implicit link with the last
              process with an output type after the signature. Otherwise,
              it will check is a corresponding link start already exists in
              the at least one process upstream of the pipeline and if so,
              it will update the ``secondary_channels`` attribute with the
              new link.

        Parameters
        ----------
        p : flowcraft.Process.Process
        """

        # Check if the current process has a start of a secondary
        # side channel
        if p.link_start:
            logger.debug("[{}] Found secondary link start: {}".format(
                p.template, p.link_start))
            for l in p.link_start:
                # If there are multiple link starts in the same lane, the
                # last one is the only one saved.
                if l in self.secondary_channels:
                    self.secondary_channels[l][p.lane] = {"p": p, "end": []}
                else:
                    self.secondary_channels[l] = {p.lane: {"p": p, "end": []}}

        # check if the current process receives a secondary side channel.
        # If so, add to the links list of that side channel
        if p.link_end:
            logger.debug("[{}] Found secondary link end: {}".format(
                p.template, p.link_end))
            for l in p.link_end:

                # Get list of lanes from the parent forks.
                parent_forks = self._get_fork_tree(p.lane)

                # Parse special case where the secondary channel links with
                # the main output of the specified type
                if l["link"].startswith("__"):
                    self._set_implicit_link(p, l)
                    continue

                # Skip if there is no match for the current link in the
                # secondary channels
                if l["link"] not in self.secondary_channels:
                    continue

                for lane in parent_forks:
                    if lane in self.secondary_channels[l["link"]]:
                        self.secondary_channels[
                            l["link"]][lane]["end"].append("{}".format(
                                "{}_{}".format(l["alias"], p.pid)))

        logger.debug("[{}] Secondary links updated: {}".format(
            p.template, self.secondary_channels))

    def _set_channels(self):
        """Sets the main channels for the pipeline

        This method will parse de the :attr:`~Process.processes` attribute
        and perform the following tasks for each process:

            - Sets the input/output channels and main input forks and adds
              them to the process's
              :attr:`flowcraft.process.Process._context`
              attribute (See
              :func:`~NextflowGenerator.set_channels`).
            - Automatically updates the main input channel of the first
              process of each lane so that they fork from the user provide
              parameters (See
              :func:`~NextflowGenerator._update_raw_input`).
            - Check for the presence of secondary channels and adds them to the
              :attr:`~NextflowGenerator.secondary_channels` attribute.

        Notes
        -----
        **On the secondary channel setup**: With this approach, there can only
        be one secondary link start for each type of secondary link. For
        instance, If there are two processes that start a secondary channel
        for the ``SIDE_max_len`` channel, only the last one will be recorded,
        and all receiving processes will get the channel from the latest
        process. Secondary channels can only link if the source process if
        downstream of the sink process in its "forking" path.
        """

        logger.debug("=====================")
        logger.debug("Setting main channels")
        logger.debug("=====================")

        for i, p in enumerate(self.processes):

            # Set main channels for the process
            logger.debug("[{}] Setting main channels with pid: {}".format(
                p.template, i))
            p.set_channels(pid=i)

            # If there is no parent lane, set the raw input channel from user
            logger.debug("{} {} {}".format(p.parent_lane, p.input_type, p.template))
            if not p.parent_lane and p.input_type:
                self._update_raw_input(p)

            self._update_extra_inputs(p)

            self._update_secondary_channels(p)

            logger.info(colored_print(
                "\tChannels set for {} \u2713".format(p.template)))

    def _set_init_process(self):
        """Sets the main raw inputs and secondary inputs on the init process

        This method will fetch the :class:`flowcraft.process.Init` process
        instance and sets the raw input (
        :func:`flowcraft.process.Init.set_raw_inputs`) for
        that process. This will handle the connection of the user parameters
        with channels that are then consumed in the pipeline.
        """

        logger.debug("========================")
        logger.debug("Setting secondary inputs")
        logger.debug("========================")

        # Get init process
        init_process = self.processes[0]
        logger.debug("Setting main raw inputs: "
                     "{}".format(self.main_raw_inputs))
        init_process.set_raw_inputs(self.main_raw_inputs)
        logger.debug("Setting extra inputs: {}".format(self.extra_inputs))
        init_process.set_extra_inputs(self.extra_inputs)

    def _set_secondary_channels(self):
        """Sets the secondary channels for the pipeline

        This will iterate over the
        :py:attr:`NextflowGenerator.secondary_channels` dictionary that is
        populated when executing
        :func:`~NextflowGenerator._update_secondary_channels` method.
        """

        logger.debug("==========================")
        logger.debug("Setting secondary channels")
        logger.debug("==========================")

        logger.debug("Setting secondary channels: {}".format(
            self.secondary_channels))

        for source, lanes in self.secondary_channels.items():

            for vals in lanes.values():

                if not vals["end"]:
                    logger.debug("[{}] No secondary links to setup".format(
                        vals["p"].template))
                    continue

                logger.debug("[{}] Setting secondary links for "
                             "source {}: {}".format(vals["p"].template,
                                                    source,
                                                    vals["end"]))

                vals["p"].set_secondary_channel(source, vals["end"])

    def _set_compiler_channels(self):
        """Wrapper method that calls functions related to compiler channels
        """

        self._set_status_channels()
        self._set_general_compilers()

    def _set_general_compilers(self):
        """Adds compiler channels to the :attr:`processes` attribute.

        This method will iterate over the pipeline's processes and check
        if any process is feeding channels to a compiler process. If so, that
        compiler process is added to the pipeline and those channels are
        linked to the compiler via some operator.
        """

        for c, c_info in self.compilers.items():

            # Instantiate compiler class object and set empty channel list
            compiler_cls = c_info["cls"](template=c_info["template"])
            c_info["channels"] = []

            for p in self.processes:
                if not any([isinstance(p, x) for x in self.skip_class]):
                    # Check if process has channels to feed to a compiler
                    if c in p.compiler:
                        # Correct channel names according to the pid of the
                        # process
                        channels = ["{}_{}".format(i, p.pid) for i in
                                    p.compiler[c]]
                        c_info["channels"].extend(channels)

            # If one ore more channels were detected, establish connections
            # and append compiler to the process list.
            if c_info["channels"]:
                compiler_cls.set_compiler_channels(c_info["channels"],
                                                   operator="join")
                self.processes.append(compiler_cls)

    def _set_status_channels(self):
        """Compiles all status channels for the status compiler process
        """

        status_inst = pc.StatusCompiler(template="status_compiler")
        report_inst = pc.ReportCompiler(template="report_compiler")

        # Compile status channels from pipeline process
        status_channels = []
        for p in [p for p in self.processes]:
            if not any([isinstance(p, x) for x in self.skip_class]):

                status_channels.extend(p.status_strs)

        if not status_channels:
            logger.debug("No status channels found. Skipping status compiler"
                         "process")
            return

        logger.debug("Setting status channels: {}".format(status_channels))

        # Check for duplicate channels. Raise exception if found.
        if len(status_channels) != len(set(status_channels)):
            raise eh.ProcessError(
                "Duplicate status channels detected. Please ensure that "
                "the 'status_channels' attributes of each process are "
                "unique. Here are the status channels:\n\n{}".format(
                    ", ".join(status_channels)
                ))

        status_inst.set_compiler_channels(status_channels)

        report_channels = ["REPORT_{}".format(x.lstrip("STATUS_")) for x in
                           status_channels]

        report_inst.set_compiler_channels(report_channels)

        self.processes.extend([status_inst, report_inst])

    @staticmethod
    def _get_resources_string(res_dict, pid):
        """ Returns the nextflow resources string from a dictionary object

        If the dictionary has at least on of the resource directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"cpus": 1, "memory": "4GB"},
                    "processB": {"cpus": 2}}

        Parameters
        ----------
        res_dict : dict
            Dictionary with the resources for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string
        """

        config_str = ""
        ignore_directives = ["container", "version"]

        for p, directives in res_dict.items():

            for d, val in directives.items():

                if d in ignore_directives:
                    continue

                config_str += '\n\t${}_{}.{} = {}'.format(p, pid, d, val)

        return config_str

    @staticmethod
    def _get_container_string(cont_dict, pid):
        """ Returns the nextflow containers string from a dictionary object

        If the dictionary has at least on of the container directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"container": "asd", "version": "1.0.0"},
                    "processB": {"container": "dsd"}}

        Parameters
        ----------
        cont_dict : dict
            Dictionary with the containers for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string
        """

        config_str = ""

        for p, directives in cont_dict.items():

            container = ""

            if "container" in directives:
                container += directives["container"]

                if "version" in directives:
                    container += ":{}".format(directives["version"])
                else:
                    container += ":latest"

            if container:
                config_str += '\n\t${}_{}.container = "{}"'.format(p, pid, container)

        return config_str

    def _get_params_string(self):
        """Returns the nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Returns
        -------
        str
            Nextflow params configuration string
        """

        params_str = ""

        for p in self.processes:

            logger.debug("[{}] Adding parameters: {}\n".format(
                p.template, p.params)
            )

            # Add an header with the template name to structure the params
            # configuration
            if p.params and p.template != "init":

                p.set_param_id("_{}".format(p.pid))
                params_str += "\n\t/*"
                params_str += "\n\tComponent '{}_{}'\n".format(p.template,
                                                               p.pid)
                params_str += "\t{}\n".format("-" * (len(p.template) + len(p.pid) + 12))
                params_str += "\t*/\n"

            for param, val in p.params.items():

                if p.template == "init":
                    param_id = param
                else:
                    param_id = "{}_{}".format(param, p.pid)

                params_str += "\t{} = {}\n".format(param_id, val["default"])

        return params_str

    def _get_merged_params_string(self):
        """Returns the merged nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Identical parameters in multiple processes will be merged into the same
        param.

        Returns
        -------
        str
            Nextflow params configuration string
        """

        params_str = ""

        for p in self.processes:

            logger.debug("[{}] Adding parameters: {}\n".format(
                p.template, p.params)
            )

            # Add an header with the template name to structure the params
            # configuration
            if p.params and p.template != "init":

                p.set_param_id("_{}".format(p.pid))
                params_str += "\n\t/*"
                params_str += "\n\tComponent '{}_{}'\n".format(p.template,
                                                               p.pid)
                params_str += "\t{}\n".format("-" * (len(p.template) + len(p.pid) + 12))
                params_str += "\t*/\n"

            for param, val in p.params.items():

                if p.template == "init":
                    param_id = param
                else:
                    param_id = "{}_{}".format(param, p.pid)

                params_str += "\t{} = {}\n".format(param_id, val["default"])

        return params_str

    def _get_merged_params_string(self):
        """Returns the merged nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Identical parameters in multiple processes will be merged into the same
        param.

        Returns
        -------
        str
            Nextflow params configuration string
        """

        params_temp = {}

        for p in self.processes:

            logger.debug("[{}] Adding parameters: {}".format(p.template,
                                                             p.params))
            for param, val in p.params.items():

                params_temp[param] = val["default"]

        config_str = "\n\t" + "\n\t".join([
            "{} = {}".format(param, val) for param, val in params_temp.items()
        ])

        return config_str

    def _get_params_help(self):

        help_list = []

        for p in self.processes:

            # Skip init process
            if p.template == "init":
                for param, val in p.params.items():
                    help_list.append("--{:25} {} (default: {})".format(
                        param, val["description"],
                        str(val["default"]).replace('"', "'")))
                continue

            # Add component header and a line break
            if p.params:
                help_list.extend(
                    ["",
                     "Component '{}_{}'".format(p.template.upper(), p.pid),
                     "-" * (len(p.template) + len(p.pid) + 13)])

            for param, val in p.params.items():
                help_list.append("--{:<25} {} (default: {})".format(
                    param + "_" + p.pid, val["description"],
                    str(val["default"]).replace('"', "'")))

        return help_list

    def _get_merged_params_help(self):
        """

        Returns
        -------

        """

        help_dict = {}
        help_list = []

        for p in self.processes:

            for param, val in p.params.items():

                if param in help_dict:
                    help_dict[param]["process"].append(p.template)
                else:
                    tpl = [p.template] if p.template != "init" else []
                    help_dict[param] = {"process": tpl,
                                        "description": val["description"]}

        # Transform process list into final template string
        for p, val in help_dict.items():
            if not val["process"]:
                val["process"] = ""
            else:
                val["process"] = "({})".format(";".join(val["process"]))
            help_list.append("--{:<25} {} {}".format(
                p, val["description"], val["process"]))

        return help_list

    def _get_manifest_string(self):
        """Returns the nextflow manifest config string to include in the
        config file from the information on the pipeline.

        Returns
        -------
        str
            Nextflow manifest configuration string
        """

        config_str = ""

        config_str += '\n\tname = "{}"'.format(self.pipeline_name)
        config_str += '\n\tmainScript = "{}"'.format(self.nf_file)

        return config_str

    @staticmethod
    def _render_config(template, context):

        tpl_dir = join(dirname(abspath(__file__)), "templates")
        tpl_path = join(tpl_dir, template)

        path, filename = split(tpl_path)

        return jinja2.Environment(
            loader=jinja2.FileSystemLoader(path or "./")
        ).get_template(filename).render(context)

    def _set_configurations(self):
        """This method will iterate over all process in the pipeline and
        populate the nextflow configuration files with the directives
        of each process in the pipeline.
        """

        logger.debug("======================")
        logger.debug("Setting configurations")
        logger.debug("======================")

        resources = ""
        containers = ""
        params = ""
        manifest = ""

        if self.merge_params:
            params += self._get_merged_params_string()
            help_list = self._get_merged_params_help()
        else:
            params += self._get_params_string()
            help_list = self._get_params_help()

        for p in self.processes:

            # Skip processes with the directives attribute populated
            if not p.directives:
                continue

            logger.debug("[{}] Adding directives: {}".format(
                p.template, p.directives))
            resources += self._get_resources_string(p.directives, p.pid)
            containers += self._get_container_string(p.directives, p.pid)

        manifest = self._get_manifest_string()

        self.resources = self._render_config("resources.config", {
            "process_info": resources
        })
        self.containers = self._render_config("containers.config", {
            "container_info": containers
        })
        self.params = self._render_config("params.config", {
            "params_info": params
        })
        self.manifest = self._render_config("manifest.config", {
            "manifest_info": manifest
        })
        self.help = self._render_config("Helper.groovy", {
            "nf_file": basename(self.nf_file),
            "help_list": help_list,
            "version": __version__,
            "pipeline_name": " ".join([x.upper() for x in self.pipeline_name])
        })
        self.user_config = self._render_config("user.config", {})

    def dag_to_file(self, dict_viz, output_file=".treeDag.json"):
        """Writes dag to output file

        Parameters
        ----------
        dict_viz: dict
            Tree like dictionary that is used to export tree data of processes
            to html file and here for the dotfile .treeDag.json

        """

        outfile_dag = open(os.path.join(dirname(self.nf_file), output_file)
                           , "w")
        outfile_dag.write(json.dumps(dict_viz))
        outfile_dag.close()

    def render_pipeline(self):
        """Write pipeline attributes to json

        This function writes the pipeline and their attributes to a json file,
        that is intended to be read by resources/pipeline_graph.html to render
        a graphical output showing the DAG.

        """

        dict_viz = {
            "name": "root",
            "children": []
        }
        last_of_us = {}

        f_tree = self._fork_tree if self._fork_tree else {1: [1]}

        for x, (k, v) in enumerate(f_tree.items()):
            for p in self.processes[1:]:

                if x == 0 and p.lane not in [k] + v:
                    continue

                if x > 0 and p.lane not in v:
                    continue

                if not p.parent_lane:
                    lst = dict_viz["children"]
                else:
                    lst = last_of_us[p.parent_lane]

                tooltip = {
                    "name": "{}_{}".format(p.template, p.pid),
                    "process": {
                        "pid": p.pid,
                        "input": p.input_type,
                        "output": p.output_type if p.output_type else "None",
                        "lane": p.lane,
                    },
                    "children": []
                }

                dir_var = ""
                for k2, v2 in p.directives.items():
                    dir_var += k2
                    for d in v2:
                        try:
                            # Remove quotes from string directives
                            directive = v2[d].replace("'", "").replace('"', '') \
                                if isinstance(v2[d], str) else v2[d]
                            dir_var += "{}: {}".format(d, directive)
                        except KeyError:
                            pass

                if dir_var:
                    tooltip["process"]["directives"] = dir_var
                else:
                    tooltip["process"]["directives"] = "N/A"

                lst.append(tooltip)

                last_of_us[p.lane] = lst[-1]["children"]

        # write to file dict_viz
        self.dag_to_file(dict_viz)

        # Write tree forking information for dotfile
        with open(os.path.join(dirname(self.nf_file),
                               ".forkTree.json"), "w") as fh:
            fh.write(json.dumps(self._fork_tree))

        # send with jinja to html resource
        return self._render_config("pipeline_graph.html", {"data": dict_viz})

    def write_configs(self, project_root):
        """Wrapper method that writes all configuration files to the pipeline
        directory
        """

        # Write resources config
        with open(join(project_root, "resources.config"), "w") as fh:
            fh.write(self.resources)

        # Write containers config
        with open(join(project_root, "containers.config"), "w") as fh:
            fh.write(self.containers)

        # Write containers config
        with open(join(project_root, "params.config"), "w") as fh:
            fh.write(self.params)

        # Write manifest config
        with open(join(project_root, "manifest.config"), "w") as fh:
            fh.write(self.manifest)

        # Write user config if not present in the project directory
        if not exists(join(project_root, "user.config")):
            with open(join(project_root, "user.config"), "w") as fh:
                fh.write(self.user_config)

        lib_dir = join(project_root, "lib")
        if not exists(lib_dir):
            os.makedirs(lib_dir)
        with open(join(lib_dir, "Helper.groovy"), "w") as fh:
            fh.write(self.help)

        # Generate the pipeline DAG
        pipeline_to_json = self.render_pipeline()
        with open(splitext(self.nf_file)[0] + ".html", "w") as fh:
            fh.write(pipeline_to_json)

    def export_params(self):
        """Export pipeline params as a JSON to stdout

        This run mode iterates over the pipeline processes and exports the
        params dictionary of each component as a JSON to stdout.
        """

        params_json = {}

        # Skip first init process
        for p in self.processes[1:]:
            params_json[p.template] = p.params

        # Flush params json to stdout
        sys.stdout.write(json.dumps(params_json))

    def export_directives(self):
        """Export pipeline directives as a JSON to stdout
        """

        directives_json = {}

        # Skip first init process
        for p in self.processes[1:]:
            directives_json[p.template] = p.directives

        # Flush params json to stdout
        sys.stdout.write(json.dumps(directives_json))

    def fetch_docker_tags(self):
        """
        Export all dockerhub tags associated with each component given by
        the -t flag.
        """

        # dict to store the already parsed components (useful when forks are
        # given to the pipeline string via -t flag
        dict_of_parsed = {}

        # fetches terminal width and subtracts 3 because we always add a
        # new line character and we want a space at the beggining and at the end
        # of each line
        terminal_width = shutil.get_terminal_size().columns - 3

        # first header
        center_string = " Selected container tags "

        # starts a list with the headers
        tags_list = [
            [
                "=" * int(terminal_width / 4),
                "{0}{1}{0}".format(
                    "=" * int(((terminal_width/2 - len(center_string)) / 2)),
                    center_string)
                ,
                "{}\n".format("=" * int(terminal_width / 4))
            ],
            ["component", "container", "tags"],
            [
                "=" * int(terminal_width / 4),
                "=" * int(terminal_width / 2),
                "=" * int(terminal_width / 4)
            ]
        ]

        # Skip first init process and iterate through the others
        for p in self.processes[1:]:
            template = p.template
            # if component has already been printed then skip and don't print
            # again
            if template in dict_of_parsed:
                continue

            # starts a list of  containers for the current process in
            # dict_of_parsed, in which each containers will be added to this
            # list once it gets parsed
            dict_of_parsed[template] = {
                "container": []
            }

            # fetch repo name from directives of each component.
            for directives in p.directives.values():
                try:
                    repo = directives["container"]
                    default_version = directives["version"]
                except KeyError:
                    # adds the default container if container key isn't present
                    # this happens for instance in integrity_coverage
                    repo = "flowcraft/flowcraft_base"
                    default_version = "1.0.0-1"
                # checks if repo_version already exists in list of the
                # containers for the current component being queried
                repo_version = repo + default_version
                if repo_version not in dict_of_parsed[template]["container"]:
                    # make the request to docker hub
                    r = requests.get(
                        "https://hub.docker.com/v2/repositories/{}/tags/"
                        .format(repo)
                    )
                    # checks the status code of the request, if it is 200 then
                    # parses docker hub entry, otherwise retrieve no tags but
                    # alerts the user
                    if r.status_code != 404:
                        # parse response content to dict and fetch results key
                        r_content = json.loads(r.content)["results"]
                        for version in r_content:
                            printed_version = (version["name"] + "*") \
                                if version["name"] == default_version \
                                else version["name"]
                            tags_list.append([template, repo, printed_version])
                    else:
                        tags_list.append([template, repo, "No DockerHub tags"])

                dict_of_parsed[template]["container"].append(repo_version)

        # iterate through each entry in tags_list and print the list of tags
        # for each component. Each entry (excluding the headers) contains
        # 3 elements (component name, container and tag version)
        for x, entry in enumerate(tags_list):
            # adds different color to the header in the first list and
            # if row is pair add one color and if is even add another (different
            # background)
            color = "blue_bold" if x < 3 else \
                ("white" if x % 2 != 0 else "0;37;40m")
            # generates a small list with the terminal width for each column,
            # this will be given to string formatting as the 3, 4 and 5 element
            final_width = [
                int(terminal_width/4),
                int(terminal_width/2),
                int(terminal_width/4)
            ]
            # writes the string to the stdout
            sys.stdout.write(
                colored_print("\n {0: <{3}} {1: ^{4}} {2: >{5}}".format(
                    *entry, *final_width), color)
            )
        # assures that the entire line gets the same color
        sys.stdout.write("\n{0: >{1}}\n".format("(* = default)",
                                                terminal_width + 3))

    def build(self):
        """Main pipeline builder

        This method is responsible for building the
        :py:attr:`NextflowGenerator.template` attribute that will contain
        the nextflow code of the pipeline.

        First it builds the header, then sets the main channels, the
        secondary inputs, secondary channels and finally the
        status channels. When the pipeline is built, is writes the code
        to a nextflow file.
        """

        logger.info(colored_print(
            "\tSuccessfully connected {} process(es) with {} "
            "fork(s) across {} lane(s) \u2713".format(
                len(self.processes[1:]), len(self._fork_tree), self.lanes)))

        # Generate regular nextflow header that sets up the shebang, imports
        # and all possible initial channels
        self._build_header()

        self._set_channels()

        self._set_init_process()

        self._set_secondary_channels()

        logger.info(colored_print(
            "\tSuccessfully set {} secondary channel(s) \u2713".format(
                len(self.secondary_channels))))

        self._set_compiler_channels()

        self._set_configurations()

        logger.info(colored_print(
            "\tFinished configurations \u2713"))

        for p in self.processes:
            self.template += "\n{}".format(p.template_str)

        self._build_footer()

        project_root = dirname(self.nf_file)

        # Write configs
        self.write_configs(project_root)

        # Write pipeline file
        with open(self.nf_file, "w") as fh:
            fh.write(self.template)

        logger.info(colored_print(
            "\tPipeline written into {} \u2713".format(self.nf_file)))


#!/usr/bin/env python3

"""
Purpose
-------

This module is intended execute metaSpades on paired-end FastQ files.

Expected input
--------------

The following variables are expected whether using NextFlow or the
:py:func:`main` executor.

- ``sample_id`` : Sample Identification string.
    - e.g.: ``'SampleA'``
- ``fastq_pair`` : Pair of FastQ file paths.
    - e.g.: ``'SampleA_1.fastq.gz SampleA_2.fastq.gz'``
- ``kmers`` : Setting for Spades kmers. Can be either ``'auto'``, \
    ``'default'`` or a user provided list.
    - e.g.: ``'auto'`` or ``'default'`` or ``'55 77 99 113 127'``

Generated output
----------------

- ``contigs.fasta`` : Main output of spades with the assembly
    - e.g.: ``contigs.fasta``
- ``spades_status`` :  Stores the status of the spades run. If it was \
    successfully executed, it stores ``'pass'``. Otherwise, it stores the\
    ``STDERR`` message.
    - e.g.: ``'pass'``

Code documentation
------------------

"""

__version__ = "1.0.1"
__build__ = "16012018"
__template__ = "metaspades-nf"

import os
import re
import subprocess

from subprocess import PIPE

from flowcraft_utils.flowcraft_base import get_logger, MainWrapper

logger = get_logger(__file__)

def __get_version_spades():

    try:

        cli = ["metaspades.py", "--version"]
        p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
        stdout, _ = p.communicate()

        version = stdout.strip().split()[-1][1:].decode("utf8")

    except Exception as e:
        logger.debug(e)
        version = "undefined"

    return {
        "program": "metaSPAdes",
        "version": version,
    }

if __file__.endswith(".command.sh"):
    SAMPLE_ID = '$sample_id'
    FASTQ_PAIR = '$fastq_pair'.split()
    MAX_LEN = int('$max_len'.strip())
    KMERS = '$kmers'.strip()
    CLEAR = '$clear'
    logger.debug("Running {} with parameters:".format(
        os.path.basename(__file__)))
    logger.debug("SAMPLE_ID: {}".format(SAMPLE_ID))
    logger.debug("FASTQ_PAIR: {}".format(FASTQ_PAIR))
    logger.debug("MAX_LEN: {}".format(MAX_LEN))
    logger.debug("KMERS: {}".format(KMERS))
    logger.debug("CLEAR: {}".format(CLEAR))

def clean_up(fastq):
    """
    Cleans the temporary fastq files. If they are symlinks, the link
    source is removed

    Parameters
    ----------
    fastq : list
        List of fastq files.
    """

    for fq in fastq:
        # Get real path of fastq files, following symlinks
        rp = os.path.realpath(fq)
        logger.debug("Removing temporary fastq file path: {}".format(rp))
        if re.match(".*/work/.{2}/.{30}/.*", rp):
            os.remove(rp)

def set_kmers(kmer_opt, max_read_len):
    """Returns a kmer list based on the provided kmer option and max read len.

    Parameters
    ----------
    kmer_opt : str
        The k-mer option. Can be either ``'auto'``, ``'default'`` or a
        sequence of space separated integers, ``'23, 45, 67'``.
    max_read_len : int
        The maximum read length of the current sample.

    Returns
    -------
    kmers : list
        List of k-mer values that will be provided to Spades.

    """

    logger.debug("Kmer option set to: {}".format(kmer_opt))

    # Check if kmer option is set to auto
    if kmer_opt == "auto":

        if max_read_len >= 175:
            kmers = [55, 77, 99, 113, 127]
        else:
            kmers = [21, 33, 55, 67, 77]

        logger.debug("Kmer range automatically selected based on max read"
                     "length of {}: {}".format(max_read_len, kmers))

    # Check if manual kmers were specified
    elif len(kmer_opt.split()) > 1:

        kmers = kmer_opt.split()
        logger.debug("Kmer range manually set to: {}".format(kmers))

    else:

        kmers = []
        logger.debug("Kmer range set to empty (will be automatically "
                     "determined by SPAdes")

    return kmers

@MainWrapper
def main(sample_id, fastq_pair, max_len, kmer, clear):
    """Main executor of the spades template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    max_len : int
        Maximum read length. This value is determined in
        :py:class:`templates.integrity_coverage`
    kmer : str
        Can be either ``'auto'``, ``'default'`` or a
        sequence of space separated integers, ``'23, 45, 67'``.

    """

    logger.info("Starting spades")

    logger.info("Setting SPAdes kmers")
    kmers = set_kmers(kmer, max_len)
    logger.info("SPAdes kmers set to: {}".format(kmers))

    cli = [
        "metaspades.py",
        "--only-assembler",
        "--threads",
        "$task.cpus",
        "-o",
        "."
    ]

    # Add kmers, if any were specified
    if kmers:
        cli += ["-k {}".format(",".join([str(x) for x in kmers]))]

    # Add FastQ files
    cli += [
        "-1",
        fastq_pair[0],
        "-2",
        fastq_pair[1]
    ]

    logger.debug("Running metaSPAdes subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
        stdout = stdout.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)
        stdout = str(stdout)

    logger.info("Finished metaSPAdes subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished metaSPAdes subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished metaSPAdes with return code: {}".format(
        p.returncode))

    with open(".status", "w") as fh:
        if p.returncode != 0:
            fh.write("error")
            return
        else:
            fh.write("pass")

    # Change the default contigs.fasta assembly name to a more informative one
    if "_trim." in fastq_pair[0]:
        sample_id += "_trim"

    assembly_file = "{}_metaspades.fasta".format(
        sample_id)
    os.rename("contigs.fasta", assembly_file)
    logger.info("Setting main assembly file to: {}".format(assembly_file))

    # Remove input fastq files when clear option is specified.
    # Only remove temporary input when the expected output exists.
    if clear == "true" and os.path.exists(assembly_file):
        clean_up(fastq_pair)

if __name__ == '__main__':

    main(SAMPLE_ID, FASTQ_PAIR, MAX_LEN, KMERS, CLEAR)


import os
import re
import sys
import json
import uuid
import signal
import socket
import hashlib
import logging
import requests

from os.path import join, abspath
from time import sleep
from pympler.asizeof import asizeof

try:
    import generator.error_handling as eh
    from generator.process_details import colored_print
    from generator.utils import get_nextflow_filepath
except ImportError:
    import flowcraft.generator.error_handling as eh
    from flowcraft.generator.process_details import colored_print
    from flowcraft.generator.utils import get_nextflow_filepath

logger = logging.getLogger("main.{}".format(__name__))

def signal_handler():
    """This function is bound to the SIGINT signal (like ctrl+c) to graciously
    exit the program and reset the curses options.
    """

    print("Exiting flowcraft report brodcast... Bye")
    sys.exit(0)

class FlowcraftReport:

    def __init__(self, report_file, trace_file=None, log_file=None,
                 watch=False, ip_addr=None):

        self.report_file = report_file
        """
        str: Path to Report JSON file.
        """

        if not ip_addr:
            self.app_address = "http://www.flowcraft.live:80/"
        else:
            self.app_address = ip_addr
            """
            str: Address of flowcraft web app
            """

        self.broadcast_address = "{}reports/broadcast/api/reports".format(
            self.app_address)

        self.refresh_rate = 1

        self.send = True
        """
        boolean: This attribute is used when the report mode is used with the
        --watch option. It will be set to False after sending a request, and
        set to True when there is a change in the pipeline reports.
        """

        self.watch = watch
        """
        boolean: When False, the reports mode will try to open the provided
        report JSON file and send it to the flowcraft service. When True,
        it will try to open the nextflow trace file instead and continuously
        compile the report JSON files from the `report` processes as they
        are created.
        """

        self.log_file = log_file
        """
        str: Path to .nextflow.log file.
        """

        self.log_sizestamp = None
        """
        str: Stores the sizestamp of the last modification of the trace file.
        This is used to parse the file only when it has changed.
        """

        self.status_info = None
        """
        str: Status of the pipeline execution. Used in the watch report mode
        and varies between 'running', 'aborted', 'complete'.
        """

        self.trace_file = trace_file
        """
        str: Path to nextflow trace file.
        """

        self.trace_sizestamp = None
        """
        str: Stores the sizestamp of the last modification of the trace file.
        This is used to parse the file only when it has changed.
        """

        self.trace_retry = 0
        """
        int: Each time the log file is not found, this counter is
        increased. Only when it matches the :attr:`MAX_RETRIES` attribute
        does it raises a FileNotFoundError.
        """

        self.stored_ids = []
        """
        list: Stores the task_ids that have already been parsed. It is used
        to skip them when parsing the trace files multiple times.
        """

        self.report_queue = []
        """
        list: Stores the paths of the report JSON files that are on queue to
        be sent to the flowcraft service. This list will be emptied when these
        JSONs are sent.
        """

        # Checks if report file is available
        self._check_required_files()

        signal.signal(signal.SIGINT, lambda *x: signal_handler())

    def _check_required_files(self):

        if not os.path.exists(self.report_file) and not self.watch:
            raise eh.ReportError("The provided report JSON file could not be"
                                 " opened: {}".format(self.report_file))

    @staticmethod
    def _header_mapping(header):
        """Parses the trace file header and retrieves the positions of each
        column key.

        Parameters
        ----------
        header : str
            The header line of nextflow's trace file

        Returns
        -------
        dict
            Mapping the column ID to its position (e.g.: {"tag":2})
        """

        return dict(
            (x.strip(), pos) for pos, x in enumerate(header.split("\t"))
        )

    @staticmethod
    def _expand_path(hash_str):
        """Expands the hash string of a process (ae/1dasjdm) into a full
        working directory

        Parameters
        ----------
        hash_str : str
            Nextflow process hash with the beggining of the work directory

        Returns
        -------
        str
            Path to working directory of the hash string
        """

        try:
            first_hash, second_hash = hash_str.split("/")
            first_hash_path = join(abspath("work"), first_hash)

            for l in os.listdir(first_hash_path):
                if l.startswith(second_hash):
                    return join(first_hash_path, l)
        except FileNotFoundError:
            return None

    def _get_report_id(self):
        """Returns a hash of the reports JSON file
        """

        if self.watch:

            # Searches for the first occurence of the nextflow pipeline
            # file name in the .nextflow.log file
            pipeline_path = get_nextflow_filepath(self.log_file)

            # Get hash from the entire pipeline file
            pipeline_hash = hashlib.md5()
            with open(pipeline_path, "rb") as fh:
                for chunk in iter(lambda: fh.read(4096), b""):
                    pipeline_hash.update(chunk)
            # Get hash from the current working dir and hostname
            workdir = os.getcwd().encode("utf8")
            hostname = socket.gethostname().encode("utf8")
            hardware_addr = str(uuid.getnode()).encode("utf8")
            dir_hash = hashlib.md5(workdir + hostname + hardware_addr)

            return pipeline_hash.hexdigest() + dir_hash.hexdigest()

        else:
            with open(self.report_file) as fh:
                report_json = json.loads(fh.read())

            metadata = report_json["data"]["results"][0]["nfMetadata"]

            try:
                report_id = metadata["scriptId"] + metadata["sessionId"]
            except KeyError:
                raise eh.ReportError("Incomplete or corrupt report JSON file "
                                     "missing the 'scriptId' and/or 'sessionId' "
                                     "metadata information")

            return report_id

    def _update_pipeline_status(self):
        """
        Parses the .nextflow.log file for signatures of pipeline status and sets
        the :attr:`status_info` attribute.
        """

        prev_status = self.status_info

        with open(self.log_file) as fh:

            for line in fh:

                if "Session aborted" in line:
                    self.status_info = "aborted"
                    self.send = True if prev_status != self.status_info \
                        else self.send
                    return

                if "Execution complete -- Goodbye" in line:
                    self.status_info = "complete"
                    self.send = True if prev_status != self.status_info \
                        else self.send
                    return

            self.status_info = "running"
            self.send = True if prev_status != self.status_info \
                else self.send

    def update_trace_watch(self):
        """Parses the nextflow trace file and retrieves the path of report JSON
        files that have not been sent to the service yet.
        """

        # Check the size stamp of the tracefile. Only proceed with the parsing
        # if it changed from the previous size.
        size_stamp = os.path.getsize(self.trace_file)
        self.trace_retry = 0
        if size_stamp and size_stamp == self.trace_sizestamp:
            return
        else:
            logger.debug("Updating trace size stamp to: {}".format(size_stamp))
            self.trace_sizestamp = size_stamp

        with open(self.trace_file) as fh:

            # Skip potential empty lines at the start of file
            header = next(fh).strip()
            while not header:
                header = next(fh).strip()

            # Get header mappings before parsing the file
            hm = self._header_mapping(header)

            for line in fh:
                # Skip empty lines
                if line.strip() == "":
                    continue

                fields = line.strip().split("\t")

                # Skip if task ID was already processes
                if fields[hm["task_id"]] in self.stored_ids:
                    continue

                if fields[hm["process"]] == "report":
                    self.report_queue.append(
                        self._expand_path(fields[hm["hash"]])
                    )
                    self.send = True

                # Add the processed trace line to the stored ids. It will be
                # skipped in future parsers
                self.stored_ids.append(fields[hm["task_id"]])

    def update_log_watch(self):
        """Parses nextflow log file and updates the run status
        """

        # Check the size stamp of the tracefile. Only proceed with the parsing
        # if it changed from the previous size.
        size_stamp = os.path.getsize(self.log_file)
        self.trace_retry = 0
        if size_stamp and size_stamp == self.log_sizestamp:
            return
        else:
            logger.debug("Updating log size stamp to: {}".format(size_stamp))
            self.log_sizestamp = size_stamp

        self._update_pipeline_status()

    def _send_live_report(self, report_id):
        """Sends a PUT request with the report JSON files currently in the
        report_queue attribute.

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        # Determines the maximum number of reports sent at the same time in
        # the same payload
        buffer_size = 100
        logger.debug("Report buffer size set to: {}".format(buffer_size))

        for i in range(0, len(self.report_queue), buffer_size):

            # Reset the report compilation batch
            reports_compilation = []

            # Iterate over report JSON batches determined by buffer_size
            for report in self.report_queue[i: i + buffer_size]:
                try:
                    report_file = [x for x in os.listdir(report)
                                   if x.endswith(".json")][0]
                except IndexError:
                    continue
                with open(join(report, report_file)) as fh:
                    reports_compilation.append(json.loads(fh.read()))

            logger.debug("Payload sent with size: {}".format(
                asizeof(json.dumps(reports_compilation))
            ))
            logger.debug("status: {}".format(self.status_info))

            try:
                requests.put(
                    self.broadcast_address,
                    json={"run_id": report_id,
                          "report_json": reports_compilation,
                          "status": self.status_info}
                )
            except requests.exceptions.ConnectionError:
                logger.error(colored_print(
                    "ERROR: Could not establish connection with server. The server"
                    " may be down or there is a problem with your internet "
                    "connection.", "red_bold"))
                sys.exit(1)

        # When there is no change in the report queue, but there is a change
        # in the run status of the pipeline
        if not self.report_queue:

            logger.debug("status: {}".format(self.status_info))

            try:
                requests.put(
                    self.broadcast_address,
                    json={"run_id": report_id,
                          "report_json": [],
                          "status": self.status_info}
                )
            except requests.exceptions.ConnectionError:
                logger.error(colored_print(
                    "ERROR: Could not establish connection with server. The"
                    " server may be down or there is a problem with your "
                    "internet connection.", "red_bold"))
                sys.exit(1)

        # Reset the report queue after sending the request
        self.report_queue = []

    def _init_live_reports(self, report_id):
        """Sends a POST request to initialize the live reports

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        logger.debug("Sending initial POST request to {} to start report live"
                     " update".format(self.broadcast_address))

        try:
            with open(".metadata.json") as fh:
                metadata = [json.load(fh)]
        except:
            metadata = []

        start_json = {
            "data": {"results": metadata}
        }

        try:
            requests.post(
                self.broadcast_address,
                json={"run_id": report_id, "report_json": start_json,
                      "status": self.status_info}
            )
        except requests.exceptions.ConnectionError:
            logger.error(colored_print(
                "ERROR: Could not establish connection with server. The server"
                " may be down or there is a problem with your internet "
                "connection.", "red_bold"))
            sys.exit(1)

    def _close_connection(self, report_id):
        """Sends a delete request for the report JSON hash

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        logger.debug(
            "Closing connection and sending DELETE request to {}".format(
                self.broadcast_address))

        try:
            r = requests.delete(self.broadcast_address,
                                json={"run_id": report_id})
            if r.status_code != 202:
                logger.error(colored_print(
                    "ERROR: There was a problem sending data to the server"
                    "with reason: {}".format(r.reason)))
        except requests.exceptions.ConnectionError:
            logger.error(colored_print(
                "ERROR: Could not establish connection with server. The server"
                " may be down or there is a problem with your internet "
                "connection.", "red_bold"))
            sys.exit(1)

    def _send_report(self, report_id):

        with open(self.report_file) as fh:
            report_json = json.loads(fh.read())

        logger.debug("Unique payload sent with size: {}".format(
            asizeof(json.dumps(report_json))
        ))

        try:
            requests.post(
                self.broadcast_address,
                json={"run_id": report_id, "report_json": report_json}
            )
        except requests.exceptions.ConnectionError:
            logger.error(colored_print(
                "ERROR: Could not establish connection with server. The server"
                " may be down or there is a problem with your internet "
                "connection.", "red_bold"))
            sys.exit(1)

    def _print_msg(self, run_id):

        report_address = "{}reports/broadcast/{}".format(self.app_address,
                                                         run_id)
        logger.info(colored_print(
            "The pipeline reports are available in the following link:",
            "green_bold"))
        logger.info("{}".format(report_address))

    def broadcast_report(self):

        logger.info(colored_print("Preparing to broacast reports...",
                                  "green_bold"))

        report_hash = self._get_report_id()

        # When in watch mode,
        if self.watch:
            logger.info(colored_print("\tFetching pipeline run status",
                                      "green_bold"))
            self._update_pipeline_status()
            logger.info(colored_print(
                "\tSending initial request to test service", "green_bold"))
            self._init_live_reports(report_hash)
            logger.info(colored_print("\tInitial parsing of trace file",
                                      "green_bold"))
            self.update_trace_watch()

            self._print_msg(report_hash)

        logger.debug("Establishing connection...")

        stay_alive = True
        _broadcast_sent = False
        try:
            while stay_alive:

                # When not in watch mode, send the report JSON once
                if not _broadcast_sent and not self.watch:
                    self._send_report(report_hash)
                    self._print_msg(report_hash)
                    _broadcast_sent = True

                # When in watch mode, continuously monitor the trace file for
                # updates
                if self.watch:
                    self.update_trace_watch()
                    self.update_log_watch()
                    # When new report JSON files are available, send then
                    # via a PUT request
                    if self.send:
                        self._send_live_report(report_hash)
                        self.send = False

                sleep(self.refresh_rate)

        except FileNotFoundError as e:
            print(e)
            logger.error(colored_print(
                "ERROR: Report JSON file is not reachable!", "red_bold"))
        except Exception as e:
            logger.exception("ERROR: " + e)
        finally:
            logger.info("Closing connection")
            self._close_connection(report_hash)


#!/usr/bin/env python3

"""
Purpose
-------

This module is intended to run FastQC on paired-end FastQ files.

Expected input
--------------

The following variables are expected whether using NextFlow or the
:py:func:`main` executor.

- ``fastq_pair`` : *Pair of FastQ file paths*
    - e.g.: ``'SampleA_1.fastq.gz SampleA_2.fastq.gz'``

Generated output
----------------

The generated output are output files that contain an object, usually a string.

- ``pair_{1,2}_data`` : File containing FastQC report at the nucleotide level\
    for each pair
    - e.g.: ``'pair_1_data'`` and ``'pair_2_data'``
- ``pair_{1,2}_summary``: File containing FastQC report for each category and\
    for each pair
    - e.g.: ``'pair_1_summary'`` and ``'pair_2_summary'``

Code documentation
------------------

"""

__version__ = "1.0.1"
__build__ = "28032018"
__template__ = "fastqc-nf"

import os
import subprocess

from subprocess import PIPE
from os.path import exists, join

from flowcraft_utils.flowcraft_base import get_logger, MainWrapper

logger = get_logger(__file__)

def __get_version_fastqc():

    try:

        cli = ["fastqc", "--version"]
        p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
        stdout, _ = p.communicate()

        version = stdout.strip().split()[1][1:].decode("utf8")

    except Exception as e:
        logger.debug(e)
        version = "undefined"

    return {
        "program": "FastQC",
        "version": version,
    }

if __file__.endswith(".command.sh"):
    FASTQ_PAIR = '$fastq_pair'.split()
    ADAPTER_FILE = '$ad'
    CPUS = '$task.cpus'
    logger.debug("Running {} with parameters:".format(
        os.path.basename(__file__)))
    logger.debug("FASTQ_PAIR: {}".format(FASTQ_PAIR))
    logger.debug("ADAPTER_FILE: {}".format(ADAPTER_FILE))
    logger.debug("CPUS: {}".format(CPUS))

def convert_adatpers(adapter_fasta):
    """Generates an adapter file for FastQC from a fasta file.

    The provided adapters file is assumed to be a simple fasta file with the
    adapter's name as header and the corresponding sequence::

        >TruSeq_Universal_Adapter
        AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT
        >TruSeq_Adapter_Index 1
        GATCGGAAGAGCACACGTCTGAACTCCAGTCACATCACGATCTCGTATGCCGTCTTCTGCTTG

    Parameters
    ----------
    adapter_fasta : str
        Path to Fasta file with adapter sequences.

    Returns
    -------
    adapter_out : str or None
        The path to the reformatted adapter file. Returns ``None`` if the
        adapters file does not exist or the path is incorrect.
    """

    adapter_out = "fastqc_adapters.tab"
    logger.debug("Setting output adapters file to: {}".format(adapter_out))

    try:

        with open(adapter_fasta) as fh, \
                open(adapter_out, "w") as adap_fh:

            for line in fh:
                if line.startswith(">"):

                    head = line[1:].strip()
                    # Get the next line with the sequence string
                    sequence = next(fh).strip()

                    adap_fh.write("{}\\t{}\\n".format(head, sequence))

        logger.info("Converted adapters file")

        return adapter_out

    # If an invalid adapters file is provided, return None.
    except FileNotFoundError:
        logger.warning("Could not find the provided adapters file: {}".format(
            adapter_fasta))
        return

@MainWrapper
def main(fastq_pair, adapter_file, cpus):
    """ Main executor of the fastq template.

    Parameters
    ----------
    fastq_pair : list
        Two element list containing the paired FastQ files.
    adapter_file : str
        Path to adapters file.
    cpus : int or str
        Number of cpu's that will be by FastQC.

    """

    logger.info("Starting fastqc")

    # If an adapter file was provided, convert it to FastQC format
    if os.path.exists(adapter_file):
        logger.info("Adapters file provided: {}".format(adapter_file))
        adapters = convert_adatpers(adapter_file)
    else:
        logger.info("Adapters file '{}' not provided or does not "
                    "exist".format(adapter_file))
        adapters = None

    # Setting command line for FastQC
    cli = [
        "fastqc",
        "--extract",
        "--nogroup",
        "--format",
        "fastq",
        "--threads",
        str(cpus)
    ]

    # Add adapters file to command line, if it exists
    if adapters:
        cli += ["--adapters", "{}".format(adapters)]

    # Add FastQ files at the end of command line
    cli += fastq_pair

    logger.debug("Running fastqc subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE, shell=False)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)

    logger.info("Finished fastqc subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished fastqc subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished fastqc with return code: {}".format(
        p.returncode))

    logger.info("Checking if FastQC output was correctly generated")
    # Check if the FastQC output was correctly generated.
    with open(".status", "w") as status_fh:
        for fastq in fastq_pair:
            fpath = join(fastq.rsplit(".", 2)[0] + "_fastqc",
                         "fastqc_data.txt")
            logger.debug("Checking path: {}".format(fpath))
            # If the FastQC output does not exist, pass the STDERR to
            # the output status channel and exit
            if not exists(fpath):
                logger.warning("Path does not exist: {}".format(fpath))
                status_fh.write("fail")
                return

            logger.debug("Found path: {}".format(fpath))

        # If the output directories exist, write 'pass' to the output status
        # channel
            status_fh.write("pass")

    logger.info("Retrieving relevant FastQC output files")

    # Both FastQC have been correctly executed. Get the relevant FastQC
    # output files for the output channel
    for i, fastq in enumerate(fastq_pair):
        # Get results for each pair
        fastqc_dir = fastq.rsplit(".", 2)[0] + "_fastqc"

        summary_file = join(fastqc_dir, "summary.txt")
        logger.debug("Retrieving summary file: {}".format(summary_file))
        fastqc_data_file = join(fastqc_dir, "fastqc_data.txt")
        logger.debug("Retrieving data file: {}".format(fastqc_data_file))

        # Rename output files to a file name that is easier to handle in the
        # output channel
        os.rename(fastqc_data_file, "pair_{}_data".format(i + 1))
        os.rename(summary_file, "pair_{}_summary".format(i + 1))

if __name__ == "__main__":

    main(FASTQ_PAIR, ADAPTER_FILE, CPUS)


#!/usr/bin/env python3

"""
Purpose
-------

This module is intended to generate a json output for mash dist results that
can be imported in pATLAS.

Expected input
--------------

The following variables are expected whether using NextFlow or the
:py:func:`main` executor.

- ``mash_output`` : String with the name of the mash screen output file.
    - e.g.: ``'fastaFileA_mashdist.txt'``

Code documentation
------------------

"""

__version__ = "1.4.0"
__build__ = "04092018"
__template__ = "mashsdist2json-nf"

import json
import os

from flowcraft_utils.flowcraft_base import get_logger, MainWrapper

logger = get_logger(__file__)

if __file__.endswith(".command.sh"):
    MASH_TXT = '$mashtxt'
    HASH_CUTOFF = '$shared_hashes'
    SAMPLE_ID = '$sample_id'
    ASSEMBLY_IN = '$fasta'
    logger.debug("Running {} with parameters:".format(
        os.path.basename(__file__)))
    logger.debug("MASH_TXT: {}".format(MASH_TXT))
    logger.debug("HASH_CUTOFF: {}".format(HASH_CUTOFF))
    logger.debug("SAMPLE_ID: {}".format(SAMPLE_ID))
    logger.debug("ASSEMBLY_IN: {}".format(ASSEMBLY_IN))

def send_to_output(master_dict, mash_output, sample_id, assembly_file):
    """Send dictionary to output json file
    This function sends master_dict dictionary to a json file if master_dict is
    populated with entries, otherwise it won't create the file

    Parameters
    ----------
    master_dict: dict
        dictionary that stores all entries for a specific query sequence
        in multi-fasta given to mash dist as input against patlas database
    last_seq: str
        string that stores the last sequence that was parsed before writing to
        file and therefore after the change of query sequence between different
        rows on the input file
    mash_output: str
        the name/path of input file to main function, i.e., the name/path of
        the mash dist output txt file.
    sample_id: str
        The name of the sample being parse to .report.json file

    Returns
    -------

    """

    plot_dict = {}

    # create a new file only if master_dict is populated
    if master_dict:
        out_file = open("{}.json".format(
            "".join(mash_output.split(".")[0])), "w")
        out_file.write(json.dumps(master_dict))
        out_file.close()

        # iterate through master_dict in order to make contigs the keys
        for k,v in master_dict.items():
            if not v[2] in plot_dict:
                plot_dict[v[2]] = [k]
            else:
                plot_dict[v[2]].append(k)

        number_hits = len(master_dict)
    else:
        number_hits = 0

    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [{
                "header": "Mash Dist",
                "table": "plasmids",
                "patlas_mashdist": master_dict,
                "value": number_hits
            }]
        }],
        "plotData": [{
            "sample": sample_id,
            "data": {
                "patlasMashDistXrange": plot_dict
            },
            "assemblyFile": assembly_file
        }]
    }

    with open(".report.json", "w") as json_report:
        json_report.write(json.dumps(json_dic, separators=(",", ":")))

@MainWrapper
def main(mash_output, hash_cutoff, sample_id, assembly_file):
    """
    Main function that allows to dump a mash dist txt file to a json file

    Parameters
    ----------
    mash_output: str
        A string with the input file.
    hash_cutoff: str
        the percentage cutoff for the percentage of shared hashes between query
        and plasmid in database that is allowed for the plasmid to be reported
        to the results outputs
    sample_id: str
        The name of the sample.
    """

    input_f = open(mash_output, "r")

    master_dict = {}

    for line in input_f:

        tab_split = line.split("\t")
        current_seq = tab_split[1].strip()
        ref_accession = "_".join(tab_split[0].strip().split("_")[0:3])
        mash_dist = tab_split[2].strip()
        hashes_list = tab_split[-1].strip().split("/")

        # creates a percentage of the shared hashes between the sample and the
        # reference
        perc_hashes = float(hashes_list[0]) / float(hashes_list[1])

        # if ref_accession already in dict, i.e., if the same accession number
        # matches more than one contig.
        if ref_accession in master_dict.keys():
            current_seq += ", {}".format(master_dict[ref_accession][-1])

        # assures that only the hashes with a given shared percentage are
        # reported to json file
        if perc_hashes > float(hash_cutoff):

            master_dict[ref_accession] = [
                round(1 - float(mash_dist), 2),
                round(perc_hashes, 2),
                current_seq
            ]

    # assures that file is closed in last iteration of the loop
    send_to_output(master_dict, mash_output, sample_id, assembly_file)

if __name__ == "__main__":

    main(MASH_TXT, HASH_CUTOFF, SAMPLE_ID, ASSEMBLY_IN)