def _set_channels(self):
        """Sets the main channels for the pipeline

        This method will parse de the :attr:`~Process.processes` attribute
        and perform the following tasks for each process:

            - Sets the input/output channels and main input forks and adds
              them to the process's
              :attr:`flowcraft.process.Process._context`
              attribute (See
              :func:`~NextflowGenerator.set_channels`).
            - Automatically updates the main input channel of the first
              process of each lane so that they fork from the user provide
              parameters (See
              :func:`~NextflowGenerator._update_raw_input`).
            - Check for the presence of secondary channels and adds them to the
              :attr:`~NextflowGenerator.secondary_channels` attribute.

        Notes
        -----
        **On the secondary channel setup**: With this approach, there can only
        be one secondary link start for each type of secondary link. For
        instance, If there are two processes that start a secondary channel
        for the ``SIDE_max_len`` channel, only the last one will be recorded,
        and all receiving processes will get the channel from the latest
        process. Secondary channels can only link if the source process if
        downstream of the sink process in its "forking" path.
        """

        logger.debug("=====================")
        logger.debug("Setting main channels")
        logger.debug("=====================")

        for i, p in enumerate(self.processes):

            # Set main channels for the process
            logger.debug("[{}] Setting main channels with pid: {}".format(
                p.template, i))
            p.set_channels(pid=i)

            # If there is no parent lane, set the raw input channel from user
            logger.debug("{} {} {}".format(p.parent_lane, p.input_type, p.template))
            if not p.parent_lane and p.input_type:
                self._update_raw_input(p)

            self._update_extra_inputs(p)

            self._update_secondary_channels(p)

            logger.info(colored_print(
                "\tChannels set for {} \u2713".format(p.template)))


def _set_init_process(self):
        """Sets the main raw inputs and secondary inputs on the init process

        This method will fetch the :class:`flowcraft.process.Init` process
        instance and sets the raw input (
        :func:`flowcraft.process.Init.set_raw_inputs`) for
        that process. This will handle the connection of the user parameters
        with channels that are then consumed in the pipeline.
        """

        logger.debug("========================")
        logger.debug("Setting secondary inputs")
        logger.debug("========================")

        # Get init process
        init_process = self.processes[0]
        logger.debug("Setting main raw inputs: "
                     "{}".format(self.main_raw_inputs))
        init_process.set_raw_inputs(self.main_raw_inputs)
        logger.debug("Setting extra inputs: {}".format(self.extra_inputs))
        init_process.set_extra_inputs(self.extra_inputs)

def
_set_init_process
(
self
)
:
logger
.
debug
(
"========================"
)
logger
.
debug
(
"Setting secondary inputs"
)
logger
.
debug
(
"========================"
)
# Get init process
init_process
=
self
.
processes
[
0
]
logger
.
debug
(
"Setting main raw inputs: "
"{}"
.
format
(
self
.
main_raw_inputs
)
)
init_process
.
set_raw_inputs
(
self
.
main_raw_inputs
)
logger
.
debug
(
"Setting extra inputs: {}"
.
format
(
self
.
extra_inputs
)
)
init_process
.
set_extra_inputs
(
self
.
extra_inputs
)


def _set_secondary_channels(self):
        """Sets the secondary channels for the pipeline

        This will iterate over the
        :py:attr:`NextflowGenerator.secondary_channels` dictionary that is
        populated when executing
        :func:`~NextflowGenerator._update_secondary_channels` method.
        """

        logger.debug("==========================")
        logger.debug("Setting secondary channels")
        logger.debug("==========================")

        logger.debug("Setting secondary channels: {}".format(
            self.secondary_channels))

        for source, lanes in self.secondary_channels.items():

            for vals in lanes.values():

                if not vals["end"]:
                    logger.debug("[{}] No secondary links to setup".format(
                        vals["p"].template))
                    continue

                logger.debug("[{}] Setting secondary links for "
                             "source {}: {}".format(vals["p"].template,
                                                    source,
                                                    vals["end"]))

                vals["p"].set_secondary_channel(source, vals["end"])

def
_set_secondary_channels
(
self
)
:
logger
.
debug
(
"=========================="
)
logger
.
debug
(
"Setting secondary channels"
)
logger
.
debug
(
"=========================="
)
logger
.
debug
(
"Setting secondary channels: {}"
.
format
(
self
.
secondary_channels
)
)
for
source
,
lanes
in
self
.
secondary_channels
.
items
(
)
:
for
vals
in
lanes
.
values
(
)
:
if
not
vals
[
"end"
]
:
logger
.
debug
(
"[{}] No secondary links to setup"
.
format
(
vals
[
"p"
]
.
template
)
)
continue
logger
.
debug
(
"[{}] Setting secondary links for "
"source {}: {}"
.
format
(
vals
[
"p"
]
.
template
,
source
,
vals
[
"end"
]
)
)
vals
[
"p"
]
.
set_secondary_channel
(
source
,
vals
[
"end"
]
)


def _set_status_channels(self):
        """Compiles all status channels for the status compiler process
        """

        status_inst = pc.StatusCompiler(template="status_compiler")
        report_inst = pc.ReportCompiler(template="report_compiler")

        # Compile status channels from pipeline process
        status_channels = []
        for p in [p for p in self.processes]:
            if not any([isinstance(p, x) for x in self.skip_class]):

                status_channels.extend(p.status_strs)

        if not status_channels:
            logger.debug("No status channels found. Skipping status compiler"
                         "process")
            return

        logger.debug("Setting status channels: {}".format(status_channels))

        # Check for duplicate channels. Raise exception if found.
        if len(status_channels) != len(set(status_channels)):
            raise eh.ProcessError(
                "Duplicate status channels detected. Please ensure that "
                "the 'status_channels' attributes of each process are "
                "unique. Here are the status channels:\n\n{}".format(
                    ", ".join(status_channels)
                ))

        status_inst.set_compiler_channels(status_channels)

        report_channels = ["REPORT_{}".format(x.lstrip("STATUS_")) for x in
                           status_channels]

        report_inst.set_compiler_channels(report_channels)

        self.processes.extend([status_inst, report_inst])


def _get_resources_string(res_dict, pid):
        """ Returns the nextflow resources string from a dictionary object

        If the dictionary has at least on of the resource directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"cpus": 1, "memory": "4GB"},
                    "processB": {"cpus": 2}}

        Parameters
        ----------
        res_dict : dict
            Dictionary with the resources for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string
        """

        config_str = ""
        ignore_directives = ["container", "version"]

        for p, directives in res_dict.items():

            for d, val in directives.items():

                if d in ignore_directives:
                    continue

                config_str += '\n\t${}_{}.{} = {}'.format(p, pid, d, val)

        return config_str

def
_get_resources_string
(
res_dict
,
pid
)
:
config_str
=
""
ignore_directives
=
[
"container"
,
"version"
]
for
p
,
directives
in
res_dict
.
items
(
)
:
for
d
,
val
in
directives
.
items
(
)
:
if
d
in
ignore_directives
:
continue
config_str
+=
'\n\t${}_{}.{} = {}'
.
format
(
p
,
pid
,
d
,
val
)
return
config_str


def _get_container_string(cont_dict, pid):
        """ Returns the nextflow containers string from a dictionary object

        If the dictionary has at least on of the container directives, these
        will be compiled for each process in the dictionary and returned
        as a string read for injection in the nextflow config file template.

        This dictionary should be::

            dict = {"processA": {"container": "asd", "version": "1.0.0"},
                    "processB": {"container": "dsd"}}

        Parameters
        ----------
        cont_dict : dict
            Dictionary with the containers for processes.
        pid : int
            Unique identified of the process

        Returns
        -------
        str
            nextflow config string
        """

        config_str = ""

        for p, directives in cont_dict.items():

            container = ""

            if "container" in directives:
                container += directives["container"]

                if "version" in directives:
                    container += ":{}".format(directives["version"])
                else:
                    container += ":latest"

            if container:
                config_str += '\n\t${}_{}.container = "{}"'.format(p, pid, container)

        return config_str

def
_get_container_string
(
cont_dict
,
pid
)
:
config_str
=
""
for
p
,
directives
in
cont_dict
.
items
(
)
:
container
=
""
if
"container"
in
directives
:
container
+=
directives
[
"container"
]
if
"version"
in
directives
:
container
+=
":{}"
.
format
(
directives
[
"version"
]
)
else
:
container
+=
":latest"
if
container
:
config_str
+=
'\n\t${}_{}.container = "{}"'
.
format
(
p
,
pid
,
container
)
return
config_str


def _get_params_string(self):
        """Returns the nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Returns
        -------
        str
            Nextflow params configuration string
        """

        params_str = ""

        for p in self.processes:

            logger.debug("[{}] Adding parameters: {}\n".format(
                p.template, p.params)
            )

            # Add an header with the template name to structure the params
            # configuration
            if p.params and p.template != "init":

                p.set_param_id("_{}".format(p.pid))
                params_str += "\n\t/*"
                params_str += "\n\tComponent '{}_{}'\n".format(p.template,
                                                               p.pid)
                params_str += "\t{}\n".format("-" * (len(p.template) + len(p.pid) + 12))
                params_str += "\t*/\n"

            for param, val in p.params.items():

                if p.template == "init":
                    param_id = param
                else:
                    param_id = "{}_{}".format(param, p.pid)

                params_str += "\t{} = {}\n".format(param_id, val["default"])

        return params_str

Returns the nextflow params string from a dictionary object .


def _get_merged_params_string(self):
        """Returns the merged nextflow params string from a dictionary object.

        The params dict should be a set of key:value pairs with the
        parameter name, and the default parameter value::

            self.params = {
                "genomeSize": 2.1,
                "minCoverage": 15
            }

        The values are then added to the string as they are. For instance,
        a ``2.1`` float will appear as ``param = 2.1`` and a
        ``"'teste'" string will appear as ``param = 'teste'`` (Note the
        string).

        Identical parameters in multiple processes will be merged into the same
        param.

        Returns
        -------
        str
            Nextflow params configuration string
        """

        params_temp = {}

        for p in self.processes:

            logger.debug("[{}] Adding parameters: {}".format(p.template,
                                                             p.params))
            for param, val in p.params.items():

                params_temp[param] = val["default"]

        config_str = "\n\t" + "\n\t".join([
            "{} = {}".format(param, val) for param, val in params_temp.items()
        ])

        return config_str

def
_get_merged_params_string
(
self
)
:
params_temp
=
{
}
for
p
in
self
.
processes
:
logger
.
debug
(
"[{}] Adding parameters: {}"
.
format
(
p
.
template
,
p
.
params
)
)
for
param
,
val
in
p
.
params
.
items
(
)
:
params_temp
[
param
]
=
val
[
"default"
]
config_str
=
"\n\t"
+
"\n\t"
.
join
(
[
"{} = {}"
.
format
(
param
,
val
)
for
param
,
val
in
params_temp
.
items
(
)
]
)
return
config_str

Returns the merged nextflow params string from a dictionary object .


def _get_manifest_string(self):
        """Returns the nextflow manifest config string to include in the
        config file from the information on the pipeline.

        Returns
        -------
        str
            Nextflow manifest configuration string
        """

        config_str = ""

        config_str += '\n\tname = "{}"'.format(self.pipeline_name)
        config_str += '\n\tmainScript = "{}"'.format(self.nf_file)

        return config_str

def
_get_manifest_string
(
self
)
:
config_str
=
""
config_str
+=
'\n\tname = "{}"'
.
format
(
self
.
pipeline_name
)
config_str
+=
'\n\tmainScript = "{}"'
.
format
(
self
.
nf_file
)
return
config_str

Returns the nextflow manifest config string to include in the config file from the information on the pipeline .


def _set_configurations(self):
        """This method will iterate over all process in the pipeline and
        populate the nextflow configuration files with the directives
        of each process in the pipeline.
        """

        logger.debug("======================")
        logger.debug("Setting configurations")
        logger.debug("======================")

        resources = ""
        containers = ""
        params = ""
        manifest = ""

        if self.merge_params:
            params += self._get_merged_params_string()
            help_list = self._get_merged_params_help()
        else:
            params += self._get_params_string()
            help_list = self._get_params_help()

        for p in self.processes:

            # Skip processes with the directives attribute populated
            if not p.directives:
                continue

            logger.debug("[{}] Adding directives: {}".format(
                p.template, p.directives))
            resources += self._get_resources_string(p.directives, p.pid)
            containers += self._get_container_string(p.directives, p.pid)

        manifest = self._get_manifest_string()

        self.resources = self._render_config("resources.config", {
            "process_info": resources
        })
        self.containers = self._render_config("containers.config", {
            "container_info": containers
        })
        self.params = self._render_config("params.config", {
            "params_info": params
        })
        self.manifest = self._render_config("manifest.config", {
            "manifest_info": manifest
        })
        self.help = self._render_config("Helper.groovy", {
            "nf_file": basename(self.nf_file),
            "help_list": help_list,
            "version": __version__,
            "pipeline_name": " ".join([x.upper() for x in self.pipeline_name])
        })
        self.user_config = self._render_config("user.config", {})

This method will iterate over all process in the pipeline and populate the nextflow configuration files with the directives of each process in the pipeline .


def dag_to_file(self, dict_viz, output_file=".treeDag.json"):
        """Writes dag to output file

        Parameters
        ----------
        dict_viz: dict
            Tree like dictionary that is used to export tree data of processes
            to html file and here for the dotfile .treeDag.json

        """

        outfile_dag = open(os.path.join(dirname(self.nf_file), output_file)
                           , "w")
        outfile_dag.write(json.dumps(dict_viz))
        outfile_dag.close()

def dag_to_file ( self , dict_viz , output_file = ".treeDag.json" ) : outfile_dag = open ( os . path . join ( dirname ( self . nf_file ) , output_file ) , "w" ) outfile_dag . write ( json . dumps ( dict_viz ) ) outfile_dag . close ( )


def render_pipeline(self):
        """Write pipeline attributes to json

        This function writes the pipeline and their attributes to a json file,
        that is intended to be read by resources/pipeline_graph.html to render
        a graphical output showing the DAG.

        """

        dict_viz = {
            "name": "root",
            "children": []
        }
        last_of_us = {}

        f_tree = self._fork_tree if self._fork_tree else {1: [1]}

        for x, (k, v) in enumerate(f_tree.items()):
            for p in self.processes[1:]:

                if x == 0 and p.lane not in [k] + v:
                    continue

                if x > 0 and p.lane not in v:
                    continue

                if not p.parent_lane:
                    lst = dict_viz["children"]
                else:
                    lst = last_of_us[p.parent_lane]

                tooltip = {
                    "name": "{}_{}".format(p.template, p.pid),
                    "process": {
                        "pid": p.pid,
                        "input": p.input_type,
                        "output": p.output_type if p.output_type else "None",
                        "lane": p.lane,
                    },
                    "children": []
                }

                dir_var = ""
                for k2, v2 in p.directives.items():
                    dir_var += k2
                    for d in v2:
                        try:
                            # Remove quotes from string directives
                            directive = v2[d].replace("'", "").replace('"', '') \
                                if isinstance(v2[d], str) else v2[d]
                            dir_var += "{}: {}".format(d, directive)
                        except KeyError:
                            pass

                if dir_var:
                    tooltip["process"]["directives"] = dir_var
                else:
                    tooltip["process"]["directives"] = "N/A"

                lst.append(tooltip)

                last_of_us[p.lane] = lst[-1]["children"]

        # write to file dict_viz
        self.dag_to_file(dict_viz)

        # Write tree forking information for dotfile
        with open(os.path.join(dirname(self.nf_file),
                               ".forkTree.json"), "w") as fh:
            fh.write(json.dumps(self._fork_tree))

        # send with jinja to html resource
        return self._render_config("pipeline_graph.html", {"data": dict_viz})


def write_configs(self, project_root):
        """Wrapper method that writes all configuration files to the pipeline
        directory
        """

        # Write resources config
        with open(join(project_root, "resources.config"), "w") as fh:
            fh.write(self.resources)

        # Write containers config
        with open(join(project_root, "containers.config"), "w") as fh:
            fh.write(self.containers)

        # Write containers config
        with open(join(project_root, "params.config"), "w") as fh:
            fh.write(self.params)

        # Write manifest config
        with open(join(project_root, "manifest.config"), "w") as fh:
            fh.write(self.manifest)

        # Write user config if not present in the project directory
        if not exists(join(project_root, "user.config")):
            with open(join(project_root, "user.config"), "w") as fh:
                fh.write(self.user_config)

        lib_dir = join(project_root, "lib")
        if not exists(lib_dir):
            os.makedirs(lib_dir)
        with open(join(lib_dir, "Helper.groovy"), "w") as fh:
            fh.write(self.help)

        # Generate the pipeline DAG
        pipeline_to_json = self.render_pipeline()
        with open(splitext(self.nf_file)[0] + ".html", "w") as fh:
            fh.write(pipeline_to_json)


def export_params(self):
        """Export pipeline params as a JSON to stdout

        This run mode iterates over the pipeline processes and exports the
        params dictionary of each component as a JSON to stdout.
        """

        params_json = {}

        # Skip first init process
        for p in self.processes[1:]:
            params_json[p.template] = p.params

        # Flush params json to stdout
        sys.stdout.write(json.dumps(params_json))

def
export_params
(
self
)
:
params_json
=
{
}
# Skip first init process
for
p
in
self
.
processes
[
1
:
]
:
params_json
[
p
.
template
]
=
p
.
params
# Flush params json to stdout
sys
.
stdout
.
write
(
json
.
dumps
(
params_json
)
)


def export_directives(self):
        """Export pipeline directives as a JSON to stdout
        """

        directives_json = {}

        # Skip first init process
        for p in self.processes[1:]:
            directives_json[p.template] = p.directives

        # Flush params json to stdout
        sys.stdout.write(json.dumps(directives_json))

def
export_directives
(
self
)
:
directives_json
=
{
}
# Skip first init process
for
p
in
self
.
processes
[
1
:
]
:
directives_json
[
p
.
template
]
=
p
.
directives
# Flush params json to stdout
sys
.
stdout
.
write
(
json
.
dumps
(
directives_json
)
)


def fetch_docker_tags(self):
        """
        Export all dockerhub tags associated with each component given by
        the -t flag.
        """

        # dict to store the already parsed components (useful when forks are
        # given to the pipeline string via -t flag
        dict_of_parsed = {}

        # fetches terminal width and subtracts 3 because we always add a
        # new line character and we want a space at the beggining and at the end
        # of each line
        terminal_width = shutil.get_terminal_size().columns - 3

        # first header
        center_string = " Selected container tags "

        # starts a list with the headers
        tags_list = [
            [
                "=" * int(terminal_width / 4),
                "{0}{1}{0}".format(
                    "=" * int(((terminal_width/2 - len(center_string)) / 2)),
                    center_string)
                ,
                "{}\n".format("=" * int(terminal_width / 4))
            ],
            ["component", "container", "tags"],
            [
                "=" * int(terminal_width / 4),
                "=" * int(terminal_width / 2),
                "=" * int(terminal_width / 4)
            ]
        ]

        # Skip first init process and iterate through the others
        for p in self.processes[1:]:
            template = p.template
            # if component has already been printed then skip and don't print
            # again
            if template in dict_of_parsed:
                continue

            # starts a list of  containers for the current process in
            # dict_of_parsed, in which each containers will be added to this
            # list once it gets parsed
            dict_of_parsed[template] = {
                "container": []
            }

            # fetch repo name from directives of each component.
            for directives in p.directives.values():
                try:
                    repo = directives["container"]
                    default_version = directives["version"]
                except KeyError:
                    # adds the default container if container key isn't present
                    # this happens for instance in integrity_coverage
                    repo = "flowcraft/flowcraft_base"
                    default_version = "1.0.0-1"
                # checks if repo_version already exists in list of the
                # containers for the current component being queried
                repo_version = repo + default_version
                if repo_version not in dict_of_parsed[template]["container"]:
                    # make the request to docker hub
                    r = requests.get(
                        "https://hub.docker.com/v2/repositories/{}/tags/"
                        .format(repo)
                    )
                    # checks the status code of the request, if it is 200 then
                    # parses docker hub entry, otherwise retrieve no tags but
                    # alerts the user
                    if r.status_code != 404:
                        # parse response content to dict and fetch results key
                        r_content = json.loads(r.content)["results"]
                        for version in r_content:
                            printed_version = (version["name"] + "*") \
                                if version["name"] == default_version \
                                else version["name"]
                            tags_list.append([template, repo, printed_version])
                    else:
                        tags_list.append([template, repo, "No DockerHub tags"])

                dict_of_parsed[template]["container"].append(repo_version)

        # iterate through each entry in tags_list and print the list of tags
        # for each component. Each entry (excluding the headers) contains
        # 3 elements (component name, container and tag version)
        for x, entry in enumerate(tags_list):
            # adds different color to the header in the first list and
            # if row is pair add one color and if is even add another (different
            # background)
            color = "blue_bold" if x < 3 else \
                ("white" if x % 2 != 0 else "0;37;40m")
            # generates a small list with the terminal width for each column,
            # this will be given to string formatting as the 3, 4 and 5 element
            final_width = [
                int(terminal_width/4),
                int(terminal_width/2),
                int(terminal_width/4)
            ]
            # writes the string to the stdout
            sys.stdout.write(
                colored_print("\n {0: <{3}} {1: ^{4}} {2: >{5}}".format(
                    *entry, *final_width), color)
            )
        # assures that the entire line gets the same color
        sys.stdout.write("\n{0: >{1}}\n".format("(* = default)",
                                                terminal_width + 3))

Export all dockerhub tags associated with each component given by the - t flag .


def build(self):
        """Main pipeline builder

        This method is responsible for building the
        :py:attr:`NextflowGenerator.template` attribute that will contain
        the nextflow code of the pipeline.

        First it builds the header, then sets the main channels, the
        secondary inputs, secondary channels and finally the
        status channels. When the pipeline is built, is writes the code
        to a nextflow file.
        """

        logger.info(colored_print(
            "\tSuccessfully connected {} process(es) with {} "
            "fork(s) across {} lane(s) \u2713".format(
                len(self.processes[1:]), len(self._fork_tree), self.lanes)))

        # Generate regular nextflow header that sets up the shebang, imports
        # and all possible initial channels
        self._build_header()

        self._set_channels()

        self._set_init_process()

        self._set_secondary_channels()

        logger.info(colored_print(
            "\tSuccessfully set {} secondary channel(s) \u2713".format(
                len(self.secondary_channels))))

        self._set_compiler_channels()

        self._set_configurations()

        logger.info(colored_print(
            "\tFinished configurations \u2713"))

        for p in self.processes:
            self.template += "\n{}".format(p.template_str)

        self._build_footer()

        project_root = dirname(self.nf_file)

        # Write configs
        self.write_configs(project_root)

        # Write pipeline file
        with open(self.nf_file, "w") as fh:
            fh.write(self.template)

        logger.info(colored_print(
            "\tPipeline written into {} \u2713".format(self.nf_file)))


def set_kmers(kmer_opt, max_read_len):
    """Returns a kmer list based on the provided kmer option and max read len.

    Parameters
    ----------
    kmer_opt : str
        The k-mer option. Can be either ``'auto'``, ``'default'`` or a
        sequence of space separated integers, ``'23, 45, 67'``.
    max_read_len : int
        The maximum read length of the current sample.

    Returns
    -------
    kmers : list
        List of k-mer values that will be provided to Spades.

    """

    logger.debug("Kmer option set to: {}".format(kmer_opt))

    # Check if kmer option is set to auto
    if kmer_opt == "auto":

        if max_read_len >= 175:
            kmers = [55, 77, 99, 113, 127]
        else:
            kmers = [21, 33, 55, 67, 77]

        logger.debug("Kmer range automatically selected based on max read"
                     "length of {}: {}".format(max_read_len, kmers))

    # Check if manual kmers were specified
    elif len(kmer_opt.split()) > 1:

        kmers = kmer_opt.split()
        logger.debug("Kmer range manually set to: {}".format(kmers))

    else:

        kmers = []
        logger.debug("Kmer range set to empty (will be automatically "
                     "determined by SPAdes")

    return kmers

Returns a kmer list based on the provided kmer option and max read len .


def main(sample_id, fastq_pair, max_len, kmer, clear):
    """Main executor of the spades template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    max_len : int
        Maximum read length. This value is determined in
        :py:class:`templates.integrity_coverage`
    kmer : str
        Can be either ``'auto'``, ``'default'`` or a
        sequence of space separated integers, ``'23, 45, 67'``.

    """

    logger.info("Starting spades")

    logger.info("Setting SPAdes kmers")
    kmers = set_kmers(kmer, max_len)
    logger.info("SPAdes kmers set to: {}".format(kmers))

    cli = [
        "metaspades.py",
        "--only-assembler",
        "--threads",
        "$task.cpus",
        "-o",
        "."
    ]

    # Add kmers, if any were specified
    if kmers:
        cli += ["-k {}".format(",".join([str(x) for x in kmers]))]

    # Add FastQ files
    cli += [
        "-1",
        fastq_pair[0],
        "-2",
        fastq_pair[1]
    ]

    logger.debug("Running metaSPAdes subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
        stdout = stdout.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)
        stdout = str(stdout)

    logger.info("Finished metaSPAdes subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished metaSPAdes subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished metaSPAdes with return code: {}".format(
        p.returncode))

    with open(".status", "w") as fh:
        if p.returncode != 0:
            fh.write("error")
            return
        else:
            fh.write("pass")

    # Change the default contigs.fasta assembly name to a more informative one
    if "_trim." in fastq_pair[0]:
        sample_id += "_trim"

    assembly_file = "{}_metaspades.fasta".format(
        sample_id)
    os.rename("contigs.fasta", assembly_file)
    logger.info("Setting main assembly file to: {}".format(assembly_file))

    # Remove input fastq files when clear option is specified.
    # Only remove temporary input when the expected output exists.
    if clear == "true" and os.path.exists(assembly_file):
        clean_up(fastq_pair)


def _get_report_id(self):
        """Returns a hash of the reports JSON file
        """

        if self.watch:

            # Searches for the first occurence of the nextflow pipeline
            # file name in the .nextflow.log file
            pipeline_path = get_nextflow_filepath(self.log_file)

            # Get hash from the entire pipeline file
            pipeline_hash = hashlib.md5()
            with open(pipeline_path, "rb") as fh:
                for chunk in iter(lambda: fh.read(4096), b""):
                    pipeline_hash.update(chunk)
            # Get hash from the current working dir and hostname
            workdir = os.getcwd().encode("utf8")
            hostname = socket.gethostname().encode("utf8")
            hardware_addr = str(uuid.getnode()).encode("utf8")
            dir_hash = hashlib.md5(workdir + hostname + hardware_addr)

            return pipeline_hash.hexdigest() + dir_hash.hexdigest()

        else:
            with open(self.report_file) as fh:
                report_json = json.loads(fh.read())

            metadata = report_json["data"]["results"][0]["nfMetadata"]

            try:
                report_id = metadata["scriptId"] + metadata["sessionId"]
            except KeyError:
                raise eh.ReportError("Incomplete or corrupt report JSON file "
                                     "missing the 'scriptId' and/or 'sessionId' "
                                     "metadata information")

            return report_id


def update_trace_watch(self):
        """Parses the nextflow trace file and retrieves the path of report JSON
        files that have not been sent to the service yet.
        """

        # Check the size stamp of the tracefile. Only proceed with the parsing
        # if it changed from the previous size.
        size_stamp = os.path.getsize(self.trace_file)
        self.trace_retry = 0
        if size_stamp and size_stamp == self.trace_sizestamp:
            return
        else:
            logger.debug("Updating trace size stamp to: {}".format(size_stamp))
            self.trace_sizestamp = size_stamp

        with open(self.trace_file) as fh:

            # Skip potential empty lines at the start of file
            header = next(fh).strip()
            while not header:
                header = next(fh).strip()

            # Get header mappings before parsing the file
            hm = self._header_mapping(header)

            for line in fh:
                # Skip empty lines
                if line.strip() == "":
                    continue

                fields = line.strip().split("\t")

                # Skip if task ID was already processes
                if fields[hm["task_id"]] in self.stored_ids:
                    continue

                if fields[hm["process"]] == "report":
                    self.report_queue.append(
                        self._expand_path(fields[hm["hash"]])
                    )
                    self.send = True

                # Add the processed trace line to the stored ids. It will be
                # skipped in future parsers
                self.stored_ids.append(fields[hm["task_id"]])

Parses the nextflow trace file and retrieves the path of report JSON files that have not been sent to the service yet .


def update_log_watch(self):
        """Parses nextflow log file and updates the run status
        """

        # Check the size stamp of the tracefile. Only proceed with the parsing
        # if it changed from the previous size.
        size_stamp = os.path.getsize(self.log_file)
        self.trace_retry = 0
        if size_stamp and size_stamp == self.log_sizestamp:
            return
        else:
            logger.debug("Updating log size stamp to: {}".format(size_stamp))
            self.log_sizestamp = size_stamp

        self._update_pipeline_status()


def _send_live_report(self, report_id):
        """Sends a PUT request with the report JSON files currently in the
        report_queue attribute.

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        # Determines the maximum number of reports sent at the same time in
        # the same payload
        buffer_size = 100
        logger.debug("Report buffer size set to: {}".format(buffer_size))

        for i in range(0, len(self.report_queue), buffer_size):

            # Reset the report compilation batch
            reports_compilation = []

            # Iterate over report JSON batches determined by buffer_size
            for report in self.report_queue[i: i + buffer_size]:
                try:
                    report_file = [x for x in os.listdir(report)
                                   if x.endswith(".json")][0]
                except IndexError:
                    continue
                with open(join(report, report_file)) as fh:
                    reports_compilation.append(json.loads(fh.read()))

            logger.debug("Payload sent with size: {}".format(
                asizeof(json.dumps(reports_compilation))
            ))
            logger.debug("status: {}".format(self.status_info))

            try:
                requests.put(
                    self.broadcast_address,
                    json={"run_id": report_id,
                          "report_json": reports_compilation,
                          "status": self.status_info}
                )
            except requests.exceptions.ConnectionError:
                logger.error(colored_print(
                    "ERROR: Could not establish connection with server. The server"
                    " may be down or there is a problem with your internet "
                    "connection.", "red_bold"))
                sys.exit(1)

        # When there is no change in the report queue, but there is a change
        # in the run status of the pipeline
        if not self.report_queue:

            logger.debug("status: {}".format(self.status_info))

            try:
                requests.put(
                    self.broadcast_address,
                    json={"run_id": report_id,
                          "report_json": [],
                          "status": self.status_info}
                )
            except requests.exceptions.ConnectionError:
                logger.error(colored_print(
                    "ERROR: Could not establish connection with server. The"
                    " server may be down or there is a problem with your "
                    "internet connection.", "red_bold"))
                sys.exit(1)

        # Reset the report queue after sending the request
        self.report_queue = []

Sends a PUT request with the report JSON files currently in the report_queue attribute .


def _init_live_reports(self, report_id):
        """Sends a POST request to initialize the live reports

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        logger.debug("Sending initial POST request to {} to start report live"
                     " update".format(self.broadcast_address))

        try:
            with open(".metadata.json") as fh:
                metadata = [json.load(fh)]
        except:
            metadata = []

        start_json = {
            "data": {"results": metadata}
        }

        try:
            requests.post(
                self.broadcast_address,
                json={"run_id": report_id, "report_json": start_json,
                      "status": self.status_info}
            )
        except requests.exceptions.ConnectionError:
            logger.error(colored_print(
                "ERROR: Could not establish connection with server. The server"
                " may be down or there is a problem with your internet "
                "connection.", "red_bold"))
            sys.exit(1)


def _close_connection(self, report_id):
        """Sends a delete request for the report JSON hash

        Parameters
        ----------
        report_id : str
            Hash of the report JSON as retrieved from :func:`~_get_report_hash`
        """

        logger.debug(
            "Closing connection and sending DELETE request to {}".format(
                self.broadcast_address))

        try:
            r = requests.delete(self.broadcast_address,
                                json={"run_id": report_id})
            if r.status_code != 202:
                logger.error(colored_print(
                    "ERROR: There was a problem sending data to the server"
                    "with reason: {}".format(r.reason)))
        except requests.exceptions.ConnectionError:
            logger.error(colored_print(
                "ERROR: Could not establish connection with server. The server"
                " may be down or there is a problem with your internet "
                "connection.", "red_bold"))
            sys.exit(1)


def convert_adatpers(adapter_fasta):
    """Generates an adapter file for FastQC from a fasta file.

    The provided adapters file is assumed to be a simple fasta file with the
    adapter's name as header and the corresponding sequence::

        >TruSeq_Universal_Adapter
        AATGATACGGCGACCACCGAGATCTACACTCTTTCCCTACACGACGCTCTTCCGATCT
        >TruSeq_Adapter_Index 1
        GATCGGAAGAGCACACGTCTGAACTCCAGTCACATCACGATCTCGTATGCCGTCTTCTGCTTG

    Parameters
    ----------
    adapter_fasta : str
        Path to Fasta file with adapter sequences.

    Returns
    -------
    adapter_out : str or None
        The path to the reformatted adapter file. Returns ``None`` if the
        adapters file does not exist or the path is incorrect.
    """

    adapter_out = "fastqc_adapters.tab"
    logger.debug("Setting output adapters file to: {}".format(adapter_out))

    try:

        with open(adapter_fasta) as fh, \
                open(adapter_out, "w") as adap_fh:

            for line in fh:
                if line.startswith(">"):

                    head = line[1:].strip()
                    # Get the next line with the sequence string
                    sequence = next(fh).strip()

                    adap_fh.write("{}\\t{}\\n".format(head, sequence))

        logger.info("Converted adapters file")

        return adapter_out

    # If an invalid adapters file is provided, return None.
    except FileNotFoundError:
        logger.warning("Could not find the provided adapters file: {}".format(
            adapter_fasta))
        return

Generates an adapter file for FastQC from a fasta file .


def main(fastq_pair, adapter_file, cpus):
    """ Main executor of the fastq template.

    Parameters
    ----------
    fastq_pair : list
        Two element list containing the paired FastQ files.
    adapter_file : str
        Path to adapters file.
    cpus : int or str
        Number of cpu's that will be by FastQC.

    """

    logger.info("Starting fastqc")

    # If an adapter file was provided, convert it to FastQC format
    if os.path.exists(adapter_file):
        logger.info("Adapters file provided: {}".format(adapter_file))
        adapters = convert_adatpers(adapter_file)
    else:
        logger.info("Adapters file '{}' not provided or does not "
                    "exist".format(adapter_file))
        adapters = None

    # Setting command line for FastQC
    cli = [
        "fastqc",
        "--extract",
        "--nogroup",
        "--format",
        "fastq",
        "--threads",
        str(cpus)
    ]

    # Add adapters file to command line, if it exists
    if adapters:
        cli += ["--adapters", "{}".format(adapters)]

    # Add FastQ files at the end of command line
    cli += fastq_pair

    logger.debug("Running fastqc subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE, shell=False)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)

    logger.info("Finished fastqc subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished fastqc subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished fastqc with return code: {}".format(
        p.returncode))

    logger.info("Checking if FastQC output was correctly generated")
    # Check if the FastQC output was correctly generated.
    with open(".status", "w") as status_fh:
        for fastq in fastq_pair:
            fpath = join(fastq.rsplit(".", 2)[0] + "_fastqc",
                         "fastqc_data.txt")
            logger.debug("Checking path: {}".format(fpath))
            # If the FastQC output does not exist, pass the STDERR to
            # the output status channel and exit
            if not exists(fpath):
                logger.warning("Path does not exist: {}".format(fpath))
                status_fh.write("fail")
                return

            logger.debug("Found path: {}".format(fpath))

        # If the output directories exist, write 'pass' to the output status
        # channel
            status_fh.write("pass")

    logger.info("Retrieving relevant FastQC output files")

    # Both FastQC have been correctly executed. Get the relevant FastQC
    # output files for the output channel
    for i, fastq in enumerate(fastq_pair):
        # Get results for each pair
        fastqc_dir = fastq.rsplit(".", 2)[0] + "_fastqc"

        summary_file = join(fastqc_dir, "summary.txt")
        logger.debug("Retrieving summary file: {}".format(summary_file))
        fastqc_data_file = join(fastqc_dir, "fastqc_data.txt")
        logger.debug("Retrieving data file: {}".format(fastqc_data_file))

        # Rename output files to a file name that is easier to handle in the
        # output channel
        os.rename(fastqc_data_file, "pair_{}_data".format(i + 1))
        os.rename(summary_file, "pair_{}_summary".format(i + 1))


def send_to_output(master_dict, mash_output, sample_id, assembly_file):
    """Send dictionary to output json file
    This function sends master_dict dictionary to a json file if master_dict is
    populated with entries, otherwise it won't create the file

    Parameters
    ----------
    master_dict: dict
        dictionary that stores all entries for a specific query sequence
        in multi-fasta given to mash dist as input against patlas database
    last_seq: str
        string that stores the last sequence that was parsed before writing to
        file and therefore after the change of query sequence between different
        rows on the input file
    mash_output: str
        the name/path of input file to main function, i.e., the name/path of
        the mash dist output txt file.
    sample_id: str
        The name of the sample being parse to .report.json file

    Returns
    -------

    """

    plot_dict = {}

    # create a new file only if master_dict is populated
    if master_dict:
        out_file = open("{}.json".format(
            "".join(mash_output.split(".")[0])), "w")
        out_file.write(json.dumps(master_dict))
        out_file.close()

        # iterate through master_dict in order to make contigs the keys
        for k,v in master_dict.items():
            if not v[2] in plot_dict:
                plot_dict[v[2]] = [k]
            else:
                plot_dict[v[2]].append(k)

        number_hits = len(master_dict)
    else:
        number_hits = 0

    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [{
                "header": "Mash Dist",
                "table": "plasmids",
                "patlas_mashdist": master_dict,
                "value": number_hits
            }]
        }],
        "plotData": [{
            "sample": sample_id,
            "data": {
                "patlasMashDistXrange": plot_dict
            },
            "assemblyFile": assembly_file
        }]
    }

    with open(".report.json", "w") as json_report:
        json_report.write(json.dumps(json_dic, separators=(",", ":")))

Send dictionary to output json file This function sends master_dict dictionary to a json file if master_dict is populated with entries otherwise it won t create the file


def main(mash_output, hash_cutoff, sample_id, assembly_file):
    """
    Main function that allows to dump a mash dist txt file to a json file

    Parameters
    ----------
    mash_output: str
        A string with the input file.
    hash_cutoff: str
        the percentage cutoff for the percentage of shared hashes between query
        and plasmid in database that is allowed for the plasmid to be reported
        to the results outputs
    sample_id: str
        The name of the sample.
    """

    input_f = open(mash_output, "r")

    master_dict = {}

    for line in input_f:

        tab_split = line.split("\t")
        current_seq = tab_split[1].strip()
        ref_accession = "_".join(tab_split[0].strip().split("_")[0:3])
        mash_dist = tab_split[2].strip()
        hashes_list = tab_split[-1].strip().split("/")

        # creates a percentage of the shared hashes between the sample and the
        # reference
        perc_hashes = float(hashes_list[0]) / float(hashes_list[1])

        # if ref_accession already in dict, i.e., if the same accession number
        # matches more than one contig.
        if ref_accession in master_dict.keys():
            current_seq += ", {}".format(master_dict[ref_accession][-1])

        # assures that only the hashes with a given shared percentage are
        # reported to json file
        if perc_hashes > float(hash_cutoff):

            master_dict[ref_accession] = [
                round(1 - float(mash_dist), 2),
                round(perc_hashes, 2),
                current_seq
            ]

    # assures that file is closed in last iteration of the loop
    send_to_output(master_dict, mash_output, sample_id, assembly_file)


def build_versions(self):
        """Writes versions JSON for a template file

        This method creates the JSON file ``.versions`` based on the metadata
        and specific functions that are present in a given template script.

        It starts by fetching the template metadata, which can be specified
        via the ``__version__``, ``__template__`` and ``__build__``
        attributes. If all of these attributes exist, it starts to populate
        a JSON/dict array (Note that the absence of any one of them will
        prevent the version from being written).

        Then, it will search the
        template scope for functions that start with the substring
        ``__set_version`` (For example ``def __set_version_fastqc()`).
        These functions should gather the version of
        an arbitrary program and return a JSON/dict object with the following
        information::

            {
                "program": <program_name>,
                "version": <version>
                "build": <build>
            }

        This JSON/dict object is then written in the ``.versions`` file.
        """

        version_storage = []

        template_version = self.context.get("__version__", None)
        template_program = self.context.get("__template__", None)
        template_build = self.context.get("__build__", None)

        if template_version and template_program and template_build:
            if self.logger:
                self.logger.debug("Adding template version: {}; {}; "
                                  "{}".format(template_program,
                                              template_version,
                                              template_build))
            version_storage.append({
                "program": template_program,
                "version": template_version,
                "build": template_build
            })

        for var, obj in self.context.items():
            if var.startswith("__get_version"):
                ver = obj()
                version_storage.append(ver)
                if self.logger:
                    self.logger.debug("Found additional software version"
                                      "{}".format(ver))

        with open(".versions", "w") as fh:
            fh.write(json.dumps(version_storage, separators=(",", ":")))

def
build_versions
(
self
)
:
version_storage
=
[
]
template_version
=
self
.
context
.
get
(
"__version__"
,
)
template_program
=
self
.
context
.
get
(
"__template__"
,
)
template_build
=
self
.
context
.
get
(
"__build__"
,
)
if
template_version
and
template_program
and
template_build
:
if
self
.
logger
:
self
.
logger
.
debug
(
"Adding template version: {}; {}; "
"{}"
.
format
(
template_program
,
template_version
,
template_build
)
)
version_storage
.
append
(
{
"program"
:
template_program
,
"version"
:
template_version
,
"build"
:
template_build
}
)
for
var
,
obj
in
self
.
context
.
items
(
)
:
if
var
.
startswith
(
"__get_version"
)
:
ver
=
obj
(
)
version_storage
.
append
(
ver
)
if
self
.
logger
:
self
.
logger
.
debug
(
"Found additional software version"
"{}"
.
format
(
ver
)
)
with
open
(
".versions"
,
"w"
)
as
fh
:
fh
.
write
(
json
.
dumps
(
version_storage
,
separators
=
(
","
,
":"
)
)
)


def main(mash_output, sample_id):
    '''
    converts top results from mash screen txt output to json format

    Parameters
    ----------
    mash_output: str
        this is a string that stores the path to this file, i.e, the name of
        the file
    sample_id: str
        sample name

    '''
    logger.info("Reading file : {}".format(mash_output))
    read_mash_output = open(mash_output)

    dic = {}
    median_list = []
    filtered_dic = {}

    logger.info("Generating dictionary and list to pre-process the final json")
    for line in read_mash_output:
        tab_split = line.split("\t")
        identity = tab_split[0]
        # shared_hashes = tab_split[1]
        median_multiplicity = tab_split[2]
        # p_value = tab_split[3]
        query_id = tab_split[4]
        # query-comment should not exist here and it is irrelevant

        # here identity is what in fact interests to report to json but
        # median_multiplicity also is important since it gives an rough
        # estimation of the coverage depth for each plasmid.
        # Plasmids should have higher coverage depth due to their increased
        # copy number in relation to the chromosome.
        dic[query_id] = [identity, median_multiplicity]
        median_list.append(float(median_multiplicity))

    output_json = open(" ".join(mash_output.split(".")[:-1]) + ".json", "w")

    # median cutoff is twice the median of all median_multiplicity values
    # reported by mash screen. In the case of plasmids, since the database
    # has 9k entries and reads shouldn't have that many sequences it seems ok...
    if len(median_list) > 0:
        # this statement assures that median_list has indeed any entries
        median_cutoff = median(median_list)
        logger.info("Generating final json to dump to a file")
        for k, v in dic.items():
            # estimated copy number
            copy_number = int(float(v[1]) / median_cutoff)
            # assure that plasmid as at least twice the median coverage depth
            if float(v[1]) > median_cutoff:
                filtered_dic["_".join(k.split("_")[0:3])] = [
                    round(float(v[0]),2),
                    copy_number
                ]
        logger.info(
            "Exported dictionary has {} entries".format(len(filtered_dic)))
    else:
        # if no entries were found raise an error
        logger.error("No matches were found using mash screen for the queried reads")

    output_json.write(json.dumps(filtered_dic))
    output_json.close()

    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [{
                "header": "Mash Screen",
                "table": "plasmids",
                "patlas_mashscreen": filtered_dic,
                "value": len(filtered_dic)
            }]
        }],
    }

    with open(".report.json", "w") as json_report:
        json_report.write(json.dumps(json_dic, separators=(",", ":")))


def colored_print(msg, color_label="white_bold"):
    """
    This function enables users to add a color to the print. It also enables
    to pass end_char to print allowing to print several strings in the same line
    in different prints.

    Parameters
    ----------
    color_string: str
        The color code to pass to the function, which enables color change as
        well as background color change.
    msg: str
        The actual text to be printed
    end_char: str
        The character in which each print should finish. By default it will be
        "\n".

    """

    if sys.stdout.encoding != "UTF-8":
        msg = "".join([i if ord(i) < 128 else "" for i in msg])

    # try except first looks for the color in COLORS dictionary, otherwise use
    # color_label as the color.
    try:
        col = COLORS[color_label]
    except KeyError:
        col = color_label

    return "\x1b[{}{}\x1b[0m".format(col, msg)

This function enables users to add a color to the print . It also enables to pass end_char to print allowing to print several strings in the same line in different prints .


def procs_dict_parser(procs_dict):
    """
    This function handles the dictionary of attributes of each Process class
    to print to stdout lists of all the components or the components which the
    user specifies in the -t flag.

    Parameters
    ----------
    procs_dict: dict
        A dictionary with the class attributes for all the components (or
        components that are used by the -t flag), that allow to create
        both the short_list and detailed_list. Dictionary example:
        {"abyss": {'input_type': 'fastq', 'output_type': 'fasta',
        'dependencies': [], 'directives': {'abyss': {'cpus': 4,
        'memory': '{ 5.GB * task.attempt }', 'container': 'flowcraft/abyss',
        'version': '2.1.1', 'scratch': 'true'}}}
    """

    logger.info(colored_print(
        "\n===== L I S T   O F   P R O C E S S E S =====\n", "green_bold"))

    #Sort to print alphabetically ordered list of processes to ease reading
    procs_dict_ordered = {k: procs_dict[k] for k in sorted(procs_dict)}

    for template, dict_proc_info in procs_dict_ordered.items():
        template_str = "=> {}".format(template)
        logger.info(colored_print(template_str, "blue_bold"))

        for info in dict_proc_info:
            info_str = "{}:".format(info)

            if isinstance(dict_proc_info[info], list):
                if not dict_proc_info[info]:
                    arg_msg = "None"
                else:
                    arg_msg = ", ".join(dict_proc_info[info])
            elif info == "directives":
                # this is used for the "directives", which is a dict
                if not dict_proc_info[info]:
                    # if dict is empty then add None to the message
                    arg_msg = "None"
                else:
                    # otherwise fetch all template names within a component
                    # and all the directives for each template to a list
                    list_msg = ["\n      {}: {}".format(
                        templt,
                        " , ".join(["{}: {}".format(dr, val)
                                    for dr, val in drs.items()]))
                                for templt, drs in dict_proc_info[info].items()
                    ]
                    # write list to a str
                    arg_msg = "".join(list_msg)
            else:
                arg_msg = dict_proc_info[info]

            logger.info("   {} {}".format(
                colored_print(info_str, "white_underline"), arg_msg
            ))

This function handles the dictionary of attributes of each Process class to print to stdout lists of all the components or the components which the user specifies in the - t flag .


def proc_collector(process_map, args, pipeline_string):
    """
    Function that collects all processes available and stores a dictionary of
    the required arguments of each process class to be passed to
    procs_dict_parser

    Parameters
    ----------
    process_map: dict
        The dictionary with the Processes currently available in flowcraft
        and their corresponding classes as values
    args: argparse.Namespace
        The arguments passed through argparser that will be access to check the
        type of list to be printed
    pipeline_string: str
        the pipeline string

    """

    arguments_list = []

    # prints a detailed list of the process class arguments
    if args.detailed_list:
        # list of attributes to be passed to proc_collector
        arguments_list += [
            "input_type",
            "output_type",
            "description",
            "dependencies",
            "conflicts",
            "directives"
        ]

    # prints a short list with each process and the corresponding description
    if args.short_list:
        arguments_list += [
            "description"
        ]

    if arguments_list:
        # dict to store only the required entries
        procs_dict = {}
        # loops between all process_map Processes
        for name, cls in process_map.items():

            # instantiates each Process class
            cls_inst = cls(template=name)

            # checks if recipe is provided
            if pipeline_string:
                if name not in pipeline_string:
                    continue

            d = {arg_key: vars(cls_inst)[arg_key] for arg_key in
                 vars(cls_inst) if arg_key in arguments_list}
            procs_dict[name] = d

        procs_dict_parser(procs_dict)

        sys.exit(0)


def guess_file_compression(file_path, magic_dict=None):
    """Guesses the compression of an input file.

    This function guesses the compression of a given file by checking for
    a binary signature at the beginning of the file. These signatures are
    stored in the :py:data:`MAGIC_DICT` dictionary. The supported compression
    formats are gzip, bzip2 and zip. If none of the signatures in this
    dictionary are found at the beginning of the file, it returns ``None``.

    Parameters
    ----------
    file_path : str
        Path to input file.
    magic_dict : dict, optional
        Dictionary containing the signatures of the compression types. The
        key should be the binary signature and the value should be the
        compression format. If left ``None``, it falls back to
        :py:data:`MAGIC_DICT`.

    Returns
    -------
    file_type : str or None
        If a compression type is detected, returns a string with the format.
        If not, returns ``None``.
    """

    if not magic_dict:
        magic_dict = MAGIC_DICT

    max_len = max(len(x) for x in magic_dict)

    with open(file_path, "rb") as f:
        file_start = f.read(max_len)

    logger.debug("Binary signature start: {}".format(file_start))

    for magic, file_type in magic_dict.items():
        if file_start.startswith(magic):
            return file_type

    return None

def
guess_file_compression
(
file_path
,
magic_dict
=
)
:
if
not
magic_dict
:
magic_dict
=
MAGIC_DICT
max_len
=
max
(
len
(
x
)
for
x
in
magic_dict
)
with
open
(
file_path
,
"rb"
)
as
f
:
file_start
=
f
.
read
(
max_len
)
logger
.
debug
(
"Binary signature start: {}"
.
format
(
file_start
)
)
for
magic
,
file_type
in
magic_dict
.
items
(
)
:
if
file_start
.
startswith
(
magic
)
:
return
file_type
return

Guesses the compression of an input file .


def get_qual_range(qual_str):
    """ Get range of the Unicode encode range for a given string of characters.

    The encoding is determined from the result of the :py:func:`ord` built-in.

    Parameters
    ----------
    qual_str : str
        Arbitrary string.

    Returns
    -------
    x : tuple
        (Minimum Unicode code, Maximum Unicode code).
    """

    vals = [ord(c) for c in qual_str]

    return min(vals), max(vals)

def get_qual_range ( qual_str ) : vals = [ ord ( c ) for c in qual_str ] return min ( vals ) , max ( vals )

Get range of the Unicode encode range for a given string of characters .


def get_encodings_in_range(rmin, rmax):
    """ Returns the valid encodings for a given encoding range.

    The encoding ranges are stored in the :py:data:`RANGES` dictionary, with
    the encoding name as a string and a list as a value containing the
    phred score and a tuple with the encoding range. For a given encoding
    range provided via the two first arguments, this function will return
    all possible encodings and phred scores.

    Parameters
    ----------
    rmin : int
        Minimum Unicode code in range.
    rmax : int
        Maximum Unicode code in range.

    Returns
    -------
    valid_encodings : list
        List of all possible encodings for the provided range.
    valid_phred : list
        List of all possible phred scores.

    """

    valid_encodings = []
    valid_phred = []

    for encoding, (phred, (emin, emax)) in RANGES.items():
        if rmin >= emin and rmax <= emax:
            valid_encodings.append(encoding)
            valid_phred.append(phred)

    return valid_encodings, valid_phred

def get_encodings_in_range ( rmin , rmax ) : valid_encodings = [ ] valid_phred = [ ] for encoding , ( phred , ( emin , emax ) ) in RANGES . items ( ) : if rmin >= emin and rmax <= emax : valid_encodings . append ( encoding ) valid_phred . append ( phred ) return valid_encodings , valid_phred

Returns the valid encodings for a given encoding range .


def parse_coverage_table(coverage_file):
    """Parses a file with coverage information into objects.

    This function parses a TSV file containing coverage results for
    all contigs in a given assembly and will build an ``OrderedDict``
    with the information about their coverage and length.  The length
    information is actually gathered from the contig header using a
    regular expression that assumes the usual header produced by Spades::

        contig_len = int(re.search("length_(.+?)_", line).group(1))

    Parameters
    ----------
    coverage_file : str
        Path to TSV file containing the coverage results.

    Returns
    -------
    coverage_dict : OrderedDict
        Contains the coverage and length information for each contig.
    total_size : int
        Total size of the assembly in base pairs.
    total_cov : int
        Sum of coverage values across all contigs.
    """

    # Stores the correspondence between a contig and the corresponding coverage
    # e.g.: {"contig_1": {"cov": 424} }
    coverage_dict = OrderedDict()
    # Stores the total coverage
    total_cov = 0

    with open(coverage_file) as fh:
        for line in fh:
            # Get contig and coverage
            contig, cov = line.strip().split()
            coverage_dict[contig] = {"cov": int(cov)}
            # Add total coverage
            total_cov += int(cov)
            logger.debug("Processing contig '{}' with coverage '{}'"
                         "".format(contig, cov))

    return coverage_dict, total_cov

Parses a file with coverage information into objects .


def filter_assembly(assembly_file, minimum_coverage, coverage_info,
                    output_file):
    """Generates a filtered assembly file.

    This function generates a filtered assembly file based on an original
    assembly and a minimum coverage threshold.

    Parameters
    ----------
    assembly_file : str
        Path to original assembly file.
    minimum_coverage : int or float
        Minimum coverage required for a contig to pass the filter.
    coverage_info : OrderedDict or dict
        Dictionary containing the coverage information for each contig.
    output_file : str
        Path where the filtered assembly file will be generated.

    """

    # This flag will determine whether sequence data should be written or
    # ignored because the current contig did not pass the minimum
    # coverage threshold
    write_flag = False

    with open(assembly_file) as fh, open(output_file, "w") as out_fh:

        for line in fh:
            if line.startswith(">"):
                # Reset write_flag
                write_flag = False
                # Get header of contig
                header = line.strip()[1:]
                # Check coverage for current contig
                contig_cov = coverage_info[header]["cov"]
                # If the contig coverage is above the threshold, write to
                # output filtered assembly
                if contig_cov >= minimum_coverage:
                    write_flag = True
                    out_fh.write(line)

            elif write_flag:
                out_fh.write(line)


def filter_bam(coverage_info, bam_file, min_coverage, output_bam):
    """Uses Samtools to filter a BAM file according to minimum coverage

    Provided with a minimum coverage value, this function will use Samtools
    to filter a BAM file. This is performed to apply the same filter to
    the BAM file as the one applied to the assembly file in
    :py:func:`filter_assembly`.

    Parameters
    ----------
    coverage_info : OrderedDict or dict
        Dictionary containing the coverage information for each contig.
    bam_file : str
        Path to the BAM file.
    min_coverage : int
        Minimum coverage required for a contig to pass the filter.
    output_bam : str
        Path to the generated filtered BAM file.
    """

    # Get list of contigs that will be kept
    contig_list = [x for x, vals in coverage_info.items()
                   if vals["cov"] >= min_coverage]

    cli = [
        "samtools",
        "view",
        "-bh",
        "-F",
        "4",
        "-o",
        output_bam,
        "-@",
        "1",
        bam_file,
    ]

    cli += contig_list

    logger.debug("Runnig samtools view subprocess with command: {}".format(
        cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
        stdout = stdout.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)
        stdout = str(stdout)

    logger.info("Finished samtools view subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished samtools view subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished samtools view with return code: {}".format(
        p.returncode))

    if not p.returncode:
        # Create index
        cli = [
            "samtools",
            "index",
            output_bam
        ]

        logger.debug("Runnig samtools index subprocess with command: "
                     "{}".format(cli))

        p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
        stdout, stderr = p.communicate()

        try:
            stderr = stderr.decode("utf8")
            stdout = stdout.decode("utf8")
        except (UnicodeDecodeError, AttributeError):
            stderr = str(stderr)
            stdout = str(stdout)

        logger.info("Finished samtools index subprocess with STDOUT:\\n"
                    "======================================\\n{}".format(
            stdout))
        logger.info("Fished samtools index subprocesswith STDERR:\\n"
                    "======================================\\n{}".format(
            stderr))
        logger.info("Finished samtools index with return code: {}".format(
            p.returncode))


def evaluate_min_coverage(coverage_opt, assembly_coverage, assembly_size):
    """ Evaluates the minimum coverage threshold from the value provided in
    the coverage_opt.

    Parameters
    ----------
    coverage_opt : str or int or float
        If set to "auto" it will try to automatically determine the coverage
        to 1/3 of the assembly size, to a minimum value of 10. If it set
        to a int or float, the specified value will be used.
    assembly_coverage : int or float
        The average assembly coverage for a genome assembly. This value
        is retrieved by the `:py:func:parse_coverage_table` function.
    assembly_size : int
        The size of the genome assembly. This value is retrieved by the
        `py:func:get_assembly_size` function.

    Returns
    -------
    x: int
        Minimum coverage threshold.

    """

    if coverage_opt == "auto":
        # Get the 1/3 value of the current assembly coverage
        min_coverage = (assembly_coverage / assembly_size) * .3
        logger.info("Minimum assembly coverage automatically set to: "
                    "{}".format(min_coverage))
        # If the 1/3 coverage is lower than 10, change it to the minimum of
        # 10
        if min_coverage < 10:
            logger.info("Minimum assembly coverage cannot be set to lower"
                        " that 10. Setting to 10")
            min_coverage = 10
    else:
        min_coverage = int(coverage_opt)
        logger.info("Minimum assembly coverage manually set to: {}".format(
            min_coverage))

    return min_coverage

Evaluates the minimum coverage threshold from the value provided in the coverage_opt .


def get_assembly_size(assembly_file):
    """Returns the number of nucleotides and the size per contig for the
    provided assembly file path

    Parameters
    ----------
    assembly_file : str
        Path to assembly file.

    Returns
    -------
    assembly_size : int
        Size of the assembly in nucleotides
    contig_size : dict
        Length of each contig (contig name as key and length as value)

    """

    assembly_size = 0
    contig_size = {}
    header = ""

    with open(assembly_file) as fh:
        for line in fh:

            # Skip empty lines
            if line.strip() == "":
                continue

            if line.startswith(">"):
                header = line.strip()[1:]
                contig_size[header] = 0

            else:
                line_len = len(line.strip())
                assembly_size += line_len
                contig_size[header] += line_len

    return assembly_size, contig_size

def
get_assembly_size
(
assembly_file
)
:
assembly_size
=
0
contig_size
=
{
}
header
=
""
with
open
(
assembly_file
)
as
fh
:
for
line
in
fh
:
# Skip empty lines
if
line
.
strip
(
)
==
""
:
continue
if
line
.
startswith
(
">"
)
:
header
=
line
.
strip
(
)
[
1
:
]
contig_size
[
header
]
=
0
else
:
line_len
=
len
(
line
.
strip
(
)
)
assembly_size
+=
line_len
contig_size
[
header
]
+=
line_len
return
assembly_size
,
contig_size


def main(sample_id, assembly_file, coverage_file, coverage_bp_file, bam_file,
         opts, gsize):
    """Main executor of the process_assembly_mapping template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    assembly_file : str
        Path to assembly file in Fasta format.
    coverage_file : str
        Path to TSV file with coverage information for each contig.
    coverage_bp_file : str
        Path to TSV file with coverage information for each base.
    bam_file : str
        Path to BAM file.
    opts : list
        List of options for processing assembly mapping.
    gsize : int
        Expected genome size

    """

    min_assembly_coverage, max_contigs = opts

    logger.info("Starting assembly mapping processing")

    # Get coverage info, total size and total coverage from the assembly
    logger.info("Parsing coverage table")
    coverage_info, a_cov = parse_coverage_table(coverage_file)
    a_size, contig_size = get_assembly_size(assembly_file)
    logger.info("Assembly processed with a total size of '{}' and coverage"
                " of '{}'".format(a_size, a_cov))
    # Get number of assembled bp after filters
    logger.info("Parsing coverage per bp table")
    coverage_bp_data = get_coverage_from_file(coverage_bp_file)

    # Assess the minimum assembly coverage
    min_coverage = evaluate_min_coverage(min_assembly_coverage, a_cov, a_size)

    # Check if filtering the assembly using the provided min_coverage will
    # reduce the final bp number to less than 80% of the estimated genome
    # size.
    # If the check below passes with True, then the filtered assembly
    # is above the 80% genome size threshold.
    filtered_assembly = "{}_filt.fasta".format(
        os.path.splitext(assembly_file)[0])
    filtered_bam = "filtered.bam"
    logger.info("Checking filtered assembly")
    if check_filtered_assembly(coverage_info, coverage_bp_data, min_coverage,
                               gsize, contig_size, int(max_contigs),
                               sample_id):
        # Filter assembly contigs based on the minimum coverage.
        logger.info("Filtered assembly passed minimum size threshold")
        logger.info("Writting filtered assembly")
        filter_assembly(assembly_file, min_coverage, coverage_info,
                        filtered_assembly)
        logger.info("Filtering BAM file according to saved contigs")
        filter_bam(coverage_info, bam_file, min_coverage, filtered_bam)
    # Could not filter the assembly as it would drop below acceptable
    # length levels. Copy the original assembly to the output assembly file
    # for compliance with the output channel
    else:
        shutil.copy(assembly_file, filtered_assembly)
        shutil.copy(bam_file, filtered_bam)
        shutil.copy(bam_file + ".bai", filtered_bam + ".bai")

    with open(".status", "w") as status_fh:
        status_fh.write("pass")

Main executor of the process_assembly_mapping template .


def convert_camel_case(name):
    """Convers a CamelCase string into a snake_case one

    Parameters
    ----------
    name : str
        An arbitrary string that may be CamelCase

    Returns
    -------
    str
        The input string converted into snake_case

    """
    s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', name)
    return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()


def collect_process_map():
    """Collects Process classes and return dict mapping templates to classes

    This function crawls through the components module and retrieves all
    classes that inherit from the Process class. Then, it converts the name
    of the classes (which should be CamelCase) to snake_case, which is used
    as the template name.

    Returns
    -------
    dict
        Dictionary mapping the template name (snake_case) to the corresponding
        process class.
    """

    process_map = {}

    prefix = "{}.".format(components.__name__)
    for importer, modname, _ in pkgutil.iter_modules(components.__path__,
                                                     prefix):

        _module = importer.find_module(modname).load_module(modname)

        _component_classes = [
            cls for cls in _module.__dict__.values() if
            isinstance(cls, type) and cls.__name__ != "Process"
        ]

        for cls in _component_classes:
            process_map[convert_camel_case(cls.__name__)] = cls

    return process_map

def collect_process_map ( ) : process_map = { } prefix = "{}." . format ( components . __name__ ) for importer , modname , _ in pkgutil . iter_modules ( components . __path__ , prefix ) : _module = importer . find_module ( modname ) . load_module ( modname ) _component_classes = [ cls for cls in _module . __dict__ . values ( ) if isinstance ( cls , type ) and cls . __name__ != "Process" ] for cls in _component_classes : process_map [ convert_camel_case ( cls . __name__ ) ] = cls return process_map


def main(newick):
    """Main executor of the process_newick template.

    Parameters
    ----------
    newick : str
        path to the newick file.

    """

    logger.info("Starting newick file processing")

    print(newick)

    tree = dendropy.Tree.get(file=open(newick, 'r'), schema="newick")

    tree.reroot_at_midpoint()

    to_write=tree.as_string("newick").strip().replace("[&R] ", '').replace(' ', '_').replace("'", "")

    with open(".report.json", "w") as json_report:
        json_dic = {
            "treeData": [{
                "trees": [
                    to_write
                ]
            }],
        }

        json_report.write(json.dumps(json_dic, separators=(",", ":")))

    with open(".status", "w") as status_fh:
        status_fh.write("pass")

def
main
(
newick
)
:
logger
.
info
(
"Starting newick file processing"
)
print
(
newick
)
tree
=
dendropy
.
Tree
.
get
(
file
=
open
(
newick
,
'r'
)
,
schema
=
"newick"
)
tree
.
reroot_at_midpoint
(
)
to_write
=
tree
.
as_string
(
"newick"
)
.
strip
(
)
.
replace
(
"[&R] "
,
''
)
.
replace
(
' '
,
'_'
)
.
replace
(
"'"
,
""
)
with
open
(
".report.json"
,
"w"
)
as
json_report
:
json_dic
=
{
"treeData"
:
[
{
"trees"
:
[
to_write
]
}
]
,
}
json_report
.
write
(
json
.
dumps
(
json_dic
,
separators
=
(
","
,
":"
)
)
)
with
open
(
".status"
,
"w"
)
as
status_fh
:
status_fh
.
write
(
"pass"
)

Main executor of the process_newick template .


def quickhull(sample):
    """ Find data points on the convex hull of a supplied data set

    Args:
        sample: data points as column vectors n x d
                    n - number samples
                    d - data dimension (should be two)

    Returns:
        a k x d matrix containint the convex hull data points
    """

    link = lambda a, b: np.concatenate((a, b[1:]))
    edge = lambda a, b: np.concatenate(([a], [b]))

    def dome(sample, base):
        h, t = base
        dists = np.dot(sample - h, np.dot(((0, -1), (1, 0)), (t - h)))
        outer = np.repeat(sample, dists > 0, axis=0)

        if len(outer):
            pivot = sample[np.argmax(dists)]
            return link(dome(outer, edge(h, pivot)),
                dome(outer, edge(pivot, t)))
        else:
            return base

    if len(sample) > 2:
        axis = sample[:, 0]
        base = np.take(sample, [np.argmin(axis), np.argmax(axis)], axis=0)
        return link(dome(sample, base),
            dome(sample, base[::-1]))
    else:
        return sample

def quickhull ( sample ) : link = lambda a , b : np . concatenate ( ( a , b [ 1 : ] ) ) edge = lambda a , b : np . concatenate ( ( [ a ] , [ b ] ) ) def dome ( sample , base ) : h , t = base dists = np . dot ( sample - h , np . dot ( ( ( 0 , - 1 ) , ( 1 , 0 ) ) , ( t - h ) ) ) outer = np . repeat ( sample , dists > 0 , axis = 0 ) if len ( outer ) : pivot = sample [ np . argmax ( dists ) ] return link ( dome ( outer , edge ( h , pivot ) ) , dome ( outer , edge ( pivot , t ) ) ) else : return base if len ( sample ) > 2 : axis = sample [ : , 0 ] base = np . take ( sample , [ np . argmin ( axis ) , np . argmax ( axis ) ] , axis = 0 ) return link ( dome ( sample , base ) , dome ( sample , base [ : : - 1 ] ) ) else : return sample


def _map_w_to_data(self):
        """ Return data points that are most similar to basis vectors W
        """

        # assign W to the next best data sample
        self._Wmapped_index = vq(self.data, self.W)
        self.Wmapped = np.zeros(self.W.shape)

        # do not directly assign, i.e. Wdist = self.data[:,sel]
        # as self might be unsorted (in non ascending order)
        # -> sorting sel would screw the matching to W if
        # self.data is stored as a hdf5 table (see h5py)
        for i, s in enumerate(self._Wmapped_index):
            self.Wmapped[:,i] = self.data[:,s]


def median_filter(X, M=8):
    """Median filter along the first axis of the feature matrix X."""
    for i in range(X.shape[1]):
        X[:, i] = filters.median_filter(X[:, i], size=M)
    return X

def median_filter ( X , M = 8 ) : for i in range ( X . shape [ 1 ] ) : X [ : , i ] = filters . median_filter ( X [ : , i ] , size = M ) return X

Median filter along the first axis of the feature matrix X .


def compute_gaussian_krnl(M):
    """Creates a gaussian kernel following Foote's paper."""
    g = signal.gaussian(M, M // 3., sym=True)
    G = np.dot(g.reshape(-1, 1), g.reshape(1, -1))
    G[M // 2:, :M // 2] = -G[M // 2:, :M // 2]
    G[:M // 2, M // 2:] = -G[:M // 2, M // 2:]
    return G

def compute_gaussian_krnl ( M ) : g = signal . gaussian ( M , M // 3. , sym = True ) G = np . dot ( g . reshape ( - 1 , 1 ) , g . reshape ( 1 , - 1 ) ) G [ M // 2 : , : M // 2 ] = - G [ M // 2 : , : M // 2 ] G [ : M // 2 , M // 2 : ] = - G [ : M // 2 , M // 2 : ] return G

Creates a gaussian kernel following Foote s paper .


def compute_ssm(X, metric="seuclidean"):
    """Computes the self-similarity matrix of X."""
    D = distance.pdist(X, metric=metric)
    D = distance.squareform(D)
    D /= D.max()
    return 1 - D

def compute_ssm ( X , metric = "seuclidean" ) : D = distance . pdist ( X , metric = metric ) D = distance . squareform ( D ) D /= D . max ( ) return 1 - D

Computes the self - similarity matrix of X .


def compute_nc(X, G):
    """Computes the novelty curve from the self-similarity matrix X and
        the gaussian kernel G."""
    N = X.shape[0]
    M = G.shape[0]
    nc = np.zeros(N)

    for i in range(M // 2, N - M // 2 + 1):
        nc[i] = np.sum(X[i - M // 2:i + M // 2, i - M // 2:i + M // 2] * G)

    # Normalize
    nc += nc.min()
    nc /= nc.max()
    return nc

def
compute_nc
(
X
,
G
)
:
N
=
X
.
shape
[
0
]
M
=
G
.
shape
[
0
]
nc
=
np
.
zeros
(
N
)
for
i
in
range
(
M
//
2
,
N
M
//
2
+
1
)
:
nc
[
i
]
=
np
.
sum
(
X
[
i
M
//
2
:
i
+
M
//
2
,
i
M
//
2
:
i
+
M
//
2
]
*
G
)
# Normalize
nc
+=
nc
.
min
(
)
nc
/=
nc
.
max
(
)
return
nc

Computes the novelty curve from the self - similarity matrix X and the gaussian kernel G .


def gaussian_filter(X, M=8, axis=0):
    """Gaussian filter along the first axis of the feature matrix X."""
    for i in range(X.shape[axis]):
        if axis == 1:
            X[:, i] = filters.gaussian_filter(X[:, i], sigma=M / 2.)
        elif axis == 0:
            X[i, :] = filters.gaussian_filter(X[i, :], sigma=M / 2.)
    return X

def gaussian_filter ( X , M = 8 , axis = 0 ) : for i in range ( X . shape [ axis ] ) : if axis == 1 : X [ : , i ] = filters . gaussian_filter ( X [ : , i ] , sigma = M / 2. ) elif axis == 0 : X [ i , : ] = filters . gaussian_filter ( X [ i , : ] , sigma = M / 2. ) return X

Gaussian filter along the first axis of the feature matrix X .


def compute_nc(X):
    """Computes the novelty curve from the structural features."""
    N = X.shape[0]
    # nc = np.sum(np.diff(X, axis=0), axis=1) # Difference between SF's

    nc = np.zeros(N)
    for i in range(N - 1):
        nc[i] = distance.euclidean(X[i, :], X[i + 1, :])

    # Normalize
    nc += np.abs(nc.min())
    nc /= float(nc.max())
    return nc

Computes the novelty curve from the structural features .


def circular_shift(X):
    """Shifts circularly the X squre matrix in order to get a
        time-lag matrix."""
    N = X.shape[0]
    L = np.zeros(X.shape)
    for i in range(N):
        L[i, :] = np.asarray([X[(i + j) % N, j] for j in range(N)])
    return L

def circular_shift ( X ) : N = X . shape [ 0 ] L = np . zeros ( X . shape ) for i in range ( N ) : L [ i , : ] = np . asarray ( [ X [ ( i + j ) % N , j ] for j in range ( N ) ] ) return L

Shifts circularly the X squre matrix in order to get a time - lag matrix .


def embedded_space(X, m, tau=1):
    """Time-delay embedding with m dimensions and tau delays."""
    N = X.shape[0] - int(np.ceil(m))
    Y = np.zeros((N, int(np.ceil(X.shape[1] * m))))
    for i in range(N):
        # print X[i:i+m,:].flatten().shape, w, X.shape
        # print Y[i,:].shape
        rem = int((m % 1) * X.shape[1])  # Reminder for float m
        Y[i, :] = np.concatenate((X[i:i + int(m), :].flatten(),
                                 X[i + int(m), :rem]))
    return Y

Time - delay embedding with m dimensions and tau delays .


def _plot_formatting(title, est_file, algo_ids, last_bound, N, output_file):
    """Formats the plot with the correct axis labels, title, ticks, and
    so on."""
    import matplotlib.pyplot as plt
    if title is None:
        title = os.path.basename(est_file).split(".")[0]
    plt.title(title)
    plt.yticks(np.arange(0, 1, 1 / float(N)) + 1 / (float(N) * 2))
    plt.gcf().subplots_adjust(bottom=0.22)
    plt.gca().set_yticklabels(algo_ids)
    plt.xlabel("Time (seconds)")
    plt.xlim((0, last_bound))
    plt.tight_layout()
    if output_file is not None:
        plt.savefig(output_file)
    plt.show()

def
_plot_formatting
(
title
,
est_file
,
algo_ids
,
last_bound
,
N
,
output_file
)
:
import
matplotlib
.
pyplot
as
plt
if
title
is
:
title
=
os
.
path
.
basename
(
est_file
)
.
split
(
"."
)
[
0
]
plt
.
title
(
title
)
plt
.
yticks
(
np
.
arange
(
0
,
1
,
1
/
float
(
N
)
)
+
1
/
(
float
(
N
)
*
2
)
)
plt
.
gcf
(
)
.
subplots_adjust
(
bottom
=
0.22
)
plt
.
gca
(
)
.
set_yticklabels
(
algo_ids
)
plt
.
xlabel
(
"Time (seconds)"
)
plt
.
xlim
(
(
0
,
last_bound
)
)
plt
.
tight_layout
(
)
if
output_file
is
not
:
plt
.
savefig
(
output_file
)
plt
.
show
(
)

Formats the plot with the correct axis labels title ticks and so on .


def plot_boundaries(all_boundaries, est_file, algo_ids=None, title=None,
                    output_file=None):
    """Plots all the boundaries.

    Parameters
    ----------
    all_boundaries: list
        A list of np.arrays containing the times of the boundaries, one array
        for each algorithm.
    est_file: str
        Path to the estimated file (JSON file)
    algo_ids : list
        List of algorithm ids to to read boundaries from.
        If None, all algorithm ids are read.
    title : str
        Title of the plot. If None, the name of the file is printed instead.
    """
    import matplotlib.pyplot as plt
    N = len(all_boundaries)  # Number of lists of boundaries
    if algo_ids is None:
        algo_ids = io.get_algo_ids(est_file)

    # Translate ids
    for i, algo_id in enumerate(algo_ids):
        algo_ids[i] = translate_ids[algo_id]
    algo_ids = ["GT"] + algo_ids

    figsize = (6, 4)
    plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')
    for i, boundaries in enumerate(all_boundaries):
        color = "b"
        if i == 0:
            color = "g"
        for b in boundaries:
            plt.axvline(b, i / float(N), (i + 1) / float(N), color=color)
        plt.axhline(i / float(N), color="k", linewidth=1)

    # Format plot
    _plot_formatting(title, est_file, algo_ids, all_boundaries[0][-1], N,
                     output_file)

def
plot_boundaries
(
all_boundaries
,
est_file
,
algo_ids
=
,
title
=
,
output_file
=
)
:
import
matplotlib
.
pyplot
as
plt
N
=
len
(
all_boundaries
)
# Number of lists of boundaries
if
algo_ids
is
:
algo_ids
=
io
.
get_algo_ids
(
est_file
)
# Translate ids
for
i
,
algo_id
in
enumerate
(
algo_ids
)
:
algo_ids
[
i
]
=
translate_ids
[
algo_id
]
algo_ids
=
[
"GT"
]
+
algo_ids
figsize
=
(
6
,
4
)
plt
.
figure
(
1
,
figsize
=
figsize
,
dpi
=
120
,
facecolor
=
'w'
,
edgecolor
=
'k'
)
for
i
,
boundaries
in
enumerate
(
all_boundaries
)
:
color
=
"b"
if
i
==
0
:
color
=
"g"
for
b
in
boundaries
:
plt
.
axvline
(
b
,
i
/
float
(
N
)
,
(
i
+
1
)
/
float
(
N
)
,
color
=
color
)
plt
.
axhline
(
i
/
float
(
N
)
,
color
=
"k"
,
linewidth
=
1
)
# Format plot
_plot_formatting
(
title
,
est_file
,
algo_ids
,
all_boundaries
[
0
]
[
1
]
,
N
,
output_file
)


def plot_labels(all_labels, gt_times, est_file, algo_ids=None, title=None,
                output_file=None):
    """Plots all the labels.

    Parameters
    ----------
    all_labels: list
        A list of np.arrays containing the labels of the boundaries, one array
        for each algorithm.
    gt_times: np.array
        Array with the ground truth boundaries.
    est_file: str
        Path to the estimated file (JSON file)
    algo_ids : list
        List of algorithm ids to to read boundaries from.
        If None, all algorithm ids are read.
    title : str
        Title of the plot. If None, the name of the file is printed instead.
    """
    import matplotlib.pyplot as plt
    N = len(all_labels)  # Number of lists of labels
    if algo_ids is None:
        algo_ids = io.get_algo_ids(est_file)

    # Translate ids
    for i, algo_id in enumerate(algo_ids):
        algo_ids[i] = translate_ids[algo_id]
    algo_ids = ["GT"] + algo_ids

    # Index the labels to normalize them
    for i, labels in enumerate(all_labels):
        all_labels[i] = mir_eval.util.index_labels(labels)[0]

    # Get color map
    cm = plt.get_cmap('gist_rainbow')
    max_label = max(max(labels) for labels in all_labels)

    # To intervals
    gt_inters = utils.times_to_intervals(gt_times)

    # Plot labels
    figsize = (6, 4)
    plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')
    for i, labels in enumerate(all_labels):
        for label, inter in zip(labels, gt_inters):
            plt.axvspan(inter[0], inter[1], ymin=i / float(N),
                        ymax=(i + 1) / float(N), alpha=0.6,
                        color=cm(label / float(max_label)))
        plt.axhline(i / float(N), color="k", linewidth=1)

    # Draw the boundary lines
    for bound in gt_times:
        plt.axvline(bound, color="g")

    # Format plot
    _plot_formatting(title, est_file, algo_ids, gt_times[-1], N,
                     output_file)

def
plot_labels
(
all_labels
,
gt_times
,
est_file
,
algo_ids
=
,
title
=
,
output_file
=
)
:
import
matplotlib
.
pyplot
as
plt
N
=
len
(
all_labels
)
# Number of lists of labels
if
algo_ids
is
:
algo_ids
=
io
.
get_algo_ids
(
est_file
)
# Translate ids
for
i
,
algo_id
in
enumerate
(
algo_ids
)
:
algo_ids
[
i
]
=
translate_ids
[
algo_id
]
algo_ids
=
[
"GT"
]
+
algo_ids
# Index the labels to normalize them
for
i
,
labels
in
enumerate
(
all_labels
)
:
all_labels
[
i
]
=
mir_eval
.
util
.
index_labels
(
labels
)
[
0
]
# Get color map
cm
=
plt
.
get_cmap
(
'gist_rainbow'
)
max_label
=
max
(
max
(
labels
)
for
labels
in
all_labels
)
# To intervals
gt_inters
=
utils
.
times_to_intervals
(
gt_times
)
# Plot labels
figsize
=
(
6
,
4
)
plt
.
figure
(
1
,
figsize
=
figsize
,
dpi
=
120
,
facecolor
=
'w'
,
edgecolor
=
'k'
)
for
i
,
labels
in
enumerate
(
all_labels
)
:
for
label
,
inter
in
zip
(
labels
,
gt_inters
)
:
plt
.
axvspan
(
inter
[
0
]
,
inter
[
1
]
,
ymin
=
i
/
float
(
N
)
,
ymax
=
(
i
+
1
)
/
float
(
N
)
,
alpha
=
0.6
,
color
=
cm
(
label
/
float
(
max_label
)
)
)
plt
.
axhline
(
i
/
float
(
N
)
,
color
=
"k"
,
linewidth
=
1
)
# Draw the boundary lines
for
bound
in
gt_times
:
plt
.
axvline
(
bound
,
color
=
"g"
)
# Format plot
_plot_formatting
(
title
,
est_file
,
algo_ids
,
gt_times
[
1
]
,
N
,
output_file
)


def plot_one_track(file_struct, est_times, est_labels, boundaries_id, labels_id,
                   title=None):
    """Plots the results of one track, with ground truth if it exists."""
    import matplotlib.pyplot as plt
    # Set up the boundaries id
    bid_lid = boundaries_id
    if labels_id is not None:
        bid_lid += " + " + labels_id
    try:
        # Read file
        jam = jams.load(file_struct.ref_file)
        ann = jam.search(namespace='segment_.*')[0]
        ref_inters, ref_labels = ann.to_interval_values()

        # To times
        ref_times = utils.intervals_to_times(ref_inters)
        all_boundaries = [ref_times, est_times]
        all_labels = [ref_labels, est_labels]
        algo_ids = ["GT", bid_lid]
    except:
        logging.warning("No references found in %s. Not plotting groundtruth"
                        % file_struct.ref_file)
        all_boundaries = [est_times]
        all_labels = [est_labels]
        algo_ids = [bid_lid]

    N = len(all_boundaries)

    # Index the labels to normalize them
    for i, labels in enumerate(all_labels):
        all_labels[i] = mir_eval.util.index_labels(labels)[0]

    # Get color map
    cm = plt.get_cmap('gist_rainbow')
    max_label = max(max(labels) for labels in all_labels)

    figsize = (8, 4)
    plt.figure(1, figsize=figsize, dpi=120, facecolor='w', edgecolor='k')
    for i, boundaries in enumerate(all_boundaries):
        color = "b"
        if i == 0:
            color = "g"
        for b in boundaries:
            plt.axvline(b, i / float(N), (i + 1) / float(N), color=color)
        if labels_id is not None:
            labels = all_labels[i]
            inters = utils.times_to_intervals(boundaries)
            for label, inter in zip(labels, inters):
                plt.axvspan(inter[0], inter[1], ymin=i / float(N),
                            ymax=(i + 1) / float(N), alpha=0.6,
                            color=cm(label / float(max_label)))
        plt.axhline(i / float(N), color="k", linewidth=1)

    # Format plot
    _plot_formatting(title, os.path.basename(file_struct.audio_file), algo_ids,
                     all_boundaries[0][-1], N, None)

Plots the results of one track with ground truth if it exists .