def _hms(s):
        """Converts a hms string into seconds.

        Parameters
        ----------
        s : str
            The hms string can be something like '20s', '1m30s' or '300ms'.

        Returns
        -------
        float
            Time in seconds.

        """

        if s == "-":
            return 0

        if s.endswith("ms"):
            return float(s.rstrip("ms")) / 1000

        fields = list(map(float, re.split("[dhms]", s)[:-1]))
        if len(fields) == 4:
            return fields[0] * 24 * 3600 + fields[1] * 3600 + fields[2] * 60 +\
                fields[3]
        if len(fields) == 3:
            return fields[0] * 3600 + fields[1] * 60 + fields[2]
        elif len(fields) == 2:
            return fields[0] * 60 + fields[1]
        else:
            return fields[0]

def _hms ( s ) : if s == "-" : return 0 if s . endswith ( "ms" ) : return float ( s . rstrip ( "ms" ) ) / 1000 fields = list ( map ( float , re . split ( "[dhms]" , s ) [ : - 1 ] ) ) if len ( fields ) == 4 : return fields [ 0 ] * 24 * 3600 + fields [ 1 ] * 3600 + fields [ 2 ] * 60 + fields [ 3 ] if len ( fields ) == 3 : return fields [ 0 ] * 3600 + fields [ 1 ] * 60 + fields [ 2 ] elif len ( fields ) == 2 : return fields [ 0 ] * 60 + fields [ 1 ] else : return fields [ 0 ]


def _size_coverter(s):
        """Converts size string into megabytes

        Parameters
        ----------
        s : str
            The size string can be '30KB', '20MB' or '1GB'

        Returns
        -------
        float
            With the size in bytes

        """

        if s.upper().endswith("KB"):
            return float(s.rstrip("KB")) / 1024

        elif s.upper().endswith(" B"):
            return float(s.rstrip("B")) / 1024 / 1024

        elif s.upper().endswith("MB"):
            return float(s.rstrip("MB"))

        elif s.upper().endswith("GB"):
            return float(s.rstrip("GB")) * 1024

        elif s.upper().endswith("TB"):
            return float(s.rstrip("TB")) * 1024 * 1024

        else:
            return float(s)

def
_size_coverter
(
s
)
:
if
s
.
upper
(
)
.
endswith
(
"KB"
)
:
return
float
(
s
.
rstrip
(
"KB"
)
)
/
1024
elif
s
.
upper
(
)
.
endswith
(
" B"
)
:
return
float
(
s
.
rstrip
(
"B"
)
)
/
1024
/
1024
elif
s
.
upper
(
)
.
endswith
(
"MB"
)
:
return
float
(
s
.
rstrip
(
"MB"
)
)
elif
s
.
upper
(
)
.
endswith
(
"GB"
)
:
return
float
(
s
.
rstrip
(
"GB"
)
)
*
1024
elif
s
.
upper
(
)
.
endswith
(
"TB"
)
:
return
float
(
s
.
rstrip
(
"TB"
)
)
*
1024
*
1024
else
:
return
float
(
s
)


def _get_pipeline_processes(self):
        """Parses the .nextflow.log file and retrieves the complete list
        of processes

        This method searches for specific signatures at the beginning of the
        .nextflow.log file::

             Apr-19 19:07:32.660 [main] DEBUG nextflow.processor
             TaskProcessor - Creating operator > report_corrupt_1_1 --
             maxForks: 4

        When a line with the .*Creating operator.* signature is found, the
        process name is retrieved and populates the :attr:`processes` attribute
        """

        with open(self.log_file) as fh:

            for line in fh:
                if re.match(".*Creating operator.*", line):
                    # Retrieves the process name from the string
                    match = re.match(".*Creating operator > (.*) --", line)
                    process = match.group(1)

                    if any([process.startswith(x) for x in self._blacklist]):
                        continue

                    if process not in self.skip_processes:
                        self.processes[match.group(1)] = {
                            "barrier": "W",
                            "submitted": set(),
                            "finished": set(),
                            "failed": set(),
                            "retry": set(),
                            "cpus": None,
                            "memory": None
                        }
                        self.process_tags[process] = {}

                # Retrieves the pipeline name from the string
                if re.match(".*Launching `.*` \[.*\] ", line):
                    tag_match = re.match(".*Launching `.*` \[(.*)\] ", line)
                    self.pipeline_tag = tag_match.group(1) if tag_match else \
                        "?"
                    name_match = re.match(".*Launching `(.*)` \[.*\] ", line)
                    self.pipeline_name = name_match.group(1) if name_match \
                        else "?"

        self.content_lines = len(self.processes)

Parses the . nextflow . log file and retrieves the complete list of processes


def _clear_inspect(self):
        """Clears inspect attributes when re-executing a pipeline"""

        self.trace_info = defaultdict(list)
        self.process_tags = {}
        self.process_stats = {}
        self.samples = []
        self.stored_ids = []
        self.stored_log_ids = []
        self.time_start = None
        self.time_stop = None
        self.execution_command = None
        self.nextflow_version = None
        self.abort_cause = None
        self._c = 0
        # Clean up of tag running status
        for p in self.processes.values():
            p["barrier"] = "W"
            for i in ["submitted", "finished", "failed", "retry"]:
                p[i] = set()

def
_clear_inspect
(
self
)
:
self
.
trace_info
=
defaultdict
(
list
)
self
.
process_tags
=
{
}
self
.
process_stats
=
{
}
self
.
samples
=
[
]
self
.
stored_ids
=
[
]
self
.
stored_log_ids
=
[
]
self
.
time_start
=
self
.
time_stop
=
self
.
execution_command
=
self
.
nextflow_version
=
self
.
abort_cause
=
self
.
_c
=
0
# Clean up of tag running status
for
p
in
self
.
processes
.
values
(
)
:
p
[
"barrier"
]
=
"W"
for
i
in
[
"submitted"
,
"finished"
,
"failed"
,
"retry"
]
:
p
[
i
]
=
set
(
)

Clears inspect attributes when re - executing a pipeline


def _update_barrier_status(self):
        """Checks whether the channels to each process have been closed.
        """

        with open(self.log_file) as fh:

            for line in fh:

                # Exit barrier update after session abort signal
                if "Session aborted" in line:
                    return

                if "<<< barrier arrive" in line:
                    # Retrieve process name from string
                    process_m = re.match(".*process: (.*)\)", line)
                    if process_m:
                        process = process_m.group(1)
                        # Updates process channel to complete
                        if process in self.processes:
                            self.processes[process]["barrier"] = "C"

Checks whether the channels to each process have been closed .


def _retrieve_log(path):
        """Method used to retrieve the contents of a log file into a list.

        Parameters
        ----------
        path

        Returns
        -------
        list or None
            Contents of the provided file, each line as a list entry
        """

        if not os.path.exists(path):
            return None

        with open(path) as fh:
            return fh.readlines()

def _retrieve_log ( path ) : if not os . path . exists ( path ) : return None with open ( path ) as fh : return fh . readlines ( )

Method used to retrieve the contents of a log file into a list .


def _assess_resource_warnings(self, process, vals):
        """Assess whether the cpu load or memory usage is above the allocation

        Parameters
        ----------
        process : str
            Process name
        vals : vals
            List of trace information for each tag of that process

        Returns
        -------
        cpu_warnings : dict
            Keys are tags and values are the excessive cpu load
        mem_warnings : dict
            Keys are tags and values are the excessive rss
        """

        cpu_warnings = {}
        mem_warnings = {}

        for i in vals:
            try:
                expected_load = float(i["cpus"]) * 100
                cpu_load = float(i["%cpu"].replace(",", ".").replace("%", ""))

                if expected_load * 0.9 > cpu_load > expected_load * 1.10:
                    cpu_warnings[i["tag"]] = {
                        "expected":  expected_load,
                        "value": cpu_load
                    }
            except (ValueError, KeyError):
                pass

            try:
                rss = self._size_coverter(i["rss"])
                mem_allocated = self._size_coverter(i["memory"])

                if rss > mem_allocated * 1.10:
                    mem_warnings[i["tag"]] = {
                        "expected": mem_allocated,
                        "value": rss
                    }
            except (ValueError, KeyError):
                pass

        return cpu_warnings, mem_warnings

def _assess_resource_warnings ( self , process , vals ) : cpu_warnings = { } mem_warnings = { } for i in vals : try : expected_load = float ( i [ "cpus" ] ) * 100 cpu_load = float ( i [ "%cpu" ] . replace ( "," , "." ) . replace ( "%" , "" ) ) if expected_load * 0.9 > cpu_load > expected_load * 1.10 : cpu_warnings [ i [ "tag" ] ] = { "expected" : expected_load , "value" : cpu_load } except ( ValueError , KeyError ) : pass try : rss = self . _size_coverter ( i [ "rss" ] ) mem_allocated = self . _size_coverter ( i [ "memory" ] ) if rss > mem_allocated * 1.10 : mem_warnings [ i [ "tag" ] ] = { "expected" : mem_allocated , "value" : rss } except ( ValueError , KeyError ) : pass return cpu_warnings , mem_warnings


def _update_process_stats(self):
        """Updates the process stats with the information from the processes

        This method is called at the end of each static parsing of the nextflow
        trace file. It re-populates the :attr:`process_stats` dictionary
        with the new stat metrics.
        """

        good_status = ["COMPLETED", "CACHED"]

        for process, vals in self.trace_info.items():

            # Update submission status of tags for each process
            vals = self._update_tag_status(process, vals)

            # Update process resources
            self._update_process_resources(process, vals)

            self.process_stats[process] = {}

            inst = self.process_stats[process]

            # Get number of completed samples
            inst["completed"] = "{}".format(
                len([x for x in vals if x["status"] in good_status]))

            # Get average time
            try:
                time_array = [self._hms(x["realtime"]) for x in vals]
                mean_time = round(sum(time_array) / len(time_array), 1)
                mean_time_str = strftime('%H:%M:%S', gmtime(mean_time))
                inst["realtime"] = mean_time_str
            # When the realtime column is not present
            except KeyError:
                inst["realtime"] = "-"

            # Get cumulative cpu/hours
            try:
                cpu_hours = [self._cpu_load_parser(
                    x["cpus"], x["%cpu"], x["realtime"]) for x in vals]
                inst["cpuhour"] = round(sum(cpu_hours), 2)
            # When the realtime, cpus or %cpus column are not present
            except KeyError:
                inst["cpuhour"] = "-"

            # Assess resource warnings
            inst["cpu_warnings"], inst["mem_warnings"] = \
                self._assess_resource_warnings(process, vals)

            # Get maximum memory
            try:
                rss_values = [self._size_coverter(x["rss"]) for x in vals
                              if x["rss"] != "-"]
                if rss_values:
                    max_rss = round(max(rss_values))
                    rss_str = self._size_compress(max_rss)
                else:
                    rss_str = "-"
                inst["maxmem"] = rss_str
            except KeyError:
                inst["maxmem"] = "-"

            # Get read size
            try:
                rchar_values = [self._size_coverter(x["rchar"]) for x in vals
                                if x["rchar"] != "-"]
                if rchar_values:
                    avg_rchar = round(sum(rchar_values) / len(rchar_values))
                    rchar_str = self._size_compress(avg_rchar)
                else:
                    rchar_str = "-"
            except KeyError:
                rchar_str = "-"
            inst["avgread"] = rchar_str

            # Get write size
            try:
                wchar_values = [self._size_coverter(x["wchar"]) for x in vals
                                if x["wchar"] != "-"]
                if wchar_values:
                    avg_wchar = round(sum(wchar_values) / len(wchar_values))
                    wchar_str = self._size_compress(avg_wchar)
                else:
                    wchar_str = "-"
            except KeyError:
                wchar_str = "-"
            inst["avgwrite"] = wchar_str


def log_parser(self):
        """Method that parses the nextflow log file once and updates the
        submitted number of samples for each process
        """

        # Check the timestamp of the log file. Only proceed with the parsing
        # if it changed from the previous time.
        size_stamp = os.path.getsize(self.log_file)
        self.log_retry = 0
        if size_stamp and size_stamp == self.log_sizestamp:
            return
        else:
            logger.debug("Updating log size stamp to: {}".format(size_stamp))
            self.log_sizestamp = size_stamp

        # Regular expression to catch four groups:
        # 1. Start timestamp
        # 2. Work directory hash
        # 3. Process name
        # 4. Tag name
        r = ".* (.*) \[.*\].*\[(.*)\].*process > (.*) \((.*)\).*"

        with open(self.log_file) as fh:

            for line in fh:
                if "Submitted process >" in line or \
                        "Re-submitted process >" in line or \
                        "Cached process >" in line:
                    m = re.match(r, line)
                    if not m:
                        continue

                    time_start = m.group(1)
                    workdir = m.group(2)
                    process = m.group(3)
                    tag = m.group(4)

                    # Skip if this line has already been parsed
                    if time_start + tag not in self.stored_log_ids:
                        self.stored_log_ids.append(time_start + tag)
                    else:
                        continue

                    # For first time processes
                    if process not in self.processes:
                        continue
                    p = self.processes[process]

                    # Skip is process/tag combination has finished or is retrying
                    if tag in list(p["finished"]) + list(p["retry"]):
                        continue

                    # Update failed process/tags when they have been re-submitted
                    if tag in list(p["failed"]) and \
                            "Re-submitted process >" in line:
                        p["retry"].add(tag)
                        self.send = True
                        continue

                    # Set process barrier to running. Check for barrier status
                    # are performed at the end of the trace parsing in the
                    # _update_barrier_status method.
                    p["barrier"] = "R"
                    if tag not in p["submitted"]:
                        p["submitted"].add(tag)
                        # Update the process_tags attribute with the new tag.
                        # Update only when the tag does not exist. This may rarely
                        # occur when the tag is parsed first in the trace file
                        if tag not in self.process_tags[process]:
                            self.process_tags[process][tag] = {
                                "workdir": self._expand_path(workdir),
                                "start": time_start
                            }
                            self.send = True
                        # When the tag is filled in the trace file parsing,
                        # the timestamp may not be present in the trace. In
                        # those cases, fill that information here.
                        elif not self.process_tags[process][tag]["start"]:
                            self.process_tags[process][tag]["start"] = time_start
                            self.send = True

        self._update_pipeline_status()


def update_inspection(self):
        """Wrapper method that calls the appropriate main updating methods of
        the inspection.

        It is meant to be used inside a loop (like while), so that it can
        continuously update the class attributes from the trace and log files.
        It already implements checks to parse these files only when they
        change, and they ignore entries that have been previously processes.
        """

        try:
            self.log_parser()
        except (FileNotFoundError, StopIteration) as e:
            logger.debug("ERROR: " + str(sys.exc_info()[0]))
            self.log_retry += 1
            if self.log_retry == self.MAX_RETRIES:
                raise e
        try:
            self.trace_parser()
        except (FileNotFoundError, StopIteration) as e:
            logger.debug("ERROR: " + str(sys.exc_info()[0]))
            self.trace_retry += 1
            if self.trace_retry == self.MAX_RETRIES:
                raise e

def
update_inspection
(
self
)
:
try
:
self
.
log_parser
(
)
except
(
FileNotFoundError
,
StopIteration
)
as
e
:
logger
.
debug
(
"ERROR: "
+
str
(
sys
.
exc_info
(
)
[
0
]
)
)
self
.
log_retry
+=
1
if
self
.
log_retry
==
self
.
MAX_RETRIES
:
raise
e
try
:
self
.
trace_parser
(
)
except
(
FileNotFoundError
,
StopIteration
)
as
e
:
logger
.
debug
(
"ERROR: "
+
str
(
sys
.
exc_info
(
)
[
0
]
)
)
self
.
trace_retry
+=
1
if
self
.
trace_retry
==
self
.
MAX_RETRIES
:
raise
e

Wrapper method that calls the appropriate main updating methods of the inspection .


def display_overview(self):
        """Displays the default pipeline inspection overview
        """

        stay_alive = True

        self.screen = curses.initscr()

        self.screen.keypad(True)
        self.screen.nodelay(-1)
        curses.cbreak()
        curses.noecho()
        curses.start_color()

        self.screen_lines = self.screen.getmaxyx()[0]
        # self.screen_width = self.screen.getmaxyx()[1]

        try:
            while stay_alive:

                # Provide functionality to certain keybindings
                self._curses_keybindings()
                # Updates main inspector attributes
                self.update_inspection()
                # Display curses interface
                self.flush_overview()

                sleep(self.refresh_rate)
        except FileNotFoundError:
            sys.stderr.write(colored_print(
                "ERROR: nextflow log and/or trace files are no longer "
                "reachable!", "red_bold"))
        except Exception as e:
            sys.stderr.write(str(e))
        finally:
            curses.nocbreak()
            self.screen.keypad(0)
            curses.echo()
            curses.endwin()


def _updown(self, direction):
        """Provides curses scroll functionality.
        """

        if direction == "up" and self.top_line != 0:
            self.top_line -= 1
        elif direction == "down" and \
                self.screen.getmaxyx()[0] + self.top_line\
                <= self.content_lines + 3:
            self.top_line += 1

def _updown ( self , direction ) : if direction == "up" and self . top_line != 0 : self . top_line -= 1 elif direction == "down" and self . screen . getmaxyx ( ) [ 0 ] + self . top_line <= self . content_lines + 3 : self . top_line += 1


def _rightleft(self, direction):
        """Provides curses horizontal padding"""

        if direction == "left" and self.padding != 0:
            self.padding -= 1

        if direction == "right" and \
                self.screen.getmaxyx()[1] + self.padding < self.max_width:
            self.padding += 1

def _rightleft ( self , direction ) : if direction == "left" and self . padding != 0 : self . padding -= 1 if direction == "right" and self . screen . getmaxyx ( ) [ 1 ] + self . padding < self . max_width : self . padding += 1


def _get_log_lines(self, n=300):
        """Returns a list with the last ``n`` lines of the nextflow log file

        Parameters
        ----------
        n : int
            Number of last lines from the log file

        Returns
        -------
        list
            List of strings with the nextflow log
        """

        with open(self.log_file) as fh:
            last_lines = fh.readlines()[-n:]

        return last_lines

def _get_log_lines ( self , n = 300 ) : with open ( self . log_file ) as fh : last_lines = fh . readlines ( ) [ - n : ] return last_lines

Returns a list with the last n lines of the nextflow log file


def _prepare_static_info(self):
        """Prepares the first batch of information, containing static
        information such as the pipeline file, and configuration files

        Returns
        -------
        dict
            Dict with the static information for the first POST request
        """

        pipeline_files = {}

        with open(join(self.workdir, self.pipeline_name)) as fh:
            pipeline_files["pipelineFile"] = fh.readlines()

        nf_config = join(self.workdir, "nextflow.config")
        if os.path.exists(nf_config):
            with open(nf_config) as fh:
                pipeline_files["configFile"] = fh.readlines()

        # Check for specific flowcraft configurations files
        configs = {
            "params.config": "paramsFile",
            "resources.config": "resourcesFile",
            "containers.config": "containersFile",
            "user.config": "userFile",
        }
        for config, key in configs.items():
            cfile = join(self.workdir, config)
            if os.path.exists(cfile):
                with open(cfile) as fh:
                    pipeline_files[key] = fh.readlines()

        return pipeline_files

Prepares the first batch of information containing static information such as the pipeline file and configuration files


def _dag_file_to_dict(self):
        """Function that opens the dotfile named .treeDag.json in the current
        working directory

        Returns
        -------
        Returns a dictionary with the dag object to be used in the post
        instance available through the method _establish_connection

        """
        try:
            dag_file = open(os.path.join(self.workdir, ".treeDag.json"))
            dag_json = json.load(dag_file)
        except (FileNotFoundError, json.decoder.JSONDecodeError):
            logger.warning(colored_print(
                "WARNING: dotfile named .treeDag.json not found or corrupted",
                "red_bold"))
            dag_json = {}

        return dag_json

Function that opens the dotfile named . treeDag . json in the current working directory


def _get_run_hash(self):
        """Gets the hash of the nextflow file"""

        # Get name and path of the pipeline from the log file
        pipeline_path = get_nextflow_filepath(self.log_file)

        # Get hash from the entire pipeline file
        pipeline_hash = hashlib.md5()
        with open(pipeline_path, "rb") as fh:
            for chunk in iter(lambda: fh.read(4096), b""):
                pipeline_hash.update(chunk)
        # Get hash from the current working dir and hostname
        workdir = self.workdir.encode("utf8")
        hostname = socket.gethostname().encode("utf8")
        hardware_addr = str(uuid.getnode()).encode("utf8")
        dir_hash = hashlib.md5(workdir + hostname + hardware_addr)

        return pipeline_hash.hexdigest() + dir_hash.hexdigest()


def get_nextflow_filepath(log_file):
    """Gets the nextflow file path from the nextflow log file. It searches for
    the nextflow run command throughout the file.

    Parameters
    ----------
    log_file : str
        Path for the .nextflow.log file

    Returns
    -------
    str
        Path for the nextflow file
    """

    with open(log_file) as fh:
        # Searches for the first occurence of the nextflow pipeline
        # file name in the .nextflow.log file
        while 1:
            line = fh.readline()
            if not line:
                # file is empty
                raise eh.LogError("Nextflow command path could not be found - Is "
                                 ".nextflow.log empty?")
            try:
                # Regex supports absolute paths and relative paths
                pipeline_path = re.match(".*\s(.*.nf).*", line) \
                    .group(1)
                return pipeline_path
            except AttributeError:
                continue

Gets the nextflow file path from the nextflow log file . It searches for the nextflow run command throughout the file .


def main(sample_id, assembly, min_size):
    """Main executor of the split_fasta template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    assembly : list
        Assembly file.
    min_size : int
        Minimum contig size.

    """

    logger.info("Starting script")

    f_open = open(assembly, "rU")

    entry = (x[1] for x in groupby(f_open, lambda line: line[0] == ">"))

    success = 0

    for header in entry:
        headerStr = header.__next__()[1:].strip()
        seq = "".join(s.strip() for s in entry.__next__())
        if len(seq) >= min_size:
            with open(sample_id + '_' + headerStr.replace(" ","_").replace("=","_") + '.fasta', "w") as output_file:
                output_file.write(">" + sample_id + "_" + headerStr.replace(" ","_").replace("=","_") + "\\n" + seq + "\\n")
                success += 1

    f_open.close()

    logger.info("{} sequences sucessfully splitted.".format(success))

def
main
(
sample_id
,
assembly
,
min_size
)
:
logger
.
info
(
"Starting script"
)
f_open
=
open
(
assembly
,
"rU"
)
entry
=
(
x
[
1
]
for
x
in
groupby
(
f_open
,
lambda
line
:
line
[
0
]
==
">"
)
)
success
=
0
for
header
in
entry
:
headerStr
=
header
.
__next__
(
)
[
1
:
]
.
strip
(
)
seq
=
""
.
join
(
s
.
strip
(
)
for
s
in
entry
.
__next__
(
)
)
if
len
(
seq
)
>=
min_size
:
with
open
(
sample_id
+
'_'
+
headerStr
.
replace
(
" "
,
"_"
)
.
replace
(
"="
,
"_"
)
+
'.fasta'
,
"w"
)
as
output_file
:
output_file
.
write
(
">"
+
sample_id
+
"_"
+
headerStr
.
replace
(
" "
,
"_"
)
.
replace
(
"="
,
"_"
)
+
"\\n"
+
seq
+
"\\n"
)
success
+=
1
f_open
.
close
(
)
logger
.
info
(
"{} sequences sucessfully splitted."
.
format
(
success
)
)

Main executor of the split_fasta template .


def main(sample_id, trace_file, workdir):
    """
    Parses a nextflow trace file, searches for processes with a specific tag
    and sends a JSON report with the relevant information

    The expected fields for the trace file are::

        0. task_id
        1. process
        2. tag
        3. status
        4. exit code
        5. start timestamp
        6. container
        7. cpus
        8. duration
        9. realtime
        10. queue
        11. cpu percentage
        12. memory percentage
        13. real memory size of the process
        14. virtual memory size of the process

    Parameters
    ----------
    trace_file : str
        Path to the nextflow trace file
    """

    # Determine the path of the stored JSON for the sample_id
    stats_suffix = ".stats.json"
    stats_path = join(workdir, sample_id + stats_suffix)
    trace_path = join(workdir, trace_file)

    logger.info("Starting pipeline status routine")

    logger.debug("Checking for previous pipeline status data")
    stats_array = get_previous_stats(stats_path)
    logger.info("Stats JSON object set to : {}".format(stats_array))

    # Search for this substring in the tags field. Only lines with this
    # tag will be processed for the reports
    tag = " getStats"
    logger.debug("Tag variable set to: {}".format(tag))

    logger.info("Starting parsing of trace file: {}".format(trace_path))
    with open(trace_path) as fh:

        header = next(fh).strip().split()
        logger.debug("Header set to: {}".format(header))

        for line in fh:
            fields = line.strip().split("\t")
            # Check if tag substring is in the tag field of the nextflow trace
            if tag in fields[2] and fields[3] == "COMPLETED":
                logger.debug(
                    "Parsing trace line with COMPLETED status: {}".format(
                        line))
                current_json = get_json_info(fields, header)

                stats_array[fields[0]] = current_json
            else:
                logger.debug(
                    "Ignoring trace line without COMPLETED status"
                    " or stats specific tag: {}".format(
                        line))

    with open(join(stats_path), "w") as fh, open(".report.json", "w") as rfh:
        fh.write(json.dumps(stats_array, separators=(",", ":")))
        rfh.write(json.dumps(stats_array, separators=(",", ":")))

Parses a nextflow trace file searches for processes with a specific tag and sends a JSON report with the relevant information


def brew_innuendo(args):
    """Brews a given list of processes according to the recipe

    Parameters
    ----------
    args : argparse.Namespace
        The arguments passed through argparser that will be used to check the
        the recipe, tasks and brew the process

    Returns
    -------
    str
        The final pipeline string, ready for the engine.
    list
        List of process strings.
    """

    # Create recipe class instance
    automatic_pipeline = Innuendo()

    if not args.tasks:
        input_processes = " ".join(
            automatic_pipeline.process_descriptions.keys())
    else:
        input_processes = args.tasks

    # Validate the provided pipeline processes
    validated = automatic_pipeline.validate_pipeline(input_processes)
    if not validated:
        sys.exit(1)
    # Get the final pipeline string
    pipeline_string = automatic_pipeline.run_auto_pipeline(input_processes)

    return pipeline_string


def brew_recipe(recipe_name):
    """Returns a pipeline string from a recipe name.

    Parameters
    ----------
    recipe_name : str
        Name of the recipe. Must match the name attribute in one of the classes
        defined in :mod:`flowcraft.generator.recipes`

    Returns
    -------
    str
        Pipeline string ready for parsing and processing by flowcraft engine
    """

    # This will iterate over all modules included in the recipes subpackage
    # It will return the import class and the module name, algon with the
    # correct prefix
    prefix = "{}.".format(recipes.__name__)
    for importer, modname, _ in pkgutil.iter_modules(recipes.__path__, prefix):

        # Import the current module
        _module = importer.find_module(modname).load_module(modname)

        # Fetch all available classes in module
        _recipe_classes = [cls for cls in _module.__dict__.values() if
                           isinstance(cls, type)]

        # Iterate over each Recipe class, and check for a match with the
        # provided recipe name.
        for cls in _recipe_classes:
            # Create instance of class to allow fetching the name attribute
            recipe_cls = cls()
            if getattr(recipe_cls, "name", None) == recipe_name:
                return recipe_cls.brew()

    logger.error(
        colored_print("Recipe name '{}' does not exist.".format(recipe_name))
    )
    sys.exit(1)

Returns a pipeline string from a recipe name .


def list_recipes(full=False):
    """Method that iterates over all available recipes and prints their
    information to the standard output

    Parameters
    ----------
    full : bool
        If true, it will provide the pipeline string along with the recipe name
    """

    logger.info(colored_print(
        "\n===== L I S T   O F   R E C I P E S =====\n",
        "green_bold"))

    # This will iterate over all modules included in the recipes subpackage
    # It will return the import class and the module name, algon with the
    # correct prefix
    prefix = "{}.".format(recipes.__name__)
    for importer, modname, _ in pkgutil.iter_modules(recipes.__path__, prefix):

        # Import the current module
        _module = importer.find_module(modname).load_module(modname)

        # Fetch all available classes in module
        _recipe_classes = [cls for cls in _module.__dict__.values() if
                           isinstance(cls, type)]

        # Iterate over each Recipe class, and check for a match with the
        # provided recipe name.
        for cls in _recipe_classes:

            recipe_cls = cls()

            if hasattr(recipe_cls, "name"):
                logger.info(colored_print("=> {}".format(recipe_cls.name), "blue_bold"))
                if full:
                    logger.info(colored_print("\t {}".format(recipe_cls.__doc__), "purple_bold"))
                    logger.info(colored_print("Pipeline string: {}\n".format(recipe_cls.pipeline_str), "yellow_bold"))

    sys.exit(0)


def validate_pipeline(pipeline_string):
        """Validate pipeline string

        Validates the pipeline string by searching for forbidden characters

        Parameters
        ----------
        pipeline_string : str
            STring with the processes provided

        Returns
        -------

        """
        if "(" in pipeline_string or ")" in pipeline_string or "|" in \
                pipeline_string:
            logger.error(
                colored_print("Please provide a valid task list!", "red_bold")
            )
            return False

        return True

def
validate_pipeline
(
pipeline_string
)
:
if
"("
in
pipeline_string
or
")"
in
pipeline_string
or
"|"
in
pipeline_string
:
logger
.
error
(
colored_print
(
"Please provide a valid task list!"
,
"red_bold"
)
)
return
False
return
True


def build_upstream(self, process_descriptions, task, all_tasks,
                       task_pipeline,
                       count_forks, total_tasks, forks):
        """Builds the upstream pipeline of the current process

        Checks for the upstream processes to the current process and
        adds them to the current pipeline fragment if they were provided in
        the process list.

        Parameters
        ----------
        process_descriptions : dict
            Information of processes input, output and if is forkable
        task : str
            Current process
        all_tasks : list
            A list of all provided processes
        task_pipeline : list
            Current pipeline fragment
        count_forks : int
            Current number of forks
        total_tasks : str
            All space separated processes
        forks : list
            Current forks
        Returns
        -------
        list : resulting pipeline fragment
        """
        if task in process_descriptions:
            if process_descriptions[task][1] is not None:
                if len(process_descriptions[task][1].split("|")) > 1:
                    local_forks = process_descriptions[task][1].split("|")

                    # Produces a new pipeline fragment for each forkable
                    #  process
                    for local_fork in local_forks:
                        if local_fork in total_tasks:
                            count_forks += 1
                            task_pipeline.insert(
                                0,
                                process_descriptions[task][1]
                            )
                            self.define_pipeline_string(
                                process_descriptions,
                                local_fork,
                                False,
                                True,
                                count_forks,
                                total_tasks,
                                forks
                            )

                    return task_pipeline
                else:
                    # Adds the process to the pipeline fragment in case it is
                    # provided in the task list
                    if process_descriptions[task][1] in total_tasks:
                        task_pipeline.insert(
                            0,
                            process_descriptions[task][1].split("|")[0]
                        )

                        # Proceeds building upstream until the input for a
                        # process is None
                        self.build_upstream(
                            process_descriptions,
                            process_descriptions[task][1].split("|")[0],
                            all_tasks,
                            task_pipeline,
                            count_forks,
                            total_tasks,
                            forks
                        )
                    else:
                        logger.error(
                            colored_print("{} not in provided protocols as "
                                          "input for {}".format(
                                process_descriptions[task][1], task), "red_bold"
                            )
                        )

                        sys.exit()

                    return task_pipeline
            else:
                return task_pipeline


def build_downstream(self, process_descriptions, task, all_tasks,
                         task_pipeline,
                         count_forks, total_tasks, forks):
        """Builds the downstream pipeline of the current process

        Checks for the downstream processes to the current process and
        adds them to the current pipeline fragment.

        Parameters
        ----------
        process_descriptions : dict
            Information of processes input, output and if is forkable
        task : str
            Current process
        all_tasks : list
            A list of all provided processes
        task_pipeline : list
            Current pipeline fragment
        count_forks : int
            Current number of forks
        total_tasks : str
            All space separated processes
        forks : list
            Current forks
        Returns
        -------
        list : resulting pipeline fragment
        """

        if task in process_descriptions:
            if process_descriptions[task][2] is not None:
                if len(process_descriptions[task][2].split("|")) > 1:
                    local_forks = process_descriptions[task][2].split("|")

                    # Adds the process to the pipeline fragment downstream
                    # and defines a new pipeline fragment for each fork.
                    # Those will only look for downstream processes
                    for local_fork in local_forks:
                        if local_fork in total_tasks:
                            count_forks += 1
                            task_pipeline.append(process_descriptions[task][2])
                            self.define_pipeline_string(
                                process_descriptions,
                                local_fork,
                                False,
                                True,
                                count_forks,
                                total_tasks,
                                forks
                            )

                    return task_pipeline
                else:
                    if process_descriptions[task][2] in total_tasks:
                        task_pipeline.append(process_descriptions[task][2].split("|")[0])

                        # Proceeds building downstream until the output for a
                        # process is None
                        self.build_downstream(
                            process_descriptions,
                            process_descriptions[task][2].split("|")[0],
                            all_tasks,
                            task_pipeline,
                            count_forks,
                            total_tasks,
                            forks
                        )

                    return task_pipeline
            else:
                return task_pipeline


def define_pipeline_string(self, process_descriptions, tasks,
                               check_upstream,
                               check_downstream, count_forks, total_tasks,
                               forks):
        """Builds the possible forks and connections between the provided
        processes

        This method loops through all the provided tasks and builds the
        upstream and downstream pipeline if required. It then returns all
        possible forks than need to be merged à posteriori`

        Parameters
        ----------
        process_descriptions : dict
            Information of processes input, output and if is forkable
        tasks : str
            Space separated processes
        check_upstream : bool
            If is to build the upstream pipeline of the current task
        check_downstream : bool
            If is to build the downstream pipeline of the current task
        count_forks : int
            Number of current forks
        total_tasks : str
            All space separated processes
        forks : list
            Current forks

        Returns
        -------
        list : List with all the possible pipeline forks
        """

        tasks_array = tasks.split()

        for task_unsplit in tasks_array:
            task = task_unsplit.split("=")[0]

            if task not in process_descriptions.keys():
                logger.error(
                    colored_print(
                        "{} not in the possible processes".format(task),
                        "red_bold"
                    )
                )

                sys.exit()
            else:
                process_split = task_unsplit.split("=")

                if len(process_split) > 1:
                    self.process_to_id[process_split[0]] = process_split[1]

            # Only uses the process if it is not already in the possible forks
            if not bool([x for x in forks if task in x]) and not bool([y for y in forks if process_descriptions[task][2] in y]):
                task_pipeline = []

                if task in process_descriptions:

                    if check_upstream:
                        task_pipeline = self.build_upstream(
                            process_descriptions,
                            task,
                            tasks_array,
                            task_pipeline,
                            count_forks,
                            total_tasks,
                            forks
                        )

                    task_pipeline.append(task)

                    if check_downstream:
                        task_pipeline = self.build_downstream(
                            process_descriptions,
                            task,
                            tasks_array,
                            task_pipeline,
                            count_forks,
                            total_tasks,
                            forks
                        )

                # Adds the pipeline fragment to the list of possible forks
                forks.append(list(OrderedDict.fromkeys(task_pipeline)))

            # Checks for task in fork. Case order of input processes is reversed
            elif bool([y for y in forks if process_descriptions[task][2] in y]):
                for fork in forks:
                    if task not in fork:
                        try:
                            dependent_index = fork.index(process_descriptions[task][2])
                            fork.insert(dependent_index, task)
                        except ValueError:
                            continue

        for i in range(0, len(forks)):
            for j in range(0, len(forks[i])):
                try:
                    if len(forks[i][j].split("|")) > 1:
                        forks[i][j] = forks[i][j].split("|")
                        tmp_fork = []
                        for s in forks[i][j]:
                            if s in total_tasks:
                                tmp_fork.append(s)

                        forks[i][j] = tmp_fork

                except AttributeError as e:
                    continue

        return forks


def run_auto_pipeline(self, tasks):
        """Main method to run the automatic pipeline creation

        This method aggregates the functions required to build the pipeline
        string that can be used as input for the workflow generator.

        Parameters
        ----------
        tasks : str
            A string with the space separated tasks to be included in the
            pipeline

        Returns
        -------
        str : String with the pipeline definition used as input for
        parse_pipeline
        """

        self.forks = self.define_pipeline_string(
            self.process_descriptions,
            tasks,
            True,
            True,
            self.count_forks,
            tasks,
            self.forks
        )

        self.pipeline_string = self.build_pipeline_string(self.forks)

        return self.pipeline_string

def run_auto_pipeline ( self , tasks ) : self . forks = self . define_pipeline_string ( self . process_descriptions , tasks , True , True , self . count_forks , tasks , self . forks ) self . pipeline_string = self . build_pipeline_string ( self . forks ) return self . pipeline_string


def _get_component_str(component, params=None, directives=None):
        """ Generates a component string based on the provided parameters and
        directives

        Parameters
        ----------
        component : str
            Component name
        params : dict
            Dictionary with parameter information
        directives : dict
            Dictionary with directives information

        Returns
        -------
        str
            Component string with the parameters and directives, ready for
            parsing by flowcraft engine
        """

        final_directives = {}

        if directives:
            final_directives = directives

        if params:
            final_directives["params"] = params

        if final_directives:
            return "{}={}".format(
                component, json.dumps(final_directives, separators=(",", ":")))
        else:
            return component

def _get_component_str ( component , params = None , directives = None ) : final_directives = { } if directives : final_directives = directives if params : final_directives [ "params" ] = params if final_directives : return "{}={}" . format ( component , json . dumps ( final_directives , separators = ( "," , ":" ) ) ) else : return component


def write_report(storage_dic, output_file, sample_id):
    """ Writes a report from multiple samples.

    Parameters
    ----------
    storage_dic : dict or :py:class:`OrderedDict`
        Storage containing the trimming statistics. See :py:func:`parse_log`
        for its generation.
    output_file : str
        Path where the output file will be generated.
    sample_id : str
        Id or name of the current sample.
    """

    with open(output_file, "w") as fh, open(".report.json", "w") as json_rep:

        # Write header
        fh.write("Sample,Total length,Total trimmed,%,5end Trim,3end Trim,"
                 "bad_reads\\n")

        # Write contents
        for sample, vals in storage_dic.items():
            fh.write("{},{}\\n".format(
                sample, ",".join([str(x) for x in vals.values()])))

            json_dic = {
                "tableRow": [{
                    "sample": sample_id,
                    "data": [
                        {"header": "trimmed",
                         "value": vals["total_trim_perc"],
                         "table": "qc",
                         "columnBar": True},
                    ]
                }],
                "plotData": [{
                    "sample": sample_id,
                    "data": {
                        "sparkline": vals["clean_len"]
                    }
                }],
                "badReads": vals["bad_reads"]
            }
            json_rep.write(json.dumps(json_dic, separators=(",", ":")))


def main(log_files):
    """ Main executor of the trimmomatic_report template.

    Parameters
    ----------
    log_files : list
        List of paths to the trimmomatic log files.
    """

    log_storage = OrderedDict()

    for log in log_files:

        log_id = log.rstrip("_trimlog.txt")

        # Populate storage of current sample
        log_storage[log_id] = parse_log(log)

        # Remove temporary trim log file
        os.remove(log)

    write_report(log_storage, "trimmomatic_report.csv", log_id)

def
main
(
log_files
)
:
log_storage
=
OrderedDict
(
)
for
log
in
log_files
:
log_id
=
log
.
rstrip
(
"_trimlog.txt"
)
# Populate storage of current sample
log_storage
[
log_id
]
=
parse_log
(
log
)
# Remove temporary trim log file
os
.
remove
(
log
)
write_report
(
log_storage
,
"trimmomatic_report.csv"
,
log_id
)

Main executor of the trimmomatic_report template .


def fix_contig_names(asseembly_path):
    """Removes whitespace from the assembly contig names

    Parameters
    ----------
    asseembly_path : path to assembly file

    Returns
    -------
    str:
        Path to new assembly file with fixed contig names
    """

    fixed_assembly = "fixed_assembly.fa"

    with open(asseembly_path) as in_hf, open(fixed_assembly, "w") as ou_fh:

        for line in in_hf:

            if line.startswith(">"):
                fixed_line = line.replace(" ", "_")
                ou_fh.write(fixed_line)
            else:
                ou_fh.write(line)

    return fixed_assembly

def
fix_contig_names
(
asseembly_path
)
:
fixed_assembly
=
"fixed_assembly.fa"
with
open
(
asseembly_path
)
as
in_hf
,
open
(
fixed_assembly
,
"w"
)
as
ou_fh
:
for
line
in
in_hf
:
if
line
.
startswith
(
">"
)
:
fixed_line
=
line
.
replace
(
" "
,
"_"
)
ou_fh
.
write
(
fixed_line
)
else
:
ou_fh
.
write
(
line
)
return
fixed_assembly


def clean_up(fastq):
    """
    Cleans the temporary fastq files. If they are symlinks, the link
    source is removed

    Parameters
    ----------
    fastq : list
        List of fastq files.
    """

    for fq in fastq:
        # Get real path of fastq files, following symlinks
        rp = os.path.realpath(fq)
        logger.debug("Removing temporary fastq file path: {}".format(rp))
        if re.match(".*/work/.{2}/.{30}/.*", rp):
            os.remove(rp)

Cleans the temporary fastq files . If they are symlinks the link source is removed


def parse_files(self, fls):
        """Public method for parsing abricate output files.

        This method is called at at class instantiation for the provided
        output files. Additional abricate output files can be added using
        this method after the class instantiation.

        Parameters
        ----------
        fls : list
            List of paths to Abricate files

        """

        for f in fls:
            # Make sure paths exists
            if os.path.exists(f):
                self._parser(f)
            else:
                logger.warning("File {} does not exist".format(f))

def
parse_files
(
self
,
fls
)
:
for
f
in
fls
:
# Make sure paths exists
if
os
.
path
.
exists
(
f
)
:
self
.
_parser
(
f
)
else
:
logger
.
warning
(
"File {} does not exist"
.
format
(
f
)
)

Public method for parsing abricate output files .


def _parser(self, fl):
        """Parser for a single abricate output file.

        This parser will scan a single Abricate output file and populate
        the :py:attr:`Abricate.storage` attribute.

        Parameters
        ----------
        fl : str
            Path to abricate output file

        Notes
        -----
        This method will populate the :py:attr:`Abricate.storage` attribute
        with all compliant lines in the abricate output file. Entries are
        inserted using an arbitrary key that is set by the
        :py:attr:`Abricate._key` attribute.

        """

        with open(fl) as fh:

            for line in fh:
                # Skip header and comment lines
                if line.startswith("#") or line.strip() == "":
                    continue

                fields = line.strip().split("\t")

                try:
                    coverage = float(fields[8])
                except ValueError:
                    coverage = None
                try:
                    identity = float(fields[9])
                except ValueError:
                    identity = None

                try:
                    accession = fields[11]
                except IndexError:
                    accession = None

                self.storage[self._key] = {
                    "log_file": os.path.basename(fl),
                    "infile": fields[0],
                    "reference": fields[1],
                    "seq_range": (int(fields[2]), int(fields[3])),
                    "gene": fields[4],
                    "accession": accession,
                    "database": fields[10],
                    "coverage": coverage,
                    "identity": identity
                }

                self._key += 1

def
_parser
(
self
,
fl
)
:
with
open
(
fl
)
as
fh
:
for
line
in
fh
:
# Skip header and comment lines
if
line
.
startswith
(
"#"
)
or
line
.
strip
(
)
==
""
:
continue
fields
=
line
.
strip
(
)
.
split
(
"\t"
)
try
:
coverage
=
float
(
fields
[
8
]
)
except
ValueError
:
coverage
=
try
:
identity
=
float
(
fields
[
9
]
)
except
ValueError
:
identity
=
try
:
accession
=
fields
[
11
]
except
IndexError
:
accession
=
self
.
storage
[
self
.
_key
]
=
{
"log_file"
:
os
.
path
.
basename
(
fl
)
,
"infile"
:
fields
[
0
]
,
"reference"
:
fields
[
1
]
,
"seq_range"
:
(
int
(
fields
[
2
]
)
,
int
(
fields
[
3
]
)
)
,
"gene"
:
fields
[
4
]
,
"accession"
:
accession
,
"database"
:
fields
[
10
]
,
"coverage"
:
coverage
,
"identity"
:
identity
}
self
.
_key
+=
1

Parser for a single abricate output file .


def iter_filter(self, filters, databases=None, fields=None,
                    filter_behavior="and"):
        """General purpose filter iterator.

        This general filter iterator allows the filtering of entries based
        on one or more custom filters. These filters must contain
        an entry of the `storage` attribute, a comparison operator, and the
        test value. For example, to filter out entries with coverage below 80::

            my_filter = ["coverage", ">=", 80]

        Filters should always be provide as a list of lists::

            iter_filter([["coverage", ">=", 80]])
            # or
            my_filters = [["coverage", ">=", 80],
                          ["identity", ">=", 50]]

            iter_filter(my_filters)

        As a convenience, a list of the desired databases can be directly
        specified using the `database` argument, which will only report
        entries for the specified databases::

            iter_filter(my_filters, databases=["plasmidfinder"])

        By default, this method will yield the complete entry record. However,
        the returned filters can be specified using the `fields` option::

            iter_filter(my_filters, fields=["reference", "coverage"])

        Parameters
        ----------
        filters : list
            List of lists with the custom filter. Each list should have three
            elements. (1) the key from the entry to be compared; (2) the
            comparison operator; (3) the test value. Example:
                ``[["identity", ">", 80]]``.
        databases : list
            List of databases that should be reported.
        fields : list
            List of fields from each individual entry that are yielded.
        filter_behavior : str
            options: ``'and'`` ``'or'``
            Sets the behaviour of the filters, if multiple filters have been
            provided. By default it is set to ``'and'``, which means that an
            entry has to pass all filters. It can be set to ``'or'``, in which
            case one one of the filters has to pass.

        yields
        ------
        dic : dict
            Dictionary object containing a :py:attr:`Abricate.storage` entry
            that passed the filters.

        """

        if filter_behavior not in ["and", "or"]:
            raise ValueError("Filter behavior must be either 'and' or 'or'")

        for dic in self.storage.values():

            # This attribute will determine whether an entry will be yielded
            # or not
            _pass = False

            # Stores the flags with the test results for each filter
            # The results will be either True or False
            flag = []

            # Filter for databases
            if databases:
                # Skip entry if not in specified database
                if dic["database"] not in databases:
                    continue

            # Apply filters
            for f in filters:
                # Get value of current filter
                val = dic[f[0]]
                if not self._test_truth(val, f[1], f[2]):
                    flag.append(False)
                else:
                    flag.append(True)

            # Test whether the entry will pass based on the test results
            # and the filter behaviour
            if filter_behavior == "and":
                if all(flag):
                    _pass = True
            elif filter_behavior == "or":
                if any(flag):
                    _pass = True

            if _pass:
                if fields:
                    yield dict((x, y) for x, y in dic.items() if x in fields)
                else:
                    yield dic


def _get_contig_id(contig_str):
        """Tries to retrieve contig id. Returns the original string if it
        is unable to retrieve the id.

        Parameters
        ----------
        contig_str : str
            Full contig string (fasta header)

        Returns
        -------
        str
            Contig id
        """

        contig_id = contig_str

        try:
            contig_id = re.search(".*NODE_([0-9]*)_.*", contig_str).group(1)
        except AttributeError:
            pass

        try:
            contig_id = re.search(".*Contig_([0-9]*)_.*", contig_str).group(1)
        except AttributeError:
            pass

        return contig_id

def _get_contig_id ( contig_str ) : contig_id = contig_str try : contig_id = re . search ( ".*NODE_([0-9]*)_.*" , contig_str ) . group ( 1 ) except AttributeError : pass try : contig_id = re . search ( ".*Contig_([0-9]*)_.*" , contig_str ) . group ( 1 ) except AttributeError : pass return contig_id

Tries to retrieve contig id . Returns the original string if it is unable to retrieve the id .


def get_plot_data(self):
        """ Generates the JSON report to plot the gene boxes

        Following the convention of the reports platform, this method returns
        a list of JSON/dict objects with the information about each entry in
        the abricate file. The information contained in this JSON is::

            {contig_id: <str>,
             seqRange: [<int>, <int>],
             gene: <str>,
             accession: <str>,
             coverage: <float>,
             identity: <float>
             }

        Note that the `seqRange` entry contains the position in the
        corresponding contig, not the absolute position in the whole assembly.

        Returns
        -------
        json_dic : list
            List of JSON/dict objects with the report data.
        """

        json_dic = {"plotData": []}
        sample_dic = {}
        sample_assembly_map = {}

        for entry in self.storage.values():

            sample_id = re.match("(.*)_abr", entry["log_file"]).groups()[0]
            if sample_id not in sample_dic:
                sample_dic[sample_id] = {}

            # Get contig ID using the same regex as in `assembly_report.py`
            # template
            contig_id = self._get_contig_id(entry["reference"])
            # Get database
            database = entry["database"]
            if database not in sample_dic[sample_id]:
                sample_dic[sample_id][database] = []

            # Update the sample-assembly correspondence dict
            if sample_id not in sample_assembly_map:
                sample_assembly_map[sample_id] = entry["infile"]

            sample_dic[sample_id][database].append(
                {"contig": contig_id,
                 "seqRange": entry["seq_range"],
                 "gene": entry["gene"].replace("'", ""),
                 "accession": entry["accession"],
                 "coverage": entry["coverage"],
                 "identity": entry["identity"],
                 },
            )

        for sample, data in sample_dic.items():
            json_dic["plotData"].append(
                {
                    "sample": sample,
                    "data": {"abricateXrange": data},
                    "assemblyFile": sample_assembly_map[sample]
                }
            )

        return json_dic


def write_report_data(self):
        """Writes the JSON report to a json file
        """

        json_plot = self.get_plot_data()
        json_table = self.get_table_data()

        json_dic = {**json_plot, **json_table}

        with open(".report.json", "w") as json_report:
            json_report.write(json.dumps(json_dic, separators=(",", ":")))

def write_report_data ( self ) : json_plot = self . get_plot_data ( ) json_table = self . get_table_data ( ) json_dic = { * * json_plot , * * json_table } with open ( ".report.json" , "w" ) as json_report : json_report . write ( json . dumps ( json_dic , separators = ( "," , ":" ) ) )


def main(sample_id, assembly_file, coverage_bp_file=None):
    """Main executor of the assembly_report template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    assembly_file : str
        Path to assembly file in Fasta format.

    """

    logger.info("Starting assembly report")
    assembly_obj = Assembly(assembly_file, sample_id)

    logger.info("Retrieving summary statistics for assembly")
    assembly_obj.get_summary_stats("{}_assembly_report.csv".format(sample_id))

    size_dist = [len(x) for x in assembly_obj.contigs.values()]
    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [
                {"header": "Contigs",
                 "value": assembly_obj.summary_info["ncontigs"],
                 "table": "assembly",
                 "columnBar": True},
                {"header": "Assembled BP",
                 "value": assembly_obj.summary_info["total_len"],
                 "table": "assembly",
                 "columnBar": True},
            ]
        }],
        "plotData": [{
            "sample": sample_id,
            "data": {
                "size_dist": size_dist
            }
        }]
    }

    if coverage_bp_file:
        try:
            window = 2000
            gc_sliding_data = assembly_obj.get_gc_sliding(window=window)
            cov_sliding_data = \
                assembly_obj.get_coverage_sliding(coverage_bp_file,
                                                  window=window)

            # Get total basepairs based on the individual coverage of each
            # contig bpx
            total_bp = sum(
                [sum(x) for x in assembly_obj.contig_coverage.values()]
            )

            # Add data to json report
            json_dic["plotData"][0]["data"]["genomeSliding"] = {
                "gcData": gc_sliding_data,
                "covData": cov_sliding_data,
                "window": window,
                "xbars": assembly_obj._get_window_labels(window),
                "assemblyFile": os.path.basename(assembly_file)
            }
            json_dic["plotData"][0]["data"]["sparkline"] = total_bp

        except:
            logger.error("Unexpected error creating sliding window data:\\n"
                         "{}".format(traceback.format_exc()))

    # Write json report
    with open(".report.json", "w") as json_report:

        json_report.write(json.dumps(json_dic, separators=(",", ":")))

    with open(".status", "w") as status_fh:
        status_fh.write("pass")

Main executor of the assembly_report template .


def _parse_assembly(self, assembly_file):
        """Parse an assembly file in fasta format.

        This is a Fasta parsing method that populates the
        :py:attr:`Assembly.contigs` attribute with data for each contig in the
         assembly.

        Parameters
        ----------
        assembly_file : str
            Path to the assembly fasta file.

        """

        with open(assembly_file) as fh:

            header = None
            logger.debug("Starting iteration of assembly file: {}".format(
                assembly_file))

            for line in fh:

                # Skip empty lines
                if not line.strip():
                    continue

                if line.startswith(">"):
                    # Add contig header to contig dictionary
                    header = line[1:].strip()
                    self.contigs[header] = []

                else:
                    # Add sequence string for the current contig
                    self.contigs[header].append(line.strip())

            # After populating the contigs dictionary, convert the values
            # list into a string sequence
            self.contigs = OrderedDict(
                (header, "".join(seq)) for header, seq in self.contigs.items())

Parse an assembly file in fasta format .


def get_summary_stats(self, output_csv=None):
        """Generates a CSV report with summary statistics about the assembly

        The calculated statistics are:

            - Number of contigs
            - Average contig size
            - N50
            - Total assembly length
            - Average GC content
            - Amount of missing data

        Parameters
        ----------
        output_csv: str
            Name of the output CSV file.
        """

        contig_size_list = []

        self.summary_info["ncontigs"] = len(self.contigs)

        for contig_id, sequence in self.contigs.items():

            logger.debug("Processing contig: {}".format(contig_id))

            # Get contig sequence size
            contig_len = len(sequence)

            # Add size for average contig size
            contig_size_list.append(contig_len)

            # Add to total assembly length
            self.summary_info["total_len"] += contig_len

            # Add to average gc
            self.summary_info["avg_gc"].append(
                sum(map(sequence.count, ["G", "C"])) / contig_len
            )

            # Add to missing data
            self.summary_info["missing_data"] += sequence.count("N")

        # Get average contig size
        logger.debug("Getting average contig size")
        self.summary_info["avg_contig_size"] = \
            sum(contig_size_list) / len(contig_size_list)

        # Get average gc content
        logger.debug("Getting average GC content")
        self.summary_info["avg_gc"] = \
            sum(self.summary_info["avg_gc"]) / len(self.summary_info["avg_gc"])

        # Get N50
        logger.debug("Getting N50")
        cum_size = 0
        for l in sorted(contig_size_list, reverse=True):
            cum_size += l
            if cum_size >= self.summary_info["total_len"] / 2:
                self.summary_info["n50"] = l
                break

        if output_csv:
            logger.debug("Writing report to csv")
            # Write summary info to CSV
            with open(output_csv, "w") as fh:
                summary_line = "{}, {}\\n".format(
                    self.sample, ",".join(
                        [str(x) for x in self.summary_info.values()]))
                fh.write(summary_line)

def
get_summary_stats
(
self
,
output_csv
=
)
:
contig_size_list
=
[
]
self
.
summary_info
[
"ncontigs"
]
=
len
(
self
.
contigs
)
for
contig_id
,
sequence
in
self
.
contigs
.
items
(
)
:
logger
.
debug
(
"Processing contig: {}"
.
format
(
contig_id
)
)
# Get contig sequence size
contig_len
=
len
(
sequence
)
# Add size for average contig size
contig_size_list
.
append
(
contig_len
)
# Add to total assembly length
self
.
summary_info
[
"total_len"
]
+=
contig_len
# Add to average gc
self
.
summary_info
[
"avg_gc"
]
.
append
(
sum
(
map
(
sequence
.
count
,
[
"G"
,
"C"
]
)
)
/
contig_len
)
# Add to missing data
self
.
summary_info
[
"missing_data"
]
+=
sequence
.
count
(
"N"
)
# Get average contig size
logger
.
debug
(
"Getting average contig size"
)
self
.
summary_info
[
"avg_contig_size"
]
=
sum
(
contig_size_list
)
/
len
(
contig_size_list
)
# Get average gc content
logger
.
debug
(
"Getting average GC content"
)
self
.
summary_info
[
"avg_gc"
]
=
sum
(
self
.
summary_info
[
"avg_gc"
]
)
/
len
(
self
.
summary_info
[
"avg_gc"
]
)
# Get N50
logger
.
debug
(
"Getting N50"
)
cum_size
=
0
for
l
in
sorted
(
contig_size_list
,
reverse
=
True
)
:
cum_size
+=
l
if
cum_size
>=
self
.
summary_info
[
"total_len"
]
/
2
:
self
.
summary_info
[
"n50"
]
=
l
break
if
output_csv
:
logger
.
debug
(
"Writing report to csv"
)
# Write summary info to CSV
with
open
(
output_csv
,
"w"
)
as
fh
:
summary_line
=
"{}, {}\\n"
.
format
(
self
.
sample
,
","
.
join
(
[
str
(
x
)
for
x
in
self
.
summary_info
.
values
(
)
]
)
)
fh
.
write
(
summary_line
)


def _get_window_labels(self, window):
        """Returns the mapping between sliding window points and their contigs,
        and the x-axis position of contig

        Parameters
        ----------
        window : int
            Size of the window.

        Returns
        -------
        xbars : list
            The x-axis position of the ending for each contig.
        labels : list
            The x-axis labels for each data point in the sliding window

        """

        # Get summary stats, if they have not yet been triggered
        if not self.summary_info:
            self.get_summary_stats()

        # Get contig boundary positon
        c = 0
        xbars = []
        for contig, seq in self.contigs.items():
            contig_id = self._get_contig_id(contig)
            self.contig_boundaries[contig_id] = [c, c + len(seq)]
            c += len(seq)
            xbars.append((contig_id, c, contig))

        return xbars

Returns the mapping between sliding window points and their contigs and the x - axis position of contig


def _gc_prop(s, length):
        """Get proportion of GC from a string

        Parameters
        ----------
        s : str
            Arbitrary string

        Returns
        -------
        x : float
            GC proportion.
        """

        gc = sum(map(s.count, ["c", "g"]))

        return gc / length

def _gc_prop ( s , length ) : gc = sum ( map ( s . count , [ "c" , "g" ] ) ) return gc / length


def get_gc_sliding(self, window=2000):
        """Calculates a sliding window of the GC content for the assembly

        Returns
        -------
        gc_res : list
            List of GC proportion floats for each data point in the sliding
            window
        """

        gc_res = []

        # Get complete sequence to calculate sliding window values
        complete_seq = "".join(self.contigs.values()).lower()

        for i in range(0, len(complete_seq), window):

            seq_window = complete_seq[i:i + window]

            # Get GC proportion
            gc_res.append(round(self._gc_prop(seq_window, len(seq_window)), 2))

        return gc_res


def main(sample_id, fastq_pair, clear):
    """Main executor of the skesa template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    clear : str
        Can be either 'true' or 'false'. If 'true', the input fastq files will
        be removed at the end of the run, IF they are in the working directory
    """

    logger.info("Starting skesa")

    # Determine output file
    if "_trim." in fastq_pair[0]:
        sample_id += "_trim"
    version = __get_version_skesa()["version"]
    output_file = "{}_skesa{}.fasta".format(sample_id, version.replace(".", ""))

    cli = [
        "skesa",
        "--fastq",
        "{},{}".format(fastq_pair[0], fastq_pair[1]),
        "--gz",
        "--use_paired_ends",
        "--cores",
        "${task.cpus}"
    ]

    logger.debug("Running Skesa subprocess with command: {}".format(cli))

    with open(output_file, "w") as fh:
        p = subprocess.Popen(cli, stdout=fh, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
        stdout = stdout.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)
        stdout = str(stdout)

    logger.info("Finished Skesa subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Fished Skesa subprocess with STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished Skesa with return code: {}".format(
        p.returncode))

    # Remove input fastq files when clear option is specified.
    # Only remove temporary input when the expected output exists.
    if clear == "true" and os.path.exists(output_file):
        clean_up(fastq_pair)

    with open(".status", "w") as fh:
        if p.returncode != 0:
            fh.write("error")
            raise SystemExit(p.returncode)
        else:
            fh.write("pass")


def write_json_report(sample_id, data1, data2):
    """Writes the report

    Parameters
    ----------
    data1
    data2

    Returns
    -------

    """

    parser_map = {
        "base_sequence_quality": ">>Per base sequence quality",
        "sequence_quality": ">>Per sequence quality scores",
        "base_gc_content": ">>Per sequence GC content",
        "base_n_content": ">>Per base N content",
        "sequence_length_dist": ">>Sequence Length Distribution",
        "per_base_sequence_content": ">>Per base sequence content"
    }

    json_dic = {
        "plotData": [{
            "sample": sample_id,
            "data": {
                "base_sequence_quality": {"status": None, "data": []},
                "sequence_quality": {"status": None, "data": []},
                "base_gc_content": {"status": None, "data": []},
                "base_n_content": {"status": None, "data": []},
                "sequence_length_dist": {"status": None, "data": []},
                "per_base_sequence_content": {"status": None, "data": []}
            }
        }]
    }

    for cat, start_str in parser_map.items():

        if cat == "per_base_sequence_content":
            fs = 1
            fe = 5
        else:
            fs = 1
            fe = 2

        report1, status1 = _get_quality_stats(data1, start_str,
                                              field_start=fs, field_end=fe)
        report2, status2 = _get_quality_stats(data2, start_str,
                                              field_start=fs, field_end=fe)

        status = None
        for i in ["fail", "warn", "pass"]:
            if i in [status1, status2]:
                status = i

        json_dic["plotData"][0]["data"][cat]["data"] = [report1, report2]
        json_dic["plotData"][0]["data"][cat]["status"] = status

    return json_dic

def
write_json_report
(
sample_id
,
data1
,
data2
)
:
parser_map
=
{
"base_sequence_quality"
:
">>Per base sequence quality"
,
"sequence_quality"
:
">>Per sequence quality scores"
,
"base_gc_content"
:
">>Per sequence GC content"
,
"base_n_content"
:
">>Per base N content"
,
"sequence_length_dist"
:
">>Sequence Length Distribution"
,
"per_base_sequence_content"
:
">>Per base sequence content"
}
json_dic
=
{
"plotData"
:
[
{
"sample"
:
sample_id
,
"data"
:
{
"base_sequence_quality"
:
{
"status"
:
,
"data"
:
[
]
}
,
"sequence_quality"
:
{
"status"
:
,
"data"
:
[
]
}
,
"base_gc_content"
:
{
"status"
:
,
"data"
:
[
]
}
,
"base_n_content"
:
{
"status"
:
,
"data"
:
[
]
}
,
"sequence_length_dist"
:
{
"status"
:
,
"data"
:
[
]
}
,
"per_base_sequence_content"
:
{
"status"
:
,
"data"
:
[
]
}
}
}
]
}
for
cat
,
start_str
in
parser_map
.
items
(
)
:
if
cat
==
"per_base_sequence_content"
:
fs
=
1
fe
=
5
else
:
fs
=
1
fe
=
2
report1
,
status1
=
_get_quality_stats
(
data1
,
start_str
,
field_start
=
fs
,
field_end
=
fe
)
report2
,
status2
=
_get_quality_stats
(
data2
,
start_str
,
field_start
=
fs
,
field_end
=
fe
)
status
=
for
i
in
[
"fail"
,
"warn"
,
"pass"
]
:
if
i
in
[
status1
,
status2
]
:
status
=
i
json_dic
[
"plotData"
]
[
0
]
[
"data"
]
[
cat
]
[
"data"
]
=
[
report1
,
report2
]
json_dic
[
"plotData"
]
[
0
]
[
"data"
]
[
cat
]
[
"status"
]
=
status
return
json_dic


def get_trim_index(biased_list):
    """Returns the trim index from a ``bool`` list

    Provided with a list of ``bool`` elements (``[False, False, True, True]``),
    this function will assess the index of the list that minimizes the number
    of True elements (biased positions) at the extremities. To do so,
    it will iterate over the boolean list and find an index position where
    there are two consecutive ``False`` elements after a ``True`` element. This
    will be considered as an optimal trim position. For example, in the
    following list::

        [True, True, False, True, True, False, False, False, False, ...]

    The optimal trim index will be the 4th position, since it is the first
    occurrence of a ``True`` element with two False elements after it.

    If the provided ``bool`` list has no ``True`` elements, then the 0 index is
    returned.

    Parameters
    ----------
    biased_list: list
        List of ``bool`` elements, where ``True`` means a biased site.

    Returns
    -------
        x : index position of the biased list for the optimal trim.

    """

    # Return index 0 if there are no biased positions
    if set(biased_list) == {False}:
        return 0

    if set(biased_list[:5]) == {False}:
        return 0

    # Iterate over the biased_list array. Keep the iteration going until
    # we find a biased position with the two following positions unbiased
    # (e.g.: True, False, False).
    # When this condition is verified, return the last biased position
    # index for subsequent trimming.
    for i, val in enumerate(biased_list):
        if val and set(biased_list[i+1:i+3]) == {False}:
            return i + 1

    # If the previous iteration could not find and index to trim, it means
    # that the whole list is basically biased. Return the length of the
    # biased_list
    return len(biased_list)


def trim_range(data_file):
    """Assess the optimal trim range for a given FastQC data file.

    This function will parse a single FastQC data file, namely the
    *'Per base sequence content'* category. It will retrieve the A/T and G/C
    content for each nucleotide position in the reads, and check whether the
    G/C and A/T proportions are between 80% and 120%. If they are, that
    nucleotide position is marked as biased for future removal.

    Parameters
    ----------
    data_file: str
        Path to FastQC data file.

    Returns
    -------
    trim_nt: list
        List containing the range with the best trimming positions for the
        corresponding FastQ file. The first element is the 5' end trim index
        and the second element is the 3' end trim index.
    """

    logger.debug("Starting trim range assessment")

    # Target string for nucleotide bias assessment
    target_nuc_bias = ">>Per base sequence content"
    logger.debug("Target string to start nucleotide bias assessment set to "
                 "{}".format(target_nuc_bias))
    # This flag will become True when gathering base proportion data
    # from file.
    gather = False

    # This variable will store a boolean array on the biased/unbiased
    # positions. Biased position will be True, while unbiased positions
    # will be False
    biased = []

    with open(data_file) as fh:

        for line in fh:
            # Start assessment of nucleotide bias
            if line.startswith(target_nuc_bias):
                # Skip comment line
                logger.debug("Found target string at line: {}".format(line))
                next(fh)
                gather = True
            # Stop assessment when reaching end of target module
            elif line.startswith(">>END_MODULE") and gather:
                logger.debug("Stopping parsing at line: {}".format(line))
                break
            elif gather:
                # Get proportions of each nucleotide
                g, a, t, c = [float(x) for x in line.strip().split()[1:]]
                # Get 'GC' and 'AT content
                gc = (g + 0.1) / (c + 0.1)
                at = (a + 0.1) / (t + 0.1)
                # Assess bias
                if 0.8 <= gc <= 1.2 and 0.8 <= at <= 1.2:
                    biased.append(False)
                else:
                    biased.append(True)

    logger.debug("Finished bias assessment with result: {}".format(biased))

    # Split biased list in half to get the 5' and 3' ends
    biased_5end, biased_3end = biased[:int(len(biased)/2)],\
        biased[int(len(biased)/2):][::-1]

    logger.debug("Getting optimal trim range from biased list")
    trim_nt = [0, 0]
    # Assess number of nucleotides to clip at 5' end
    trim_nt[0] = get_trim_index(biased_5end)
    logger.debug("Optimal trim range at 5' end set to: {}".format(trim_nt[0]))
    # Assess number of nucleotides to clip at 3' end
    trim_nt[1] = len(biased) - get_trim_index(biased_3end)
    logger.debug("Optimal trim range at 3' end set to: {}".format(trim_nt[1]))

    return trim_nt

Assess the optimal trim range for a given FastQC data file .


def get_sample_trim(p1_data, p2_data):
    """Get the optimal read trim range from data files of paired FastQ reads.

    Given the FastQC data report files for paired-end FastQ reads, this
    function will assess the optimal trim range for the 3' and 5' ends of
    the paired-end reads. This assessment will be based on the *'Per sequence
    GC content'*.

    Parameters
    ----------
    p1_data: str
        Path to FastQC data report file from pair 1
    p2_data: str
        Path to FastQC data report file from pair 2

    Returns
    -------
    optimal_5trim: int
        Optimal trim index for the 5' end of the reads
    optima_3trim: int
        Optimal trim index for the 3' end of the reads

    See Also
    --------
    trim_range

    """

    sample_ranges = [trim_range(x) for x in [p1_data, p2_data]]

    # Get the optimal trim position for 5' end
    optimal_5trim = max([x[0] for x in sample_ranges])
    # Get optimal trim position for 3' end
    optimal_3trim = min([x[1] for x in sample_ranges])

    return optimal_5trim, optimal_3trim

Get the optimal read trim range from data files of paired FastQ reads .


def get_summary(summary_file):
    """Parses a FastQC summary report file and returns it as a dictionary.

    This function parses a typical FastQC summary report file, retrieving
    only the information on the first two columns. For instance, a line could
    be::

        'PASS	Basic Statistics	SH10762A_1.fastq.gz'

    This parser will build a dictionary with the string in the second column
    as a key and the QC result as the value. In this case, the returned
    ``dict`` would be something like::

        {"Basic Statistics": "PASS"}

    Parameters
    ----------
    summary_file: str
        Path to FastQC summary report.

    Returns
    -------
    summary_info: :py:data:`OrderedDict`
        Returns the information of the FastQC summary report as an ordered
        dictionary, with the categories as strings and the QC result as values.

    """

    summary_info = OrderedDict()
    logger.debug("Retrieving summary information from file: {}".format(
        summary_file))

    with open(summary_file) as fh:
        for line in fh:
            # Skip empty lines
            if not line.strip():
                continue
            # Populate summary info
            fields = [x.strip() for x in line.split("\t")]
            summary_info[fields[1]] = fields[0]

    logger.debug("Retrieved summary information from file: {}".format(
        summary_info))

    return summary_info

Parses a FastQC summary report file and returns it as a dictionary .


def check_summary_health(summary_file, **kwargs):
    """Checks the health of a sample from the FastQC summary file.

    Parses the FastQC summary file and tests whether the sample is good
    or not. There are four categories that cannot fail, and two that
    must pass in order for the sample pass this check. If the sample fails
    the quality checks, a list with the failing categories is also returned.

    Categories that cannot fail::

        fail_sensitive = [
            "Per base sequence quality",
            "Overrepresented sequences",
            "Sequence Length Distribution",
            "Per sequence GC content"
        ]

    Categories that must pass::

        must_pass = [
            "Per base N content",
            "Adapter Content"
        ]

    Parameters
    ----------
    summary_file: str
        Path to FastQC summary file.

    Returns
    -------
    x : bool
        Returns ``True`` if the sample passes all tests. ``False`` if not.
    summary_info : list
        A list with the FastQC categories that failed the tests. Is empty
        if the sample passes all tests.
    """

    # Store the summary categories that cannot fail. If they fail, do not
    # proceed with this sample
    fail_sensitive = kwargs.get("fail_sensitive", [
        "Per base sequence quality",
        "Overrepresented sequences",
        "Sequence Length Distribution",
        "Per sequence GC content"
    ])
    logger.debug("Fail sensitive categories: {}".format(fail_sensitive))

    # Store summary categories that must pass. If they do not, do not proceed
    # with that sample
    must_pass = kwargs.get("must_pass", [
        "Per base N content",
        "Adapter Content"
    ])
    logger.debug("Must pass categories: {}".format(must_pass))

    warning_fail_sensitive = kwargs.get("warning_fail_sensitive", [
        "Per base sequence quality",
        "Overrepresented sequences",

    ])

    warning_must_pass = kwargs.get("warning_must_pass", [
        "Per base sequence content"
    ])

    # Get summary dictionary
    summary_info = get_summary(summary_file)

    # This flag will change to False if one of the tests fails
    health = True
    # List of failing categories
    failed = []
    # List of warning categories
    warning = []

    for cat, test in summary_info.items():

        logger.debug("Assessing category {} with result {}".format(cat, test))

        # FAILURES
        # Check for fail sensitive
        if cat in fail_sensitive and test == "FAIL":
            health = False
            failed.append("{}:{}".format(cat, test))
            logger.error("Category {} failed a fail sensitive "
                         "category".format(cat))

        # Check for must pass
        if cat in must_pass and test != "PASS":
            health = False
            failed.append("{}:{}".format(cat, test))
            logger.error("Category {} failed a must pass category".format(
                cat))

        # WARNINGS
        # Check for fail sensitive
        if cat in warning_fail_sensitive and test == "FAIL":
            warning.append("Failed category: {}".format(cat))
            logger.warning("Category {} flagged at a fail sensitive "
                           "category".format(cat))

        if cat in warning_must_pass and test != "PASS":
            warning.append("Did not pass category: {}".format(cat))
            logger.warning("Category {} flagged at a must pass "
                           "category".format(cat))

    # Passed all tests
    return health, failed, warning

Checks the health of a sample from the FastQC summary file .


def parse_log(self, bowtie_log):
        """Parse a bowtie log file.

        This is a bowtie log parsing method that populates the
        :py:attr:`self.n_reads, self.align_0x, self.align_1x, self.align_mt1x and self.overall_rate` attributes with
        data from the log file.

        Disclamer: THIS METHOD IS HORRIBLE BECAUSE THE BOWTIE LOG IS HORRIBLE.

        The insertion of data on the attribytes is done by the
        :py:meth:`set_attribute method.

        Parameters
        ----------
        bowtie_log : str
            Path to the boetie log file.

       """

        print("is here!")

        # Regexes - thanks to https://github.com/ewels/MultiQC/blob/master/multiqc/modules/bowtie2/bowtie2.py
        regexes = {
            'unpaired': {
                'unpaired_aligned_none': r"(\\d+) \\([\\d\\.]+%\\) aligned 0 times",
                'unpaired_aligned_one': r"(\\d+) \\([\\d\\.]+%\\) aligned exactly 1 time",
                'unpaired_aligned_multi': r"(\\d+) \\([\\d\\.]+%\\) aligned >1 times"
            },
            'paired': {
                'paired_aligned_none': r"(\\d+) \\([\\d\\.]+%\\) aligned concordantly 0 times",
                'paired_aligned_one': r"(\\d+) \\([\\d\\.]+%\\) aligned concordantly exactly 1 time",
                'paired_aligned_multi': r"(\\d+) \\([\\d\\.]+%\\) aligned concordantly >1 times",
                'paired_aligned_discord_one': r"(\\d+) \\([\\d\\.]+%\\) aligned discordantly 1 time",
                'paired_aligned_discord_multi': r"(\\d+) \\([\\d\\.]+%\\) aligned discordantly >1 times",
                'paired_aligned_mate_one': r"(\\d+) \\([\\d\\.]+%\\) aligned exactly 1 time",
                'paired_aligned_mate_multi': r"(\\d+) \\([\\d\\.]+%\\) aligned >1 times",
                'paired_aligned_mate_none': r"(\\d+) \\([\\d\\.]+%\\) aligned 0 times"
            }
        }

        #Missing parser for unpaired (not implemented in flowcraft yet)

        with open(bowtie_log, "r") as f:
            #Go through log file line by line
            for l in f:

                print(l)

                #total reads
                total = re.search(r"(\\d+) reads; of these:", l)
                print(total)
                if total:
                    print(total)
                    self.set_n_reads(total.group(1))

                # Paired end reads aka the pain
                paired = re.search(r"(\\d+) \\([\\d\\.]+%\\) were paired; of these:", l)
                if paired:
                    paired_total = int(paired.group(1))

                    paired_numbers = {}

                    # Do nested loop whilst we have this level of indentation
                    l = f.readline()
                    while l.startswith('    '):
                        for k, r in regexes['paired'].items():
                            match = re.search(r, l)
                            if match:
                                paired_numbers[k] = int(match.group(1))
                        l = f.readline()

                    align_zero_times = paired_numbers['paired_aligned_none'] + paired_numbers['paired_aligned_mate_none']
                    if align_zero_times:
                        self.set_align_0x(align_zero_times)

                    align_one_time = paired_numbers['paired_aligned_one'] + paired_numbers['paired_aligned_mate_one']
                    if align_one_time:
                        self.set_align_1x(align_one_time)

                    align_more_than_one_time = paired_numbers['paired_aligned_multi'] + paired_numbers['paired_aligned_mate_multi']
                    if align_more_than_one_time:
                        self.set_align_mt1x(align_more_than_one_time)

                # Overall alignment rate
                overall = re.search(r"([\\d\\.]+)% overall alignment rate", l)
                if overall:
                    self.overall_rate = float(overall.group(1))


def _parse_process_name(name_str):
        """Parses the process string and returns the process name and its
        directives

        Process strings my contain directive information with the following
        syntax::

            proc_name={'directive':'val'}

        This method parses this string and returns the process name as a
        string and the directives information as a dictionary.

        Parameters
        ----------
        name_str : str
            Raw string with process name and, potentially, directive
            information

        Returns
        -------
        str
            Process name
        dict or None
            Process directives
        """

        directives = None

        fields = name_str.split("=")
        process_name = fields[0]

        if len(fields) == 2:
            _directives = fields[1].replace("'", '"')
            try:
                directives = json.loads(_directives)
            except json.decoder.JSONDecodeError:
                raise eh.ProcessError(
                    "Could not parse directives for process '{}'. The raw"
                    " string is: {}\n"
                    "Possible causes include:\n"
                    "\t1. Spaces inside directives\n"
                    "\t2. Missing '=' symbol before directives\n"
                    "\t3. Missing quotes (' or \") around directives\n"
                    "A valid example: process_name={{'cpus':'2'}}".format(
                        process_name, name_str))

        return process_name, directives


def _add_dependency(self, p, template, inlane, outlane, pid):
        """Automatically Adds a dependency of a process.

        This method adds a template to the process list attribute as a
        dependency. It will adapt the input lane, output lane and process
        id of the process that depends on it.

        Parameters
        ----------
        p : Process
            Process class that contains the dependency.
        template : str
            Template name of the dependency.
        inlane : int
            Input lane.
        outlane : int
            Output lane.
        pid : int
            Process ID.
        """

        dependency_proc = self.process_map[template](template=template)

        if dependency_proc.input_type != p.input_type:
            logger.error("Cannot automatically add dependency with different"
                         " input type. Input type of process '{}' is '{}."
                         " Input type of dependency '{}' is '{}'".format(
                            p.template, p.input_type, template,
                            dependency_proc.input_type))

        input_suf = "{}_{}_dep".format(inlane, pid)
        output_suf = "{}_{}_dep".format(outlane, pid)
        dependency_proc.set_main_channel_names(input_suf, output_suf, outlane)

        # To insert the dependency process before the current process, we'll
        # need to move the input channel name of the later to the former, and
        # set a new connection between the dependency and the process.
        dependency_proc.input_channel = p.input_channel
        p.input_channel = dependency_proc.output_channel

        # If the current process was the first in the pipeline, change the
        # lanes so that the dependency becomes the first process
        if not p.parent_lane:
            p.parent_lane = outlane
            dependency_proc.parent_lane = None
        else:
            dependency_proc.parent_lane = inlane
            p.parent_lane = outlane

        self.processes.append(dependency_proc)

Automatically Adds a dependency of a process .


def _search_tree_backwards(self, template, parent_lanes):
        """Searches the process tree backwards in search of a provided process

        The search takes into consideration the provided parent lanes and
        searches only those

        Parameters
        ----------
        template : str
            Name of the process template attribute being searched
        parent_lanes : list
            List of integers with the parent lanes to be searched

        Returns
        -------
        bool
            Returns True when the template is found. Otherwise returns False.
        """

        for p in self.processes[::-1]:

            # Ignore process in different lanes
            if p.lane not in parent_lanes:
                continue

            # template found
            if p.template == template:
                return True

        return False

def
_search_tree_backwards
(
self
,
template
,
parent_lanes
)
:
for
p
in
self
.
processes
[
:
:
1
]
:
# Ignore process in different lanes
if
p
.
lane
not
in
parent_lanes
:
continue
# template found
if
p
.
template
==
template
:
return
True
return
False


def _build_header(self):
        """Adds the header template to the master template string
        """

        logger.debug("===============")
        logger.debug("Building header")
        logger.debug("===============")
        self.template += hs.header

def
_build_header
(
self
)
:
logger
.
debug
(
"==============="
)
logger
.
debug
(
"Building header"
)
logger
.
debug
(
"==============="
)
self
.
template
+=
hs
.
header


def _build_footer(self):
        """Adds the footer template to the master template string"""

        logger.debug("===============")
        logger.debug("Building header")
        logger.debug("===============")
        self.template += fs.footer

def
_build_footer
(
self
)
:
logger
.
debug
(
"==============="
)
logger
.
debug
(
"Building header"
)
logger
.
debug
(
"==============="
)
self
.
template
+=
fs
.
footer