def to_python(self, value):
        """
        Convert a string from a form into an Enum value.
        """
        if value is None:
            return value
        if isinstance(value, self.enum):
            return value
        return self.enum[value]

def to_python ( self , value ) : if value is None : return value if isinstance ( value , self . enum ) : return value return self . enum [ value ]

Convert a string from a form into an Enum value .


def get_prep_value(self, value):
        """
        Convert an Enum value into a string for the database
        """
        if value is None:
            return None
        if isinstance(value, self.enum):
            return value.name
        raise ValueError("Unknown value {value:r} of type {cls}".format(
            value=value, cls=type(value)))

def
get_prep_value
(
self
,
value
)
:
if
value
is
:
return
if
isinstance
(
value
,
self
.
enum
)
:
return
value
.
name
raise
ValueError
(
"Unknown value {value:r} of type {cls}"
.
format
(
value
=
value
,
cls
=
type
(
value
)
)
)


def _resolve_path(obj, path):
    """path is a mul of coord or a coord"""
    if obj.__class__ not in path.context.accept:
        result = set()
        for ctx in path.context.accept:
            result |= {e for u in obj[ctx] for e in _resolve_path(u, path)}

        return result

    if isinstance(obj, Text):
        if path.index is not None:
            return {obj.children[path.index]}

        return set(obj.children)

    if isinstance(obj, (Fact, Theory)):
        return _resolve_path_tree_graph(obj.tree_graph, path)

    if isinstance(obj, Topic):
        if path.kind == 'r':
            if path.index is not None:
                return {obj.root[path.index]}
            return set(obj.root)
        else:
            if path.index is not None:
                return {obj.flexing[path.index]}
            return set(obj.flexing)

def _resolve_path ( obj , path ) : if obj . __class__ not in path . context . accept : result = set ( ) for ctx in path . context . accept : result |= { e for u in obj [ ctx ] for e in _resolve_path ( u , path ) } return result if isinstance ( obj , Text ) : if path . index is not None : return { obj . children [ path . index ] } return set ( obj . children ) if isinstance ( obj , ( Fact , Theory ) ) : return _resolve_path_tree_graph ( obj . tree_graph , path ) if isinstance ( obj , Topic ) : if path . kind == 'r' : if path . index is not None : return { obj . root [ path . index ] } return set ( obj . root ) else : if path . index is not None : return { obj . flexing [ path . index ] } return set ( obj . flexing )


def project_usls_on_dictionary(usls, allowed_terms=None):
    """`usls` is an iterable of usl.

    return a mapping term -> usl list
    """

    cells_to_usls = defaultdict(set)
    tables = set()

    for u in usls:
        for t in u.objects(Term):
            for c in t.singular_sequences:
                # This is the first time we meet the cell c
                if not cells_to_usls[c]:
                    tables.update(c.relations.contained)

                cells_to_usls[c].add(u)

    if allowed_terms:
        allowed_terms = set(allowed_terms)
        tables = tables & allowed_terms
        cells_to_usls = {c: l for c, l in cells_to_usls.items() if c in allowed_terms}

    tables_to_usls = {
        table: list(set(u for c in table.singular_sequences for u in cells_to_usls[c]))
            for table in tables if not isinstance(table, TableSet)
    }

    return tables_to_usls


def mean(self):
        """Returns the mean value."""
        if self.counter.value > 0:
            return self.sum.value / self.counter.value
        return 0.0

def mean ( self ) : if self . counter . value > 0 : return self . sum . value / self . counter . value return 0.0


def mark(self, value=1):
        """Record an event with the meter. By default it will record one event.

        :param value: number of event to record
        """
        self.counter += value
        self.m1_rate.update(value)
        self.m5_rate.update(value)
        self.m15_rate.update(value)

def mark ( self , value = 1 ) : self . counter += value self . m1_rate . update ( value ) self . m5_rate . update ( value ) self . m15_rate . update ( value )

Record an event with the meter . By default it will record one event .


def mean_rate(self):
        """
        Returns the mean rate of the events since the start of the process.
        """
        if self.counter.value == 0:
            return 0.0
        else:
            elapsed = time() - self.start_time
            return self.counter.value / elapsed

def mean_rate ( self ) : if self . counter . value == 0 : return 0.0 else : elapsed = time ( ) - self . start_time return self . counter . value / elapsed

Returns the mean rate of the events since the start of the process .


def mark(self, value=1):
        """Record an event with the derive.

        :param value: counter value to record
        """
        last = self.last.get_and_set(value)
        if last <= value:
            value = value - last
        super(Derive, self).mark(value)

def mark ( self , value = 1 ) : last = self . last . get_and_set ( value ) if last <= value : value = value - last super ( Derive , self ) . mark ( value )


def send_metric(self, name, metric):
        """Send metric and its snapshot."""
        config = SERIALIZER_CONFIG[class_name(metric)]

        mmap(
            self._buffered_send_metric,
            self.serialize_metric(
                metric,
                name,
                config['keys'],
                config['serialized_type']
            )
        )

        if hasattr(metric, 'snapshot') and config.get('snapshot_keys'):
            mmap(
                self._buffered_send_metric,
                self.serialize_metric(
                    metric.snapshot,
                    name,
                    config['snapshot_keys'],
                    config['serialized_type']
                )
            )

def send_metric ( self , name , metric ) : config = SERIALIZER_CONFIG [ class_name ( metric ) ] mmap ( self . _buffered_send_metric , self . serialize_metric ( metric , name , config [ 'keys' ] , config [ 'serialized_type' ] ) ) if hasattr ( metric , 'snapshot' ) and config . get ( 'snapshot_keys' ) : mmap ( self . _buffered_send_metric , self . serialize_metric ( metric . snapshot , name , config [ 'snapshot_keys' ] , config [ 'serialized_type' ] ) )


def serialize_metric(self, metric, m_name, keys, m_type):
        """Serialize and send available measures of a metric."""

        return [
            self.format_metric_string(m_name, getattr(metric, key), m_type)
            for key in keys
        ]

def serialize_metric ( self , metric , m_name , keys , m_type ) : return [ self . format_metric_string ( m_name , getattr ( metric , key ) , m_type ) for key in keys ]

Serialize and send available measures of a metric .


def format_metric_string(self, name, value, m_type):
        """Compose a statsd compatible string for a metric's measurement."""

        # NOTE(romcheg): This serialized metric template is based on
        #                statsd's documentation.
        template = '{name}:{value}|{m_type}\n'

        if self.prefix:
            name = "{prefix}.{m_name}".format(prefix=self.prefix, m_name=name)

        return template.format(name=name, value=value, m_type=m_type)

#                statsd's documentation.

Compose a statsd compatible string for a metric s measurement .


def _buffered_send_metric(self, metric_str):
        """Add a metric to the buffer."""

        self.batch_count += 1

        self.batch_buffer += metric_str

        # NOTE(romcheg): Send metrics if the number of metrics in the buffer
        #                has reached the threshold for sending.
        if self.batch_count >= self.batch_size:
            self._send()


def get(self, section, option, **kwargs):
        """
        Get method that raises MissingSetting if the value was unset.

        This differs from the SafeConfigParser which may raise either a
        NoOptionError or a NoSectionError.

        We take extra **kwargs because the Python 3.5 configparser extends the
        get method signature and it calls self with those parameters.

            def get(self, section, option, *, raw=False, vars=None,
                    fallback=_UNSET):
        """
        try:
            ret = super(ExactOnlineConfig, self).get(section, option, **kwargs)
        except (NoOptionError, NoSectionError):
            raise MissingSetting(option, section)

        return ret

def get ( self , section , option , * * kwargs ) : try : ret = super ( ExactOnlineConfig , self ) . get ( section , option , * * kwargs ) except ( NoOptionError , NoSectionError ) : raise MissingSetting ( option , section ) return ret

Get method that raises MissingSetting if the value was unset .


def _json_safe(data):
    """
    json.loads wants an unistr in Python3. Convert it.
    """
    if not hasattr(data, 'encode'):
        try:
            data = data.decode('utf-8')
        except UnicodeDecodeError:
            raise ValueError(
                'Expected valid UTF8 for JSON data, got %r' % (data,))
    return data

json . loads wants an unistr in Python3 . Convert it .


def get_or_set_default(self, section, option, value):
        """
        Base method to fetch values and to set defaults in case they
        don't exist.
        """
        try:
            ret = self.get(section, option)
        except MissingSetting:
            self.set(section, option, value)
            ret = value

        return ret

def get_or_set_default ( self , section , option , value ) : try : ret = self . get ( section , option ) except MissingSetting : self . set ( section , option , value ) ret = value return ret

Base method to fetch values and to set defaults in case they don t exist .


def get_ledger_code_to_guid_map(self, codes):
        """
        Convert set of human codes and to a dict of code to exactonline
        guid mappings.

        Example::

            ret = inv.get_ledger_code_to_guid_map(['1234', '5555'])
            ret == {'1234': '<guid1_from_exactonline_ledgeraccounts>',
                    '5555': '<guid2_from_exactonline_ledgeraccounts>'}
        """
        if codes:
            codes = set(str(i) for i in codes)
            ledger_ids = self._api.ledgeraccounts.filter(code__in=codes)
            ret = dict((str(i['Code']), i['ID']) for i in ledger_ids)
            found = set(ret.keys())
            missing = (codes - found)
            if missing:
                raise UnknownLedgerCodes(missing)
            return ret
        return {}

def get_ledger_code_to_guid_map ( self , codes ) : if codes : codes = set ( str ( i ) for i in codes ) ledger_ids = self . _api . ledgeraccounts . filter ( code__in = codes ) ret = dict ( ( str ( i [ 'Code' ] ) , i [ 'ID' ] ) for i in ledger_ids ) found = set ( ret . keys ( ) ) missing = ( codes - found ) if missing : raise UnknownLedgerCodes ( missing ) return ret return { }

Convert set of human codes and to a dict of code to exactonline guid mappings .


def get_divisions(self):
        """
        Get the "current" division and return a dictionary of divisions
        so the user can select the right one.
        """
        ret = self.rest(GET('v1/current/Me?$select=CurrentDivision'))
        current_division = ret[0]['CurrentDivision']
        assert isinstance(current_division, int)

        urlbase = 'v1/%d/' % (current_division,)
        resource = urljoin(urlbase, 'hrm/Divisions?$select=Code,Description')
        ret = self.rest(GET(resource))

        choices = dict((i['Code'], i['Description']) for i in ret)
        return choices, current_division

Get the current division and return a dictionary of divisions so the user can select the right one .


def map_exact2foreign_invoice_numbers(self, exact_invoice_numbers=None):
        """
        Optionally supply a list of ExactOnline invoice numbers.

        Returns a dictionary of ExactOnline invoice numbers to foreign
        (YourRef) invoice numbers.
        """
        # Quick, select all. Not the most nice to the server though.
        if exact_invoice_numbers is None:
            ret = self.filter(select='InvoiceNumber,YourRef')
            return dict((i['InvoiceNumber'], i['YourRef']) for i in ret)

        # Slower, select what we want to know. More work for us.
        exact_to_foreign_map = {}

        # Do it in batches. If we append 300 InvoiceNumbers at once, we
        # get a 12kB URI. (If the list is empty, we skip the entire
        # forloop and correctly return the empty dict.)
        exact_invoice_numbers = list(set(exact_invoice_numbers))  # unique
        for offset in range(0, len(exact_invoice_numbers), 40):
            batch = exact_invoice_numbers[offset:(offset + 40)]
            filter_ = ' or '.join(
                'InvoiceNumber eq %s' % (i,) for i in batch)
            assert filter_  # if filter was empty, we'd get all!
            ret = self.filter(filter=filter_, select='InvoiceNumber,YourRef')
            exact_to_foreign_map.update(
                dict((i['InvoiceNumber'], i['YourRef']) for i in ret))

        # Any values we missed?
        for exact_invoice_number in exact_invoice_numbers:
            if exact_invoice_number not in exact_to_foreign_map:
                exact_to_foreign_map[exact_invoice_number] = None

        return exact_to_foreign_map

Optionally supply a list of ExactOnline invoice numbers .


def solve(grid):
    """
    solve a Sudoku grid inplace
    """
    clauses = sudoku_clauses()
    for i in range(1, 10):
        for j in range(1, 10):
            d = grid[i - 1][j - 1]
            # For each digit already known, a clause (with one literal).
            # Note:
            #     We could also remove all variables for the known cells
            #     altogether (which would be more efficient).  However, for
            #     the sake of simplicity, we decided not to do that.
            if d:
                clauses.append([v(i, j, d)])

    # solve the SAT problem
    sol = set(pycosat.solve(clauses))

    def read_cell(i, j):
        # return the digit of cell i, j according to the solution
        for d in range(1, 10):
            if v(i, j, d) in sol:
                return d

    for i in range(1, 10):
        for j in range(1, 10):
            grid[i - 1][j - 1] = read_cell(i, j)


def view(injector):
    """Create Django class-based view from injector class."""

    handler = create_handler(View, injector)
    apply_http_methods(handler, injector)
    return injector.let(as_view=handler.as_view)

def view ( injector ) : handler = create_handler ( View , injector ) apply_http_methods ( handler , injector ) return injector . let ( as_view = handler . as_view )

Create Django class - based view from injector class .


def form_view(injector):
    """Create Django form processing class-based view from injector class."""

    handler = create_handler(FormView, injector)
    apply_form_methods(handler, injector)
    return injector.let(as_view=handler.as_view)

def form_view ( injector ) : handler = create_handler ( FormView , injector ) apply_form_methods ( handler , injector ) return injector . let ( as_view = handler . as_view )

Create Django form processing class - based view from injector class .


def method_view(injector):
    """Create Flask method based dispatching view from injector class."""

    handler = create_handler(MethodView)
    apply_http_methods(handler, injector)
    return injector.let(as_view=handler.as_view)

def method_view ( injector ) : handler = create_handler ( MethodView ) apply_http_methods ( handler , injector ) return injector . let ( as_view = handler . as_view )

Create Flask method based dispatching view from injector class .


def api_view(injector):
    """Create DRF class-based API view from injector class."""

    handler = create_handler(APIView, injector)
    apply_http_methods(handler, injector)
    apply_api_view_methods(handler, injector)
    return injector.let(as_view=handler.as_view)

def api_view ( injector ) : handler = create_handler ( APIView , injector ) apply_http_methods ( handler , injector ) apply_api_view_methods ( handler , injector ) return injector . let ( as_view = handler . as_view )

Create DRF class - based API view from injector class .


def generic_api_view(injector):
    """Create DRF generic class-based API view from injector class."""

    handler = create_handler(GenericAPIView, injector)
    apply_http_methods(handler, injector)
    apply_api_view_methods(handler, injector)
    apply_generic_api_view_methods(handler, injector)
    return injector.let(as_view=handler.as_view)

def generic_api_view ( injector ) : handler = create_handler ( GenericAPIView , injector ) apply_http_methods ( handler , injector ) apply_api_view_methods ( handler , injector ) apply_generic_api_view_methods ( handler , injector ) return injector . let ( as_view = handler . as_view )

Create DRF generic class - based API view from injector class .


def model_view_set(injector):
    """Create DRF model view set from injector class."""

    handler = create_handler(ModelViewSet, injector)
    apply_api_view_methods(handler, injector)
    apply_generic_api_view_methods(handler, injector)
    apply_model_view_set_methods(handler, injector)
    return injector.let(as_viewset=lambda: handler)

def model_view_set ( injector ) : handler = create_handler ( ModelViewSet , injector ) apply_api_view_methods ( handler , injector ) apply_generic_api_view_methods ( handler , injector ) apply_model_view_set_methods ( handler , injector ) return injector . let ( as_viewset = lambda : handler )

Create DRF model view set from injector class .


def stream_from_fd(fd, loop):
    """Recieve a streamer for a given file descriptor."""
    reader = asyncio.StreamReader(loop=loop)
    protocol = asyncio.StreamReaderProtocol(reader, loop=loop)
    waiter = asyncio.futures.Future(loop=loop)

    transport = UnixFileDescriptorTransport(
        loop=loop,
        fileno=fd,
        protocol=protocol,
        waiter=waiter,
    )

    try:
        yield from waiter
    except Exception:
        transport.close()

    if loop.get_debug():
        logger.debug("Read fd %r connected: (%r, %r)", fd, transport, protocol)
    return reader, transport

def
stream_from_fd
(
fd
,
loop
)
:
reader
=
asyncio
.
StreamReader
(
loop
=
loop
)
protocol
=
asyncio
.
StreamReaderProtocol
(
reader
,
loop
=
loop
)
waiter
=
asyncio
.
futures
.
Future
(
loop
=
loop
)
transport
=
UnixFileDescriptorTransport
(
loop
=
loop
,
fileno
=
fd
,
protocol
=
protocol
,
waiter
=
waiter
,
)
try
:
yield
from
waiter
except
Exception
:
transport
.
close
(
)
if
loop
.
get_debug
(
)
:
logger
.
debug
(
"Read fd %r connected: (%r, %r)"
,
fd
,
transport
,
protocol
)
return
reader
,
transport

Recieve a streamer for a given file descriptor .


def _read_ready(self):
        """Called by the event loop whenever the fd is ready for reading."""

        try:
            data = os.read(self._fileno, self.max_size)
        except InterruptedError:
            # No worries ;)
            pass
        except OSError as exc:
            # Some OS-level problem, crash.
            self._fatal_error(exc, "Fatal read error on file descriptor read")
        else:
            if data:
                self._protocol.data_received(data)
            else:
                # We reached end-of-file.
                if self._loop.get_debug():
                    logger.info("%r was closed by the kernel", self)
                self._closing = False
                self.pause_reading()
                self._loop.call_soon(self._protocol.eof_received)
                self._loop.call_soon(self._call_connection_lost, None)

Called by the event loop whenever the fd is ready for reading .


def _close(self, error=None):
        """Actual closing code, both from manual close and errors."""
        self._closing = True
        self.pause_reading()
        self._loop.call_soon(self._call_connection_lost, error)

def _close ( self , error = None ) : self . _closing = True self . pause_reading ( ) self . _loop . call_soon ( self . _call_connection_lost , error )

Actual closing code both from manual close and errors .


def _call_connection_lost(self, error):
        """Finalize closing."""
        try:
            self._protocol.connection_lost(error)
        finally:
            os.close(self._fileno)
            self._fileno = None
            self._protocol = None
            self._loop = None

def _call_connection_lost ( self , error ) : try : self . _protocol . connection_lost ( error ) finally : os . close ( self . _fileno ) self . _fileno = None self . _protocol = None self . _loop = None


def watch(self, path, flags, *, alias=None):
        """Add a new watching rule."""
        if alias is None:
            alias = path
        if alias in self.requests:
            raise ValueError("A watch request is already scheduled for alias %s" % alias)
        self.requests[alias] = (path, flags)
        if self._fd is not None:
            # We've started, register the watch immediately.
            self._setup_watch(alias, path, flags)


def unwatch(self, alias):
        """Stop watching a given rule."""
        if alias not in self.descriptors:
            raise ValueError("Unknown watch alias %s; current set is %r" % (alias, list(self.descriptors.keys())))
        wd = self.descriptors[alias]
        errno = LibC.inotify_rm_watch(self._fd, wd)
        if errno != 0:
            raise IOError("Failed to close watcher %d: errno=%d" % (wd, errno))
        del self.descriptors[alias]
        del self.requests[alias]
        del self.aliases[wd]


def _setup_watch(self, alias, path, flags):
        """Actual rule setup."""
        assert alias not in self.descriptors, "Registering alias %s twice!" % alias
        wd = LibC.inotify_add_watch(self._fd, path, flags)
        if wd < 0:
            raise IOError("Error setting up watch on %s with flags %s: wd=%s" % (
                path, flags, wd))
        self.descriptors[alias] = wd
        self.aliases[wd] = alias


def setup(self, loop):
        """Start the watcher, registering new watches if any."""
        self._loop = loop

        self._fd = LibC.inotify_init()
        for alias, (path, flags) in self.requests.items():
            self._setup_watch(alias, path, flags)

        # We pass ownership of the fd to the transport; it will close it.
        self._stream, self._transport = yield from aioutils.stream_from_fd(self._fd, loop)

Start the watcher registering new watches if any .


def get_event(self):
        """Fetch an event.

        This coroutine will swallow events for removed watches.
        """
        while True:
            prefix = yield from self._stream.readexactly(PREFIX.size)
            if prefix == b'':
                # We got closed, return None.
                return
            wd, flags, cookie, length = PREFIX.unpack(prefix)
            path = yield from self._stream.readexactly(length)

            # All async performed, time to look at the event's content.
            if wd not in self.aliases:
                # Event for a removed watch, skip it.
                continue

            decoded_path = struct.unpack('%ds' % length, path)[0].rstrip(b'\x00').decode('utf-8')
            return Event(
                flags=flags,
                cookie=cookie,
                name=decoded_path,
                alias=self.aliases[wd],
            )


def touch(self):
        """
        Respond to ``nsqd`` that you need more time to process the message.
        """
        assert not self._has_responded
        self.trigger(event.TOUCH, message=self)

def touch ( self ) : assert not self . _has_responded self . trigger ( event . TOUCH , message = self )

Respond to nsqd that you need more time to process the message .


def success(self):
        """Update the timer to reflect a successfull call"""
        if self.interval == 0.0:
            return
        self.short_interval -= self.short_unit
        self.long_interval -= self.long_unit
        self.short_interval = max(self.short_interval, Decimal(0))
        self.long_interval = max(self.long_interval, Decimal(0))
        self.update_interval()

def success ( self ) : if self . interval == 0.0 : return self . short_interval -= self . short_unit self . long_interval -= self . long_unit self . short_interval = max ( self . short_interval , Decimal ( 0 ) ) self . long_interval = max ( self . long_interval , Decimal ( 0 ) ) self . update_interval ( )


def failure(self):
        """Update the timer to reflect a failed call"""
        self.short_interval += self.short_unit
        self.long_interval += self.long_unit
        self.short_interval = min(self.short_interval, self.max_short_timer)
        self.long_interval = min(self.long_interval, self.max_long_timer)
        self.update_interval()

def failure ( self ) : self . short_interval += self . short_unit self . long_interval += self . long_unit self . short_interval = min ( self . short_interval , self . max_short_timer ) self . long_interval = min ( self . long_interval , self . max_long_timer ) self . update_interval ( )


def close(self):
        """
        Closes all connections stops all periodic callbacks
        """
        for conn in self.conns.values():
            conn.close()

        self.redist_periodic.stop()
        if self.query_periodic is not None:
            self.query_periodic.stop()

def close ( self ) : for conn in self . conns . values ( ) : conn . close ( ) self . redist_periodic . stop ( ) if self . query_periodic is not None : self . query_periodic . stop ( )


def is_starved(self):
        """
        Used to identify when buffered messages should be processed and responded to.

        When max_in_flight > 1 and you're batching messages together to perform work
        is isn't possible to just compare the len of your list of buffered messages against
        your configured max_in_flight (because max_in_flight may not be evenly divisible
        by the number of producers you're connected to, ie. you might never get that many
        messages... it's a *max*).

        Example::

            def message_handler(self, nsq_msg, reader):
                # buffer messages
                if reader.is_starved():
                    # perform work

            reader = nsq.Reader(...)
            reader.set_message_handler(functools.partial(message_handler, reader=reader))
            nsq.run()
        """
        for conn in itervalues(self.conns):
            if conn.in_flight > 0 and conn.in_flight >= (conn.last_rdy * 0.85):
                return True
        return False

def is_starved ( self ) : for conn in itervalues ( self . conns ) : if conn . in_flight > 0 and conn . in_flight >= ( conn . last_rdy * 0.85 ) : return True return False

Used to identify when buffered messages should be processed and responded to .


def connect_to_nsqd(self, host, port):
        """
        Adds a connection to ``nsqd`` at the specified address.

        :param host: the address to connect to
        :param port: the port to connect to
        """
        assert isinstance(host, string_types)
        assert isinstance(port, int)

        conn = AsyncConn(host, port, **self.conn_kwargs)
        conn.on('identify', self._on_connection_identify)
        conn.on('identify_response', self._on_connection_identify_response)
        conn.on('auth', self._on_connection_auth)
        conn.on('auth_response', self._on_connection_auth_response)
        conn.on('error', self._on_connection_error)
        conn.on('close', self._on_connection_close)
        conn.on('ready', self._on_connection_ready)
        conn.on('message', self._on_message)
        conn.on('heartbeat', self._on_heartbeat)
        conn.on('backoff', functools.partial(self._on_backoff_resume, success=False))
        conn.on('resume', functools.partial(self._on_backoff_resume, success=True))
        conn.on('continue', functools.partial(self._on_backoff_resume, success=None))

        if conn.id in self.conns:
            return

        # only attempt to re-connect once every 10s per destination
        # this throttles reconnects to failed endpoints
        now = time.time()
        last_connect_attempt = self.connection_attempts.get(conn.id)
        if last_connect_attempt and last_connect_attempt > now - 10:
            return
        self.connection_attempts[conn.id] = now

        logger.info('[%s:%s] connecting to nsqd', conn.id, self.name)
        conn.connect()

        return conn

Adds a connection to nsqd at the specified address .


def query_lookupd(self):
        """
        Trigger a query of the configured ``nsq_lookupd_http_addresses``.
        """
        endpoint = self.lookupd_http_addresses[self.lookupd_query_index]
        self.lookupd_query_index = (self.lookupd_query_index + 1) % len(self.lookupd_http_addresses)

        # urlsplit() is faulty if scheme not present
        if '://' not in endpoint:
            endpoint = 'http://' + endpoint

        scheme, netloc, path, query, fragment = urlparse.urlsplit(endpoint)

        if not path or path == "/":
            path = "/lookup"

        params = parse_qs(query)
        params['topic'] = self.topic
        query = urlencode(_utf8_params(params), doseq=1)
        lookupd_url = urlparse.urlunsplit((scheme, netloc, path, query, fragment))

        req = tornado.httpclient.HTTPRequest(
            lookupd_url, method='GET',
            headers={'Accept': 'application/vnd.nsq; version=1.0'},
            connect_timeout=self.lookupd_connect_timeout,
            request_timeout=self.lookupd_request_timeout)
        callback = functools.partial(self._finish_query_lookupd, lookupd_url=lookupd_url)
        self.http_client.fetch(req, callback=callback)

Trigger a query of the configured nsq_lookupd_http_addresses .


def set_max_in_flight(self, max_in_flight):
        """Dynamically adjust the reader max_in_flight. Set to 0 to immediately disable a Reader"""
        assert isinstance(max_in_flight, int)
        self.max_in_flight = max_in_flight

        if max_in_flight == 0:
            # set RDY 0 to all connections
            for conn in itervalues(self.conns):
                if conn.rdy > 0:
                    logger.debug('[%s:%s] rdy: %d -> 0', conn.id, self.name, conn.rdy)
                    self._send_rdy(conn, 0)
            self.total_rdy = 0
        else:
            self.need_rdy_redistributed = True
            self._redistribute_rdy_state()

def
set_max_in_flight
(
self
,
max_in_flight
)
:
assert
isinstance
(
max_in_flight
,
int
)
self
.
max_in_flight
=
max_in_flight
if
max_in_flight
==
0
:
# set RDY 0 to all connections
for
conn
in
itervalues
(
self
.
conns
)
:
if
conn
.
rdy
>
0
:
logger
.
debug
(
'[%s:%s] rdy: %d -> 0'
,
conn
.
id
,
self
.
name
,
conn
.
rdy
)
self
.
_send_rdy
(
conn
,
0
)
self
.
total_rdy
=
0
else
:
self
.
need_rdy_redistributed
=
True
self
.
_redistribute_rdy_state
(
)

Dynamically adjust the reader max_in_flight . Set to 0 to immediately disable a Reader


def giving_up(self, message):
        """
        Called when a message has been received where ``msg.attempts > max_tries``

        This is useful to subclass and override to perform a task (such as writing to disk, etc.)

        :param message: the :class:`nsq.Message` received
        """
        logger.warning('[%s] giving up on message %s after %d tries (max:%d) %r',
                       self.name, message.id, message.attempts, self.max_tries, message.body)

Called when a message has been received where msg . attempts > max_tries


def on(self, name, callback):
        """
        Listen for the named event with the specified callback.

        :param name: the name of the event
        :type name: string

        :param callback: the callback to execute when the event is triggered
        :type callback: callable
        """
        assert callable(callback), 'callback is not callable'
        if callback in self.__listeners[name]:
            raise DuplicateListenerError
        self.__listeners[name].append(callback)

def
on
(
self
,
name
,
callback
)
:
assert
callable
(
callback
)
,
'callback is not callable'
if
callback
in
self
.
__listeners
[
name
]
:
raise
DuplicateListenerError
self
.
__listeners
[
name
]
.
append
(
callback
)

Listen for the named event with the specified callback .


def off(self, name, callback):
        """
        Stop listening for the named event via the specified callback.

        :param name: the name of the event
        :type name: string

        :param callback: the callback that was originally used
        :type callback: callable
        """
        if callback not in self.__listeners[name]:
            raise InvalidListenerError
        self.__listeners[name].remove(callback)

def off ( self , name , callback ) : if callback not in self . __listeners [ name ] : raise InvalidListenerError self . __listeners [ name ] . remove ( callback )

Stop listening for the named event via the specified callback .


def trigger(self, name, *args, **kwargs):
        """
        Execute the callbacks for the listeners on the specified event with the
        supplied arguments.

        All extra arguments are passed through to each callback.

        :param name: the name of the event
        :type name: string
        """
        for ev in self.__listeners[name]:
            ev(*args, **kwargs)

def trigger ( self , name , * args , * * kwargs ) : for ev in self . __listeners [ name ] : ev ( * args , * * kwargs )

Execute the callbacks for the listeners on the specified event with the supplied arguments .


def pub(self, topic, msg, callback=None):
        """
        publish a message to nsq

        :param topic: nsq topic
        :param msg: message body (bytes)
        :param callback: function which takes (conn, data) (data may be nsq.Error)
        """
        self._pub('pub', topic, msg, callback=callback)

def pub ( self , topic , msg , callback = None ) : self . _pub ( 'pub' , topic , msg , callback = callback )


def set_feature_transform(self, mode='polynomial', degree=1):

        '''
        Transform data feature to high level
        '''

        if self.status != 'load_train_data':
            print("Please load train data first.")
            return self.train_X

        self.feature_transform_mode = mode
        self.feature_transform_degree = degree

        self.train_X = self.train_X[:, 1:]

        self.train_X = utility.DatasetLoader.feature_transform(
            self.train_X,
            self.feature_transform_mode,
            self.feature_transform_degree
        )

        return self.train_X

def
set_feature_transform
(
self
,
mode
=
'polynomial'
,
degree
=
1
)
:
if
self
.
status
!=
'load_train_data'
:
print
(
"Please load train data first."
)
return
self
.
train_X
self
.
feature_transform_mode
=
mode
self
.
feature_transform_degree
=
degree
self
.
train_X
=
self
.
train_X
[
:
,
1
:
]
self
.
train_X
=
utility
.
DatasetLoader
.
feature_transform
(
self
.
train_X
,
self
.
feature_transform_mode
,
self
.
feature_transform_degree
)
return
self
.
train_X


def prediction(self, input_data='', mode='test_data'):

        '''
        Make prediction
        input test data
        output the prediction
        '''

        prediction = {}

        if (self.status != 'train'):
            print("Please load train data and init W then train the W first.")
            return prediction

        if (input_data == ''):
            print("Please input test data for prediction.")
            return prediction

        if mode == 'future_data':
            data = input_data.split()
            input_data_x = [float(v) for v in data]
            input_data_x = utility.DatasetLoader.feature_transform(
                np.array(input_data_x).reshape(1, -1),
                self.feature_transform_mode,
                self.feature_transform_degree
            )
            input_data_x = np.ravel(input_data_x)
            prediction = self.score_function(input_data_x, self.W)
            return {"input_data_x": input_data_x, "input_data_y": None, "prediction": prediction}
        else:
            data = input_data.split()
            input_data_x = [float(v) for v in data[:-1]]
            input_data_x = utility.DatasetLoader.feature_transform(
                np.array(input_data_x).reshape(1, -1),
                self.feature_transform_mode,
                self.feature_transform_degree
            )
            input_data_x = np.ravel(input_data_x)
            input_data_y = float(data[-1])
            prediction = self.score_function(input_data_x, self.W)
            return {"input_data_x": input_data_x, "input_data_y": input_data_y, "prediction": prediction}


def theta(self, s):

        '''
        Theta sigmoid function
        '''

        s = np.where(s < -709, -709, s)

        return 1 / (1 + np.exp((-1) * s))

def theta ( self , s ) : s = np . where ( s < - 709 , - 709 , s ) return 1 / ( 1 + np . exp ( ( - 1 ) * s ) )


def parse_log(log_file):
    """Retrieves some statistics from a single Trimmomatic log file.

    This function parses Trimmomatic's log file and stores some trimming
    statistics in an :py:class:`OrderedDict` object. This object contains
    the following keys:

        - ``clean_len``: Total length after trimming.
        - ``total_trim``: Total trimmed base pairs.
        - ``total_trim_perc``: Total trimmed base pairs in percentage.
        - ``5trim``: Total base pairs trimmed at 5' end.
        - ``3trim``: Total base pairs trimmed at 3' end.

    Parameters
    ----------
    log_file : str
        Path to trimmomatic log file.

    Returns
    -------
    x : :py:class:`OrderedDict`
        Object storing the trimming statistics.

    """

    template = OrderedDict([
        # Total length after trimming
        ("clean_len", 0),
        # Total trimmed base pairs
        ("total_trim", 0),
        # Total trimmed base pairs in percentage
        ("total_trim_perc", 0),
        # Total trimmed at 5' end
        ("5trim", 0),
        # Total trimmed at 3' end
        ("3trim", 0),
        # Bad reads (completely trimmed)
        ("bad_reads", 0)
    ])

    with open(log_file) as fh:

        for line in fh:
            # This will split the log fields into:
            # 0. read length after trimming
            # 1. amount trimmed from the start
            # 2. last surviving base
            # 3. amount trimmed from the end
            fields = [int(x) for x in line.strip().split()[-4:]]

            if not fields[0]:
                template["bad_reads"] += 1

            template["5trim"] += fields[1]
            template["3trim"] += fields[3]
            template["total_trim"] += fields[1] + fields[3]
            template["clean_len"] += fields[0]

        total_len = template["clean_len"] + template["total_trim"]

        if total_len:
            template["total_trim_perc"] = round(
                (template["total_trim"] / total_len) * 100, 2)
        else:
            template["total_trim_perc"] = 0

    return template

Retrieves some statistics from a single Trimmomatic log file .


def clean_up(fastq_pairs, clear):
    """Cleans the working directory of unwanted temporary files"""

    # Find unpaired fastq files
    unpaired_fastq = [f for f in os.listdir(".")
                      if f.endswith("_U.fastq.gz")]

    # Remove unpaired fastq files, if any
    for fpath in unpaired_fastq:
        os.remove(fpath)

    # Expected output to assess whether it is safe to remove temporary input
    expected_out = [f for f in os.listdir(".") if f.endswith("_trim.fastq.gz")]

    if clear == "true" and len(expected_out) == 2:
        for fq in fastq_pairs:
            # Get real path of fastq files, following symlinks
            rp = os.path.realpath(fq)
            logger.debug("Removing temporary fastq file path: {}".format(rp))
            if re.match(".*/work/.{2}/.{30}/.*", rp):
                os.remove(rp)


def merge_default_adapters():
    """Merges the default adapters file in the trimmomatic adapters directory

    Returns
    -------
    str
        Path with the merged adapters file.
    """

    default_adapters = [os.path.join(ADAPTERS_PATH, x) for x in
                        os.listdir(ADAPTERS_PATH)]
    filepath = os.path.join(os.getcwd(), "default_adapters.fasta")

    with open(filepath, "w") as fh, \
            fileinput.input(default_adapters) as in_fh:
        for line in in_fh:
            fh.write("{}{}".format(line, "\\n"))

    return filepath

def merge_default_adapters ( ) : default_adapters = [ os . path . join ( ADAPTERS_PATH , x ) for x in os . listdir ( ADAPTERS_PATH ) ] filepath = os . path . join ( os . getcwd ( ) , "default_adapters.fasta" ) with open ( filepath , "w" ) as fh , fileinput . input ( default_adapters ) as in_fh : for line in in_fh : fh . write ( "{}{}" . format ( line , "\\n" ) ) return filepath


def main(sample_id, fastq_pair, trim_range, trim_opts, phred, adapters_file,
         clear):
    """ Main executor of the trimmomatic template.

    Parameters
    ----------
    sample_id : str
        Sample Identification string.
    fastq_pair : list
        Two element list containing the paired FastQ files.
    trim_range : list
        Two element list containing the trimming range.
    trim_opts : list
        Four element list containing several trimmomatic options:
        [*SLIDINGWINDOW*; *LEADING*; *TRAILING*; *MINLEN*]
    phred : int
        Guessed phred score for the sample. The phred score is a generated
        output from :py:class:`templates.integrity_coverage`.
    adapters_file : str
        Path to adapters file. If not provided, or the path is not available,
        it will use the default adapters from Trimmomatic will be used
    clear : str
        Can be either 'true' or 'false'. If 'true', the input fastq files will
        be removed at the end of the run, IF they are in the working directory
    """

    logger.info("Starting trimmomatic")

    # Create base CLI
    cli = [
        "java",
        "-Xmx{}".format("$task.memory"[:-1].lower().replace(" ", "")),
        "-jar",
        TRIM_PATH.strip(),
        "PE",
        "-threads",
        "$task.cpus"
    ]

    # If the phred encoding was detected, provide it
    try:
        # Check if the provided PHRED can be converted to int
        phred = int(phred)
        phred_flag = "-phred{}".format(str(phred))
        cli += [phred_flag]
    # Could not detect phred encoding. Do not add explicit encoding to
    # trimmomatic and let it guess
    except ValueError:
        pass

    # Add input samples to CLI
    cli += fastq_pair

    # Add output file names
    output_names = []
    for i in range(len(fastq_pair)):
        output_names.append("{}_{}_trim.fastq.gz".format(
            SAMPLE_ID, str(i + 1)))
        output_names.append("{}_{}_U.fastq.gz".format(
            SAMPLE_ID, str(i + 1)))
    cli += output_names

    if trim_range != ["None"]:
        cli += [
            "CROP:{}".format(trim_range[1]),
            "HEADCROP:{}".format(trim_range[0]),
        ]

    if os.path.exists(adapters_file):
        logger.debug("Using the provided adapters file '{}'".format(
            adapters_file))
    else:
        logger.debug("Adapters file '{}' not provided or does not exist. Using"
                     " default adapters".format(adapters_file))
        adapters_file = merge_default_adapters()

    cli += [
        "ILLUMINACLIP:{}:3:30:10:6:true".format(adapters_file)
    ]

    #create log file im temporary dir to avoid issues when running on a docker container in macOS
    logfile = os.path.join(tempfile.mkdtemp(prefix='tmp'), "{}_trimlog.txt".format(sample_id))

    # Add trimmomatic options
    cli += [
        "SLIDINGWINDOW:{}".format(trim_opts[0]),
        "LEADING:{}".format(trim_opts[1]),
        "TRAILING:{}".format(trim_opts[2]),
        "MINLEN:{}".format(trim_opts[3]),
        "TOPHRED33",
        "-trimlog",
        logfile
    ]

    logger.debug("Running trimmomatic subprocess with command: {}".format(cli))

    p = subprocess.Popen(cli, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()

    # Attempt to decode STDERR output from bytes. If unsuccessful, coerce to
    # string
    try:
        stderr = stderr.decode("utf8")
    except (UnicodeDecodeError, AttributeError):
        stderr = str(stderr)

    logger.info("Finished trimmomatic subprocess with STDOUT:\\n"
                "======================================\\n{}".format(stdout))
    logger.info("Finished trimmomatic subprocesswith STDERR:\\n"
                "======================================\\n{}".format(stderr))
    logger.info("Finished trimmomatic with return code: {}".format(
        p.returncode))

    trimmomatic_log(logfile, sample_id)

    if p.returncode == 0 and os.path.exists("{}_1_trim.fastq.gz".format(
            SAMPLE_ID)):
        clean_up(fastq_pair, clear)

    # Check if trimmomatic ran successfully. If not, write the error message
    # to the status channel and exit.
    with open(".status", "w") as status_fh:
        if p.returncode != 0:
            status_fh.write("fail")
            return
        else:
            status_fh.write("pass")

Main executor of the trimmomatic template .


def depth_file_reader(depth_file):
    """
    Function that parse samtools depth file and creates 3 dictionaries that
    will be useful to make the outputs of this script, both the tabular file
    and the json file that may be imported by pATLAS

    Parameters
    ----------
    depth_file: textIO
        the path to depth file for each sample

    Returns
    -------
    depth_dic_coverage: dict
            dictionary with the coverage per position for each plasmid
    """

    # dict to store the mean coverage for each reference
    depth_dic_coverage = {}

    for line in depth_file:
        tab_split = line.split()  # split by any white space
        reference = "_".join(tab_split[0].strip().split("_")[0:3])  # store
        # only the gi for the reference
        position = tab_split[1]
        num_reads_align = float(tab_split[2].rstrip())

        if reference not in depth_dic_coverage:
            depth_dic_coverage[reference] = {}

        depth_dic_coverage[reference][position] = num_reads_align

    logger.info("Finished parsing depth file.")
    depth_file.close()

    logger.debug("Size of dict_cov: {} kb".format(
        asizeof(depth_dic_coverage)/1024))

    return depth_dic_coverage

Function that parse samtools depth file and creates 3 dictionaries that will be useful to make the outputs of this script both the tabular file and the json file that may be imported by pATLAS


def main(depth_file, json_dict, cutoff, sample_id):
    """
    Function that handles the inputs required to parse depth files from bowtie
    and dumps a dict to a json file that can be imported into pATLAS.

    Parameters
    ----------
    depth_file: str
         the path to depth file for each sample
    json_dict: str
        the file that contains the dictionary with keys and values for
        accessions
        and their respective lengths
    cutoff: str
        the cutoff used to trim the unwanted matches for the minimum coverage
        results from mapping. This value may range between 0 and 1.
    sample_id: str
        the id of the sample being parsed

    """

    # check for the appropriate value for the cutoff value for coverage results
    logger.debug("Cutoff value: {}. Type: {}".format(cutoff, type(cutoff)))
    try:
        cutoff_val = float(cutoff)
        if cutoff_val < 0.4:
            logger.warning("This cutoff value will generate a high volume of "
                           "plot data. Therefore '.report.json' can be too big")
    except ValueError:
        logger.error("Cutoff value should be a string such as: '0.6'. "
                     "The outputted value: {}. Make sure to provide an "
                     "appropriate value for --cov_cutoff".format(cutoff))
        sys.exit(1)

    # loads dict from file, this file is provided in docker image

    plasmid_length = json.load(open(json_dict))
    if plasmid_length:
        logger.info("Loaded dictionary of plasmid lengths")
    else:
        logger.error("Something went wrong and plasmid lengths dictionary"
                     "could not be loaded. Check if process received this"
                     "param successfully.")
        sys.exit(1)

    # read depth file
    depth_file_in = open(depth_file)

    # first reads the depth file and generates dictionaries to handle the input
    # to a simpler format
    logger.info("Reading depth file and creating dictionary to dump.")
    depth_dic_coverage = depth_file_reader(depth_file_in)
    percentage_bases_covered, dict_cov = generate_jsons(depth_dic_coverage,
                                                        plasmid_length,
                                                        cutoff_val)

    if percentage_bases_covered and dict_cov:
        logger.info("percentage_bases_covered length: {}".format(
            str(len(percentage_bases_covered))))
        logger.info("dict_cov length: {}".format(str(len(dict_cov))))
    else:
        logger.error("Both dicts that dump to JSON file or .report.json are "
                     "empty.")

    # then dump do file
    logger.info("Dumping to {}".format("{}_mapping.json".format(depth_file)))
    with open("{}_mapping.json".format(depth_file), "w") as output_json:
        output_json.write(json.dumps(percentage_bases_covered))

    json_dic = {
        "tableRow": [{
            "sample": sample_id,
            "data": [{
                "header": "Mapping",
                "table": "plasmids",
                "patlas_mapping": percentage_bases_covered,
                "value": len(percentage_bases_covered)
            }]
        }],
        "sample": sample_id,
        "patlas_mapping": percentage_bases_covered,
        "plotData": [{
            "sample": sample_id,
            "data": {
                "patlasMappingSliding": dict_cov
            },
        }]
    }

    logger.debug("Size of dict_cov: {} kb".format(asizeof(json_dic)/1024))
    logger.info("Writing to .report.json")
    with open(".report.json", "w") as json_report:
        json_report.write(json.dumps(json_dic, separators=(",", ":")))

Function that handles the inputs required to parse depth files from bowtie and dumps a dict to a json file that can be imported into pATLAS .


def _set_template(self, template):
        """Sets the path to the appropriate jinja template file

        When a Process instance is initialized, this method will fetch
        the location of the appropriate template file, based on the
        ``template`` argument. It will raise an exception is the template
        file is not found. Otherwise, it will set the
        :py:attr:`Process.template_path` attribute.
        """

        # Set template directory
        tpl_dir = join(dirname(abspath(__file__)), "templates")

        # Set template file path
        tpl_path = join(tpl_dir, template + ".nf")

        if not os.path.exists(tpl_path):
            raise eh.ProcessError(
                "Template {} does not exist".format(tpl_path))

        self._template_path = join(tpl_dir, template + ".nf")

def
_set_template
(
self
,
template
)
:
# Set template directory
tpl_dir
=
join
(
dirname
(
abspath
(
__file__
)
)
,
"templates"
)
# Set template file path
tpl_path
=
join
(
tpl_dir
,
template
+
".nf"
)
if
not
os
.
path
.
exists
(
tpl_path
)
:
raise
eh
.
ProcessError
(
"Template {} does not exist"
.
format
(
tpl_path
)
)
self
.
_template_path
=
join
(
tpl_dir
,
template
+
".nf"
)


def set_main_channel_names(self, input_suffix, output_suffix, lane):
        """Sets the main channel names based on the provide input and
        output channel suffixes. This is performed when connecting processes.

        Parameters
        ----------
        input_suffix : str
            Suffix added to the input channel. Should be based on the lane
            and an arbitrary unique id
        output_suffix : str
            Suffix added to the output channel. Should be based on the lane
            and an arbitrary unique id
        lane : int
            Sets the lane of the process.
        """

        self.input_channel = "{}_in_{}".format(self.template, input_suffix)
        self.output_channel = "{}_out_{}".format(self.template, output_suffix)
        self.lane = lane

def set_main_channel_names ( self , input_suffix , output_suffix , lane ) : self . input_channel = "{}_in_{}" . format ( self . template , input_suffix ) self . output_channel = "{}_out_{}" . format ( self . template , output_suffix ) self . lane = lane

Sets the main channel names based on the provide input and output channel suffixes . This is performed when connecting processes .


def get_user_channel(self, input_channel, input_type=None):
        """Returns the main raw channel for the process

        Provided with at least a channel name, this method returns the raw
        channel name and specification (the nextflow string definition)
        for the process. By default, it will fork from the raw input of
        the process' :attr:`~Process.input_type` attribute. However, this
        behaviour can be overridden by providing the ``input_type`` argument.

        If the specified or inferred input type exists in the
        :attr:`~Process.RAW_MAPPING` dictionary, the channel info dictionary
        will be retrieved along with the specified input channel. Otherwise,
        it will return None.

        An example of the returned dictionary is::

             {"input_channel": "myChannel",
             "params": "fastq",
             "channel": "IN_fastq_raw",
             "channel_str":"IN_fastq_raw = Channel.fromFilePairs(params.fastq)"
            }

        Returns
        -------
        dict or None
            Dictionary with the complete raw channel info. None if no
            channel is found.
        """

        res = {"input_channel": input_channel}

        itype = input_type if input_type else self.input_type

        if itype in self.RAW_MAPPING:

            channel_info = self.RAW_MAPPING[itype]

            return {**res, **channel_info}

def get_user_channel ( self , input_channel , input_type = None ) : res = { "input_channel" : input_channel } itype = input_type if input_type else self . input_type if itype in self . RAW_MAPPING : channel_info = self . RAW_MAPPING [ itype ] return { * * res , * * channel_info }


def render(template, context):
        """Wrapper to the jinja2 render method from a template file

        Parameters
        ----------
        template : str
            Path to template file.
        context : dict
            Dictionary with kwargs context to populate the template
        """

        path, filename = os.path.split(template)

        return jinja2.Environment(
            loader=jinja2.FileSystemLoader(path or './')
        ).get_template(filename).render(context)

def render ( template , context ) : path , filename = os . path . split ( template ) return jinja2 . Environment ( loader = jinja2 . FileSystemLoader ( path or './' ) ) . get_template ( filename ) . render ( context )


def template_str(self):
        """Class property that returns a populated template string

        This property allows the template of a particular process to be
        dynamically generated and returned when doing ``Process.template_str``.

        Returns
        -------
        x : str
            String with the complete and populated process template

        """

        if not self._context:
            raise eh.ProcessError("Channels must be setup first using the "
                                  "set_channels method")

        logger.debug("Setting context for template {}: {}".format(
            self.template, self._context
        ))

        x = self.render(self._template_path, self._context)
        return x


def set_channels(self, **kwargs):
        """ General purpose method that sets the main channels

        This method will take a variable number of keyword arguments to
        set the :py:attr:`Process._context` attribute with the information
        on the main channels for the process. This is done by appending
        the process ID (:py:attr:`Process.pid`) attribute to the input,
        output and status channel prefix strings. In the output channel,
        the process ID is incremented by 1 to allow the connection with the
        channel in the next process.

        The ``**kwargs`` system for setting the :py:attr:`Process._context`
        attribute also provides additional flexibility. In this way,
        individual processes can provide additional information not covered
        in this method, without changing it.

        Parameters
        ----------
        kwargs : dict
            Dictionary with the keyword arguments for setting up the template
            context
        """

        if not self.pid:
            self.pid = "{}_{}".format(self.lane, kwargs.get("pid"))

        for i in self.status_channels:
            if i.startswith("STATUS_"):
                self.status_strs.append("{}_{}".format(i, self.pid))
            else:
                self.status_strs.append("STATUS_{}_{}".format(i, self.pid))

        if self.main_forks:
            logger.debug("Setting main fork channels: {}".format(
                self.main_forks))
            operator = "set" if len(self.main_forks) == 1 else "into"
            self.forks = ["\n{}.{}{{ {} }}\n".format(
                self.output_channel, operator, ";".join(self.main_forks))]

        self._context = {**kwargs, **{"input_channel": self.input_channel,
                                      "output_channel": self.output_channel,
                                      "template": self.template,
                                      "forks": "\n".join(self.forks),
                                      "pid": self.pid}}

def
set_channels
(
self
,
*
*
kwargs
)
:
if
not
self
.
pid
:
self
.
pid
=
"{}_{}"
.
format
(
self
.
lane
,
kwargs
.
get
(
"pid"
)
)
for
i
in
self
.
status_channels
:
if
i
.
startswith
(
"STATUS_"
)
:
self
.
status_strs
.
append
(
"{}_{}"
.
format
(
i
,
self
.
pid
)
)
else
:
self
.
status_strs
.
append
(
"STATUS_{}_{}"
.
format
(
i
,
self
.
pid
)
)
if
self
.
main_forks
:
logger
.
debug
(
"Setting main fork channels: {}"
.
format
(
self
.
main_forks
)
)
operator
=
"set"
if
len
(
self
.
main_forks
)
==
1
else
"into"
self
.
forks
=
[
"\n{}.{}{{ {} }}\n"
.
format
(
self
.
output_channel
,
operator
,
";"
.
join
(
self
.
main_forks
)
)
]
self
.
_context
=
{
*
*
kwargs
,
*
*
{
"input_channel"
:
self
.
input_channel
,
"output_channel"
:
self
.
output_channel
,
"template"
:
self
.
template
,
"forks"
:
"\n"
.
join
(
self
.
forks
)
,
"pid"
:
self
.
pid
}
}


def update_main_forks(self, sink):
        """Updates the forks attribute with the sink channel destination

        Parameters
        ----------
        sink : str
            Channel onto which the main input will be forked to

        """

        if not self.main_forks:
            self.main_forks = [self.output_channel]
            self.output_channel = "_{}".format(self.output_channel)
        self.main_forks.append(sink)

        # fork_lst = self.forks + self.main_forks
        operator = "set" if len(self.main_forks) == 1 else "into"
        self.forks = ["\n{}.{}{{ {} }}\n".format(
            self.output_channel, operator, ";".join(self.main_forks))]

        self._context = {**self._context,
                         **{"forks": "".join(self.forks),
                            "output_channel": self.output_channel}}


def set_secondary_channel(self, source, channel_list):
        """ General purpose method for setting a secondary channel

        This method allows a given source channel to be forked into one or
        more channels and sets those forks in the :py:attr:`Process.forks`
        attribute. Both the source and the channels in the ``channel_list``
        argument must be the final channel strings,  which means that this
        method should be called only after setting the main channels.

        If the source is not a main channel, this will simply create a fork
        or set for every channel in the ``channel_list`` argument list::

            SOURCE_CHANNEL_1.into{SINK_1;SINK_2}

        If the source is a main channel, this will apply some changes to
        the output channel of the process, to avoid overlapping main output
        channels.  For instance, forking the main output channel for process
        2 would create a ``MAIN_2.into{...}``. The issue here is that the
        ``MAIN_2`` channel is expected as the input of the next process, but
        now is being used to create the fork. To solve this issue, the output
        channel is modified into ``_MAIN_2``, and the fork is set to
        the channels provided channels plus the ``MAIN_2`` channel::

            _MAIN_2.into{MAIN_2;MAIN_5;...}

        Parameters
        ----------
        source : str
            String with the name of the source channel
        channel_list : list
            List of channels that will receive a fork of the secondary
            channel
        """

        logger.debug("Setting secondary channel for source '{}': {}".format(
            source, channel_list))

        source = "{}_{}".format(source, self.pid)

        # Removes possible duplicate channels, when the fork is terminal
        channel_list = sorted(list(set(channel_list)))

        # When there is only one channel to fork into, use the 'set' operator
        # instead of 'into'
        op = "set" if len(channel_list) == 1 else "into"
        self.forks.append("\n{}.{}{{ {} }}\n".format(
            source, op, ";".join(channel_list)))

        logger.debug("Setting forks attribute to: {}".format(self.forks))
        self._context = {**self._context, **{"forks": "\n".join(self.forks)}}


def update_attributes(self, attr_dict):
        """Updates the directives attribute from a dictionary object.

        This will only update the directives for processes that have been
        defined in the subclass.

        Parameters
        ----------
        attr_dict : dict
            Dictionary containing the attributes that will be used to update
            the process attributes and/or directives.

        """

        # Update directives
        # Allowed attributes to write
        valid_directives = ["pid", "ignore_type", "ignore_pid", "extra_input",
                            "group", "input_type"]

        for attribute, val in attr_dict.items():

            # If the attribute has a valid directive key, update that
            # directive
            if attribute in valid_directives and hasattr(self, attribute):
                setattr(self, attribute, val)

            # The params attribute is special, in the sense that it provides
            # information for the self.params attribute.
            elif attribute == "params":
                for name, value in val.items():
                    if name in self.params:
                        self.params[name]["default"] = value
                    else:
                        raise eh.ProcessError(
                            "The parameter name '{}' does not exist for "
                            "component '{}'".format(name, self.template))

            else:
                for p in self.directives:
                    self.directives[p][attribute] = val

Updates the directives attribute from a dictionary object .


def set_compiler_channels(self, channel_list, operator="mix"):
        """General method for setting the input channels for the status process

        Given a list of status channels that are gathered during the pipeline
        construction, this method will automatically set the input channel
        for the status process. This makes use of the ``mix`` channel operator
        of nextflow for multiple channels::

            STATUS_1.mix(STATUS_2,STATUS_3,...)

        This will set the ``status_channels`` key for the ``_context``
        attribute of the process.

        Parameters
        ----------
        channel_list : list
            List of strings with the final name of the status channels
        operator : str
            Specifies the operator used to join the compiler channels.
            Available options are 'mix'and 'join'.
        """

        if not channel_list:
            raise eh.ProcessError("At least one status channel must be "
                                  "provided to include this process in the "
                                  "pipeline")

        if len(channel_list) == 1:
            logger.debug("Setting only one status channel: {}".format(
                channel_list[0]))
            self._context = {"compile_channels": channel_list[0]}

        else:

            first_status = channel_list[0]

            if operator == "mix":
                lst = ",".join(channel_list[1:])

                s = "{}.mix({})".format(first_status, lst)

            elif operator == "join":

                s = first_status
                for ch in channel_list[1:]:
                    s += ".join({})".format(ch)

                s += ".map{ ot -> [ ot[0], ot[1..-1] ] }"

            logger.debug("Status channel string: {}".format(s))

            self._context = {"compile_channels": s}


def set_raw_inputs(self, raw_input):
        """Sets the main input channels of the pipeline and their forks.

        The ``raw_input`` dictionary input should contain one entry for each
        input type (fastq, fasta, etc). The corresponding value should be a
        dictionary/json with the following key:values:

        - ``channel``: Name of the raw input channel (e.g.: channel1)
        - ``channel_str``: The nextflow definition of the channel and
           eventual checks (e.g.: channel1 = Channel.fromPath(param))
        - ``raw_forks``: A list of channels to which the channel name will
          for to.

        Each new type of input parameter is automatically added to the
        :attr:`params` attribute, so that they are automatically collected
        for the pipeline description and help.

        Parameters
        ----------
        raw_input : dict
            Contains an entry for each input type with the channel name,
            channel string and forks.
        """

        logger.debug("Setting raw inputs using raw input dict: {}".format(
            raw_input))

        primary_inputs = []

        for input_type, el in raw_input.items():

            primary_inputs.append(el["channel_str"])

            # Update the process' parameters with the raw input
            raw_channel = self.RAW_MAPPING[input_type]
            self.params[input_type] = {
                "default": raw_channel["default_value"],
                "description": raw_channel["description"]
            }

            op = "set" if len(el["raw_forks"]) == 1 else "into"

            self.forks.append("\n{}.{}{{ {} }}\n".format(
                el["channel"], op, ";".join(el["raw_forks"])
            ))

        logger.debug("Setting raw inputs: {}".format(primary_inputs))
        logger.debug("Setting forks attribute to: {}".format(self.forks))
        self._context = {**self._context,
                         **{"forks": "\n".join(self.forks),
                            "main_inputs": "\n".join(primary_inputs)}}

Sets the main input channels of the pipeline and their forks .


def set_secondary_inputs(self, channel_dict):
        """ Adds secondary inputs to the start of the pipeline.

        This channels are inserted into the pipeline file as they are
        provided in the values of the argument.

        Parameters
        ----------
        channel_dict : dict
            Each entry should be <parameter>: <channel string>.
        """

        logger.debug("Setting secondary inputs: {}".format(channel_dict))

        secondary_input_str = "\n".join(list(channel_dict.values()))
        self._context = {**self._context,
                         **{"secondary_inputs": secondary_input_str}}

def
set_secondary_inputs
(
self
,
channel_dict
)
:
logger
.
debug
(
"Setting secondary inputs: {}"
.
format
(
channel_dict
)
)
secondary_input_str
=
"\n"
.
join
(
list
(
channel_dict
.
values
(
)
)
)
self
.
_context
=
{
*
*
self
.
_context
,
*
*
{
"secondary_inputs"
:
secondary_input_str
}
}

Adds secondary inputs to the start of the pipeline .


def set_extra_inputs(self, channel_dict):
        """Sets the initial definition of the extra input channels.

        The ``channel_dict`` argument should contain the input type and
        destination channel of each parameter (which is the key)::

            channel_dict = {
                "param1": {
                    "input_type": "fasta"
                    "channels": ["abricate_2_3", "chewbbaca_3_4"]
                }
            }

        Parameters
        ----------
        channel_dict : dict
            Dictionary with the extra_input parameter as key, and a dictionary
            as a value with the input_type and destination channels
        """

        extra_inputs = []

        for param, info in channel_dict.items():

            # Update the process' parameters with the raw input
            raw_channel = self.RAW_MAPPING[info["input_type"]]
            self.params[param] = {
                "default": raw_channel["default_value"],
                "description": raw_channel["description"]
            }

            channel_name = "IN_{}_extraInput".format(param)
            channel_str = self.RAW_MAPPING[info["input_type"]]["channel_str"]
            extra_inputs.append("{} = {}".format(channel_name,
                                                 channel_str.format(param)))

            op = "set" if len(info["channels"]) == 1 else "into"
            extra_inputs.append("{}.{}{{ {} }}".format(
                channel_name, op, ";".join(info["channels"])))

        self._context = {
            **self._context,
            **{"extra_inputs": "\n".join(extra_inputs)}
        }

Sets the initial definition of the extra input channels .


def _parse_coverage(header_str):
        """Attempts to retrieve the coverage value from the header string.

        It splits the header by "_" and then screens the list backwards in
        search of the first float value. This will be interpreted as the
        coverage value. If it cannot find a float value, it returns None.
        This search methodology is based on the strings of assemblers
        like spades and skesa that put the mean kmer coverage for each
        contig in its corresponding fasta header.

        Parameters
        ----------
        header_str : str
            String

        Returns
        -------
        float or None
            The coverage value for the contig. None if it cannot find the
            value in the provide string.
        """

        cov = None
        for i in header_str.split("_")[::-1]:
            try:
                cov = float(i)
                break
            except ValueError:
                continue

        return cov

def _parse_coverage ( header_str ) : cov = None for i in header_str . split ( "_" ) [ : : - 1 ] : try : cov = float ( i ) break except ValueError : continue return cov

Attempts to retrieve the coverage value from the header string .


def _parse_assembly(self, assembly_file):
        """Parse an assembly fasta file.

        This is a Fasta parsing method that populates the
        :py:attr:`~Assembly.contigs` attribute with data for each contig in the
        assembly.

        The insertion of data on the self.contigs is done by the
        :py:meth:`Assembly._populate_contigs` method, which also calculates
        GC content and proportions.

        Parameters
        ----------
        assembly_file : str
            Path to the assembly fasta file.

        """

        # Temporary storage of sequence data
        seq_temp = []
        # Id counter for contig that will serve as key in self.contigs
        contig_id = 0
        # Initialize kmer coverage and header
        cov, header = None, None

        with open(assembly_file) as fh:

            logger.debug("Starting iteration of assembly file: {}".format(
                assembly_file))
            for line in fh:
                # Skip empty lines
                if not line.strip():
                    continue
                else:
                    # Remove whitespace surrounding line for further processing
                    line = line.strip()

                if line.startswith(">"):
                    # If a sequence has already been populated, save the
                    # previous contig information
                    if seq_temp:
                        # Use join() to convert string list into the full
                        # contig string. This is generally much more efficient
                        # than successively concatenating strings.
                        seq = "".join(seq_temp)

                        logger.debug("Populating contig with contig_id '{}', "
                                     "header '{}' and cov '{}'".format(
                                        contig_id, header, cov))
                        self._populate_contigs(contig_id, header, cov, seq)

                        # Reset temporary sequence storage
                        seq_temp = []
                        contig_id += 1

                    header = line[1:]
                    cov = self._parse_coverage(line)

                else:
                    seq_temp.append(line)

            # Populate last contig entry
            logger.debug("Populating contig with contig_id '{}', "
                         "header '{}' and cov '{}'".format(
                            contig_id, header, cov))
            seq = "".join(seq_temp)
            self._populate_contigs(contig_id, header, cov, seq)


def _get_gc_content(sequence, length):
        """Get GC content and proportions.

        Parameters
        ----------
        sequence : str
            The complete sequence of the contig.
        length : int
            The length of the sequence contig.

        Returns
        -------
        x : dict
            Dictionary with the at/gc/n counts and proportions

        """

        # Get AT/GC/N counts
        at = sum(map(sequence.count, ["A", "T"]))
        gc = sum(map(sequence.count, ["G", "C"]))
        n = length - (at + gc)

        # Get AT/GC/N proportions
        at_prop = at / length
        gc_prop = gc / length
        n_prop = n / length

        return {"at": at, "gc": gc, "n": n,
                "at_prop": at_prop, "gc_prop": gc_prop, "n_prop": n_prop}

def
_get_gc_content
(
sequence
,
length
)
:
# Get AT/GC/N counts
at
=
sum
(
map
(
sequence
.
count
,
[
"A"
,
"T"
]
)
)
gc
=
sum
(
map
(
sequence
.
count
,
[
"G"
,
"C"
]
)
)
n
=
length
(
at
+
gc
)
# Get AT/GC/N proportions
at_prop
=
at
/
length
gc_prop
=
gc
/
length
n_prop
=
n
/
length
return
{
"at"
:
at
,
"gc"
:
gc
,
"n"
:
n
,
"at_prop"
:
at_prop
,
"gc_prop"
:
gc_prop
,
"n_prop"
:
n_prop
}


def filter_contigs(self, *comparisons):
        """Filters the contigs of the assembly according to user provided\
        comparisons.

        The comparisons must be a list of three elements with the
        :py:attr:`~Assembly.contigs` key, operator and test value. For
        example, to filter contigs with a minimum length of 250, a comparison
        would be::

            self.filter_contigs(["length", ">=", 250])

        The filtered contig ids will be stored in the
        :py:attr:`~Assembly.filtered_ids` list.

        The result of the test for all contigs will be stored in the
        :py:attr:`~Assembly.report` dictionary.

        Parameters
        ----------
        comparisons : list
            List with contig key, operator and value to test.

        """

        # Reset list of filtered ids
        self.filtered_ids = []
        self.report = {}

        gc_filters = [
            ["gc_prop", ">=", self.min_gc],
            ["gc_prop", "<=", 1 - self.min_gc]
        ]

        self.filters = list(comparisons) + gc_filters

        logger.debug("Filtering contigs using filters: {}".format(
            self.filters))

        for contig_id, contig in self.contigs.items():
            for key, op, value in list(comparisons) + gc_filters:
                if not self._test_truth(contig[key], op, value):
                    self.filtered_ids.append(contig_id)
                    self.report[contig_id] = "{}/{}/{}".format(key,
                                                               contig[key],
                                                               value)
                    break
                else:
                    self.report[contig_id] = "pass"

def
filter_contigs
(
self
,
*
comparisons
)
:
# Reset list of filtered ids
self
.
filtered_ids
=
[
]
self
.
report
=
{
}
gc_filters
=
[
[
"gc_prop"
,
">="
,
self
.
min_gc
]
,
[
"gc_prop"
,
"<="
,
1
self
.
min_gc
]
]
self
.
filters
=
list
(
comparisons
)
+
gc_filters
logger
.
debug
(
"Filtering contigs using filters: {}"
.
format
(
self
.
filters
)
)
for
contig_id
,
contig
in
self
.
contigs
.
items
(
)
:
for
key
,
op
,
value
in
list
(
comparisons
)
+
gc_filters
:
if
not
self
.
_test_truth
(
contig
[
key
]
,
op
,
value
)
:
self
.
filtered_ids
.
append
(
contig_id
)
self
.
report
[
contig_id
]
=
"{}/{}/{}"
.
format
(
key
,
contig
[
key
]
,
value
)
break
else
:
self
.
report
[
contig_id
]
=
"pass"

Filters the contigs of the assembly according to user provided \ comparisons .


def get_assembly_length(self):
        """Returns the length of the assembly, without the filtered contigs.

        Returns
        -------
        x : int
            Total length of the assembly.

        """

        return sum(
            [vals["length"] for contig_id, vals in self.contigs.items()
             if contig_id not in self.filtered_ids])

def get_assembly_length ( self ) : return sum ( [ vals [ "length" ] for contig_id , vals in self . contigs . items ( ) if contig_id not in self . filtered_ids ] )

Returns the length of the assembly without the filtered contigs .


def write_assembly(self, output_file, filtered=True):
        """Writes the assembly to a new file.

        The ``filtered`` option controls whether the new assembly will be
        filtered or not.

        Parameters
        ----------
        output_file : str
            Name of the output assembly file.
        filtered : bool
            If ``True``, does not include filtered ids.
        """

        logger.debug("Writing the filtered assembly into: {}".format(
            output_file))
        with open(output_file, "w") as fh:

            for contig_id, contig in self.contigs.items():
                if contig_id not in self.filtered_ids and filtered:
                    fh.write(">{}_{}\\n{}\\n".format(self.sample,
                                                     contig["header"],
                                                     contig["sequence"]))


def write_report(self, output_file):
        """Writes a report with the test results for the current assembly

        Parameters
        ----------
        output_file : str
            Name of the output assembly file.

        """

        logger.debug("Writing the assembly report into: {}".format(
            output_file))
        with open(output_file, "w") as fh:

            for contig_id, vals in self.report.items():
                fh.write("{}, {}\\n".format(contig_id, vals))

def
write_report
(
self
,
output_file
)
:
logger
.
debug
(
"Writing the assembly report into: {}"
.
format
(
output_file
)
)
with
open
(
output_file
,
"w"
)
as
fh
:
for
contig_id
,
vals
in
self
.
report
.
items
(
)
:
fh
.
write
(
"{}, {}\\n"
.
format
(
contig_id
,
vals
)
)


def remove_inner_forks(text):
    """Recursively removes nested brackets

    This function is used to remove nested brackets from fork strings using
    regular expressions

    Parameters
    ----------
    text: str
        The string that contains brackets with inner forks to be removed

    Returns
    -------
    text: str
        the string with only the processes that are not in inner forks, thus
        the processes that belong to a given fork.

    """

    n = 1  # run at least once for one level of fork
    # Then this loop assures that all brackets will get removed in a nested
    # structure
    while n:
        # this removes non-nested brackets
        text, n = re.subn(r'\([^()]*\)', '', text)

    return text


def inner_fork_insanity_checks(pipeline_string):
    """
    This function performs two sanity checks in the pipeline string. The first
    check, assures that each fork contains a lane token '|', while the second
    check looks for duplicated processes within the same fork.

    Parameters
    ----------
    pipeline_string: str
         String with the definition of the pipeline, e.g.::
             'processA processB processC(ProcessD | ProcessE)'

    """

    # first lets get all forks to a list.
    list_of_forks = []  # stores forks
    left_indexes = []  # stores indexes of left brackets

    # iterate through the string looking for '(' and ')'.
    for pos, char in enumerate(pipeline_string):
        if char == FORK_TOKEN:
            # saves pos to left_indexes list
            left_indexes.append(pos)
        elif char == CLOSE_TOKEN and len(left_indexes) > 0:
            # saves fork to list_of_forks
            list_of_forks.append(pipeline_string[left_indexes[-1] + 1: pos])
            # removes last bracket from left_indexes list
            left_indexes = left_indexes[:-1]

    # sort list in descending order of number of forks
    list_of_forks.sort(key=lambda x: x.count(FORK_TOKEN), reverse=True)

    # Now, we can iterate through list_of_forks and check for errors in each
    # fork
    for fork in list_of_forks:
        # remove inner forks for these checks since each fork has its own entry
        # in list_of_forks. Note that each fork is now sorted in descending
        # order which enables to remove sequentially the string for the fork
        # potentially with more inner forks
        for subfork in list_of_forks:
            # checks if subfork is contained in fork and if they are different,
            # avoiding to remove itself
            if subfork in list_of_forks and subfork != fork:
                # removes inner forks. Note that string has no spaces
                fork_simplified = fork.replace("({})".format(subfork), "")
            else:
                fork_simplified = fork

        # Checks if there is no fork separator character '|' within each fork
        if not len(fork_simplified.split(LANE_TOKEN)) > 1:
            raise SanityError("One of the forks doesn't have '|' "
                              "separator between the processes to fork. This is"
                              " the prime suspect: '({})'".format(fork))

This function performs two sanity checks in the pipeline string . The first check assures that each fork contains a lane token | while the second check looks for duplicated processes within the same fork .


def insanity_checks(pipeline_str):
    """Wrapper that performs all sanity checks on the pipeline string

    Parameters
    ----------
    pipeline_str : str
        String with the pipeline definition
    """

    # Gets rid of all spaces in string
    p_string = pipeline_str.replace(" ", "").strip()

    # some of the check functions use the pipeline_str as the user provided but
    # the majority uses the parsed p_string.
    checks = [
        [p_string, [
            empty_tasks,
            brackets_but_no_lanes,
            brackets_insanity_check,
            lane_char_insanity_check,
            final_char_insanity_check,
            fork_procs_insanity_check,
            start_proc_insanity_check,
            late_proc_insanity_check
        ]],
        [pipeline_str, [
            inner_fork_insanity_checks
        ]]
    ]

    # executes sanity checks in pipeline string before parsing it.
    for param, func_list in checks:
        for func in func_list:
            func(param)


def parse_pipeline(pipeline_str):
    """Parses a pipeline string into a list of dictionaries with the connections
     between processes

    Parameters
    ----------
    pipeline_str : str
        String with the definition of the pipeline, e.g.::
            'processA processB processC(ProcessD | ProcessE)'

    Returns
    -------
    pipeline_links : list

    """

    if os.path.exists(pipeline_str):
        logger.debug("Found pipeline file: {}".format(pipeline_str))
        with open(pipeline_str) as fh:
            pipeline_str = "".join([x.strip() for x in fh.readlines()])

    logger.info(colored_print("Resulting pipeline string:\n"))
    logger.info(colored_print(pipeline_str + "\n"))

    # Perform pipeline insanity checks
    insanity_checks(pipeline_str)

    logger.debug("Parsing pipeline string: {}".format(pipeline_str))

    pipeline_links = []
    lane = 1

    # Add unique identifiers to each process to allow a correct connection
    # between forks with same processes
    pipeline_str_modified, identifiers_to_tags = add_unique_identifiers(
        pipeline_str)

    # Get number of forks in the pipeline
    nforks = pipeline_str_modified.count(FORK_TOKEN)
    logger.debug("Found {} fork(s)".format(nforks))

    # If there are no forks, connect the pipeline as purely linear
    if not nforks:
        logger.debug("Detected linear pipeline string : {}".format(
            pipeline_str))
        linear_pipeline = ["__init__"] + pipeline_str_modified.split()
        pipeline_links.extend(linear_connection(linear_pipeline, lane))
        # Removes unique identifiers used for correctly assign fork parents with
        #  a possible same process name
        pipeline_links = remove_unique_identifiers(identifiers_to_tags,
                                                   pipeline_links)
        return pipeline_links

    for i in range(nforks):

        logger.debug("Processing fork {} in lane {}".format(i, lane))
        # Split the pipeline at each fork start position. fields[-1] will
        # hold the process after the fork. fields[-2] will hold the processes
        # before the fork.
        fields = pipeline_str_modified.split(FORK_TOKEN, i + 1)

        # Get the processes before the fork. This may be empty when the
        # fork is at the beginning of the pipeline.
        previous_process = fields[-2].split(LANE_TOKEN)[-1].split()
        logger.debug("Previous processes string: {}".format(fields[-2]))
        logger.debug("Previous processes list: {}".format(previous_process))
        # Get lanes after the fork
        next_lanes = get_lanes(fields[-1])
        logger.debug("Next lanes object: {}".format(next_lanes))
        # Get the immediate targets of the fork
        fork_sink = [x[0] for x in next_lanes]
        logger.debug("The fork sinks into the processes: {}".format(fork_sink))

        # The first fork is a special case, where the processes before AND
        # after the fork (until the start of another fork) are added to
        # the ``pipeline_links`` variable. Otherwise, only the processes
        # after the fork will be added
        if i == 0:
            # If there are no previous process, the fork is at the beginning
            # of the pipeline string. In this case, inject the special
            # "init" process.
            if not previous_process:
                previous_process = ["__init__"]
                lane = 0
            else:
                previous_process = ["__init__"] + previous_process

            # Add the linear modules before the fork
            pipeline_links.extend(
                linear_connection(previous_process, lane))

        fork_source = previous_process[-1]
        logger.debug("Fork source is set to: {}".format(fork_source))
        fork_lane = get_source_lane(previous_process, pipeline_links)
        logger.debug("Fork lane is set to: {}".format(fork_lane))
        # Add the forking modules
        pipeline_links.extend(
            fork_connection(fork_source, fork_sink, fork_lane, lane))
        # Add the linear connections in the subsequent lanes
        pipeline_links.extend(
            linear_lane_connection(next_lanes, lane))

        lane += len(fork_sink)

    pipeline_links = remove_unique_identifiers(identifiers_to_tags,
                                               pipeline_links)
    return pipeline_links


def get_source_lane(fork_process, pipeline_list):
    """Returns the lane of the last process that matches fork_process

    Parameters
    ----------
    fork_process : list
        List of processes before the fork.
    pipeline_list : list
        List with the pipeline connection dictionaries.

    Returns
    -------
    int
        Lane of the last process that matches fork_process
    """

    fork_source = fork_process[-1]
    fork_sig = [x for x in fork_process if x != "__init__"]

    for position, p in enumerate(pipeline_list[::-1]):

        if p["output"]["process"] == fork_source:

            lane = p["output"]["lane"]
            logger.debug("Possible source match found in position {} in lane"
                         " {}".format(position, lane))
            lane_sequence = [x["output"]["process"] for x in pipeline_list
                             if x["output"]["lane"] == lane]
            logger.debug("Testing lane sequence '{}' against fork signature"
                         " '{}'".format(lane_sequence, fork_sig))
            if lane_sequence == fork_sig:
                return p["output"]["lane"]

    return 0


def get_lanes(lanes_str):
    """From a raw pipeline string, get a list of lanes from the start
    of the current fork.

    When the pipeline is being parsed, it will be split at every fork
    position. The string at the right of the fork position will be provided
    to this function. It's job is to retrieve the lanes that result
    from that fork, ignoring any nested forks.

    Parameters
    ----------
    lanes_str : str
        Pipeline string after a fork split

    Returns
    -------
    lanes : list
        List of lists, with the list of processes for each lane

    """

    logger.debug("Parsing lanes from raw string: {}".format(lanes_str))

    # Temporarily stores the lanes string after removal of nested forks
    parsed_lanes = ""
    # Flag used to determined whether the cursor is inside or outside the
    # right fork
    infork = 0
    for i in lanes_str:

        # Nested fork started
        if i == FORK_TOKEN:
            infork += 1
        # Nested fork stopped
        if i == CLOSE_TOKEN:
            infork -= 1

        if infork < 0:
            break

        # Save only when in the right fork
        if infork == 0:
            # Ignore forking syntax tokens
            if i not in [FORK_TOKEN, CLOSE_TOKEN]:
                parsed_lanes += i

    return [x.split() for x in parsed_lanes.split(LANE_TOKEN)]

From a raw pipeline string get a list of lanes from the start of the current fork .


def linear_connection(plist, lane):
    """Connects a linear list of processes into a list of dictionaries

    Parameters
    ----------
    plist : list
        List with process names. This list should contain at least two entries.
    lane : int
        Corresponding lane of the processes

    Returns
    -------
    res : list
        List of dictionaries with the links between processes
    """

    logger.debug(
        "Establishing linear connection with processes: {}".format(plist))

    res = []
    previous = None

    for p in plist:
        # Skip first process
        if not previous:
            previous = p
            continue

        res.append({
            "input": {
                "process": previous,
                "lane": lane
            },
            "output": {
                "process": p,
                "lane": lane
            }
        })
        previous = p

    return res


def fork_connection(source, sink, source_lane, lane):
    """Makes the connection between a process and the first processes in the
    lanes to which it forks.

    The ``lane`` argument should correspond to the lane of the source process.
    For each lane in ``sink``, the lane counter will increase.

    Parameters
    ----------
    source : str
        Name of the process that is forking
    sink : list
        List of the processes where the source will fork to. Each element
        corresponds to the start of a lane.
    source_lane : int
        Lane of the forking process
    lane : int
        Lane of the source process

    Returns
    -------
    res : list
        List of dictionaries with the links between processes
    """

    logger.debug("Establishing forking of source '{}' into processes"
                 " '{}'. Source lane set to '{}' and lane set to '{}'".format(
                    source, sink, source_lane, lane))

    res = []
    # Increase the lane counter for the first lane
    lane_counter = lane + 1

    for p in sink:
        res.append({
            "input": {
                "process": source,
                "lane": source_lane
            },
            "output": {
                "process": p,
                "lane": lane_counter
            }
        })
        lane_counter += 1

    return res

Makes the connection between a process and the first processes in the lanes to which it forks .


def add_unique_identifiers(pipeline_str):
    """Returns the pipeline string with unique identifiers and a dictionary with
     references between the unique keys and the original values

    Parameters
    ----------
    pipeline_str : str
        Pipeline string

    Returns
    -------
    str
        Pipeline string with unique identifiers
    dict
        Match between process unique values and original names
    """

    # Add space at beginning and end of pipeline to allow regex mapping of final
    # process in linear pipelines
    pipeline_str_modified = " {} ".format(pipeline_str)

    # Regex to get all process names. Catch all words without spaces and that
    # are not fork tokens or pipes
    reg_find_proc = r"[^\s{}{}{}]+".format(LANE_TOKEN, FORK_TOKEN, CLOSE_TOKEN)
    process_names = re.findall(reg_find_proc, pipeline_str_modified)

    identifiers_to_tags = {}
    """
    dict: Matches new process names (identifiers) with original process
    names
    """

    new_process_names = []
    """
    list: New process names used to replace in the pipeline string
    """

    # Assigns the new process names by appending a numeric id at the end of
    # the process name
    for index, val in enumerate(process_names):
        if "=" in val:
            parts = val.split("=")
            new_id = "{}_{}={}".format(parts[0], index, parts[1])
        else:
            new_id = "{}_{}".format(val, index)

        # add new process with id
        new_process_names.append(new_id)
        # makes a match between new process name and original process name
        identifiers_to_tags[new_id] = val

    # Add space between forks, pipes and the process names for the replace
    # regex to work
    match_result = lambda match: " {} ".format(match.group())

    # force to add a space between each token so that regex modification can
    # be applied
    find = r'[{}{}{}]+'.format(FORK_TOKEN, LANE_TOKEN, CLOSE_TOKEN)
    pipeline_str_modified = re.sub(find, match_result, pipeline_str_modified)

    # Replace original process names by the unique identifiers
    for index, val in enumerate(process_names):
        # regex to replace process names with non assigned process ids
        # escape characters are required to match to the dict keys
        # (identifiers_to_tags), since python keys with escape characters
        # must be escaped
        find = r'{}[^_]'.format(val).replace("\\", "\\\\")
        pipeline_str_modified = re.sub(find, new_process_names[index] + " ",
                                       pipeline_str_modified, 1)

    return pipeline_str_modified, identifiers_to_tags


def remove_unique_identifiers(identifiers_to_tags, pipeline_links):
    """Removes unique identifiers and add the original process names to the
    already parsed pipelines

    Parameters
    ----------
    identifiers_to_tags : dict
        Match between unique process identifiers and process names
    pipeline_links: list
        Parsed pipeline list with unique identifiers

    Returns
    -------
    list
        Pipeline list with original identifiers
    """

    # Replaces the unique identifiers by the original process names
    for index, val in enumerate(pipeline_links):
        if val["input"]["process"] != "__init__":
            val["input"]["process"] = identifiers_to_tags[
                val["input"]["process"]]
        if val["output"]["process"] != "__init__":
            val["output"]["process"] = identifiers_to_tags[
                val["output"]["process"]]

    return pipeline_links


def _check_required_files(self):
        """Checks whetner the trace and log files are available
        """

        if not os.path.exists(self.trace_file):
            raise eh.InspectionError("The provided trace file could not be "
                                     "opened: {}".format(self.trace_file))

        if not os.path.exists(self.log_file):
            raise eh.InspectionError("The .nextflow.log files could not be "
                                     "opened. Are you sure you are in a "
                                     "nextflow project directory?")

def
_check_required_files
(
self
)
:
if
not
os
.
path
.
exists
(
self
.
trace_file
)
:
raise
eh
.
InspectionError
(
"The provided trace file could not be "
"opened: {}"
.
format
(
self
.
trace_file
)
)
if
not
os
.
path
.
exists
(
self
.
log_file
)
:
raise
eh
.
InspectionError
(
"The .nextflow.log files could not be "
"opened. Are you sure you are in a "
"nextflow project directory?"
)


def _header_mapping(header):
        """Parses the trace file header and retrieves the positions of each
        column key.

        Parameters
        ----------
        header : str
            The header line of nextflow's trace file

        Returns
        -------
        dict
            Mapping the column ID to its position (e.g.: {"tag":2})
        """

        return dict(
            (x.strip(), pos) for pos, x in enumerate(header.split("\t"))
        )

def _header_mapping ( header ) : return dict ( ( x . strip ( ) , pos ) for pos , x in enumerate ( header . split ( "\t" ) ) )

Parses the trace file header and retrieves the positions of each column key .