def like(self, **kwargs):
        '''
        When provided with keyword arguments of the form ``col=pattern``, this
        will limit the entities returned to those that include the provided
        pattern. Note that 'like' queries require that the ``prefix=True``
        option must have been provided as part of the column definition.

        Patterns allow for 4 wildcard characters, whose semantics are as
        follows:

            * *?* - will match 0 or 1 of any character
            * *\** - will match 0 or more of any character
            * *+* - will match 1 or more of any character
            * *!* - will match exactly 1 of any character

        As an example, imagine that you have enabled the required prefix
        matching on your ``User.email`` column. And lets say that you want to
        find everyone with an email address that contains the name 'frank'
        before the ``@`` sign. You can use either of the following patterns
        to discover those users.

            * *\*frank\*@*
            * *\*frank\*@*

        .. note:: Like queries implicitly start at the beginning of strings
          checked, so if you want to match a pattern that doesn't start at
          the beginning of a string, you should prefix it with one of the
          wildcard characters (like ``*`` as we did with the 'frank' pattern).
        '''
        new = []
        for k, v in kwargs.items():
            v = self._check(k, v, 'like')
            new.append(Pattern(k, v))
        return self.replace(filters=self._filters+tuple(new))

def like ( self , * * kwargs ) : new = [ ] for k , v in kwargs . items ( ) : v = self . _check ( k , v , 'like' ) new . append ( Pattern ( k , v ) ) return self . replace ( filters = self . _filters + tuple ( new ) )

When provided with keyword arguments of the form col = pattern this will limit the entities returned to those that include the provided pattern . Note that like queries require that the prefix = True option must have been provided as part of the column definition .


def cached_result(self, timeout):
        '''
        This will execute the query, returning the key where a ZSET of your
        results will be stored for pagination, further operations, etc.

        The timeout must be a positive integer number of seconds for which to
        set the expiration time on the key (this is to ensure that any cached
        query results are eventually deleted, unless you make the explicit
        step to use the PERSIST command).

        .. note:: Limit clauses are ignored and not passed.

        Usage::

            ukey = User.query.endswith(email='@gmail.com').cached_result(30)
            for i in xrange(0, conn.zcard(ukey), 100):
                # refresh the expiration
                conn.expire(ukey, 30)
                users = User.get(conn.zrange(ukey, i, i+99))
                ...
        '''
        if not (self._filters or self._order_by):
            raise QueryError("You are missing filter or order criteria")
        timeout = int(timeout)
        if timeout < 1:
            raise QueryError("You must specify a timeout >= 1, you gave %r"%timeout)
        return self._model._gindex.search(
            _connect(self._model), self._filters, self._order_by, timeout=timeout)

This will execute the query returning the key where a ZSET of your results will be stored for pagination further operations etc .


def first(self):
        '''
        Returns only the first result from the query, if any.
        '''
        lim = [0, 1]
        if self._limit:
            lim[0] = self._limit[0]
        if not self._filters and not self._order_by:
            for ent in self:
                return ent
            return None
        ids = self.limit(*lim)._search()
        if ids:
            return self._model.get(ids[0])
        return None

def first ( self ) : lim = [ 0 , 1 ] if self . _limit : lim [ 0 ] = self . _limit [ 0 ] if not self . _filters and not self . _order_by : for ent in self : return ent return None ids = self . limit ( * lim ) . _search ( ) if ids : return self . _model . get ( ids [ 0 ] ) return None

Returns only the first result from the query if any .


def delete(self, blocksize=100):
        '''
        Will delete the entities that match at the time the query is executed.

        Used like::

            MyModel.query.filter(email=...).delete()
            MyModel.query.endswith(email='@host.com').delete()

        .. warning:: can't be used on models on either side of a ``OneToMany``,
            ``ManyToOne``, or ``OneToOne`` relationship.
        '''

        from .columns import MODELS_REFERENCED
        if not self._model._no_fk or self._model._namespace in MODELS_REFERENCED:
            raise QueryError("Can't delete entities of models with foreign key relationships")

        de = []
        i = 0
        for result in self.iter_result(pagesize=blocksize):
            de.append(result)
            i += 1
            if i >= blocksize:
                session.delete(de) # one round-trip to delete "chunk" items
                del de[:]
                i = 0

        if de:
            session.delete(de)

Will delete the entities that match at the time the query is executed .


def _on_delete(ent):
    '''
    This function handles all on_delete semantics defined on OneToMany columns.

    This function only exists because 'cascade' is *very* hard to get right.
    '''
    seen_d = set([ent._pk])
    to_delete = [ent]
    seen_s = set()
    to_save = []

    def _set_default(ent, attr, de=NULL):
        pk = ent._pk
        if pk in seen_d:
            # going to be deleted, don't need to modify
            return

        col = ent.__class__._columns[attr]
        de = de if de is not NULL else col._default
        if de in (None, NULL):
            setattr(ent, attr, None)
        elif callable(col._default):
            setattr(ent, attr, col._default())
        else:
            setattr(ent, attr, col._default)

        if pk not in seen_s:
            seen_s.add(pk)
            to_save.append(ent)

    for self in to_delete:
        for tbl, attr, action in MODELS_REFERENCED.get(self._namespace, ()):
            if action == 'no action':
                continue

            refs = MODELS[tbl].get_by(**{attr: self.id})
            if not refs:
                continue

            if action == 'restrict':
                # raise the exception here for a better traceback
                raise _restrict(self, attr, refs)
            elif action == 'set null':
                for ref in refs:
                    _set_default(ref, attr, None)
                continue
            elif action == 'set default':
                for ref in refs:
                    _set_default(ref, attr)
                continue

            # otherwise col._on_delete == 'cascade'
            for ent in (refs if isinstance(refs, list) else [refs]):
                if ent._pk not in seen_d:
                    seen_d.add(ent._pk)
                    to_delete.append(ent)

    # If we got here, then to_delete includes all items to delete. Let's delete
    # them!
    for self in to_delete:
        self.delete(skip_on_delete_i_really_mean_it=SKIP_ON_DELETE)
    for self in to_save:
        # Careful not to resurrect deleted entities
        if self._pk not in seen_d:
            self.save()

This function handles all on_delete semantics defined on OneToMany columns .


def redis_prefix_lua(conn, dest, index, prefix, is_first, pattern=None):
    '''
    Performs the actual prefix, suffix, and pattern match operations.
    '''
    tkey = '%s:%s'%(index.partition(':')[0], uuid.uuid4())
    start, end = _start_end(prefix)
    return _redis_prefix_lua(conn,
        [dest, tkey, index],
        [start, end, pattern or prefix, int(pattern is not None), int(bool(is_first))]
    )

def redis_prefix_lua ( conn , dest , index , prefix , is_first , pattern = None ) : tkey = '%s:%s' % ( index . partition ( ':' ) [ 0 ] , uuid . uuid4 ( ) ) start , end = _start_end ( prefix ) return _redis_prefix_lua ( conn , [ dest , tkey , index ] , [ start , end , pattern or prefix , int ( pattern is not None ) , int ( bool ( is_first ) ) ] )

Performs the actual prefix suffix and pattern match operations .


def estimate_work_lua(conn, index, prefix):
    '''
    Estimates the total work necessary to calculate the prefix match over the
    given index with the provided prefix.
    '''
    if index.endswith(':idx'):
        args = [] if not prefix else list(prefix)
        if args:
            args[0] = '-inf' if args[0] is None else repr(float(args[0]))
            args[1] = 'inf' if args[1] is None else repr(float(args[1]))
        return _estimate_work_lua(conn, [index], args, force_eval=True)
    elif index.endswith(':geo'):
        return _estimate_work_lua(conn, [index], filter(None, [prefix]), force_eval=True)

    start, end = _start_end(prefix)
    return _estimate_work_lua(conn, [index], [start, '(' + end], force_eval=True)

def estimate_work_lua ( conn , index , prefix ) : if index . endswith ( ':idx' ) : args = [ ] if not prefix else list ( prefix ) if args : args [ 0 ] = '-inf' if args [ 0 ] is None else repr ( float ( args [ 0 ] ) ) args [ 1 ] = 'inf' if args [ 1 ] is None else repr ( float ( args [ 1 ] ) ) return _estimate_work_lua ( conn , [ index ] , args , force_eval = True ) elif index . endswith ( ':geo' ) : return _estimate_work_lua ( conn , [ index ] , filter ( None , [ prefix ] ) , force_eval = True ) start , end = _start_end ( prefix ) return _estimate_work_lua ( conn , [ index ] , [ start , '(' + end ] , force_eval = True )

Estimates the total work necessary to calculate the prefix match over the given index with the provided prefix .


def search(self, conn, filters, order_by, offset=None, count=None, timeout=None):
        '''
        Search for model ids that match the provided filters.

        Arguments:

            * *filters* - A list of filters that apply to the search of one of
              the following two forms:

                1. ``'column:string'`` - a plain string will match a word in a
                   text search on the column

                .. note:: Read the documentation about the ``Query`` object
                  for what is actually passed during text search

                2. ``('column', min, max)`` - a numeric column range search,
                   between min and max (inclusive by default)

                .. note:: Read the documentation about the ``Query`` object
                  for information about open-ended ranges

                3. ``['column:string1', 'column:string2']`` - will match any
                   of the provided words in a text search on the column

                4. ``Prefix('column', 'prefix')`` - will match prefixes of
                   words in a text search on the column

                5. ``Suffix('column', 'suffix')`` - will match suffixes of
                   words in a text search on the column

                6. ``Pattern('column', 'pattern')`` - will match patterns over
                   words in a text search on the column

            * *order_by* - A string that names the numeric column by which to
              sort the results by. Prefixing with '-' will return results in
              descending order

            .. note:: While you can technically pass a non-numeric index as an
              *order_by* clause, the results will basically be to order the
              results by string comparison of the ids (10 will come before 2).

            .. note:: If you omit the ``order_by`` argument, results will be
              ordered by the last filter. If the last filter was a text
              filter, see the previous note. If the last filter was numeric,
              then results will be ordered by that result.

            * *offset* - A numeric starting offset for results
            * *count* - The maximum number of results to return from the query
        '''
        # prepare the filters
        pipe, intersect, temp_id = self._prepare(conn, filters)

        # handle ordering
        if order_by:
            reverse = order_by and order_by.startswith('-')
            order_clause = '%s:%s:idx'%(self.namespace, order_by.lstrip('-'))
            intersect(temp_id, {temp_id:0, order_clause: -1 if reverse else 1})

        # handle returning the temporary result key
        if timeout is not None:
            pipe.expire(temp_id, timeout)
            pipe.execute()
            return temp_id

        offset = offset if offset is not None else 0
        end = (offset + count - 1) if count and count > 0 else -1
        pipe.zrange(temp_id, offset, end)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

Search for model ids that match the provided filters .


def count(self, conn, filters):
        '''
        Returns the count of the items that match the provided filters.

        For the meaning of what the ``filters`` argument means, see the
        ``.search()`` method docs.
        '''
        pipe, intersect, temp_id = self._prepare(conn, filters)
        pipe.zcard(temp_id)
        pipe.delete(temp_id)
        return pipe.execute()[-2]

def count ( self , conn , filters ) : pipe , intersect , temp_id = self . _prepare ( conn , filters ) pipe . zcard ( temp_id ) pipe . delete ( temp_id ) return pipe . execute ( ) [ - 2 ]

Returns the count of the items that match the provided filters .


def _connect(obj):
    '''
    Tries to get the _conn attribute from a model. Barring that, gets the
    global default connection using other methods.
    '''
    from .columns import MODELS
    if isinstance(obj, MODELS['Model']):
        obj = obj.__class__
    if hasattr(obj, '_conn'):
        return obj._conn
    if hasattr(obj, 'CONN'):
        return obj.CONN
    return get_connection()

def _connect ( obj ) : from . columns import MODELS if isinstance ( obj , MODELS [ 'Model' ] ) : obj = obj . __class__ if hasattr ( obj , '_conn' ) : return obj . _conn if hasattr ( obj , 'CONN' ) : return obj . CONN return get_connection ( )

Tries to get the _conn attribute from a model . Barring that gets the global default connection using other methods .


def FULL_TEXT(val):
    '''
    This is a basic full-text index keygen function. Words are lowercased, split
    by whitespace, and stripped of punctuation from both ends before an inverted
    index is created for term searching.
    '''
    if isinstance(val, float):
        val = repr(val)
    elif val in (None, ''):
        return None
    elif not isinstance(val, six.string_types):
        if six.PY3 and isinstance(val, bytes):
            val = val.decode('latin-1')
        else:
            val = str(val)
    r = sorted(set([x for x in [s.lower().strip(string.punctuation) for s in val.split()] if x]))
    if not isinstance(val, str):  # unicode on py2k
        return [s.encode('utf-8') for s in r]
    return r

def
FULL_TEXT
(
val
)
:
if
isinstance
(
val
,
float
)
:
val
=
repr
(
val
)
elif
val
in
(
,
''
)
:
return
elif
not
isinstance
(
val
,
six
.
string_types
)
:
if
six
.
PY3
and
isinstance
(
val
,
bytes
)
:
val
=
val
.
decode
(
'latin-1'
)
else
:
val
=
str
(
val
)
r
=
sorted
(
set
(
[
x
for
x
in
[
s
.
lower
(
)
.
strip
(
string
.
punctuation
)
for
s
in
val
.
split
(
)
]
if
x
]
)
)
if
not
isinstance
(
val
,
str
)
:
# unicode on py2k
return
[
s
.
encode
(
'utf-8'
)
for
s
in
r
]
return
r

This is a basic full - text index keygen function . Words are lowercased split by whitespace and stripped of punctuation from both ends before an inverted index is created for term searching .


def refresh_indices(model, block_size=100):
    '''
    This utility function will iterate over all entities of a provided model,
    refreshing their indices. This is primarily useful after adding an index
    on a column.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of entities you want to fetch from
          Redis at a time, defaulting to 100

    This function will yield its progression through re-indexing all of your
    entities.

    Example use::

        for progress, total in refresh_indices(MyModel, block_size=200):
            print "%s of %s"%(progress, total)

    .. note:: This uses the session object to handle index refresh via calls to
      ``.commit()``. If you have any outstanding entities known in the
      session, they will be committed.
    '''
    conn = _connect(model)
    max_id = int(conn.get('%s:%s:'%(model._namespace, model._pkey)) or '0')
    block_size = max(block_size, 10)
    for i in range(1, max_id+1, block_size):
        # fetches entities, keeping a record in the session
        models = model.get(list(range(i, i+block_size)))
        models # for pyflakes
        # re-save un-modified data, resulting in index-only updates
        session.commit(all=True)
        yield min(i+block_size, max_id), max_id

This utility function will iterate over all entities of a provided model refreshing their indices . This is primarily useful after adding an index on a column .


def clean_old_index(model, block_size=100, **kwargs):
    '''
    This utility function will clean out old index data that was accidentally
    left during item deletion in rom versions <= 0.27.0 . You should run this
    after you have upgraded all of your clients to version 0.28.0 or later.

    Arguments:

        * *model* - the model whose entities you want to reindex
        * *block_size* - the maximum number of items to check at a time
          defaulting to 100

    This function will yield its progression through re-checking all of the
    data that could be left over.

    Example use::

        for progress, total in clean_old_index(MyModel, block_size=200):
            print "%s of %s"%(progress, total)
    '''

    conn = _connect(model)
    version = list(map(int, conn.info()['redis_version'].split('.')[:2]))
    has_hscan = version >= [2, 8]
    pipe = conn.pipeline(True)
    prefix = '%s:'%model._namespace
    index = prefix + ':'
    block_size = max(block_size, 10)

    force_hscan = kwargs.get('force_hscan', False)
    if (has_hscan or force_hscan) and force_hscan is not None:
        max_id = conn.hlen(index)
        cursor = None
        scanned = 0
        while cursor != b'0':
            cursor, remove = _scan_index_lua(conn, [index, prefix], [cursor or '0', block_size, 0, 0])
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            scanned += block_size
            if scanned > max_id:
                max_id = scanned + 1
            yield scanned, max_id

        # need to scan over unique indexes :/
        for uniq in chain(model._unique, model._cunique):
            name = uniq if isinstance(uniq, six.string_types) else ':'.join(uniq)
            idx = prefix + name + ':uidx'

            cursor = None
            while cursor != b'0':
                cursor, remove = _scan_index_lua(conn, [idx, prefix], [cursor or '0', block_size, 1, 0])
                if remove:
                    conn.hdel(idx, *remove)

                scanned += block_size
                if scanned > max_id:
                    max_id = scanned + 1
                yield scanned, max_id
    else:
        if model._unique or model._cunique:
            if has_hscan:
                warnings.warn("You have disabled the use of HSCAN to clean up indexes, this will prevent unique index cleanup", stacklevel=2)
            else:
                warnings.warn("Unique indexes cannot be cleaned up in Redis versions prior to 2.8", stacklevel=2)

        max_id = int(conn.get('%s%s:'%(prefix, model._pkey)) or '0')
        for i in range(1, max_id+1, block_size):
            ids = list(range(i, min(i+block_size, max_id+1)))
            for id in ids:
                pipe.exists(prefix + str(id))
                pipe.hexists(index, id)

            result = iter(pipe.execute())
            remove = [id for id, ent, ind in zip(ids, result, result) if ind and not ent]
            if remove:
                _clean_index_lua(conn, [model._namespace], remove)

            yield min(i+block_size, max_id-1), max_id

    yield max_id, max_id

This utility function will clean out old index data that was accidentally left during item deletion in rom versions < = 0 . 27 . 0 . You should run this after you have upgraded all of your clients to version 0 . 28 . 0 or later .


def add(self, obj):
        '''
        Adds an entity to the session.
        '''
        if self.null_session:
            return
        self._init()
        pk = obj._pk
        if not pk.endswith(':None'):
            self.known[pk] = obj
            self.wknown[pk] = obj

def add ( self , obj ) : if self . null_session : return self . _init ( ) pk = obj . _pk if not pk . endswith ( ':None' ) : self . known [ pk ] = obj self . wknown [ pk ] = obj


def get(self, pk):
        '''
        Fetches an entity from the session based on primary key.
        '''
        self._init()
        return self.known.get(pk) or self.wknown.get(pk)

def get ( self , pk ) : self . _init ( ) return self . known . get ( pk ) or self . wknown . get ( pk )

Fetches an entity from the session based on primary key .


def redis_writer_lua(conn, pkey, namespace, id, unique, udelete, delete,
                     data, keys, scored, prefix, suffix, geo, old_data, is_delete):
    '''
    ... Actually write data to Redis. This is an internal detail. Please don't
    call me directly.
    '''
    ldata = []
    for pair in data.items():
        ldata.extend(pair)

    for item in prefix:
        item.append(_prefix_score(item[-1]))
    for item in suffix:
        item.append(_prefix_score(item[-1]))

    data = [json.dumps(x, default=_fix_bytes) for x in
            (unique, udelete, delete, ldata, keys, scored, prefix, suffix, geo, is_delete, old_data)]
    result = _redis_writer_lua(conn, [], [namespace, id] + data)

    if isinstance(conn, _Pipeline):
        # we're in a pipelined write situation, don't parse the pipeline :P
        return

    if six.PY3:
        result = result.decode()

    result = json.loads(result)
    if 'unique' in result:
        result = result['unique']
        raise UniqueKeyViolation(
            "Value %r for %s:%s:uidx not distinct (failed for pk=%s)"%(
                unique[result], namespace, result, id),
            namespace, id)

    if 'race' in result:
        result = result['race']
        if pkey in result:
            raise EntityDeletedError(
                "Entity %s:%s deleted by another writer; use .save(force=True) to re-save"%(
                    namespace, id),
                namespace, id)

        raise DataRaceError(
            "%s:%s Column(s) %r updated by another writer, write aborted!"%(
                namespace, id, result),
            namespace, id)

... Actually write data to Redis . This is an internal detail . Please don t call me directly .


def save(self, full=False, force=False):
        '''
        Saves the current entity to Redis. Will only save changed data by
        default, but you can force a full save by passing ``full=True``.

        If the underlying entity was deleted and you want to re-save the entity,
        you can pass ``force=True`` to force a full re-save of the entity.
        '''
        # handle the pre-commit hooks
        was_new = self._new
        if was_new:
            self._before_insert()
        else:
            self._before_update()

        new = self.to_dict()
        ret, data = self._apply_changes(
            self._last, new, full or self._new or force, is_new=self._new or force)
        self._last = data
        self._new = False
        self._modified = False
        self._deleted = False
        # handle the post-commit hooks
        if was_new:
            self._after_insert()
        else:
            self._after_update()
        return ret

def
save
(
self
,
full
=
False
,
force
=
False
)
:
# handle the pre-commit hooks
was_new
=
self
.
_new
if
was_new
:
self
.
_before_insert
(
)
else
:
self
.
_before_update
(
)
new
=
self
.
to_dict
(
)
ret
,
data
=
self
.
_apply_changes
(
self
.
_last
,
new
,
full
or
self
.
_new
or
force
,
is_new
=
self
.
_new
or
force
)
self
.
_last
=
data
self
.
_new
=
False
self
.
_modified
=
False
self
.
_deleted
=
False
# handle the post-commit hooks
if
was_new
:
self
.
_after_insert
(
)
else
:
self
.
_after_update
(
)
return
ret

Saves the current entity to Redis . Will only save changed data by default but you can force a full save by passing full = True .


def delete(self, **kwargs):
        '''
        Deletes the entity immediately. Also performs any on_delete operations
        specified as part of column definitions.
        '''
        if kwargs.get('skip_on_delete_i_really_mean_it') is not SKIP_ON_DELETE:
            # handle the pre-commit hook
            self._before_delete()
            # handle any foreign key references + cascade options
            _on_delete(self)

        session.forget(self)
        self._apply_changes(self._last, {}, delete=True, _conn=kwargs.get('_conn'))
        self._modified = True
        self._deleted = True
        # handle the post-commit hooks
        if kwargs.get('skip_on_delete_i_really_mean_it') is not SKIP_ON_DELETE:
            self._after_delete()

Deletes the entity immediately . Also performs any on_delete operations specified as part of column definitions .


def get(cls, ids):
        '''
        Will fetch one or more entities of this type from the session or
        Redis.

        Used like::

            MyModel.get(5)
            MyModel.get([1, 6, 2, 4])

        Passing a list or a tuple will return multiple entities, in the same
        order that the ids were passed.
        '''
        conn = _connect(cls)
        # prepare the ids
        single = not isinstance(ids, (list, tuple, set, frozenset))
        if single:
            ids = [ids]
        pks = ['%s:%s'%(cls._namespace, id) for id in map(int, ids)]
        # get from the session, if possible
        out = list(map(session.get, pks))
        # if we couldn't get an instance from the session, load from Redis
        if None in out:
            pipe = conn.pipeline(True)
            idxs = []
            # Fetch missing data
            for i, data in enumerate(out):
                if data is None:
                    idxs.append(i)
                    pipe.hgetall(pks[i])
            # Update output list
            for i, data in zip(idxs, pipe.execute()):
                if data:
                    if six.PY3:
                        data = dict((k.decode(), v.decode()) for k, v in data.items())
                    out[i] = cls(_loading=True, **data)
            # Get rid of missing models
            out = [x for x in out if x]
        if single:
            return out[0] if out else None
        return out

Will fetch one or more entities of this type from the session or Redis .


def register(cls, type, reduce_func):
        """Attach a reducer function to a given type in the dispatch table."""
        if sys.version_info < (3,):
            # Python 2 pickler dispatching is not explicitly customizable.
            # Let us use a closure to workaround this limitation.
            def dispatcher(cls, obj):
                reduced = reduce_func(obj)
                cls.save_reduce(obj=obj, *reduced)
            cls.dispatch_table[type] = dispatcher
        else:
            cls.dispatch_table[type] = reduce_func

Attach a reducer function to a given type in the dispatch table .


def _sem_open(name, value=None):
    """ Construct or retrieve a semaphore with the given name

    If value is None, try to retrieve an existing named semaphore.
    Else create a new semaphore with the given value
    """
    if value is None:
        handle = pthread.sem_open(ctypes.c_char_p(name), 0)
    else:
        handle = pthread.sem_open(ctypes.c_char_p(name), SEM_OFLAG, SEM_PERM,
                                  ctypes.c_int(value))

    if handle == SEM_FAILURE:
        e = ctypes.get_errno()
        if e == errno.EEXIST:
            raise FileExistsError("a semaphore named %s already exists" % name)
        elif e == errno.ENOENT:
            raise FileNotFoundError('cannot find semaphore named %s' % name)
        elif e == errno.ENOSYS:
            raise NotImplementedError('No semaphore implementation on this '
                                      'system')
        else:
            raiseFromErrno()

    return handle

def
_sem_open
(
name
,
value
=
)
:
if
value
is
:
handle
=
pthread
.
sem_open
(
ctypes
.
c_char_p
(
name
)
,
0
)
else
:
handle
=
pthread
.
sem_open
(
ctypes
.
c_char_p
(
name
)
,
SEM_OFLAG
,
SEM_PERM
,
ctypes
.
c_int
(
value
)
)
if
handle
==
SEM_FAILURE
:
e
=
ctypes
.
get_errno
(
)
if
e
==
errno
.
EEXIST
:
raise
FileExistsError
(
"a semaphore named %s already exists"
%
name
)
elif
e
==
errno
.
ENOENT
:
raise
FileNotFoundError
(
'cannot find semaphore named %s'
%
name
)
elif
e
==
errno
.
ENOSYS
:
raise
NotImplementedError
(
'No semaphore implementation on this '
'system'
)
else
:
raiseFromErrno
(
)
return
handle


def cpu_count():
    """Return the number of CPUs the current process can use.

    The returned number of CPUs accounts for:
     * the number of CPUs in the system, as given by
       ``multiprocessing.cpu_count``;
     * the CPU affinity settings of the current process
       (available with Python 3.4+ on some Unix systems);
     * CFS scheduler CPU bandwidth limit (available on Linux only, typically
       set by docker and similar container orchestration systems);
     * the value of the LOKY_MAX_CPU_COUNT environment variable if defined.
    and is given as the minimum of these constraints.
    It is also always larger or equal to 1.
    """
    import math

    try:
        cpu_count_mp = mp.cpu_count()
    except NotImplementedError:
        cpu_count_mp = 1

    # Number of available CPUs given affinity settings
    cpu_count_affinity = cpu_count_mp
    if hasattr(os, 'sched_getaffinity'):
        try:
            cpu_count_affinity = len(os.sched_getaffinity(0))
        except NotImplementedError:
            pass

    # CFS scheduler CPU bandwidth limit
    # available in Linux since 2.6 kernel
    cpu_count_cfs = cpu_count_mp
    cfs_quota_fname = "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"
    cfs_period_fname = "/sys/fs/cgroup/cpu/cpu.cfs_period_us"
    if os.path.exists(cfs_quota_fname) and os.path.exists(cfs_period_fname):
        with open(cfs_quota_fname, 'r') as fh:
            cfs_quota_us = int(fh.read())
        with open(cfs_period_fname, 'r') as fh:
            cfs_period_us = int(fh.read())

        if cfs_quota_us > 0 and cfs_period_us > 0:
            # Make sure this quantity is an int as math.ceil returns a
            # float in python2.7. (See issue #165)
            cpu_count_cfs = int(math.ceil(cfs_quota_us / cfs_period_us))

    # User defined soft-limit passed as an loky specific environment variable.
    cpu_count_loky = int(os.environ.get('LOKY_MAX_CPU_COUNT', cpu_count_mp))
    aggregate_cpu_count = min(cpu_count_mp, cpu_count_affinity, cpu_count_cfs,
                              cpu_count_loky)
    return max(aggregate_cpu_count, 1)

Return the number of CPUs the current process can use .


def _sendback_result(result_queue, work_id, result=None, exception=None):
    """Safely send back the given result or exception"""
    try:
        result_queue.put(_ResultItem(work_id, result=result,
                                     exception=exception))
    except BaseException as e:
        exc = _ExceptionWithTraceback(e)
        result_queue.put(_ResultItem(work_id, exception=exc))

def _sendback_result ( result_queue , work_id , result = None , exception = None ) : try : result_queue . put ( _ResultItem ( work_id , result = result , exception = exception ) ) except BaseException as e : exc = _ExceptionWithTraceback ( e ) result_queue . put ( _ResultItem ( work_id , exception = exc ) )


def _process_worker(call_queue, result_queue, initializer, initargs,
                    processes_management_lock, timeout, worker_exit_lock,
                    current_depth):
    """Evaluates calls from call_queue and places the results in result_queue.

    This worker is run in a separate process.

    Args:
        call_queue: A ctx.Queue of _CallItems that will be read and
            evaluated by the worker.
        result_queue: A ctx.Queue of _ResultItems that will written
            to by the worker.
        initializer: A callable initializer, or None
        initargs: A tuple of args for the initializer
        process_management_lock: A ctx.Lock avoiding worker timeout while some
            workers are being spawned.
        timeout: maximum time to wait for a new item in the call_queue. If that
            time is expired, the worker will shutdown.
        worker_exit_lock: Lock to avoid flagging the executor as broken on
            workers timeout.
        current_depth: Nested parallelism level, to avoid infinite spawning.
    """
    if initializer is not None:
        try:
            initializer(*initargs)
        except BaseException:
            _base.LOGGER.critical('Exception in initializer:', exc_info=True)
            # The parent will notice that the process stopped and
            # mark the pool broken
            return

    # set the global _CURRENT_DEPTH mechanism to limit recursive call
    global _CURRENT_DEPTH
    _CURRENT_DEPTH = current_depth
    _process_reference_size = None
    _last_memory_leak_check = None
    pid = os.getpid()

    mp.util.debug('Worker started with timeout=%s' % timeout)
    while True:
        try:
            call_item = call_queue.get(block=True, timeout=timeout)
            if call_item is None:
                mp.util.info("Shutting down worker on sentinel")
        except queue.Empty:
            mp.util.info("Shutting down worker after timeout %0.3fs"
                         % timeout)
            if processes_management_lock.acquire(block=False):
                processes_management_lock.release()
                call_item = None
            else:
                mp.util.info("Could not acquire processes_management_lock")
                continue
        except BaseException as e:
            previous_tb = traceback.format_exc()
            try:
                result_queue.put(_RemoteTraceback(previous_tb))
            except BaseException:
                # If we cannot format correctly the exception, at least print
                # the traceback.
                print(previous_tb)
            sys.exit(1)
        if call_item is None:
            # Notify queue management thread about clean worker shutdown
            result_queue.put(pid)
            with worker_exit_lock:
                return
        try:
            r = call_item()
        except BaseException as e:
            exc = _ExceptionWithTraceback(e)
            result_queue.put(_ResultItem(call_item.work_id, exception=exc))
        else:
            _sendback_result(result_queue, call_item.work_id, result=r)
            del r

        # Free the resource as soon as possible, to avoid holding onto
        # open files or shared memory that is not needed anymore
        del call_item

        if _USE_PSUTIL:
            if _process_reference_size is None:
                # Make reference measurement after the first call
                _process_reference_size = _get_memory_usage(pid, force_gc=True)
                _last_memory_leak_check = time()
                continue
            if time() - _last_memory_leak_check > _MEMORY_LEAK_CHECK_DELAY:
                mem_usage = _get_memory_usage(pid)
                _last_memory_leak_check = time()
                if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE:
                    # Memory usage stays within bounds: everything is fine.
                    continue

                # Check again memory usage; this time take the measurement
                # after a forced garbage collection to break any reference
                # cycles.
                mem_usage = _get_memory_usage(pid, force_gc=True)
                _last_memory_leak_check = time()
                if mem_usage - _process_reference_size < _MAX_MEMORY_LEAK_SIZE:
                    # The GC managed to free the memory: everything is fine.
                    continue

                # The process is leaking memory: let the master process
                # know that we need to start a new worker.
                mp.util.info("Memory leak detected: shutting down worker")
                result_queue.put(pid)
                with worker_exit_lock:
                    return
        else:
            # if psutil is not installed, trigger gc.collect events
            # regularly to limit potential memory leaks due to reference cycles
            if ((_last_memory_leak_check is None) or
                    (time() - _last_memory_leak_check >
                     _MEMORY_LEAK_CHECK_DELAY)):
                gc.collect()
                _last_memory_leak_check = time()

Evaluates calls from call_queue and places the results in result_queue .


def _add_call_item_to_queue(pending_work_items,
                            running_work_items,
                            work_ids,
                            call_queue):
    """Fills call_queue with _WorkItems from pending_work_items.

    This function never blocks.

    Args:
        pending_work_items: A dict mapping work ids to _WorkItems e.g.
            {5: <_WorkItem...>, 6: <_WorkItem...>, ...}
        work_ids: A queue.Queue of work ids e.g. Queue([5, 6, ...]). Work ids
            are consumed and the corresponding _WorkItems from
            pending_work_items are transformed into _CallItems and put in
            call_queue.
        call_queue: A ctx.Queue that will be filled with _CallItems
            derived from _WorkItems.
    """
    while True:
        if call_queue.full():
            return
        try:
            work_id = work_ids.get(block=False)
        except queue.Empty:
            return
        else:
            work_item = pending_work_items[work_id]

            if work_item.future.set_running_or_notify_cancel():
                running_work_items += [work_id]
                call_queue.put(_CallItem(work_id,
                                         work_item.fn,
                                         work_item.args,
                                         work_item.kwargs),
                               block=True)
            else:
                del pending_work_items[work_id]
                continue

def _add_call_item_to_queue ( pending_work_items , running_work_items , work_ids , call_queue ) : while True : if call_queue . full ( ) : return try : work_id = work_ids . get ( block = False ) except queue . Empty : return else : work_item = pending_work_items [ work_id ] if work_item . future . set_running_or_notify_cancel ( ) : running_work_items += [ work_id ] call_queue . put ( _CallItem ( work_id , work_item . fn , work_item . args , work_item . kwargs ) , block = True ) else : del pending_work_items [ work_id ] continue


def _ensure_executor_running(self):
        """ensures all workers and management thread are running
        """
        with self._processes_management_lock:
            if len(self._processes) != self._max_workers:
                self._adjust_process_count()
            self._start_queue_management_thread()

def _ensure_executor_running ( self ) : with self . _processes_management_lock : if len ( self . _processes ) != self . _max_workers : self . _adjust_process_count ( ) self . _start_queue_management_thread ( )


def wrap_non_picklable_objects(obj, keep_wrapper=True):
    """Wrapper for non-picklable object to use cloudpickle to serialize them.

    Note that this wrapper tends to slow down the serialization process as it
    is done with cloudpickle which is typically slower compared to pickle. The
    proper way to solve serialization issues is to avoid defining functions and
    objects in the main scripts and to implement __reduce__ functions for
    complex classes.
    """
    if not cloudpickle:
        raise ImportError("could not import cloudpickle. Please install "
                          "cloudpickle to allow extended serialization. "
                          "(`pip install cloudpickle`).")

    # If obj is a  class, create a CloudpickledClassWrapper which instantiates
    # the object internally and wrap it directly in a CloudpickledObjectWrapper
    if inspect.isclass(obj):
        class CloudpickledClassWrapper(CloudpickledObjectWrapper):
            def __init__(self, *args, **kwargs):
                self._obj = obj(*args, **kwargs)
                self._keep_wrapper = keep_wrapper

        CloudpickledClassWrapper.__name__ = obj.__name__
        return CloudpickledClassWrapper

    # If obj is an instance of a class, just wrap it in a regular
    # CloudpickledObjectWrapper
    return _wrap_non_picklable_objects(obj, keep_wrapper=keep_wrapper)

Wrapper for non - picklable object to use cloudpickle to serialize them .


def start(self, initializer=None, initargs=()):
        '''Spawn a server process for this manager object'''
        assert self._state.value == State.INITIAL

        if (initializer is not None
                and not hasattr(initializer, '__call__')):
            raise TypeError('initializer must be a callable')

        # pipe over which we will retrieve address of server
        reader, writer = mp.Pipe(duplex=False)

        # spawn process which runs a server
        self._process = Process(
            target=type(self)._run_server,
            args=(self._registry, self._address, bytes(self._authkey),
                  self._serializer, writer, initializer, initargs),
        )
        ident = ':'.join(str(i) for i in self._process._identity)
        self._process.name = type(self).__name__ + '-' + ident
        self._process.start()

        # get address of server
        writer.close()
        self._address = reader.recv()
        reader.close()

        # register a finalizer
        self._state.value = State.STARTED
        self.shutdown = mp.util.Finalize(
            self, type(self)._finalize_manager,
            args=(self._process, self._address, self._authkey,
                  self._state, self._Client),
            exitpriority=0
        )


def DupFd(fd):
    '''Return a wrapper for an fd.'''
    popen_obj = get_spawning_popen()
    if popen_obj is not None:
        return popen_obj.DupFd(popen_obj.duplicate_for_child(fd))
    elif HAVE_SEND_HANDLE and sys.version_info[:2] > (3, 3):
        from multiprocessing import resource_sharer
        return resource_sharer.DupFd(fd)
    else:
        raise TypeError(
            'Cannot pickle connection object. This object can only be '
            'passed when spawning a new process'
        )


def get_reusable_executor(max_workers=None, context=None, timeout=10,
                          kill_workers=False, reuse="auto",
                          job_reducers=None, result_reducers=None,
                          initializer=None, initargs=()):
    """Return the current ReusableExectutor instance.

    Start a new instance if it has not been started already or if the previous
    instance was left in a broken state.

    If the previous instance does not have the requested number of workers, the
    executor is dynamically resized to adjust the number of workers prior to
    returning.

    Reusing a singleton instance spares the overhead of starting new worker
    processes and importing common python packages each time.

    ``max_workers`` controls the maximum number of tasks that can be running in
    parallel in worker processes. By default this is set to the number of
    CPUs on the host.

    Setting ``timeout`` (in seconds) makes idle workers automatically shutdown
    so as to release system resources. New workers are respawn upon submission
    of new tasks so that ``max_workers`` are available to accept the newly
    submitted tasks. Setting ``timeout`` to around 100 times the time required
    to spawn new processes and import packages in them (on the order of 100ms)
    ensures that the overhead of spawning workers is negligible.

    Setting ``kill_workers=True`` makes it possible to forcibly interrupt
    previously spawned jobs to get a new instance of the reusable executor
    with new constructor argument values.

    The ``job_reducers`` and ``result_reducers`` are used to customize the
    pickling of tasks and results send to the executor.

    When provided, the ``initializer`` is run first in newly spawned
    processes with argument ``initargs``.
    """
    with _executor_lock:
        global _executor, _executor_kwargs
        executor = _executor

        if max_workers is None:
            if reuse is True and executor is not None:
                max_workers = executor._max_workers
            else:
                max_workers = cpu_count()
        elif max_workers <= 0:
            raise ValueError(
                "max_workers must be greater than 0, got {}."
                .format(max_workers))

        if isinstance(context, STRING_TYPE):
            context = get_context(context)
        if context is not None and context.get_start_method() == "fork":
            raise ValueError("Cannot use reusable executor with the 'fork' "
                             "context")

        kwargs = dict(context=context, timeout=timeout,
                      job_reducers=job_reducers,
                      result_reducers=result_reducers,
                      initializer=initializer, initargs=initargs)
        if executor is None:
            mp.util.debug("Create a executor with max_workers={}."
                          .format(max_workers))
            executor_id = _get_next_executor_id()
            _executor_kwargs = kwargs
            _executor = executor = _ReusablePoolExecutor(
                _executor_lock, max_workers=max_workers,
                executor_id=executor_id, **kwargs)
        else:
            if reuse == 'auto':
                reuse = kwargs == _executor_kwargs
            if (executor._flags.broken or executor._flags.shutdown
                    or not reuse):
                if executor._flags.broken:
                    reason = "broken"
                elif executor._flags.shutdown:
                    reason = "shutdown"
                else:
                    reason = "arguments have changed"
                mp.util.debug(
                    "Creating a new executor with max_workers={} as the "
                    "previous instance cannot be reused ({})."
                    .format(max_workers, reason))
                executor.shutdown(wait=True, kill_workers=kill_workers)
                _executor = executor = _executor_kwargs = None
                # Recursive call to build a new instance
                return get_reusable_executor(max_workers=max_workers,
                                             **kwargs)
            else:
                mp.util.debug("Reusing existing executor with max_workers={}."
                              .format(executor._max_workers))
                executor._resize(max_workers)

    return executor

Return the current ReusableExectutor instance .


def _wait_job_completion(self):
        """Wait for the cache to be empty before resizing the pool."""
        # Issue a warning to the user about the bad effect of this usage.
        if len(self._pending_work_items) > 0:
            warnings.warn("Trying to resize an executor with running jobs: "
                          "waiting for jobs completion before resizing.",
                          UserWarning)
            mp.util.debug("Executor {} waiting for jobs completion before"
                          " resizing".format(self.executor_id))
        # Wait for the completion of the jobs
        while len(self._pending_work_items) > 0:
            time.sleep(1e-3)

Wait for the cache to be empty before resizing the pool .


def get_preparation_data(name, init_main_module=True):
    '''
    Return info about parent needed by child to unpickle process object
    '''
    _check_not_importing_main()
    d = dict(
        log_to_stderr=util._log_to_stderr,
        authkey=bytes(process.current_process().authkey),
    )

    if util._logger is not None:
        d['log_level'] = util._logger.getEffectiveLevel()
        if len(util._logger.handlers) > 0:
            h = util._logger.handlers[0]
            d['log_fmt'] = h.formatter._fmt

    sys_path = [p for p in sys.path]
    try:
        i = sys_path.index('')
    except ValueError:
        pass
    else:
        sys_path[i] = process.ORIGINAL_DIR

    d.update(
        name=name,
        sys_path=sys_path,
        sys_argv=sys.argv,
        orig_dir=process.ORIGINAL_DIR,
        dir=os.getcwd()
    )

    if sys.platform != "win32":
        # Pass the semaphore_tracker pid to avoid re-spawning it in every child
        from . import semaphore_tracker
        semaphore_tracker.ensure_running()
        d['tracker_pid'] = semaphore_tracker._semaphore_tracker._pid

    # Figure out whether to initialise main in the subprocess as a module
    # or through direct execution (or to leave it alone entirely)
    if init_main_module:
        main_module = sys.modules['__main__']
        try:
            main_mod_name = getattr(main_module.__spec__, "name", None)
        except BaseException:
            main_mod_name = None
        if main_mod_name is not None:
            d['init_main_from_name'] = main_mod_name
        elif sys.platform != 'win32' or (not WINEXE and not WINSERVICE):
            main_path = getattr(main_module, '__file__', None)
            if main_path is not None:
                if (not os.path.isabs(main_path) and
                        process.ORIGINAL_DIR is not None):
                    main_path = os.path.join(process.ORIGINAL_DIR, main_path)
                d['init_main_from_path'] = os.path.normpath(main_path)
                # Compat for python2.7
                d['main_path'] = d['init_main_from_path']

    return d


def prepare(data):
    '''
    Try to get current process ready to unpickle process object
    '''
    if 'name' in data:
        process.current_process().name = data['name']

    if 'authkey' in data:
        process.current_process().authkey = data['authkey']

    if 'log_to_stderr' in data and data['log_to_stderr']:
        util.log_to_stderr()

    if 'log_level' in data:
        util.get_logger().setLevel(data['log_level'])

    if 'log_fmt' in data:
        import logging
        util.get_logger().handlers[0].setFormatter(
            logging.Formatter(data['log_fmt'])
        )

    if 'sys_path' in data:
        sys.path = data['sys_path']

    if 'sys_argv' in data:
        sys.argv = data['sys_argv']

    if 'dir' in data:
        os.chdir(data['dir'])

    if 'orig_dir' in data:
        process.ORIGINAL_DIR = data['orig_dir']

    if 'tracker_pid' in data:
        from . import semaphore_tracker
        semaphore_tracker._semaphore_tracker._pid = data["tracker_pid"]

    if 'init_main_from_name' in data:
        _fixup_main_from_name(data['init_main_from_name'])
    elif 'init_main_from_path' in data:
        _fixup_main_from_path(data['init_main_from_path'])

def prepare ( data ) : if 'name' in data : process . current_process ( ) . name = data [ 'name' ] if 'authkey' in data : process . current_process ( ) . authkey = data [ 'authkey' ] if 'log_to_stderr' in data and data [ 'log_to_stderr' ] : util . log_to_stderr ( ) if 'log_level' in data : util . get_logger ( ) . setLevel ( data [ 'log_level' ] ) if 'log_fmt' in data : import logging util . get_logger ( ) . handlers [ 0 ] . setFormatter ( logging . Formatter ( data [ 'log_fmt' ] ) ) if 'sys_path' in data : sys . path = data [ 'sys_path' ] if 'sys_argv' in data : sys . argv = data [ 'sys_argv' ] if 'dir' in data : os . chdir ( data [ 'dir' ] ) if 'orig_dir' in data : process . ORIGINAL_DIR = data [ 'orig_dir' ] if 'tracker_pid' in data : from . import semaphore_tracker semaphore_tracker . _semaphore_tracker . _pid = data [ "tracker_pid" ] if 'init_main_from_name' in data : _fixup_main_from_name ( data [ 'init_main_from_name' ] ) elif 'init_main_from_path' in data : _fixup_main_from_path ( data [ 'init_main_from_path' ] )


def close_fds(keep_fds):  # pragma: no cover
    """Close all the file descriptors except those in keep_fds."""

    # Make sure to keep stdout and stderr open for logging purpose
    keep_fds = set(keep_fds).union([1, 2])

    # We try to retrieve all the open fds
    try:
        open_fds = set(int(fd) for fd in os.listdir('/proc/self/fd'))
    except FileNotFoundError:
        import resource
        max_nfds = resource.getrlimit(resource.RLIMIT_NOFILE)[0]
        open_fds = set(fd for fd in range(3, max_nfds))
        open_fds.add(0)

    for i in open_fds - keep_fds:
        try:
            os.close(i)
        except OSError:
            pass

Close all the file descriptors except those in keep_fds .


def _recursive_terminate_without_psutil(process):
    """Terminate a process and its descendants.
    """
    try:
        _recursive_terminate(process.pid)
    except OSError as e:
        warnings.warn("Failed to kill subprocesses on this platform. Please"
                      "install psutil: https://github.com/giampaolo/psutil")
        # In case we cannot introspect the children, we fall back to the
        # classic Process.terminate.
        process.terminate()
    process.join()

Terminate a process and its descendants .


def _recursive_terminate(pid):
    """Recursively kill the descendants of a process before killing it.
    """

    if sys.platform == "win32":
        # On windows, the taskkill function with option `/T` terminate a given
        # process pid and its children.
        try:
            subprocess.check_output(
                ["taskkill", "/F", "/T", "/PID", str(pid)],
                stderr=None)
        except subprocess.CalledProcessError as e:
            # In windows, taskkill return 1 for permission denied and 128, 255
            # for no process found.
            if e.returncode not in [1, 128, 255]:
                raise
            elif e.returncode == 1:
                # Try to kill the process without its descendants if taskkill
                # was denied permission. If this fails too, with an error
                # different from process not found, let the top level function
                # raise a warning and retry to kill the process.
                try:
                    os.kill(pid, signal.SIGTERM)
                except OSError as e:
                    if e.errno != errno.ESRCH:
                        raise

    else:
        try:
            children_pids = subprocess.check_output(
                ["pgrep", "-P", str(pid)],
                stderr=None
            )
        except subprocess.CalledProcessError as e:
            # `ps` returns 1 when no child process has been found
            if e.returncode == 1:
                children_pids = b''
            else:
                raise

        # Decode the result, split the cpid and remove the trailing line
        children_pids = children_pids.decode().split('\n')[:-1]
        for cpid in children_pids:
            cpid = int(cpid)
            _recursive_terminate(cpid)

        try:
            os.kill(pid, signal.SIGTERM)
        except OSError as e:
            # if OSError is raised with [Errno 3] no such process, the process
            # is already terminated, else, raise the error and let the top
            # level function raise a warning and retry to kill the process.
            if e.errno != errno.ESRCH:
                raise

Recursively kill the descendants of a process before killing it .


def get_exitcodes_terminated_worker(processes):
    """Return a formated string with the exitcodes of terminated workers.

    If necessary, wait (up to .25s) for the system to correctly set the
    exitcode of one terminated worker.
    """
    patience = 5

    # Catch the exitcode of the terminated workers. There should at least be
    # one. If not, wait a bit for the system to correctly set the exitcode of
    # the terminated worker.
    exitcodes = [p.exitcode for p in list(processes.values())
                 if p.exitcode is not None]
    while len(exitcodes) == 0 and patience > 0:
        patience -= 1
        exitcodes = [p.exitcode for p in list(processes.values())
                     if p.exitcode is not None]
        time.sleep(.05)

    return _format_exitcodes(exitcodes)

Return a formated string with the exitcodes of terminated workers .


def _format_exitcodes(exitcodes):
    """Format a list of exit code with names of the signals if possible"""
    str_exitcodes = ["{}({})".format(_get_exitcode_name(e), e)
                     for e in exitcodes if e is not None]
    return "{" + ", ".join(str_exitcodes) + "}"

def
_format_exitcodes
(
exitcodes
)
:
str_exitcodes
=
[
"{}({})"
.
format
(
_get_exitcode_name
(
e
)
,
e
)
for
e
in
exitcodes
if
e
is
not
]
return
"{"
+
", "
.
join
(
str_exitcodes
)
+
"}"


def main(fd, verbose=0):
    '''Run semaphore tracker.'''
    # protect the process from ^C and "killall python" etc
    signal.signal(signal.SIGINT, signal.SIG_IGN)
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    if _HAVE_SIGMASK:
        signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS)

    for f in (sys.stdin, sys.stdout):
        try:
            f.close()
        except Exception:
            pass

    if verbose:  # pragma: no cover
        sys.stderr.write("Main semaphore tracker is running\n")
        sys.stderr.flush()

    cache = set()
    try:
        # keep track of registered/unregistered semaphores
        with os.fdopen(fd, 'rb') as f:
            for line in f:
                try:
                    cmd, name = line.strip().split(b':')
                    if cmd == b'REGISTER':
                        name = name.decode('ascii')
                        cache.add(name)
                        if verbose:  # pragma: no cover
                            sys.stderr.write("[SemaphoreTracker] register {}\n"
                                             .format(name))
                            sys.stderr.flush()
                    elif cmd == b'UNREGISTER':
                        name = name.decode('ascii')
                        cache.remove(name)
                        if verbose:  # pragma: no cover
                            sys.stderr.write("[SemaphoreTracker] unregister {}"
                                             ": cache({})\n"
                                             .format(name, len(cache)))
                            sys.stderr.flush()
                    elif cmd == b'PROBE':
                        pass
                    else:
                        raise RuntimeError('unrecognized command %r' % cmd)
                except BaseException:
                    try:
                        sys.excepthook(*sys.exc_info())
                    except BaseException:
                        pass
    finally:
        # all processes have terminated; cleanup any remaining semaphores
        if cache:
            try:
                warnings.warn('semaphore_tracker: There appear to be %d '
                              'leaked semaphores to clean up at shutdown' %
                              len(cache))
            except Exception:
                pass
        for name in cache:
            # For some reason the process which created and registered this
            # semaphore has failed to unregister it. Presumably it has died.
            # We therefore unlink it.
            try:
                try:
                    sem_unlink(name)
                    if verbose:  # pragma: no cover
                        sys.stderr.write("[SemaphoreTracker] unlink {}\n"
                                         .format(name))
                        sys.stderr.flush()
                except Exception as e:
                    warnings.warn('semaphore_tracker: %s: %r' % (name, e))
            finally:
                pass

    if verbose:  # pragma: no cover
        sys.stderr.write("semaphore tracker shut down\n")
        sys.stderr.flush()


def ensure_running(self):
        '''Make sure that semaphore tracker process is running.

        This can be run from any process.  Usually a child process will use
        the semaphore created by its parent.'''
        with self._lock:
            if self._fd is not None:
                # semaphore tracker was launched before, is it still running?
                if self._check_alive():
                    # => still alive
                    return
                # => dead, launch it again
                os.close(self._fd)
                try:
                    # Clean-up to avoid dangling processes.
                    os.waitpid(self._pid, 0)
                except OSError:
                    # The process was terminated or is a child from an ancestor
                    # of the current process.
                    pass
                self._fd = None
                self._pid = None

                warnings.warn('semaphore_tracker: process died unexpectedly, '
                              'relaunching.  Some semaphores might leak.')

            fds_to_pass = []
            try:
                fds_to_pass.append(sys.stderr.fileno())
            except Exception:
                pass

            r, w = os.pipe()
            cmd = 'from {} import main; main({}, {})'.format(
                main.__module__, r, VERBOSE)
            try:
                fds_to_pass.append(r)
                # process will out live us, so no need to wait on pid
                exe = spawn.get_executable()
                args = [exe] + util._args_from_interpreter_flags()
                # In python 3.3, there is a bug which put `-RRRRR..` instead of
                # `-R` in args. Replace it to get the correct flags.
                # See https://github.com/python/cpython/blob/3.3/Lib/subprocess.py#L488
                if sys.version_info[:2] <= (3, 3):
                    import re
                    for i in range(1, len(args)):
                        args[i] = re.sub("-R+", "-R", args[i])
                args += ['-c', cmd]
                util.debug("launching Semaphore tracker: {}".format(args))
                # bpo-33613: Register a signal mask that will block the
                # signals.  This signal mask will be inherited by the child
                # that is going to be spawned and will protect the child from a
                # race condition that can make the child die before it
                # registers signal handlers for SIGINT and SIGTERM. The mask is
                # unregistered after spawning the child.
                try:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_BLOCK,
                                               _IGNORED_SIGNALS)
                    pid = spawnv_passfds(exe, args, fds_to_pass)
                finally:
                    if _HAVE_SIGMASK:
                        signal.pthread_sigmask(signal.SIG_UNBLOCK,
                                               _IGNORED_SIGNALS)
            except BaseException:
                os.close(w)
                raise
            else:
                self._fd = w
                self._pid = pid
            finally:
                os.close(r)

Make sure that semaphore tracker process is running .


def event_processor(self, frame, event, arg):
        'A simple event processor that prints out events.'
        out = self.debugger.intf[-1].output
        lineno = frame.f_lineno
        filename = self.core.canonic_filename(frame)
        filename = self.core.filename(filename)
        if not out:
            print("%s - %s:%d" % (event, filename, lineno))
        else:
            out.write("%s - %s:%d" % (event, filename, lineno))
            if arg is not None:
                out.writeline(', %s ' % repr(arg))
            else:
                out.writeline('')
                pass
            pass
        return self.event_processor

def
event_processor
(
self
,
frame
,
event
,
arg
)
:
out
=
self
.
debugger
.
intf
[
1
]
.
output
lineno
=
frame
.
f_lineno
filename
=
self
.
core
.
canonic_filename
(
frame
)
filename
=
self
.
core
.
filename
(
filename
)
if
not
out
:
print
(
"%s - %s:%d"
%
(
event
,
filename
,
lineno
)
)
else
:
out
.
write
(
"%s - %s:%d"
%
(
event
,
filename
,
lineno
)
)
if
arg
is
not
:
out
.
writeline
(
', %s '
%
repr
(
arg
)
)
else
:
out
.
writeline
(
''
)
pass
pass
return
self
.
event_processor

A simple event processor that prints out events .


def run(self, args):
        """Program counter."""
        mainfile = self.core.filename(None)
        if self.core.is_running():
            curframe = self.proc.curframe
            if curframe:
                line_no = inspect.getlineno(curframe)
                offset  = curframe.f_lasti
                self.msg("PC offset is %d." % offset)
                offset = max(offset, 0)
                code = curframe.f_code
                co_code = code.co_code
                disassemble_bytes(self.msg, self.msg_nocr,
                                  co_code, offset, line_no, line_no-1, line_no+1,
                                  constants=code.co_consts, cells=code.co_cellvars,
                                  varnames=code.co_varnames, freevars=code.co_freevars,
                                  linestarts=dict(findlinestarts(code)),
                                  end_offset=offset+10)
                pass
            pass
        else:
            if mainfile:
                part1 = "Python program '%s'" % mainfile
                msg   = "is not currently running. "
                self.msg(Mmisc.wrapped_lines(part1, msg,
                                             self.settings['width']))
            else:
                self.msg('No Python program is currently running.')
                pass
            self.msg(self.core.execution_status)
            pass
        return False


def interact(banner=None, readfunc=None, my_locals=None, my_globals=None):
    """Almost a copy of code.interact
    Closely emulate the interactive Python interpreter.

    This is a backwards compatible interface to the InteractiveConsole
    class.  When readfunc is not specified, it attempts to import the
    readline module to enable GNU readline if it is available.

    Arguments (all optional, all default to None):

    banner -- passed to InteractiveConsole.interact()
    readfunc -- if not None, replaces InteractiveConsole.raw_input()
    local -- passed to InteractiveInterpreter.__init__()

    """
    console = code.InteractiveConsole(my_locals, filename='<trepan>')
    console.runcode = lambda code_obj: runcode(console, code_obj)
    setattr(console, 'globals', my_globals)
    if readfunc is not None:
        console.raw_input = readfunc
    else:
        try:
            import readline
        except ImportError:
            pass
    console.interact(banner)
    pass

def interact ( banner = None , readfunc = None , my_locals = None , my_globals = None ) : console = code . InteractiveConsole ( my_locals , filename = '<trepan>' ) console . runcode = lambda code_obj : runcode ( console , code_obj ) setattr ( console , 'globals' , my_globals ) if readfunc is not None : console . raw_input = readfunc else : try : import readline except ImportError : pass console . interact ( banner ) pass

Almost a copy of code . interact Closely emulate the interactive Python interpreter .


def arg_split(s, posix=False):
    """Split a command line's arguments in a shell-like manner returned
    as a list of lists. Use ';;' with white space to indicate separate
    commands.

    This is a modified version of the standard library's shlex.split()
    function, but with a default of posix=False for splitting, so that quotes
    in inputs are respected.
    """

    args_list = [[]]
    if isinstance(s, bytes):
        s = s.decode("utf-8")
    lex = shlex.shlex(s, posix=posix)

    lex.whitespace_split = True
    args = list(lex)
    for arg in args:
        if ';;' == arg:
            args_list.append([])
        else:
            args_list[-1].append(arg)
            pass
        pass
    return args_list

def arg_split ( s , posix = False ) : args_list = [ [ ] ] if isinstance ( s , bytes ) : s = s . decode ( "utf-8" ) lex = shlex . shlex ( s , posix = posix ) lex . whitespace_split = True args = list ( lex ) for arg in args : if ';;' == arg : args_list . append ( [ ] ) else : args_list [ - 1 ] . append ( arg ) pass pass return args_list

Split a command line s arguments in a shell - like manner returned as a list of lists . Use ;; with white space to indicate separate commands .


def get_stack(f, t, botframe, proc_obj=None):
    """Return a stack of frames which the debugger will use for in
    showing backtraces and in frame switching. As such various frame
    that are really around may be excluded unless we are debugging the
    sebugger. Also we will add traceback frame on top if that
    exists."""
    exclude_frame = lambda f: False
    if proc_obj:
        settings = proc_obj.debugger.settings
        if not settings['dbg_trepan']:
            exclude_frame = lambda f: \
                proc_obj.core.ignore_filter.is_included(f)
            pass
        pass
    stack = []
    if t and t.tb_frame is f:
        t = t.tb_next
    while f is not None:
        if exclude_frame(f): break  # See commented alternative below
        stack.append((f, f.f_lineno))
        # bdb has:
        # if f is botframe: break
        f = f.f_back
        pass
    stack.reverse()
    i = max(0, len(stack) - 1)
    while t is not None:
        stack.append((t.tb_frame, t.tb_lineno))
        t = t.tb_next
        pass
    return stack, i

def
get_stack
(
f
,
t
,
botframe
,
proc_obj
=
)
:
exclude_frame
=
lambda
f
:
False
if
proc_obj
:
settings
=
proc_obj
.
debugger
.
settings
if
not
settings
[
'dbg_trepan'
]
:
exclude_frame
=
lambda
f
:
proc_obj
.
core
.
ignore_filter
.
is_included
(
f
)
pass
pass
stack
=
[
]
if
t
and
t
.
tb_frame
is
f
:
t
=
t
.
tb_next
while
f
is
not
:
if
exclude_frame
(
f
)
:
break
# See commented alternative below
stack
.
append
(
(
f
,
f
.
f_lineno
)
)
# bdb has:
# if f is botframe: break
f
=
f
.
f_back
pass
stack
.
reverse
(
)
i
=
max
(
0
,
len
(
stack
)
1
)
while
t
is
not
:
stack
.
append
(
(
t
.
tb_frame
,
t
.
tb_lineno
)
)
t
=
t
.
tb_next
pass
return
stack
,
i

Return a stack of frames which the debugger will use for in showing backtraces and in frame switching . As such various frame that are really around may be excluded unless we are debugging the sebugger . Also we will add traceback frame on top if that exists .


def run_hooks(obj, hooks, *args):
    """Run each function in `hooks' with args"""
    for hook in hooks:
        if hook(obj, *args): return True
        pass
    return False

def run_hooks ( obj , hooks , * args ) : for hook in hooks : if hook ( obj , * args ) : return True pass return False


def forget(self):
        """ Remove memory of state variables set in the command processor """
        self.stack       = []
        self.curindex    = 0
        self.curframe    = None
        self.thread_name = None
        self.frame_thread_name = None
        return

def forget ( self ) : self . stack = [ ] self . curindex = 0 self . curframe = None self . thread_name = None self . frame_thread_name = None return


def get_int_noerr(self, arg):
        """Eval arg and it is an integer return the value. Otherwise
        return None"""
        if self.curframe:
            g = self.curframe.f_globals
            l = self.curframe.f_locals
        else:
            g = globals()
            l = locals()
            pass
        try:
            val = int(eval(arg, g, l))
        except (SyntaxError, NameError, ValueError, TypeError):
            return None
        return val

def get_int_noerr ( self , arg ) : if self . curframe : g = self . curframe . f_globals l = self . curframe . f_locals else : g = globals ( ) l = locals ( ) pass try : val = int ( eval ( arg , g , l ) ) except ( SyntaxError , NameError , ValueError , TypeError ) : return None return val

Eval arg and it is an integer return the value . Otherwise return None


def get_int(self, arg, min_value=0, default=1, cmdname=None,
                at_most=None):
        """If no argument use the default. If arg is a an integer between
        least min_value and at_most, use that. Otherwise report an error.
        If there's a stack frame use that in evaluation."""

        if arg is None: return default
        default = self.get_int_noerr(arg)
        if default is None:
            if cmdname:
                self.errmsg(("Command '%s' expects an integer; "
                             + "got: %s.") % (cmdname, str(arg)))
            else:
                self.errmsg('Expecting a positive integer, got: %s'
                            % str(arg))
                pass
            return None
            pass
        if default < min_value:
            if cmdname:
                self.errmsg(("Command '%s' expects an integer at least" +
                             ' %d; got: %d.')
                            % (cmdname, min_value, default))
            else:
                self.errmsg(("Expecting a positive integer at least" +
                             ' %d; got: %d')
                            % (min_value, default))
                pass
            return None
        elif at_most and default > at_most:
            if cmdname:
                self.errmsg(("Command '%s' expects an integer at most" +
                             ' %d; got: %d.')
                            % (cmdname, at_most, default))
            else:
                self.errmsg(("Expecting an integer at most %d; got: %d")
                            % (at_most, default))
                pass
            pass
        return default

If no argument use the default . If arg is a an integer between least min_value and at_most use that . Otherwise report an error . If there s a stack frame use that in evaluation .


def process_commands(self):
        """Handle debugger commands."""
        if self.core.execution_status != 'No program':
            self.setup()
            self.location()
            pass
        leave_loop = run_hooks(self, self.preloop_hooks)
        self.continue_running = False

        while not leave_loop:
            try:
                run_hooks(self, self.precmd_hooks)
                # bdb had a True return to leave loop.
                # A more straight-forward way is to set
                # instance variable self.continue_running.
                leave_loop = self.process_command()
                if leave_loop or self.continue_running: break
            except EOFError:
                # If we have stacked interfaces, pop to the next
                # one.  If this is the last one however, we'll
                # just stick with that.  FIXME: Possibly we should
                # check to see if we are interactive.  and not
                # leave if that's the case. Is this the right
                # thing?  investigate and fix.
                if len(self.debugger.intf) > 1:
                    del self.debugger.intf[-1]
                    self.last_command = ''
                else:
                    if self.debugger.intf[-1].output:
                        self.debugger.intf[-1].output.writeline('Leaving')
                        raise Mexcept.DebuggerQuit
                        pass
                    break
                pass
            pass
        return run_hooks(self, self.postcmd_hooks)


def queue_startfile(self, cmdfile):
        """Arrange for file of debugger commands to get read in the
        process-command loop."""
        expanded_cmdfile = os.path.expanduser(cmdfile)
        is_readable = Mfile.readable(expanded_cmdfile)
        if is_readable:
            self.cmd_queue.append('source ' + expanded_cmdfile)
        elif is_readable is None:
            self.errmsg("source file '%s' doesn't exist" % expanded_cmdfile)
        else:
            self.errmsg("source file '%s' is not readable" %
                        expanded_cmdfile)
            pass
        return

def
queue_startfile
(
self
,
cmdfile
)
:
expanded_cmdfile
=
os
.
path
.
expanduser
(
cmdfile
)
is_readable
=
Mfile
.
readable
(
expanded_cmdfile
)
if
is_readable
:
self
.
cmd_queue
.
append
(
'source '
+
expanded_cmdfile
)
elif
is_readable
is
:
self
.
errmsg
(
"source file '%s' doesn't exist"
%
expanded_cmdfile
)
else
:
self
.
errmsg
(
"source file '%s' is not readable"
%
expanded_cmdfile
)
pass
return

Arrange for file of debugger commands to get read in the process - command loop .


def next_token(str, start_pos):
    """Find the next token in str string from start_pos, we return
    the token and the next blank position after the token or
    str.size if this is the last token. Tokens are delimited by
    white space."""
    look_at = str[start_pos:]
    match = re.search('\S', look_at)
    if match:
        pos = match.start()
    else:
        pos = 0
        pass
    next_nonblank_pos = start_pos + pos
    next_match = re.search('\s', str[next_nonblank_pos:])
    if next_match:
        next_blank_pos = next_nonblank_pos + next_match.start()
    else:
        next_blank_pos = len(str)
        pass
    return [next_blank_pos, str[next_nonblank_pos:next_blank_pos+1].rstrip()]

def next_token ( str , start_pos ) : look_at = str [ start_pos : ] match = re . search ( '\S' , look_at ) if match : pos = match . start ( ) else : pos = 0 pass next_nonblank_pos = start_pos + pos next_match = re . search ( '\s' , str [ next_nonblank_pos : ] ) if next_match : next_blank_pos = next_nonblank_pos + next_match . start ( ) else : next_blank_pos = len ( str ) pass return [ next_blank_pos , str [ next_nonblank_pos : next_blank_pos + 1 ] . rstrip ( ) ]

Find the next token in str string from start_pos we return the token and the next blank position after the token or str . size if this is the last token . Tokens are delimited by white space .


def errmsg(self, msg, prefix="** "):
        """Common routine for reporting debugger error messages.
           """
        #  self.verbose shows lines so we don't have to duplicate info
        #  here. Perhaps there should be a 'terse' mode to never show
        #  position info.
        if not self.verbose:
            location = ("%s:%s: Error in source command file"
                        % (self.script_name, self.input_lineno))
            msg = "%s%s:\n%s%s" %(prefix, location, prefix, msg)
        else:
            msg = "%s%s" %(prefix, msg)
            pass
        self.msg(msg)
        if self.abort_on_error:
            raise EOFError
        return

Common routine for reporting debugger error messages .


def read_command(self, prompt=''):
        '''Script interface to read a command. `prompt' is a parameter for
        compatibilty and is ignored.'''
        self.input_lineno += 1
        line = self.readline()
        if self.verbose:
            location = "%s line %s" % (self.script_name, self.input_lineno)
            self.msg('+ %s: %s' % (location, line))
            pass
        # Do something with history?
        return line

def
read_command
(
self
,
prompt
=
''
)
:
self
.
input_lineno
+=
1
line
=
self
.
readline
(
)
if
self
.
verbose
:
location
=
"%s line %s"
%
(
self
.
script_name
,
self
.
input_lineno
)
self
.
msg
(
'+ %s: %s'
%
(
location
,
line
)
)
pass
# Do something with history?
return
line

Script interface to read a command . prompt is a parameter for compatibilty and is ignored .


def close(self):
        """ Closes both input and output """
        self.state = 'closing'
        if self.input:
            self.input.close()
            pass
        if self.output:
            self.output.close()
            pass
        self.state = 'disconnnected'
        return

def close ( self ) : self . state = 'closing' if self . input : self . input . close ( ) pass if self . output : self . output . close ( ) pass self . state = 'disconnnected' return


def disassemble(msg, msg_nocr, section, co, lasti=-1, start_line=-1,
                end_line=None, relative_pos=False, highlight='light',
                start_offset=0, end_offset=None):
    """Disassemble a code object."""
    return disassemble_bytes(msg, msg_nocr, co.co_code, lasti, co.co_firstlineno,
                             start_line, end_line, relative_pos,
                        co.co_varnames, co.co_names, co.co_consts,
                        co.co_cellvars, co.co_freevars,
                        dict(findlinestarts(co)), highlight,
                        start_offset=start_offset, end_offset=end_offset)

def disassemble ( msg , msg_nocr , section , co , lasti = - 1 , start_line = - 1 , end_line = None , relative_pos = False , highlight = 'light' , start_offset = 0 , end_offset = None ) : return disassemble_bytes ( msg , msg_nocr , co . co_code , lasti , co . co_firstlineno , start_line , end_line , relative_pos , co . co_varnames , co . co_names , co . co_consts , co . co_cellvars , co . co_freevars , dict ( findlinestarts ( co ) ) , highlight , start_offset = start_offset , end_offset = end_offset )


def disassemble_bytes(orig_msg, orig_msg_nocr, code, lasti=-1, cur_line=0,
                      start_line=-1, end_line=None, relative_pos=False,
                      varnames=(), names=(), constants=(), cells=(),
                      freevars=(), linestarts={}, highlight='light',
                      start_offset=0, end_offset=None):
    """Disassemble byte string of code. If end_line is negative
    it counts the number of statement linestarts to use."""
    statement_count = 10000
    if end_line is None:
        end_line = 10000
    elif relative_pos:
        end_line += start_line -1
        pass

    labels = findlabels(code)

    null_print = lambda x: None
    if start_line > cur_line:
        msg_nocr = null_print
        msg = null_print
    else:
        msg_nocr = orig_msg_nocr
        msg = orig_msg

    for instr in get_instructions_bytes(code, opc, varnames, names,
                                        constants, cells, linestarts):
        offset = instr.offset
        if end_offset and offset > end_offset:
            break

        if instr.starts_line:
            if offset:
                msg("")

            cur_line = instr.starts_line
            if (start_line and ((start_line > cur_line) or
                                start_offset and start_offset > offset)) :
                msg_nocr = null_print
                msg = null_print
            else:
                statement_count -= 1
                msg_nocr = orig_msg_nocr
                msg = orig_msg
                pass
            if ((cur_line > end_line) or
                (end_offset and offset > end_offset)):
                break
            msg_nocr(format_token(Mformat.LineNumber,
                                  "%4d" % cur_line,
                                  highlight=highlight))
        else:
            if start_offset and offset and start_offset <= offset:
                msg_nocr = orig_msg_nocr
                msg = orig_msg
                pass
            msg_nocr('    ')

        if offset == lasti: msg_nocr(format_token(Mformat.Arrow, '-->',
                                                  highlight=highlight))
        else: msg_nocr('   ')
        if offset in labels: msg_nocr(format_token(Mformat.Arrow, '>>',
                                                   highlight=highlight))
        else: msg_nocr('  ')
        msg_nocr(repr(offset).rjust(4))
        msg_nocr(' ')
        msg_nocr(format_token(Mformat.Opcode,
                              instr.opname.ljust(20),
                              highlight=highlight))
        msg_nocr(repr(instr.arg).ljust(10))
        msg_nocr(' ')
        # Show argval?
        msg(format_token(Mformat.Name,
                         instr.argrepr.ljust(20),
                         highlight=highlight))
        pass

    return code, offset

def
disassemble_bytes
(
orig_msg
,
orig_msg_nocr
,
code
,
lasti
=
1
,
cur_line
=
0
,
start_line
=
1
,
end_line
=
,
relative_pos
=
False
,
varnames
=
(
)
,
names
=
(
)
,
constants
=
(
)
,
cells
=
(
)
,
freevars
=
(
)
,
linestarts
=
{
}
,
highlight
=
'light'
,
start_offset
=
0
,
end_offset
=
)
:
statement_count
=
10000
if
end_line
is
:
end_line
=
10000
elif
relative_pos
:
end_line
+=
start_line
1
pass
labels
=
findlabels
(
code
)
null_print
=
lambda
x
:
if
start_line
>
cur_line
:
msg_nocr
=
null_print
msg
=
null_print
else
:
msg_nocr
=
orig_msg_nocr
msg
=
orig_msg
for
instr
in
get_instructions_bytes
(
code
,
opc
,
varnames
,
names
,
constants
,
cells
,
linestarts
)
:
offset
=
instr
.
offset
if
end_offset
and
offset
>
end_offset
:
break
if
instr
.
starts_line
:
if
offset
:
msg
(
""
)
cur_line
=
instr
.
starts_line
if
(
start_line
and
(
(
start_line
>
cur_line
)
or
start_offset
and
start_offset
>
offset
)
)
:
msg_nocr
=
null_print
msg
=
null_print
else
:
statement_count
-=
1
msg_nocr
=
orig_msg_nocr
msg
=
orig_msg
pass
if
(
(
cur_line
>
end_line
)
or
(
end_offset
and
offset
>
end_offset
)
)
:
break
msg_nocr
(
format_token
(
Mformat
.
LineNumber
,
"%4d"
%
cur_line
,
highlight
=
highlight
)
)
else
:
if
start_offset
and
offset
and
start_offset
<=
offset
:
msg_nocr
=
orig_msg_nocr
msg
=
orig_msg
pass
msg_nocr
(
'    '
)
if
offset
==
lasti
:
msg_nocr
(
format_token
(
Mformat
.
Arrow
,
'-->'
,
highlight
=
highlight
)
)
else
:
msg_nocr
(
'   '
)
if
offset
in
labels
:
msg_nocr
(
format_token
(
Mformat
.
Arrow
,
'>>'
,
highlight
=
highlight
)
)
else
:
msg_nocr
(
'  '
)
msg_nocr
(
repr
(
offset
)
.
rjust
(
4
)
)
msg_nocr
(
' '
)
msg_nocr
(
format_token
(
Mformat
.
Opcode
,
instr
.
opname
.
ljust
(
20
)
,
highlight
=
highlight
)
)
msg_nocr
(
repr
(
instr
.
arg
)
.
ljust
(
10
)
)
msg_nocr
(
' '
)
# Show argval?
msg
(
format_token
(
Mformat
.
Name
,
instr
.
argrepr
.
ljust
(
20
)
,
highlight
=
highlight
)
)
pass
return
code
,
offset

Disassemble byte string of code . If end_line is negative it counts the number of statement linestarts to use .


def count_frames(frame, count_start=0):
    "Return a count of the number of frames"
    count = -count_start
    while frame:
        count += 1
        frame = frame.f_back
    return count

def count_frames ( frame , count_start = 0 ) : count = - count_start while frame : count += 1 frame = frame . f_back return count


def get_call_function_name(frame):
    """If f_back is looking at a call function, return
    the name for it. Otherwise return None"""
    f_back = frame.f_back
    if not f_back: return None
    if 'CALL_FUNCTION' != Mbytecode.op_at_frame(f_back): return None

    co         = f_back.f_code
    code       = co.co_code
    # labels     = dis.findlabels(code)
    linestarts = dict(dis.findlinestarts(co))
    offset     = f_back.f_lasti
    while offset >= 0:
        if offset in linestarts:
            op = code[offset]
            offset += 1
            arg = code[offset]
            # FIXME: put this code in xdis
            extended_arg = 0
            while True:
                if PYTHON_VERSION >= 3.6:
                    if op == opc.EXTENDED_ARG:
                        extended_arg += (arg << 8)
                        continue
                    arg = code[offset] + extended_arg
                    # FIXME: Python 3.6.0a1 is 2, for 3.6.a3 we have 1
                else:
                    if op == opc.EXTENDED_ARG:
                        extended_arg += (arg << 256)
                        continue
                    arg = code[offset] + code[offset+1]*256 + extended_arg
                break

            return co.co_names[arg]
        offset -= 1
        pass
    return None

If f_back is looking at a call function return the name for it . Otherwise return None


def print_stack_trace(proc_obj, count=None, color='plain', opts={}):
    "Print count entries of the stack trace"
    if count is None:
        n=len(proc_obj.stack)
    else:
        n=min(len(proc_obj.stack), count)
    try:
        for i in range(n):
            print_stack_entry(proc_obj, i, color=color, opts=opts)
    except KeyboardInterrupt:
        pass
    return

def print_stack_trace ( proc_obj , count = None , color = 'plain' , opts = { } ) : if count is None : n = len ( proc_obj . stack ) else : n = min ( len ( proc_obj . stack ) , count ) try : for i in range ( n ) : print_stack_entry ( proc_obj , i , color = color , opts = opts ) except KeyboardInterrupt : pass return


def lookup(self, subcmd_prefix):
        """Find subcmd in self.subcmds"""
        for subcmd_name in list(self.subcmds.keys()):
            if subcmd_name.startswith(subcmd_prefix) \
               and len(subcmd_prefix) >= \
               self.subcmds[subcmd_name].__class__.min_abbrev:
                return self.subcmds[subcmd_name]
            pass
        return None

def lookup ( self , subcmd_prefix ) : for subcmd_name in list ( self . subcmds . keys ( ) ) : if subcmd_name . startswith ( subcmd_prefix ) and len ( subcmd_prefix ) >= self . subcmds [ subcmd_name ] . __class__ . min_abbrev : return self . subcmds [ subcmd_name ] pass return None


def short_help(self, subcmd_cb, subcmd_name, label=False):
        """Show short help for a subcommand."""
        entry = self.lookup(subcmd_name)
        if entry:
            if label:
                prefix = entry.name
            else:
                prefix = ''
                pass
            if hasattr(entry, 'short_help'):
                if prefix: prefix += ' -- '
                self.cmd_obj.msg(prefix + entry.short_help)
                pass
            pass
        else:
            self.undefined_subcmd("help", subcmd_name)
            pass
        return

def
short_help
(
self
,
subcmd_cb
,
subcmd_name
,
label
=
False
)
:
entry
=
self
.
lookup
(
subcmd_name
)
if
entry
:
if
label
:
prefix
=
entry
.
name
else
:
prefix
=
''
pass
if
hasattr
(
entry
,
'short_help'
)
:
if
prefix
:
prefix
+=
' -- '
self
.
cmd_obj
.
msg
(
prefix
+
entry
.
short_help
)
pass
pass
else
:
self
.
undefined_subcmd
(
"help"
,
subcmd_name
)
pass
return


def add(self, subcmd_cb):
        """Add subcmd to the available subcommands for this object.
        It will have the supplied docstring, and subcmd_cb will be called
        when we want to run the command. min_len is the minimum length
        allowed to abbreviate the command. in_list indicates with the
        show command will be run when giving a list of all sub commands
        of this object. Some commands have long output like "show commands"
        so we might not want to show that.
        """
        subcmd_name = subcmd_cb.name
        self.subcmds[subcmd_name] = subcmd_cb

        # We keep a list of subcommands to assist command completion
        self.cmdlist.append(subcmd_name)

Add subcmd to the available subcommands for this object . It will have the supplied docstring and subcmd_cb will be called when we want to run the command . min_len is the minimum length allowed to abbreviate the command . in_list indicates with the show command will be run when giving a list of all sub commands of this object . Some commands have long output like show commands so we might not want to show that .


def run(self, subcmd_name, arg):
        """Run subcmd_name with args using obj for the environent"""
        entry=self.lookup(subcmd_name)
        if entry:
            entry['callback'](arg)
        else:
            self.cmdproc.undefined_cmd(entry.__class__.name, subcmd_name)
            pass
        return

def run ( self , subcmd_name , arg ) : entry = self . lookup ( subcmd_name ) if entry : entry [ 'callback' ] ( arg ) else : self . cmdproc . undefined_cmd ( entry . __class__ . name , subcmd_name ) pass return


def debug(dbg_opts=None, start_opts=None, post_mortem=True,
          step_ignore=1, level=0):
    """
Enter the debugger.

Parameters
----------

level : how many stack frames go back. Usually it will be
the default 0. But sometimes though there may be calls in setup to the debugger
that you may want to skip.

step_ignore : how many line events to ignore after the
debug() call. 0 means don't even wait for the debug() call to finish.

param dbg_opts : is an optional "options" dictionary that gets fed
trepan.Debugger(); `start_opts' are the optional "options"
dictionary that gets fed to trepan.Debugger.core.start().

Use like this:

.. code-block:: python

    ... # Possibly some Python code
    import trepan.api # Needed only once
    ... # Possibly some more Python code
    trepan.api.debug() # You can wrap inside conditional logic too
    pass  # Stop will be here.
    # Below is code you want to use the debugger to do things.
    ....  # more Python code
    # If you get to a place in the program where you aren't going
    # want to debug any more, but want to remove debugger trace overhead:
    trepan.api.stop()

Parameter "level" specifies how many stack frames go back. Usually it will be
the default 0. But sometimes though there may be calls in setup to the debugger
that you may want to skip.

Parameter "step_ignore" specifies how many line events to ignore after the
debug() call. 0 means don't even wait for the debug() call to finish.

In situations where you want an immediate stop in the "debug" call
rather than the statement following it ("pass" above), add parameter
step_ignore=0 to debug() like this::

    import trepan.api  # Needed only once
    # ... as before
    trepan.api.debug(step_ignore=0)
    # ... as before

Module variable _debugger_obj_ from module trepan.debugger is used as
the debugger instance variable; it can be subsequently used to change
settings or alter behavior. It should be of type Debugger (found in
module trepan). If not, it will get changed to that type::

   $ python
   >>> from trepan.debugger import debugger_obj
   >>> type(debugger_obj)
   <type 'NoneType'>
   >>> import trepan.api
   >>> trepan.api.debug()
   ...
   (Trepan) c
   >>> from trepan.debugger import debugger_obj
   >>> debugger_obj
   <trepan.debugger.Debugger instance at 0x7fbcacd514d0>
   >>>

If however you want your own separate debugger instance, you can
create it from the debugger _class Debugger()_ from module
trepan.debugger::

  $ python
  >>> from trepan.debugger import Debugger
  >>> dbgr = Debugger()  # Add options as desired
  >>> dbgr
  <trepan.debugger.Debugger instance at 0x2e25320>

`dbg_opts' is an optional "options" dictionary that gets fed
trepan.Debugger(); `start_opts' are the optional "options"
dictionary that gets fed to trepan.Debugger.core.start().
"""
    if not isinstance(Mdebugger.debugger_obj, Mdebugger.Trepan):
        Mdebugger.debugger_obj = Mdebugger.Trepan(dbg_opts)
        Mdebugger.debugger_obj.core.add_ignore(debug, stop)
        pass
    core = Mdebugger.debugger_obj.core
    frame = sys._getframe(0+level)
    core.set_next(frame)
    if start_opts and 'startup-profile' in start_opts and start_opts['startup-profile']:
        dbg_initfiles = start_opts['startup-profile']
        from trepan import options
        options.add_startup_file(dbg_initfiles)
        for init_cmdfile in dbg_initfiles:
            core.processor.queue_startfile(init_cmdfile)

    if not core.is_started():
        core.start(start_opts)
        pass
    if post_mortem:
        debugger_on_post_mortem()
        pass
    if 0 == step_ignore:
        frame                   = sys._getframe(1+level)
        core.stop_reason        = 'at a debug() call'
        old_trace_hook_suspend  = core.trace_hook_suspend
        core.trace_hook_suspend = True
        core.processor.event_processor(frame, 'line', None)
        core.trace_hook_suspend = old_trace_hook_suspend
    else:
        core.step_ignore = step_ignore-1
        pass
    return

def
debug
(
dbg_opts
=
,
start_opts
=
,
post_mortem
=
True
,
step_ignore
=
1
,
level
=
0
)
:
if
not
isinstance
(
Mdebugger
.
debugger_obj
,
Mdebugger
.
Trepan
)
:
Mdebugger
.
debugger_obj
=
Mdebugger
.
Trepan
(
dbg_opts
)
Mdebugger
.
debugger_obj
.
core
.
add_ignore
(
debug
,
stop
)
pass
core
=
Mdebugger
.
debugger_obj
.
core
frame
=
sys
.
_getframe
(
0
+
level
)
core
.
set_next
(
frame
)
if
start_opts
and
'startup-profile'
in
start_opts
and
start_opts
[
'startup-profile'
]
:
dbg_initfiles
=
start_opts
[
'startup-profile'
]
from
trepan
import
options
options
.
add_startup_file
(
dbg_initfiles
)
for
init_cmdfile
in
dbg_initfiles
:
core
.
processor
.
queue_startfile
(
init_cmdfile
)
if
not
core
.
is_started
(
)
:
core
.
start
(
start_opts
)
pass
if
post_mortem
:
debugger_on_post_mortem
(
)
pass
if
0
==
step_ignore
:
frame
=
sys
.
_getframe
(
1
+
level
)
core
.
stop_reason
=
'at a debug() call'
old_trace_hook_suspend
=
core
.
trace_hook_suspend
core
.
trace_hook_suspend
=
True
core
.
processor
.
event_processor
(
frame
,
'line'
,
)
core
.
trace_hook_suspend
=
old_trace_hook_suspend
else
:
core
.
step_ignore
=
step_ignore
1
pass
return


def show_category(self, category, args):
        """Show short help for all commands in `category'."""
        n2cmd = self.proc.commands
        names = list(n2cmd.keys())
        if len(args) == 1 and args[0] == '*':
            self.section("Commands in class %s:" % category)
            cmds = [cmd for cmd in names if category == n2cmd[cmd].category]
            cmds.sort()
            self.msg_nocr(self.columnize_commands(cmds))
            return

        self.msg("%s.\n" % categories[category])
        self.section("List of commands:")
        names.sort()
        for name in names:  # Foo! iteritems() doesn't do sorting
            if category != n2cmd[name].category: continue
            self.msg("%-13s -- %s" % (name, n2cmd[name].short_help,))
            pass
        return

def
show_category
(
self
,
category
,
args
)
:
n2cmd
=
self
.
proc
.
commands
names
=
list
(
n2cmd
.
keys
(
)
)
if
len
(
args
)
==
1
and
args
[
0
]
==
'*'
:
self
.
section
(
"Commands in class %s:"
%
category
)
cmds
=
[
cmd
for
cmd
in
names
if
category
==
n2cmd
[
cmd
]
.
category
]
cmds
.
sort
(
)
self
.
msg_nocr
(
self
.
columnize_commands
(
cmds
)
)
return
self
.
msg
(
"%s.\n"
%
categories
[
category
]
)
self
.
section
(
"List of commands:"
)
names
.
sort
(
)
for
name
in
names
:
# Foo! iteritems() doesn't do sorting
if
category
!=
n2cmd
[
name
]
.
category
:
continue
self
.
msg
(
"%-13s -- %s"
%
(
name
,
n2cmd
[
name
]
.
short_help
,
)
)
pass
return

Show short help for all commands in category .