def remove_expired_nodes(self, node_ids=None):
        """
        Removes all expired nodes from the nodelist.  If a set of node_ids is
        passed in, those ids are checked to ensure they haven't been refreshed
        prior to a lock being acquired.

        Should only be run with a lock.

        :param list node_ids: optional, a list of node_ids to remove.  They
            will be verified to ensure they haven't been refreshed.

        """
        nodes = self.find_expired_nodes(node_ids)
        if nodes:
            self.conn.client.hdel(self.nodelist_key, *nodes)

def remove_expired_nodes ( self , node_ids = None ) : nodes = self . find_expired_nodes ( node_ids ) if nodes : self . conn . client . hdel ( self . nodelist_key , * nodes )

Removes all expired nodes from the nodelist . If a set of node_ids is passed in those ids are checked to ensure they haven t been refreshed prior to a lock being acquired .


def remove_node(self, node_id=None):
        """
        Removes a particular node from the nodelist.

        :param string node_id: optional, the process id of the node to remove
        """
        if not node_id:
            node_id = self.conn.id

        self.conn.client.hdel(self.nodelist_key, node_id)

def remove_node ( self , node_id = None ) : if not node_id : node_id = self . conn . id self . conn . client . hdel ( self . nodelist_key , node_id )

Removes a particular node from the nodelist .


def get_last_updated(self, node_id=None):
        """
        Returns the time a particular node has been last refreshed.

        :param string node_id: optional, the connection id of the node to retrieve

        :rtype: int
        :returns: Returns a unix timestamp if it exists, otherwise None
        """
        if not node_id:
            node_id = self.conn.id

        dt = self.conn.client.hget(self.nodelist_key, node_id)
        return int(dt) if dt else None

def get_last_updated ( self , node_id = None ) : if not node_id : node_id = self . conn . id dt = self . conn . client . hget ( self . nodelist_key , node_id ) return int ( dt ) if dt else None

Returns the time a particular node has been last refreshed .


def get_all_nodes(self):
        """
        Returns all nodes in the hash with the time they were last refreshed
        as a dictionary.

        :rtype: dict(string, int)
        :returns: A dictionary of strings and corresponding timestamps

        """
        nodes = self.conn.client.hgetall(self.nodelist_key)
        return {node_id: int(dt) for (node_id, dt) in nodes.items()}

def get_all_nodes ( self ) : nodes = self . conn . client . hgetall ( self . nodelist_key ) return { node_id : int ( dt ) for ( node_id , dt ) in nodes . items ( ) }

Returns all nodes in the hash with the time they were last refreshed as a dictionary .


def refresh_session(self):
        """
        Update the session for this node. Specifically; lock on the reflist,
        then update the time  this node acquired the reference.

        This method should only be called while the reference is locked.
        """
        expired_nodes = self.nodelist.find_expired_nodes()
        if expired_nodes:
            self.nodelist.remove_expired_nodes(expired_nodes)
        self.nodelist.refresh_session()

def refresh_session ( self ) : expired_nodes = self . nodelist . find_expired_nodes ( ) if expired_nodes : self . nodelist . remove_expired_nodes ( expired_nodes ) self . nodelist . refresh_session ( )

Update the session for this node . Specifically ; lock on the reflist then update the time this node acquired the reference .


def increment_times_modified(self):
        """
        Increments the number of times this resource has been modified by all
        processes.
        """
        rc = self.conn.client.incr(self.times_modified_key)
        self.conn.client.pexpire(self.times_modified_key,
                                 phonon.s_to_ms(TTL))

def increment_times_modified ( self ) : rc = self . conn . client . incr ( self . times_modified_key ) self . conn . client . pexpire ( self . times_modified_key , phonon . s_to_ms ( TTL ) )

Increments the number of times this resource has been modified by all processes .


def dereference(self, callback=None, args=None, kwargs=None):
        """
        This method should only be called while the reference is locked.

        Decrements the reference count for the resource. If this process holds
        the only reference at the time we finish dereferencing it; True is
        returned. Operating on the resource after it has been dereferenced is
        undefined behavior.

        Dereference queries the value stored in the backend, if any, iff (if
        and only if) this instance is the last reference to that resource. e.g.
        self.count() == 0

        :param function callback: A function to execute iff it's determined
            this process holds the only reference to the resource. When there
            is a failure communicating with the backend in the cleanup step the
            callback function will be called an additional time for that
            failure and each subsequent one thereafter. Ensure your callback
            handles this properly.
        :param tuple args: Positional arguments to pass your callback.
        :param dict kwargs: keyword arguments to pass your callback.

        :returns: Whether or not there are no more references among all
            processes. True if this was the last reference. False otherwise.
        :rtype: bool
        """
        if args is None:
            args = tuple()
        if kwargs is None:
            kwargs = {}

        client = self.conn.client

        should_execute = False
        if self.force_expiry:
            should_execute = True

        if not should_execute:
            self.nodelist.remove_node(self.conn.id)
            self.nodelist.remove_expired_nodes()

            updated_refcount = client.incr(self.refcount_key, -1)
            should_execute = (updated_refcount <= 0)  # When we force expiry this will be -1

        try:
            if callable(callback) and should_execute:
                callback(*args, **kwargs)
        finally:
            if should_execute:
                client.delete(self.resource_key,
                              self.nodelist.nodelist_key,
                              self.times_modified_key,
                              self.refcount_key)

            self.conn.remove_from_registry(self.resource_key)
        return should_execute

def
dereference
(
self
,
callback
=
,
args
=
,
kwargs
=
)
:
if
args
is
:
args
=
tuple
(
)
if
kwargs
is
:
kwargs
=
{
}
client
=
self
.
conn
.
client
should_execute
=
False
if
self
.
force_expiry
:
should_execute
=
True
if
not
should_execute
:
self
.
nodelist
.
remove_node
(
self
.
conn
.
id
)
self
.
nodelist
.
remove_expired_nodes
(
)
updated_refcount
=
client
.
incr
(
self
.
refcount_key
,
1
)
should_execute
=
(
updated_refcount
<=
0
)
# When we force expiry this will be -1
try
:
if
callable
(
callback
)
and
should_execute
:
callback
(
*
args
,
*
*
kwargs
)
finally
:
if
should_execute
:
client
.
delete
(
self
.
resource_key
,
self
.
nodelist
.
nodelist_key
,
self
.
times_modified_key
,
self
.
refcount_key
)
self
.
conn
.
remove_from_registry
(
self
.
resource_key
)
return
should_execute

This method should only be called while the reference is locked .


def delimit(values, delimiter=', '):
    "Returns a list of tokens interleaved with the delimiter."
    toks = []

    if not values:
        return toks

    if not isinstance(delimiter, (list, tuple)):
        delimiter = [delimiter]

    last = len(values) - 1

    for i, value in enumerate(values):
        toks.append(value)

        if i < last:
            toks.extend(delimiter)

    return toks

def
delimit
(
values
,
delimiter
=
', '
)
:
toks
=
[
]
if
not
values
:
return
toks
if
not
isinstance
(
delimiter
,
(
list
,
tuple
)
)
:
delimiter
=
[
delimiter
]
last
=
len
(
values
)
1
for
i
,
value
in
enumerate
(
values
)
:
toks
.
append
(
value
)
if
i
<
last
:
toks
.
extend
(
delimiter
)
return
toks

Returns a list of tokens interleaved with the delimiter .


def check(path, start, now):
    """check which processes need to be restarted

    :params path: a twisted.python.filepath.FilePath with configurations
    :params start: when the checker started running
    :params now: current time
    :returns: list of strings
    """
    return [child.basename() for child in path.children()
            if _isbad(child, start, now)]

def check ( path , start , now ) : return [ child . basename ( ) for child in path . children ( ) if _isbad ( child , start , now ) ]


def merge(self, status: 'Status[Input, Output]') -> 'Status[Input, Output]':
        """Merge the failure message from another status into this one.

        Whichever status represents parsing that has gone the farthest is
        retained. If both statuses have gone the same distance, then the
        expected values from both are retained.

        Args:
            status: The status to merge into this one.

        Returns:
            This ``Status`` which may have ``farthest`` and ``expected``
            updated accordingly.
        """
        if status is None or status.farthest is None:
            # No new message; simply return unchanged
            pass
        elif self.farthest is None:
            # No current message to compare to; use the message from status
            self.farthest = status.farthest
            self.expected = status.expected
        elif status.farthest.position < self.farthest.position:
            # New message is not farther; keep current message
            pass
        elif status.farthest.position > self.farthest.position:
            # New message is farther than current message; replace with new message
            self.farthest = status.farthest
            self.expected = status.expected
        else:
            # New message and current message are equally far; merge messages
            self.expected = status.expected + self.expected

        return self

Merge the failure message from another status into this one .


def exists(value):
    "Query to test if a value exists."
    if not isinstance(value, Token):
        raise TypeError('value must be a token')

    if not hasattr(value, 'identifier'):
        raise TypeError('value must support an identifier')

    if not value.identifier:
        value = value.__class__(**value.__dict__)
        value.identifier = 'v'

    ident = Identifier(value.identifier)

    return Query([
        OptionalMatch(value),
        Return(Predicate(ident, 'IS NOT NULL')),
        Limit(1),
    ])

def
exists
(
value
)
:
if
not
isinstance
(
value
,
Token
)
:
raise
TypeError
(
'value must be a token'
)
if
not
hasattr
(
value
,
'identifier'
)
:
raise
TypeError
(
'value must support an identifier'
)
if
not
value
.
identifier
:
value
=
value
.
__class__
(
*
*
value
.
__dict__
)
value
.
identifier
=
'v'
ident
=
Identifier
(
value
.
identifier
)
return
Query
(
[
OptionalMatch
(
value
)
,
Return
(
Predicate
(
ident
,
'IS NOT NULL'
)
)
,
Limit
(
1
)
,
]
)


def get(value):
    "Query to get the value."
    if not isinstance(value, Token):
        raise TypeError('value must be a token')

    if not hasattr(value, 'identifier'):
        raise TypeError('value must support an identifier')

    if not value.identifier:
        value = value.__class__(**value.__dict__)
        value.identifier = 'v'

    ident = Identifier(value.identifier)

    return Query([
        Match(value),
        Return(ident)
    ])

def
get
(
value
)
:
if
not
isinstance
(
value
,
Token
)
:
raise
TypeError
(
'value must be a token'
)
if
not
hasattr
(
value
,
'identifier'
)
:
raise
TypeError
(
'value must support an identifier'
)
if
not
value
.
identifier
:
value
=
value
.
__class__
(
*
*
value
.
__dict__
)
value
.
identifier
=
'v'
ident
=
Identifier
(
value
.
identifier
)
return
Query
(
[
Match
(
value
)
,
Return
(
ident
)
]
)


def constant(x: A) -> Callable[..., A]:
    """Produce a function that always returns a supplied value.

    Args:
        x: Any object.

    Returns:
        A function that accepts any number of positional and keyword arguments, discards them, and returns ``x``.
    """

    def constanted(*args, **kwargs):
        return x

    return constanted

def constant ( x : A ) -> Callable [ ... , A ] : def constanted ( * args , * * kwargs ) : return x return constanted

Produce a function that always returns a supplied value .


def splat(f: Callable[..., A]) -> Callable[[Iterable], A]:
    """Convert a function taking multiple arguments into a function taking a single iterable argument.

    Args:
        f: Any function

    Returns:
        A function that accepts a single iterable argument. Each element of this iterable argument is passed as an
        argument to ``f``.

    Example:
        $ def f(a, b, c):
        $     return a + b + c
        $
        $ f(1, 2, 3)  # 6
        $ g = splat(f)
        $ g([1, 2, 3])  # 6
    """

    def splatted(args):
        return f(*args)

    return splatted

def splat ( f : Callable [ ... , A ] ) -> Callable [ [ Iterable ] , A ] : def splatted ( args ) : return f ( * args ) return splatted

Convert a function taking multiple arguments into a function taking a single iterable argument .


def unsplat(f: Callable[[Iterable], A]) -> Callable[..., A]:
    """Convert a function taking a single iterable argument into a function taking multiple arguments.

    Args:
        f: Any function taking a single iterable argument

    Returns:
        A function that accepts multiple arguments. Each argument of this function is passed as an element of an
        iterable to ``f``.

    Example:
        $ def f(a):
        $     return a[0] + a[1] + a[2]
        $
        $ f([1, 2, 3])  # 6
        $ g = unsplat(f)
        $ g(1, 2, 3)  # 6
    """

    def unsplatted(*args):
        return f(args)

    return unsplatted

def unsplat ( f : Callable [ [ Iterable ] , A ] ) -> Callable [ ... , A ] : def unsplatted ( * args ) : return f ( args ) return unsplatted

Convert a function taking a single iterable argument into a function taking multiple arguments .


def runProcess(args, timeout, grace, reactor):
    """Run a process, return a deferred that fires when it is done

    :params args: Process arguments
    :params timeout: Time before terminating process
    :params grace: Time before killing process after terminating it
    :params reactor: IReactorProcess and IReactorTime
    :returns: deferred that fires with success when the process ends,
              or fails if there was a problem spawning/terminating
              the process
    """
    deferred = defer.Deferred()
    protocol = ProcessProtocol(deferred)
    process = reactor.spawnProcess(protocol, args[0], args, env=os.environ)

    def _logEnded(err):
        err.trap(tierror.ProcessDone, tierror.ProcessTerminated)
        print(err.value)
    deferred.addErrback(_logEnded)

    def _cancelTermination(dummy):
        for termination in terminations:
            if termination.active():
                termination.cancel()
    deferred.addCallback(_cancelTermination)
    terminations = []
    terminations.append(reactor.callLater(timeout, process.signalProcess,
                                          "TERM"))
    terminations.append(reactor.callLater(timeout+grace,
                                          process.signalProcess, "KILL"))
    return deferred

def runProcess ( args , timeout , grace , reactor ) : deferred = defer . Deferred ( ) protocol = ProcessProtocol ( deferred ) process = reactor . spawnProcess ( protocol , args [ 0 ] , args , env = os . environ ) def _logEnded ( err ) : err . trap ( tierror . ProcessDone , tierror . ProcessTerminated ) print ( err . value ) deferred . addErrback ( _logEnded ) def _cancelTermination ( dummy ) : for termination in terminations : if termination . active ( ) : termination . cancel ( ) deferred . addCallback ( _cancelTermination ) terminations = [ ] terminations . append ( reactor . callLater ( timeout , process . signalProcess , "TERM" ) ) terminations . append ( reactor . callLater ( timeout + grace , process . signalProcess , "KILL" ) ) return deferred

Run a process return a deferred that fires when it is done


def makeService(opts):
    """Make scheduler service

    :params opts: dict-like object.
       keys: frequency, args, timeout, grace
    """
    ser = tainternet.TimerService(opts['frequency'], runProcess, opts['args'],
                                  opts['timeout'], opts['grace'], tireactor)
    ret = service.MultiService()
    ser.setName('scheduler')
    ser.setServiceParent(ret)
    heart.maybeAddHeart(ret)
    return ret

def makeService ( opts ) : ser = tainternet . TimerService ( opts [ 'frequency' ] , runProcess , opts [ 'args' ] , opts [ 'timeout' ] , opts [ 'grace' ] , tireactor ) ret = service . MultiService ( ) ser . setName ( 'scheduler' ) ser . setServiceParent ( ret ) heart . maybeAddHeart ( ret ) return ret


def completely_parse_reader(parser: Parser[Input, Output], reader: Reader[Input]) -> Result[Output]:
    """Consume reader and return Success only on complete consumption.

    This is a helper function for ``parse`` methods, which return ``Success``
    when the input is completely consumed and ``Failure`` with an appropriate
    message otherwise.

    Args:
        parser: The parser doing the consuming
        reader: The input being consumed

    Returns:
        A parsing ``Result``
    """
    result = (parser << eof).consume(reader)

    if isinstance(result, Continue):
        return Success(result.value)
    else:
        used = set()
        unique_expected = []
        for expected_lambda in result.expected:
            expected = expected_lambda()
            if expected not in used:
                used.add(expected)
                unique_expected.append(expected)

        return Failure(result.farthest.expected_error(' or '.join(unique_expected)))

def
completely_parse_reader
(
parser
:
Parser
[
Input
,
Output
]
,
reader
:
Reader
[
Input
]
)
->
Result
[
Output
]
:
result
=
(
parser
<<
eof
)
.
consume
(
reader
)
if
isinstance
(
result
,
Continue
)
:
return
Success
(
result
.
value
)
else
:
used
=
set
(
)
unique_expected
=
[
]
for
expected_lambda
in
result
.
expected
:
expected
=
expected_lambda
(
)
if
expected
not
in
used
:
used
.
add
(
expected
)
unique_expected
.
append
(
expected
)
return
Failure
(
result
.
farthest
.
expected_error
(
' or '
.
join
(
unique_expected
)
)
)

Consume reader and return Success only on complete consumption .


def lit(literal: Sequence[Input], *literals: Sequence[Sequence[Input]]) -> Parser:
    """Match a literal sequence.

    In the `TextParsers`` context, this matches the literal string
    provided. In the ``GeneralParsers`` context, this matches a sequence of
    input.

    If multiple literals are provided, they are treated as alternatives. e.g.
    ``lit('+', '-')`` is the same as ``lit('+') | lit('-')``.

    Args:
        literal: A literal to match
        *literals: Alternative literals to match

    Returns:
        A ``LiteralParser`` in the ``GeneralContext``, a ``LiteralStringParser``
        in the ``TextParsers`` context, and an ``AlternativeParser`` if multiple
        arguments are provided.
    """
    if len(literals) > 0:
        return AlternativeParser(options.handle_literal(literal), *map(options.handle_literal, literals))
    else:
        return options.handle_literal(literal)

def lit ( literal : Sequence [ Input ] , * literals : Sequence [ Sequence [ Input ] ] ) -> Parser : if len ( literals ) > 0 : return AlternativeParser ( options . handle_literal ( literal ) , * map ( options . handle_literal , literals ) ) else : return options . handle_literal ( literal )


def opt(parser: Union[Parser, Sequence[Input]]) -> OptionalParser:
    """Optionally match a parser.

    An ``OptionalParser`` attempts to match ``parser``. If it succeeds, it
    returns a list of length one with the value returned by the parser as the
    only element. If it fails, it returns an empty list.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return OptionalParser(parser)

def opt ( parser : Union [ Parser , Sequence [ Input ] ] ) -> OptionalParser : if isinstance ( parser , str ) : parser = lit ( parser ) return OptionalParser ( parser )


def rep1(parser: Union[Parser, Sequence[Input]]) -> RepeatedOnceParser:
    """Match a parser one or more times repeatedly.

    This matches ``parser`` multiple times in a row. If it matches as least
    once, it returns a list of values from each time ``parser`` matched. If it
    does not match ``parser`` at all, it fails.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return RepeatedOnceParser(parser)

def rep1 ( parser : Union [ Parser , Sequence [ Input ] ] ) -> RepeatedOnceParser : if isinstance ( parser , str ) : parser = lit ( parser ) return RepeatedOnceParser ( parser )

Match a parser one or more times repeatedly .


def rep(parser: Union[Parser, Sequence[Input]]) -> RepeatedParser:
    """Match a parser zero or more times repeatedly.

    This matches ``parser`` multiple times in a row. A list is returned
    containing the value from each match. If there are no matches, an empty list
    is returned.

    Args:
        parser: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    return RepeatedParser(parser)

def rep ( parser : Union [ Parser , Sequence [ Input ] ] ) -> RepeatedParser : if isinstance ( parser , str ) : parser = lit ( parser ) return RepeatedParser ( parser )

Match a parser zero or more times repeatedly .


def rep1sep(parser: Union[Parser, Sequence[Input]], separator: Union[Parser, Sequence[Input]]) \
        -> RepeatedOnceSeparatedParser:
    """Match a parser one or more times separated by another parser.

    This matches repeated sequences of ``parser`` separated by ``separator``.
    If there is at least one match, a list containing the values of the
    ``parser`` matches is returned. The values from ``separator`` are discarded.
    If it does not match ``parser`` at all, it fails.

    Args:
        parser: Parser or literal
        separator: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    if isinstance(separator, str):
        separator = lit(separator)
    return RepeatedOnceSeparatedParser(parser, separator)

def rep1sep ( parser : Union [ Parser , Sequence [ Input ] ] , separator : Union [ Parser , Sequence [ Input ] ] ) -> RepeatedOnceSeparatedParser : if isinstance ( parser , str ) : parser = lit ( parser ) if isinstance ( separator , str ) : separator = lit ( separator ) return RepeatedOnceSeparatedParser ( parser , separator )

Match a parser one or more times separated by another parser .


def repsep(parser: Union[Parser, Sequence[Input]], separator: Union[Parser, Sequence[Input]]) \
        -> RepeatedSeparatedParser:
    """Match a parser zero or more times separated by another parser.

    This matches repeated sequences of ``parser`` separated by ``separator``. A
    list is returned containing the value from each match of ``parser``. The
    values from ``separator`` are discarded. If there are no matches, an empty
    list is returned.

    Args:
        parser: Parser or literal
        separator: Parser or literal
    """
    if isinstance(parser, str):
        parser = lit(parser)
    if isinstance(separator, str):
        separator = lit(separator)
    return RepeatedSeparatedParser(parser, separator)

def repsep ( parser : Union [ Parser , Sequence [ Input ] ] , separator : Union [ Parser , Sequence [ Input ] ] ) -> RepeatedSeparatedParser : if isinstance ( parser , str ) : parser = lit ( parser ) if isinstance ( separator , str ) : separator = lit ( separator ) return RepeatedSeparatedParser ( parser , separator )

Match a parser zero or more times separated by another parser .


def check(settings, states, location):
    """Check all processes"""
    children = {child.basename(): child for child in location.children()}
    last = set(states)
    current = set(children)
    gone = last - current
    added = current - last
    for name in gone:
        states[name].close()
        del states[name]
    for name in added:
        states[name] = State(location=children[name], settings=settings)
    return [name for name, state in six.iteritems(states) if state.check()]

def check ( settings , states , location ) : children = { child . basename ( ) : child for child in location . children ( ) } last = set ( states ) current = set ( children ) gone = last - current added = current - last for name in gone : states [ name ] . close ( ) del states [ name ] for name in added : states [ name ] = State ( location = children [ name ] , settings = settings ) return [ name for name , state in six . iteritems ( states ) if state . check ( ) ]


def close(self):
        """Discard data and cancel all calls.

        Instance cannot be reused after closing.
        """
        if self.closed:
            raise ValueError("Cannot close a closed state")
        if self.call is not None:
            self.call.cancel()
        self.closed = True

def
close
(
self
)
:
if
self
.
closed
:
raise
ValueError
(
"Cannot close a closed state"
)
if
self
.
call
is
not
:
self
.
call
.
cancel
(
)
self
.
closed
=
True


def check(self):
        """Check the state of HTTP"""
        if self.closed:
            raise ValueError("Cannot check a closed state")
        self._maybeReset()
        if self.url is None:
            return False
        return self._maybeCheck()

def
check
(
self
)
:
if
self
.
closed
:
raise
ValueError
(
"Cannot check a closed state"
)
self
.
_maybeReset
(
)
if
self
.
url
is
:
return
False
return
self
.
_maybeCheck
(
)


def maybeAddHeart(master):
    """Add a heart to a service collection

    Add a heart to a service.IServiceCollector if
    the heart is not None.

    :params master: a service.IServiceCollector
    """
    heartSer = makeService()
    if heartSer is None:
        return
    heartSer.setName('heart')
    heartSer.setServiceParent(master)

def maybeAddHeart ( master ) : heartSer = makeService ( ) if heartSer is None : return heartSer . setName ( 'heart' ) heartSer . setServiceParent ( master )


def wrapHeart(service):
    """Wrap a service in a MultiService with a heart"""
    master = taservice.MultiService()
    service.setServiceParent(master)
    maybeAddHeart(master)
    return master

def wrapHeart ( service ) : master = taservice . MultiService ( ) service . setServiceParent ( master ) maybeAddHeart ( master ) return master


def freeze_from_checkpoint(input_checkpoint, output_file_path, output_node_names):
    """Freeze and shrink the graph based on a checkpoint and the output node names."""
    check_input_checkpoint(input_checkpoint)

    output_node_names = output_node_names_string_as_list(output_node_names)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        freeze_graph.freeze_graph_with_def_protos(input_graph_def=sess.graph_def, input_saver_def=None,
                                                  input_checkpoint=input_checkpoint,
                                                  output_node_names=','.join(output_node_names),
                                                  restore_op_name='save/restore_all',
                                                  filename_tensor_name='save/Const:0', output_graph=output_file_path,
                                                  clear_devices=True, initializer_nodes='')

Freeze and shrink the graph based on a checkpoint and the output node names .


def freeze(sess, output_file_path, output_node_names):
    """Freeze and shrink the graph based on a session and the output node names."""
    with TemporaryDirectory() as temp_dir_name:
        checkpoint_path = os.path.join(temp_dir_name, 'model.ckpt')
        tf.train.Saver().save(sess, checkpoint_path)

        freeze_from_checkpoint(checkpoint_path, output_file_path, output_node_names)

def freeze ( sess , output_file_path , output_node_names ) : with TemporaryDirectory ( ) as temp_dir_name : checkpoint_path = os . path . join ( temp_dir_name , 'model.ckpt' ) tf . train . Saver ( ) . save ( sess , checkpoint_path ) freeze_from_checkpoint ( checkpoint_path , output_file_path , output_node_names )

Freeze and shrink the graph based on a session and the output node names .


def save_graph_only(sess, output_file_path, output_node_names, as_text=False):
    """Save a small version of the graph based on a session and the output node names."""
    for node in sess.graph_def.node:
        node.device = ''
    graph_def = graph_util.extract_sub_graph(sess.graph_def, output_node_names)
    output_dir, output_filename = os.path.split(output_file_path)
    graph_io.write_graph(graph_def, output_dir, output_filename, as_text=as_text)

def save_graph_only ( sess , output_file_path , output_node_names , as_text = False ) : for node in sess . graph_def . node : node . device = '' graph_def = graph_util . extract_sub_graph ( sess . graph_def , output_node_names ) output_dir , output_filename = os . path . split ( output_file_path ) graph_io . write_graph ( graph_def , output_dir , output_filename , as_text = as_text )

Save a small version of the graph based on a session and the output node names .


def save_graph_only_from_checkpoint(input_checkpoint, output_file_path, output_node_names, as_text=False):
    """Save a small version of the graph based on a checkpoint and the output node names."""
    check_input_checkpoint(input_checkpoint)

    output_node_names = output_node_names_string_as_list(output_node_names)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        save_graph_only(sess, output_file_path, output_node_names, as_text=as_text)

def save_graph_only_from_checkpoint ( input_checkpoint , output_file_path , output_node_names , as_text = False ) : check_input_checkpoint ( input_checkpoint ) output_node_names = output_node_names_string_as_list ( output_node_names ) with tf . Session ( ) as sess : restore_from_checkpoint ( sess , input_checkpoint ) save_graph_only ( sess , output_file_path , output_node_names , as_text = as_text )

Save a small version of the graph based on a checkpoint and the output node names .


def save_weights_from_checkpoint(input_checkpoint, output_path, conv_var_names=None, conv_transpose_var_names=None):
    """Save the weights of the trainable variables given a checkpoint, each one in a different file in output_path."""
    check_input_checkpoint(input_checkpoint)

    with tf.Session() as sess:
        restore_from_checkpoint(sess, input_checkpoint)
        save_weights(sess, output_path, conv_var_names=conv_var_names,
                     conv_transpose_var_names=conv_transpose_var_names)

def save_weights_from_checkpoint ( input_checkpoint , output_path , conv_var_names = None , conv_transpose_var_names = None ) : check_input_checkpoint ( input_checkpoint ) with tf . Session ( ) as sess : restore_from_checkpoint ( sess , input_checkpoint ) save_weights ( sess , output_path , conv_var_names = conv_var_names , conv_transpose_var_names = conv_transpose_var_names )

Save the weights of the trainable variables given a checkpoint each one in a different file in output_path .


def restore_from_checkpoint(sess, input_checkpoint):
    """Return a TensorFlow saver from a checkpoint containing the metagraph."""
    saver = tf.train.import_meta_graph('{}.meta'.format(input_checkpoint))
    saver.restore(sess, input_checkpoint)
    return saver

def restore_from_checkpoint ( sess , input_checkpoint ) : saver = tf . train . import_meta_graph ( '{}.meta' . format ( input_checkpoint ) ) saver . restore ( sess , input_checkpoint ) return saver

Return a TensorFlow saver from a checkpoint containing the metagraph .


def parse(cls, parser, token):
        """
        Parse the tag, instantiate the class.

        :type parser: django.template.base.Parser
        :type token: django.template.base.Token
        """
        tag_name, args, kwargs = parse_token_kwargs(
            parser, token,
            allowed_kwargs=cls.allowed_kwargs,
            compile_args=cls.compile_args,
            compile_kwargs=cls.compile_kwargs
        )
        cls.validate_args(tag_name, *args, **kwargs)
        if cls.end_tag_name:
            kwargs['nodelist'] = parser.parse((cls.end_tag_name,))
            parser.delete_first_token()

        return cls(tag_name, *args, **kwargs)

def parse ( cls , parser , token ) : tag_name , args , kwargs = parse_token_kwargs ( parser , token , allowed_kwargs = cls . allowed_kwargs , compile_args = cls . compile_args , compile_kwargs = cls . compile_kwargs ) cls . validate_args ( tag_name , * args , * * kwargs ) if cls . end_tag_name : kwargs [ 'nodelist' ] = parser . parse ( ( cls . end_tag_name , ) ) parser . delete_first_token ( ) return cls ( tag_name , * args , * * kwargs )


def render_tag(self, context, *tag_args, **tag_kwargs):
        """
        Render the tag, with all arguments resolved to their actual values.
        """
        raise NotImplementedError("{0}.render_tag() is not implemented!".format(self.__class__.__name__))

def
render_tag
(
self
,
context
,
*
tag_args
,
*
*
tag_kwargs
)
:
raise
NotImplementedError
(
"{0}.render_tag() is not implemented!"
.
format
(
self
.
__class__
.
__name__
)
)

Render the tag with all arguments resolved to their actual values .


def validate_args(cls, tag_name, *args, **kwargs):
        """
        Validate the syntax of the template tag.
        """
        if cls.min_args is not None and len(args) < cls.min_args:
            if cls.min_args == 1:
                raise TemplateSyntaxError("'{0}' tag requires at least {1} argument".format(tag_name, cls.min_args))
            else:
                raise TemplateSyntaxError("'{0}' tag requires at least {1} arguments".format(tag_name, cls.min_args))

        if cls.max_args is not None and len(args) > cls.max_args:
            if cls.max_args == 0:
                if cls.allowed_kwargs:
                    raise TemplateSyntaxError("'{0}' tag only allows keywords arguments, for example {1}=\"...\".".format(tag_name, cls.allowed_kwargs[0]))
                else:
                    raise TemplateSyntaxError("'{0}' tag doesn't support any arguments".format(tag_name))
            elif cls.max_args == 1:
                raise TemplateSyntaxError("'{0}' tag only allows {1} argument.".format(tag_name, cls.max_args))
            else:
                raise TemplateSyntaxError("'{0}' tag only allows {1} arguments.".format(tag_name, cls.max_args))

Validate the syntax of the template tag .


def get_context_data(self, parent_context, *tag_args, **tag_kwargs):
        """
        Return the context data for the included template.
        """
        raise NotImplementedError("{0}.get_context_data() is not implemented.".format(self.__class__.__name__))

Return the context data for the included template .


def parse(cls, parser, token):
        """
        Parse the "as var" syntax.
        """
        bits, as_var = parse_as_var(parser, token)
        tag_name, args, kwargs = parse_token_kwargs(parser, bits, ('template',) + cls.allowed_kwargs, compile_args=cls.compile_args, compile_kwargs=cls.compile_kwargs)

        # Pass through standard chain
        cls.validate_args(tag_name, *args)
        return cls(tag_name, as_var, *args, **kwargs)

def
parse
(
cls
,
parser
,
token
)
:
bits
,
as_var
=
parse_as_var
(
parser
,
token
)
tag_name
,
args
,
kwargs
=
parse_token_kwargs
(
parser
,
bits
,
(
'template'
,
)
+
cls
.
allowed_kwargs
,
compile_args
=
cls
.
compile_args
,
compile_kwargs
=
cls
.
compile_kwargs
)
# Pass through standard chain
cls
.
validate_args
(
tag_name
,
*
args
)
return
cls
(
tag_name
,
as_var
,
*
args
,
*
*
kwargs
)


def get_context_data(self, parent_context, *tag_args, **tag_kwargs):
        """
        Return the context data for the inclusion tag.

        Returns ``{'value': self.get_value(parent_context, *tag_args, **tag_kwargs)}`` by default.
        """
        if 'template' not in self.allowed_kwargs:
            # The overwritten get_value() doesn't have to take care of our customly inserted tag parameters,
            # It can safely assume passing **tag_kwargs to another function.
            tag_kwargs.pop('template', None)

        return {
            self.context_value_name: self.get_value(parent_context, *tag_args, **tag_kwargs)
        }

Return the context data for the inclusion tag .


def caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name='Graph',
                                conversion_out_dir_path=None, use_padding_same=False):
    """Create a TensorFlow Session from a Caffe model."""
    try:
        # noinspection PyUnresolvedReferences
        from caffeflow import convert
    except ImportError:
        raise Exception("caffeflow package needs to be installed to freeze Caffe models. Check out the README file.")

    with (dummy_context_mgr(conversion_out_dir_path) or util.TemporaryDirectory()) as dir_path:
        params_values_output_path = os.path.join(dir_path, 'params_values.npy')
        network_output_path = os.path.join(dir_path, 'network.py')

        convert.convert(caffe_def_path, caffemodel_path, params_values_output_path, network_output_path, False,
                        use_padding_same=use_padding_same)

        network_module = imp.load_source('module.name', network_output_path)
        network_class = getattr(network_module, graph_name)
        network = network_class(inputs)

        sess = tf.Session()

        network.load(params_values_output_path, sess)

        return sess

Create a TensorFlow Session from a Caffe model .


def freeze(caffe_def_path, caffemodel_path, inputs, output_file_path, output_node_names, graph_name='Graph',
           conversion_out_dir_path=None, checkpoint_out_path=None, use_padding_same=False):
    """Freeze and shrink the graph based on a Caffe model, the input tensors and the output node names."""
    with caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name=graph_name,
                                     conversion_out_dir_path=conversion_out_dir_path,
                                     use_padding_same=use_padding_same) as sess:
        saver = tf.train.Saver()

        with (dummy_context_mgr(checkpoint_out_path) or util.TemporaryDirectory()) as temp_dir_path:
            checkpoint_path = checkpoint_out_path or os.path.join(temp_dir_path, 'pose.ckpt')
            saver.save(sess, checkpoint_path)

            output_node_names = util.output_node_names_string_as_list(output_node_names)

            tf_freeze.freeze_from_checkpoint(checkpoint_path, output_file_path, output_node_names)

def freeze ( caffe_def_path , caffemodel_path , inputs , output_file_path , output_node_names , graph_name = 'Graph' , conversion_out_dir_path = None , checkpoint_out_path = None , use_padding_same = False ) : with caffe_to_tensorflow_session ( caffe_def_path , caffemodel_path , inputs , graph_name = graph_name , conversion_out_dir_path = conversion_out_dir_path , use_padding_same = use_padding_same ) as sess : saver = tf . train . Saver ( ) with ( dummy_context_mgr ( checkpoint_out_path ) or util . TemporaryDirectory ( ) ) as temp_dir_path : checkpoint_path = checkpoint_out_path or os . path . join ( temp_dir_path , 'pose.ckpt' ) saver . save ( sess , checkpoint_path ) output_node_names = util . output_node_names_string_as_list ( output_node_names ) tf_freeze . freeze_from_checkpoint ( checkpoint_path , output_file_path , output_node_names )

Freeze and shrink the graph based on a Caffe model the input tensors and the output node names .


def save_graph_only(caffe_def_path, caffemodel_path, inputs, output_file_path, output_node_names, graph_name='Graph',
                    use_padding_same=False):
    """Save a small version of the graph based on a Caffe model, the input tensors and the output node names."""
    with caffe_to_tensorflow_session(caffe_def_path, caffemodel_path, inputs, graph_name=graph_name,
                                     use_padding_same=use_padding_same) as sess:
        tf_freeze.save_graph_only(sess, output_file_path, output_node_names)

Save a small version of the graph based on a Caffe model the input tensors and the output node names .


def make_rows(num_columns, seq):
	"""
	Make a sequence into rows of num_columns columns.

	>>> tuple(make_rows(2, [1, 2, 3, 4, 5]))
	((1, 4), (2, 5), (3, None))
	>>> tuple(make_rows(3, [1, 2, 3, 4, 5]))
	((1, 3, 5), (2, 4, None))
	"""
	# calculate the minimum number of rows necessary to fit the list in
	# num_columns Columns
	num_rows, partial = divmod(len(seq), num_columns)
	if partial:
		num_rows += 1
	# break the seq into num_columns of length num_rows
	try:
		result = more_itertools.grouper(seq, num_rows)
	except TypeError:
		# more_itertools before 6.x
		result = more_itertools.grouper(num_rows, seq)
	# result is now a list of columns... transpose it to return a list
	# of rows
	return zip(*result)

Make a sequence into rows of num_columns columns .


def grouper_nofill_str(n, iterable):
	"""
	Take a sequence and break it up into chunks of the specified size.
	The last chunk may be smaller than size.

	This works very similar to grouper_nofill, except
	it works with strings as well.

	>>> tuple(grouper_nofill_str(3, 'foobarbaz'))
	('foo', 'bar', 'baz')

	You can still use it on non-strings too if you like.

	>>> tuple(grouper_nofill_str(42, []))
	()

	>>> tuple(grouper_nofill_str(3, list(range(10))))
	([0, 1, 2], [3, 4, 5], [6, 7, 8], [9])
	"""
	res = more_itertools.chunked(iterable, n)
	if isinstance(iterable, six.string_types):
		res = (''.join(item) for item in res)
	return res

def grouper_nofill_str ( n , iterable ) : res = more_itertools . chunked ( iterable , n ) if isinstance ( iterable , six . string_types ) : res = ( '' . join ( item ) for item in res ) return res

Take a sequence and break it up into chunks of the specified size . The last chunk may be smaller than size .


def every_other(iterable):
	"""
	Yield every other item from the iterable

	>>> ' '.join(every_other('abcdefg'))
	'a c e g'
	"""
	items = iter(iterable)
	while True:
		try:
			yield next(items)
			next(items)
		except StopIteration:
			return

def every_other ( iterable ) : items = iter ( iterable ) while True : try : yield next ( items ) next ( items ) except StopIteration : return


def remove_duplicates(iterable, key=None):
	"""
	Given an iterable with items that may come in as sequential duplicates,
	remove those duplicates.

	Unlike unique_justseen, this function does not remove triplicates.

	>>> ' '.join(remove_duplicates('abcaabbccaaabbbcccbcbc'))
	'a b c a b c a a b b c c b c b c'
	>>> ' '.join(remove_duplicates('aaaabbbbb'))
	'a a b b b'
	"""
	return itertools.chain.from_iterable(six.moves.map(
		every_other, six.moves.map(
			operator.itemgetter(1),
			itertools.groupby(iterable, key)
		)))

def remove_duplicates ( iterable , key = None ) : return itertools . chain . from_iterable ( six . moves . map ( every_other , six . moves . map ( operator . itemgetter ( 1 ) , itertools . groupby ( iterable , key ) ) ) )

Given an iterable with items that may come in as sequential duplicates remove those duplicates .


def peek(iterable):
	"""
	Get the next value from an iterable, but also return an iterable
	that will subsequently return that value and the rest of the
	original iterable.

	>>> l = iter([1,2,3])
	>>> val, l = peek(l)
	>>> val
	1
	>>> list(l)
	[1, 2, 3]
	"""
	peeker, original = itertools.tee(iterable)
	return next(peeker), original

def peek ( iterable ) : peeker , original = itertools . tee ( iterable ) return next ( peeker ) , original

Get the next value from an iterable but also return an iterable that will subsequently return that value and the rest of the original iterable .


def takewhile_peek(predicate, iterable):
	"""
	Like takewhile, but takes a peekable iterable and doesn't
	consume the non-matching item.

	>>> items = Peekable(range(10))
	>>> is_small = lambda n: n < 4

	>>> small_items = takewhile_peek(is_small, items)

	>>> list(small_items)
	[0, 1, 2, 3]

	>>> list(items)
	[4, 5, 6, 7, 8, 9]

	>>> empty = takewhile_peek(is_small, Peekable([]))
	>>> list(empty)
	[]

	>>> items = Peekable([3])
	>>> small_items = takewhile_peek(is_small, items)
	>>> list(small_items)
	[3]
	>>> list(items)
	[]

	>>> items = Peekable([4])
	>>> small_items = takewhile_peek(is_small, items)
	>>> list(small_items)
	[]
	>>> list(items)
	[4]
	"""
	while True:
		try:
			if not predicate(iterable.peek()):
				break
			yield next(iterable)
		except StopIteration:
			break

def takewhile_peek ( predicate , iterable ) : while True : try : if not predicate ( iterable . peek ( ) ) : break yield next ( iterable ) except StopIteration : break

Like takewhile but takes a peekable iterable and doesn t consume the non - matching item .


def partition_items(count, bin_size):
	"""
	Given the total number of items, determine the number of items that
	can be added to each bin with a limit on the bin size.

	So if you want to partition 11 items into groups of 3, you'll want
	three of three and one of two.

	>>> partition_items(11, 3)
	[3, 3, 3, 2]

	But if you only have ten items, you'll have two groups of three and
	two of two.

	>>> partition_items(10, 3)
	[3, 3, 2, 2]
	"""
	num_bins = int(math.ceil(count / float(bin_size)))
	bins = [0] * num_bins
	for i in range(count):
		bins[i % num_bins] += 1
	return bins

def partition_items ( count , bin_size ) : num_bins = int ( math . ceil ( count / float ( bin_size ) ) ) bins = [ 0 ] * num_bins for i in range ( count ) : bins [ i % num_bins ] += 1 return bins

Given the total number of items determine the number of items that can be added to each bin with a limit on the bin size .


def always_iterable(item):
	"""
	Given an object, always return an iterable. If the item is not
	already iterable, return a tuple containing only the item. If item is
	None, an empty iterable is returned.

	>>> always_iterable([1,2,3])
	<list_iterator...>
	>>> always_iterable('foo')
	<tuple_iterator...>
	>>> always_iterable(None)
	<tuple_iterator...>
	>>> always_iterable(range(10))
	<range_iterator...>
	>>> def _test_func(): yield "I'm iterable"
	>>> print(next(always_iterable(_test_func())))
	I'm iterable

	Although mappings are iterable, treat each like a singleton, as
	it's more like an object than a sequence.

	>>> next(always_iterable(dict(a=1)))
	{'a': 1}
	"""
	base_types = six.text_type, bytes, collections.abc.Mapping
	return more_itertools.always_iterable(item, base_type=base_types)

def always_iterable ( item ) : base_types = six . text_type , bytes , collections . abc . Mapping return more_itertools . always_iterable ( item , base_type = base_types )

Given an object always return an iterable . If the item is not already iterable return a tuple containing only the item . If item is None an empty iterable is returned .


def suppress_exceptions(callables, *exceptions):
	"""
	Call each callable in callables, suppressing any exceptions supplied. If
	no exception classes are supplied, all Exceptions will be suppressed.

	>>> import functools
	>>> c1 = functools.partial(int, 'a')
	>>> c2 = functools.partial(int, '10')
	>>> list(suppress_exceptions((c1, c2)))
	[10]
	>>> list(suppress_exceptions((c1, c2), KeyError))
	Traceback (most recent call last):
	...
	ValueError: invalid literal for int() with base 10: 'a'
	"""
	if not exceptions:
		exceptions = Exception,
	for callable in callables:
		try:
			yield callable()
		except exceptions:
			pass

def suppress_exceptions ( callables , * exceptions ) : if not exceptions : exceptions = Exception , for callable in callables : try : yield callable ( ) except exceptions : pass

Call each callable in callables suppressing any exceptions supplied . If no exception classes are supplied all Exceptions will be suppressed .


def duplicates(*iterables, **kwargs):
	"""
	Yield duplicate items from any number of sorted iterables of items

	>>> items_a = [1, 2, 3]
	>>> items_b = [0, 3, 4, 5, 6]
	>>> list(duplicates(items_a, items_b))
	[(3, 3)]

	It won't behave as you expect if the iterables aren't ordered

	>>> items_b.append(1)
	>>> list(duplicates(items_a, items_b))
	[(3, 3)]
	>>> list(duplicates(items_a, sorted(items_b)))
	[(1, 1), (3, 3)]

	This function is most interesting when it's operating on a key
	of more complex objects.

	>>> items_a = [dict(email='joe@example.com', id=1)]
	>>> items_b = [dict(email='joe@example.com', id=2), dict(email='other')]
	>>> dupe, = duplicates(items_a, items_b, key=operator.itemgetter('email'))
	>>> dupe[0]['email'] == dupe[1]['email'] == 'joe@example.com'
	True
	>>> dupe[0]['id']
	1
	>>> dupe[1]['id']
	2
	"""
	key = kwargs.pop('key', lambda x: x)
	assert not kwargs
	zipped = more_itertools.collate(*iterables, key=key)
	grouped = itertools.groupby(zipped, key=key)
	groups = (
		tuple(g)
		for k, g in grouped
	)

	def has_dupes(group):
		return len(group) > 1
	return filter(has_dupes, groups)

def duplicates ( * iterables , * * kwargs ) : key = kwargs . pop ( 'key' , lambda x : x ) assert not kwargs zipped = more_itertools . collate ( * iterables , key = key ) grouped = itertools . groupby ( zipped , key = key ) groups = ( tuple ( g ) for k , g in grouped ) def has_dupes ( group ) : return len ( group ) > 1 return filter ( has_dupes , groups )


def assert_ordered(iterable, key=lambda x: x, comp=operator.le):
	"""
	Assert that for all items in the iterable, they're in order based on comp

	>>> list(assert_ordered(range(5)))
	[0, 1, 2, 3, 4]
	>>> list(assert_ordered(range(5), comp=operator.ge))
	Traceback (most recent call last):
	...
	AssertionError: 0 < 1
	>>> list(assert_ordered(range(5, 0, -1), key=operator.neg))
	[5, 4, 3, 2, 1]
	"""
	err_tmpl = (
		"{pair[0]} > {pair[1]}" if comp is operator.le else
		"{pair[0]} < {pair[1]}" if comp is operator.ge else
		"not {comp} {pair}"
	)
	for pair in more_itertools.pairwise(iterable):
		keyed = tuple(map(key, pair))
		assert comp(*keyed), err_tmpl.format(**locals())
		yield pair[0]
	yield pair[1]

def
assert_ordered
(
iterable
,
key
=
lambda
x
:
x
,
comp
=
operator
.
le
)
:
err_tmpl
=
(
"{pair[0]} > {pair[1]}"
if
comp
is
operator
.
le
else
"{pair[0]} < {pair[1]}"
if
comp
is
operator
.
ge
else
"not {comp} {pair}"
)
for
pair
in
more_itertools
.
pairwise
(
iterable
)
:
keyed
=
tuple
(
map
(
key
,
pair
)
)
assert
comp
(
*
keyed
)
,
err_tmpl
.
format
(
*
*
locals
(
)
)
yield
pair
[
0
]
yield
pair
[
1
]

Assert that for all items in the iterable they re in order based on comp


def _swap_on_miss(partition_result):
	"""
	Given a partition_dict result, if the partition missed, swap
	the before and after.
	"""
	before, item, after = partition_result
	return (before, item, after) if item else (after, item, before)

def _swap_on_miss ( partition_result ) : before , item , after = partition_result return ( before , item , after ) if item else ( after , item , before )

Given a partition_dict result if the partition missed swap the before and after .


def partition_dict(items, key):
	"""
	Given an ordered dictionary of items and a key in that dict,
	return an ordered dict of items before, the keyed item, and
	an ordered dict of items after.

	>>> od = collections.OrderedDict(zip(range(5), 'abcde'))
	>>> before, item, after = partition_dict(od, 3)
	>>> before
	OrderedDict([(0, 'a'), (1, 'b'), (2, 'c')])
	>>> item
	'd'
	>>> after
	OrderedDict([(4, 'e')])

	Like string.partition, if the key is not found in the items,
	the before will contain all items, item will be None, and
	after will be an empty iterable.

	>>> before, item, after = partition_dict(od, -1)
	>>> before
	OrderedDict([(0, 'a'), ..., (4, 'e')])
	>>> item
	>>> list(after)
	[]
	"""
	def unmatched(pair):
		test_key, item, = pair
		return test_key != key

	items_iter = iter(items.items())
	item = items.get(key)
	left = collections.OrderedDict(itertools.takewhile(unmatched, items_iter))
	right = collections.OrderedDict(items_iter)
	return left, item, right

def partition_dict ( items , key ) : def unmatched ( pair ) : test_key , item , = pair return test_key != key items_iter = iter ( items . items ( ) ) item = items . get ( key ) left = collections . OrderedDict ( itertools . takewhile ( unmatched , items_iter ) ) right = collections . OrderedDict ( items_iter ) return left , item , right

Given an ordered dictionary of items and a key in that dict return an ordered dict of items before the keyed item and an ordered dict of items after .


def get_first_n_queues(self, n):
		"""
		Run through the sequence until n queues are created and return
		them. If fewer are created, return those plus empty iterables to
		compensate.
		"""
		try:
			while len(self.queues) < n:
				self.__fetch__()
		except StopIteration:
			pass
		values = list(self.queues.values())
		missing = n - len(values)
		values.extend(iter([]) for n in range(missing))
		return values

def get_first_n_queues ( self , n ) : try : while len ( self . queues ) < n : self . __fetch__ ( ) except StopIteration : pass values = list ( self . queues . values ( ) ) missing = n - len ( values ) values . extend ( iter ( [ ] ) for n in range ( missing ) ) return values

Run through the sequence until n queues are created and return them . If fewer are created return those plus empty iterables to compensate .


def reset(self):
		"""
		Resets the iterator to the start.

		Any remaining values in the current iteration are discarded.
		"""
		self.__iterator, self.__saved = itertools.tee(self.__saved)

def reset ( self ) : self . __iterator , self . __saved = itertools . tee ( self . __saved )


def parse_as_var(parser, token):
    """
    Parse the remainder of the token, to find a "as varname" statement.

    :param parser: The "parser" object that ``@register.tag`` provides.
    :type parser: :class:`~django.template.Parser`
    :param token: The "token" object that ``@register.tag`` provides.
    :type token: :class:`~django.template.Token` or splitted bits
    """
    if isinstance(token, Token):
        bits = token.split_contents()
    else:
        bits = token

    as_var = None
    if len(bits) > 2 and bits[-2] == 'as':
        bits = bits[:]
        as_var = bits.pop()
        bits.pop()  # as keyword

    return bits, as_var

def
parse_as_var
(
parser
,
token
)
:
if
isinstance
(
token
,
Token
)
:
bits
=
token
.
split_contents
(
)
else
:
bits
=
token
as_var
=
if
len
(
bits
)
>
2
and
bits
[
2
]
==
'as'
:
bits
=
bits
[
:
]
as_var
=
bits
.
pop
(
)
bits
.
pop
(
)
# as keyword
return
bits
,
as_var

Parse the remainder of the token to find a as varname statement .


def template_tag(library, name):
    """
    Decorator to register class tags

    :param library: The template tag library, typically instantiated as ``register = Library()``.
    :type library: :class:`~django.template.Library`
    :param name: The name of the template tag
    :type name: str

    Example:

    .. code-block:: python

        @template_tag(register, 'my_tag')
        class MyTag(BaseNode):
            pass
    """
    def _inner(cls):
        if hasattr(cls, 'parse'):
            compile_function = cls.parse
        else:
            # Hope that it's either a function, or cls with __init__(self, parser, token) method.
            compile_function = cls

        library.tag(name, compile_function)
        return cls  # Return the class body to keep it in the namespace of the module

    return _inner


def descendant(self, chain_path):
        """ A descendant is a child many steps down.
        """
        public_child = self.hdkeychain
        chain_step_bytes = 4
        max_bits_per_step = 2**31
        chain_steps = [
            int(chain_path[i:i+chain_step_bytes*2], 16) % max_bits_per_step
            for i in range(0, len(chain_path), chain_step_bytes*2)
        ]
        for step in chain_steps:
            public_child = public_child.get_child(step)

        return PublicKeychain(public_child)

def descendant ( self , chain_path ) : public_child = self . hdkeychain chain_step_bytes = 4 max_bits_per_step = 2 ** 31 chain_steps = [ int ( chain_path [ i : i + chain_step_bytes * 2 ] , 16 ) % max_bits_per_step for i in range ( 0 , len ( chain_path ) , chain_step_bytes * 2 ) ] for step in chain_steps : public_child = public_child . get_child ( step ) return PublicKeychain ( public_child )

A descendant is a child many steps down .


def fetch_sqlite_master(self):
        """
        Get sqlite_master table information as a list of dictionaries.

        :return: sqlite_master table information.
        :rtype: list

        :Sample Code:
            .. code:: python

                from sqliteschema import SQLiteSchemaExtractor

                print(json.dumps(SQLiteSchemaExtractor("sample.sqlite").fetch_sqlite_master(), indent=4))

        :Output:
            .. code-block:: json

                [
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE TABLE 'sample_table' ('a' INTEGER, 'b' REAL, 'c' TEXT, 'd' REAL, 'e' TEXT)",
                        "type": "table",
                        "name": "sample_table",
                        "rootpage": 2
                    },
                    {
                        "tbl_name": "sample_table",
                        "sql": "CREATE INDEX sample_table_a_index ON sample_table('a')",
                        "type": "index",
                        "name": "sample_table_a_index",
                        "rootpage": 3
                    }
                ]
        """

        sqlite_master_record_list = []
        result = self.__cur.execute(
            "SELECT {:s} FROM sqlite_master".format(", ".join(self._SQLITE_MASTER_ATTR_NAME_LIST))
        )

        for record in result.fetchall():
            sqlite_master_record_list.append(
                dict(
                    [
                        [attr_name, item]
                        for attr_name, item in zip(self._SQLITE_MASTER_ATTR_NAME_LIST, record)
                    ]
                )
            )

        return sqlite_master_record_list

def
fetch_sqlite_master
(
self
)
:
sqlite_master_record_list
=
[
]
result
=
self
.
__cur
.
execute
(
"SELECT {:s} FROM sqlite_master"
.
format
(
", "
.
join
(
self
.
_SQLITE_MASTER_ATTR_NAME_LIST
)
)
)
for
record
in
result
.
fetchall
(
)
:
sqlite_master_record_list
.
append
(
dict
(
[
[
attr_name
,
item
]
for
attr_name
,
item
in
zip
(
self
.
_SQLITE_MASTER_ATTR_NAME_LIST
,
record
)
]
)
)
return
sqlite_master_record_list

Get sqlite_master table information as a list of dictionaries .


def object_iter(obj, parent=None, parent_key=None, idx=None,
                siblings=None):
    """Yields each node of object graph in postorder."""

    obj_node = Node(value=obj, parent=parent, parent_key=parent_key,
                siblings=siblings, idx=idx)

    if isinstance(obj, list):
        _siblings = len(obj)
        for i, elem in enumerate(obj):
            for node in object_iter(elem, obj_node, None, i + 1, _siblings):
                yield node
    elif isinstance(obj, collections.Mapping):
        for key in obj:
            for node in object_iter(obj[key], obj_node, key):
                yield node
    yield obj_node

def object_iter ( obj , parent = None , parent_key = None , idx = None , siblings = None ) : obj_node = Node ( value = obj , parent = parent , parent_key = parent_key , siblings = siblings , idx = idx ) if isinstance ( obj , list ) : _siblings = len ( obj ) for i , elem in enumerate ( obj ) : for node in object_iter ( elem , obj_node , None , i + 1 , _siblings ) : yield node elif isinstance ( obj , collections . Mapping ) : for key in obj : for node in object_iter ( obj [ key ] , obj_node , key ) : yield node yield obj_node

Yields each node of object graph in postorder .


def select(selector, obj):
    """Appy selector to obj and return matching nodes.

    If only one node is found, return it, otherwise return a list of matches.
    Returns False on syntax error. None if no results found.
    """

    parser = Parser(obj)
    try:
        return parser.parse(selector)
    except SelectorSyntaxError as e:
        log.exception(e)
        return False

def select ( selector , obj ) : parser = Parser ( obj ) try : return parser . parse ( selector ) except SelectorSyntaxError as e : log . exception ( e ) return False

Appy selector to obj and return matching nodes .


def parse(self, selector):
        """Accept a list of tokens. Returns matched nodes of self.obj."""
        log.debug(self.obj)
        tokens = lex(selector)

        if self.peek(tokens, 'operator') == '*':
            self.match(tokens, 'operator')
            results = list(object_iter(self.obj))
        else:
            results = self.selector_production(tokens)

        results = [node.value for node in results]
        # single results should be returned as a primitive
        if len(results) == 1:
            return results[0]
        elif not len(results):
            return None
        return results

Accept a list of tokens . Returns matched nodes of self . obj .


def selector_production(self, tokens):
        """Production for a full selector."""

        validators = []
        # the following productions should return predicate functions.

        if self.peek(tokens, 'type'):
            type_ = self.match(tokens, 'type')
            validators.append(self.type_production(type_))

        if self.peek(tokens, 'identifier'):
            key = self.match(tokens, 'identifier')
            validators.append(self.key_production(key))

        if self.peek(tokens, 'pclass'):
            pclass = self.match(tokens, 'pclass')
            validators.append(self.pclass_production(pclass))

        if self.peek(tokens, 'nth_func'):
            nth_func = self.match(tokens, 'nth_func')
            validators.append(self.nth_child_production(nth_func, tokens))

        if self.peek(tokens, 'pclass_func'):
            pclass_func = self.match(tokens, 'pclass_func')
            validators.append(self.pclass_func_production(pclass_func, tokens))

        if not len(validators):
            raise SelectorSyntaxError('no selector recognized.')

        # apply validators from a selector expression to self.obj
        results = self._match_nodes(validators, self.obj)

        if self.peek(tokens, 'operator'):
            operator = self.match(tokens, 'operator')
            rvals = self.selector_production(tokens)
            if operator == ',':
                results.extend(rvals)
            elif operator == '>':
                results = self.parents(results, rvals)
            elif operator == '~':
                results = self.siblings(results, rvals)
            elif operator == ' ':
                results = self.ancestors(results, rvals)
            else:
                raise SelectorSyntaxError("unrecognized operator '%s'"
                                          % operator)
        else:
            if len(tokens):
                rvals = self.selector_production(tokens)
                results = self.ancestors(results, rvals)

        return results


def parents(self, lhs, rhs):
        """Find nodes in rhs which have parents in lhs."""

        return [node for node in rhs if node.parent in lhs]

def parents ( self , lhs , rhs ) : return [ node for node in rhs if node . parent in lhs ]

Find nodes in rhs which have parents in lhs .


def ancestors(self, lhs, rhs):
        """Return nodes from rhs which have ancestors in lhs."""

        def _search(node):
            if node in lhs:
                return True
            if not node.parent:
                return False
            return _search(node.parent)

        return [node for node in rhs if _search(node)]

def ancestors ( self , lhs , rhs ) : def _search ( node ) : if node in lhs : return True if not node . parent : return False return _search ( node . parent ) return [ node for node in rhs if _search ( node ) ]

Return nodes from rhs which have ancestors in lhs .


def siblings(self, lhs, rhs):
        """Find nodes in rhs having common parents in lhs."""
        parents = [node.parent for node in lhs]

        return [node for node in rhs if node.parent in parents]

def siblings ( self , lhs , rhs ) : parents = [ node . parent for node in lhs ] return [ node for node in rhs if node . parent in parents ]

Find nodes in rhs having common parents in lhs .


def nth_child_production(self, lexeme, tokens):
        """Parse args and pass them to pclass_func_validator."""

        args = self.match(tokens, 'expr')

        pat = self.nth_child_pat.match(args)

        if pat.group(5):
            a = 2
            b = 1 if pat.group(5) == 'odd' else 0
        elif pat.group(6):
            a = 0
            b = int(pat.group(6))
        else:
            sign = pat.group(1) if pat.group(1) else '+'
            coef = pat.group(2) if pat.group(2) else '1'
            a = eval(sign + coef)
            b = eval(pat.group(3) + pat.group(4)) if pat.group(3) else 0

        reverse = False
        if lexeme == 'nth-last-child':
            reverse = True

        def validate(node):
            """This crazy function taken from jsonselect.js:444."""

            if not node.siblings:
                return False

            idx = node.idx - 1
            tot = node.siblings

            if reverse:
                idx = tot - idx
            else:
                idx += 1

            if a == 0:
                m = b == idx
            else:
                mod = (idx - b) % a
                m = not mod and (idx * a + b) >= 0
            return m

        return validate

Parse args and pass them to pclass_func_validator .


def _match_nodes(self, validators, obj):
        """Apply each validator in validators to each node in obj.

        Return each node in obj which matches all validators.
        """

        results = []
        for node in object_iter(obj):
            if all([validate(node) for validate in validators]):
                results.append(node)
        return results

def _match_nodes ( self , validators , obj ) : results = [ ] for node in object_iter ( obj ) : if all ( [ validate ( node ) for validate in validators ] ) : results . append ( node ) return results

Apply each validator in validators to each node in obj .


def ping(dst, count, inter=0.2, maxwait=1000, size=64):
    """Sends ICMP echo requests to destination `dst` `count` times.
    Returns a deferred which fires when responses are finished.
    """
    def _then(result, p):
        p.stopListening()
        return result

    d = defer.Deferred()
    p = ICMPPort(0, ICMPPing(d, dst, count, inter, maxwait, size), "", 8192, reactor)
    p.startListening()

    return d.addCallback(_then, p)

def ping ( dst , count , inter = 0.2 , maxwait = 1000 , size = 64 ) : def _then ( result , p ) : p . stopListening ( ) return result d = defer . Deferred ( ) p = ICMPPort ( 0 , ICMPPing ( d , dst , count , inter , maxwait , size ) , "" , 8192 , reactor ) p . startListening ( ) return d . addCallback ( _then , p )

Sends ICMP echo requests to destination dst count times . Returns a deferred which fires when responses are finished .


def getBody(self, url, method='GET', headers={}, data=None, socket=None):
        """Make an HTTP request and return the body
        """

        if not 'User-Agent' in headers:
            headers['User-Agent'] = ['Tensor HTTP checker']

        return self.request(url, method, headers, data, socket)

def
getBody
(
self
,
url
,
method
=
'GET'
,
headers
=
{
}
,
data
=
,
socket
=
)
:
if
not
'User-Agent'
in
headers
:
headers
[
'User-Agent'
]
=
[
'Tensor HTTP checker'
]
return
self
.
request
(
url
,
method
,
headers
,
data
,
socket
)


def expire(self, age):
        """Expire any items in the cache older than `age` seconds"""
        now = time.time()
        cache = self._acquire_cache()

        expired = [k for k, v in cache.items() if (now - v[0]) > age]

        for k in expired:
            if k in cache:
                del cache[k]
            if k in self.store:
                del self.store[k]

        self._write_cache(cache)

def expire ( self , age ) : now = time . time ( ) cache = self . _acquire_cache ( ) expired = [ k for k , v in cache . items ( ) if ( now - v [ 0 ] ) > age ] for k in expired : if k in cache : del cache [ k ] if k in self . store : del self . store [ k ] self . _write_cache ( cache )

Expire any items in the cache older than age seconds


def set(self, k, v):
        """Set a key `k` to value `v`"""
        self.store[k] = (time.time(), v)
        self._persist()

def set ( self , k , v ) : self . store [ k ] = ( time . time ( ) , v ) self . _persist ( )


def get(self, k):
        """Returns key contents, and modify time"""
        if self._changed():
            self._read()

        if k in self.store:
            return tuple(self.store[k])
        else:
            return None

def get ( self , k ) : if self . _changed ( ) : self . _read ( ) if k in self . store : return tuple ( self . store [ k ] ) else : return None


def contains(self, k):
        """Return True if key `k` exists"""
        if self._changed():
            self._read()
        return k in self.store.keys()

def contains ( self , k ) : if self . _changed ( ) : self . _read ( ) return k in self . store . keys ( )


def chain_check(cls, timestamp: int) -> bool:
        """
        Given a record timestamp, verify the chain integrity.

        :param timestamp: UNIX time / POSIX time / Epoch time
        :return: 'True' if the timestamp fits the chain. 'False' otherwise.
        """

        # Creation is messy.
        # You want genius, you get madness; two sides of the same coin.
        # ... I'm sure this can be cleaned up. However, let's test it first.

        record = cls.get_record(timestamp)

        if isinstance(record, NistBeaconValue) is False:
            # Don't you dare try to play me
            return False

        prev_record = cls.get_previous(record.timestamp)
        next_record = cls.get_next(record.timestamp)

        if prev_record is None and next_record is None:
            # Uh, how did you manage to do this?
            # I'm not even mad, that's amazing.
            return False

        if (
                isinstance(prev_record, NistBeaconValue) and
                isinstance(next_record, NistBeaconValue)
        ):
            # Majority case, somewhere in the middle of the chain
            # True if:
            #   - All three records have proper signatures
            #   - The requested record's previous output equals previous
            #   - The next possible record's previous output equals the record
            return (
                record.valid_signature and
                prev_record.valid_signature and
                next_record.valid_signature and
                record.previous_output_value == prev_record.output_value and
                next_record.previous_output_value == record.output_value
            )

        if (
                prev_record is None and
                isinstance(next_record, NistBeaconValue)
        ):
            # Edge case, this was potentially the first record of all time
            return (
                record.valid_signature and
                next_record.valid_signature and
                cls._INIT_RECORD == record and
                next_record.previous_output_value == record.output_value
            )

        if (
                isinstance(prev_record, NistBeaconValue) and
                next_record is None
        ):
            # Edge case, this was potentially the latest and greatest
            return (
                record.valid_signature and
                prev_record.valid_signature and
                record.previous_output_value == prev_record.output_value
            )

Given a record timestamp verify the chain integrity .


def from_json(cls, input_json: str) -> 'NistBeaconValue':
        """
        Convert a string of JSON which represents a NIST randomness beacon
        value into a 'NistBeaconValue' object.

        :param input_json: JSON to build a 'Nist RandomnessBeaconValue' from
        :return: A 'NistBeaconValue' object, 'None' otherwise
        """

        try:
            data_dict = json.loads(input_json)
        except ValueError:
            return None

        # Our required values are "must haves". This makes it simple
        # to verify we loaded everything out of JSON correctly.
        required_values = {
            cls._KEY_FREQUENCY: None,
            cls._KEY_OUTPUT_VALUE: None,
            cls._KEY_PREVIOUS_OUTPUT_VALUE: None,
            cls._KEY_SEED_VALUE: None,
            cls._KEY_SIGNATURE_VALUE: None,
            cls._KEY_STATUS_CODE: None,
            cls._KEY_TIMESTAMP: None,
            cls._KEY_VERSION: None,
        }

        for key in required_values:
            if key in data_dict:
                required_values[key] = data_dict[key]

        # Confirm that the required values are set, and not 'None'
        if None in required_values.values():
            return None

        # We have all the required values, return a node object
        return cls(
            version=required_values[cls._KEY_VERSION],
            frequency=int(required_values[cls._KEY_FREQUENCY]),
            timestamp=int(required_values[cls._KEY_TIMESTAMP]),
            seed_value=required_values[cls._KEY_SEED_VALUE],
            previous_output_value=required_values[
                cls._KEY_PREVIOUS_OUTPUT_VALUE
            ],
            signature_value=required_values[cls._KEY_SIGNATURE_VALUE],
            output_value=required_values[cls._KEY_OUTPUT_VALUE],
            status_code=required_values[cls._KEY_STATUS_CODE],
        )

Convert a string of JSON which represents a NIST randomness beacon value into a NistBeaconValue object .


def from_xml(cls, input_xml: str) -> 'NistBeaconValue':
        """
        Convert a string of XML which represents a NIST Randomness Beacon value
        into a 'NistBeaconValue' object.

        :param input_xml: XML to build a 'NistBeaconValue' from
        :return: A 'NistBeaconValue' object, 'None' otherwise
        """

        invalid_result = None

        understood_namespaces = {
            'nist-0.1': 'http://beacon.nist.gov/record/0.1/',
        }

        # Our required values are "must haves". This makes it simple
        # to verify we loaded everything out of XML correctly.
        required_values = {
            cls._KEY_FREQUENCY: None,
            cls._KEY_OUTPUT_VALUE: None,
            cls._KEY_PREVIOUS_OUTPUT_VALUE: None,
            cls._KEY_SEED_VALUE: None,
            cls._KEY_SIGNATURE_VALUE: None,
            cls._KEY_STATUS_CODE: None,
            cls._KEY_TIMESTAMP: None,
            cls._KEY_VERSION: None,
        }

        # First attempt to load the xml, return 'None' on ParseError
        try:
            tree = ElementTree.ElementTree(ElementTree.fromstring(input_xml))
        except ElementTree.ParseError:
            return invalid_result

        # Using the required values, let's load the xml values in
        for key in required_values:
            discovered_element = tree.find(
                "{0}:{1}".format('nist-0.1', key),
                namespaces=understood_namespaces,
            )

            if not isinstance(discovered_element, ElementTree.Element):
                continue

            # Bad pylint message - https://github.com/PyCQA/pylint/issues/476
            # pylint: disable=no-member
            required_values[key] = discovered_element.text

        # Confirm that the required values are set, and not 'None'
        if None in required_values.values():
            return invalid_result

        # We have all the required values, return a node object
        return cls(
            version=required_values[cls._KEY_VERSION],
            frequency=int(required_values[cls._KEY_FREQUENCY]),
            timestamp=int(required_values[cls._KEY_TIMESTAMP]),
            seed_value=required_values[cls._KEY_SEED_VALUE],
            previous_output_value=required_values[
                cls._KEY_PREVIOUS_OUTPUT_VALUE
            ],
            signature_value=required_values[cls._KEY_SIGNATURE_VALUE],
            output_value=required_values[cls._KEY_OUTPUT_VALUE],
            status_code=required_values[cls._KEY_STATUS_CODE],
        )

Convert a string of XML which represents a NIST Randomness Beacon value into a NistBeaconValue object .


def rendered_content(self):
        """Returns a 'minified' version of the javascript content"""
        template = self.resolve_template(self.template_name)
        if django.VERSION[1] < 8:
            if template.name.endswith('.min'):
                return super(MinifiedJsTemplateResponse, self).rendered_content
        else:
            if template.template.name.endswith('.min'):
                return super(MinifiedJsTemplateResponse, self).rendered_content
        # if no minified template exists, minify the response
        content = super(MinifiedJsTemplateResponse, self).rendered_content
        content = jsmin.jsmin(content)
        return content

Returns a minified version of the javascript content


def get_fn(self, fn, max_lines=None):
        """Passes each parsed log line to `fn`
        This is a better idea than storing a giant log file in memory
        """
        stat = os.stat(self.logfile)

        if (stat.st_ino == self.lastInode) and (stat.st_size == self.lastSize):
            # Nothing new
            return []

        # Handle rollover and rotations vaguely
        if (stat.st_ino != self.lastInode) or (stat.st_size < self.lastSize):
            self.lastSize = 0

        fi = open(self.logfile, 'rt')
        fi.seek(self.lastSize)

        self.lastInode = stat.st_ino

        lines = 0

        for i in fi:
            lines += 1
            if max_lines and (lines > max_lines):
                self.storeLast()
                fi.close()
                return

            if '\n' in i:
                self.lastSize += len(i)
                if self.parser:
                    line = self.parser(i.strip('\n'))
                else:
                    line = i.strip('\n')

                fn(line)

        self.storeLast()

        fi.close()

def
get_fn
(
self
,
fn
,
max_lines
=
)
:
stat
=
os
.
stat
(
self
.
logfile
)
if
(
stat
.
st_ino
==
self
.
lastInode
)
and
(
stat
.
st_size
==
self
.
lastSize
)
:
# Nothing new
return
[
]
# Handle rollover and rotations vaguely
if
(
stat
.
st_ino
!=
self
.
lastInode
)
or
(
stat
.
st_size
<
self
.
lastSize
)
:
self
.
lastSize
=
0
fi
=
open
(
self
.
logfile
,
'rt'
)
fi
.
seek
(
self
.
lastSize
)
self
.
lastInode
=
stat
.
st_ino
lines
=
0
for
i
in
fi
:
lines
+=
1
if
max_lines
and
(
lines
>
max_lines
)
:
self
.
storeLast
(
)
fi
.
close
(
)
return
if
'\n'
in
i
:
self
.
lastSize
+=
len
(
i
)
if
self
.
parser
:
line
=
self
.
parser
(
i
.
strip
(
'\n'
)
)
else
:
line
=
i
.
strip
(
'\n'
)
fn
(
line
)
self
.
storeLast
(
)
fi
.
close
(
)

Passes each parsed log line to fn This is a better idea than storing a giant log file in memory


def get(self, max_lines=None):
        """Returns a big list of all log lines since the last run
        """
        rows = []

        self.get_fn(lambda row: rows.append(row), max_lines=max_lines)

        return rows

def get ( self , max_lines = None ) : rows = [ ] self . get_fn ( lambda row : rows . append ( row ) , max_lines = max_lines ) return rows


def validate_token(self, token, expected_data=None):
        """Validate secret link token.

        :param token: Token value.
        :param expected_data: A dictionary of key/values that must be present
            in the data part of the token (i.e. included via ``extra_data`` in
            ``create_token``).
        """
        try:
            # Load token and remove random data.
            data = self.load_token(token)

            # Compare expected data with data in token.
            if expected_data:
                for k in expected_data:
                    if expected_data[k] != data["data"].get(k):
                        return None
            return data
        except BadData:
            return None


def engine(self):
        """Get cryptographic engine."""
        if not hasattr(self, '_engine'):
            from cryptography.fernet import Fernet
            from cryptography.hazmat.backends import default_backend
            from cryptography.hazmat.primitives import hashes

            digest = hashes.Hash(hashes.SHA256(), backend=default_backend())
            digest.update(current_app.config['SECRET_KEY'].encode('utf8'))
            fernet_key = urlsafe_b64encode(digest.finalize())
            self._engine = Fernet(fernet_key)
        return self._engine

def engine ( self ) : if not hasattr ( self , '_engine' ) : from cryptography . fernet import Fernet from cryptography . hazmat . backends import default_backend from cryptography . hazmat . primitives import hashes digest = hashes . Hash ( hashes . SHA256 ( ) , backend = default_backend ( ) ) digest . update ( current_app . config [ 'SECRET_KEY' ] . encode ( 'utf8' ) ) fernet_key = urlsafe_b64encode ( digest . finalize ( ) ) self . _engine = Fernet ( fernet_key ) return self . _engine


def compat_validate_token(cls, *args, **kwargs):
        """Multiple algorithm-compatible token validation."""
        data = None
        for algorithm in SUPPORTED_DIGEST_ALGORITHMS:
            data = cls(algorithm_name=algorithm).validate_token(
                *args, **kwargs)
            if not data:  # move to next algorithm
                continue
        return data

def
compat_validate_token
(
cls
,
*
args
,
*
*
kwargs
)
:
data
=
for
algorithm
in
SUPPORTED_DIGEST_ALGORITHMS
:
data
=
cls
(
algorithm_name
=
algorithm
)
.
validate_token
(
*
args
,
*
*
kwargs
)
if
not
data
:
# move to next algorithm
continue
return
data

Multiple algorithm - compatible token validation .


def create_token(cls, obj_id, data, expires_at=None):
        """Create the secret link token."""
        if expires_at:
            s = TimedSecretLinkSerializer(expires_at=expires_at)
        else:
            s = SecretLinkSerializer()

        return s.create_token(obj_id, data)

def create_token ( cls , obj_id , data , expires_at = None ) : if expires_at : s = TimedSecretLinkSerializer ( expires_at = expires_at ) else : s = SecretLinkSerializer ( ) return s . create_token ( obj_id , data )


def Counter32(a, b, delta):
    """32bit counter aggregator with wrapping
    """
    if b < a:
        c = 4294967295 - a
        return (c + b) / float(delta)

    return (b - a) / float(delta)

def Counter32 ( a , b , delta ) : if b < a : c = 4294967295 - a return ( c + b ) / float ( delta ) return ( b - a ) / float ( delta )


def Counter64(a, b, delta):
    """64bit counter aggregator with wrapping
    """
    if b < a:
        c = 18446744073709551615 - a
        return (c + b) / float(delta)

    return (b - a) / float(delta)

def Counter64 ( a , b , delta ) : if b < a : c = 18446744073709551615 - a return ( c + b ) / float ( delta ) return ( b - a ) / float ( delta )


def average_duration(total_duration, visits):
    """ Method to calculate and format an average duration safely """
    if not visits:
        seconds = 0
    else:
        seconds = int(round(total_duration / Decimal(visits)))
    duration = timedelta(seconds=seconds)
    return str(duration)

def average_duration ( total_duration , visits ) : if not visits : seconds = 0 else : seconds = int ( round ( total_duration / Decimal ( visits ) ) ) duration = timedelta ( seconds = seconds ) return str ( duration )


def setupOutputs(self, config):
        """Setup output processors"""

        if self.proto == 'tcp':
            defaultOutput = {
                'output': 'tensor.outputs.riemann.RiemannTCP',
                'server': self.server,
                'port': self.port
            }
        else:
            defaultOutput = {
                'output': 'tensor.outputs.riemann.RiemannUDP',
                'server': self.server,
                'port': self.port
            }

        outputs = config.get('outputs', [defaultOutput])

        for output in outputs:
            if not ('debug' in output):
                output['debug'] = self.debug

            cl = output['output'].split('.')[-1]                # class
            path = '.'.join(output['output'].split('.')[:-1])   # import path

            # Import the module and construct the output object
            outputObj = getattr(
                importlib.import_module(path), cl)(output, self)

            name = output.get('name', None)

            # Add the output to our routing hash
            if name in self.outputs:
                self.outputs[name].append(outputObj)
            else:
                self.outputs[name] = [outputObj]

            # connect the output
            reactor.callLater(0, outputObj.createClient)


def setupSources(self, config):
        """Sets up source objects from the given config"""
        sources = config.get('sources', [])

        for source in sources:
            src = self.createSource(source)
            self.setupTriggers(source, src)

            self.sources.append(src)

def setupSources ( self , config ) : sources = config . get ( 'sources' , [ ] ) for source in sources : src = self . createSource ( source ) self . setupTriggers ( source , src ) self . sources . append ( src )


def sendEvent(self, source, events):
        """Callback that all event sources call when they have a new event
        or list of events
        """

        if isinstance(events, list):
            self.eventCounter += len(events)
        else:
            self.eventCounter += 1
            events = [events]

        queue = self._aggregateQueue(events)

        if queue:
            if (source in self.critical) or (source in self.warn):
                self.setStates(source, queue)

            self.routeEvent(source, queue)

        queue = []

        self.lastEvents[source] = time.time()

def sendEvent ( self , source , events ) : if isinstance ( events , list ) : self . eventCounter += len ( events ) else : self . eventCounter += 1 events = [ events ] queue = self . _aggregateQueue ( events ) if queue : if ( source in self . critical ) or ( source in self . warn ) : self . setStates ( source , queue ) self . routeEvent ( source , queue ) queue = [ ] self . lastEvents [ source ] = time . time ( )


def sourceWatchdog(self):
        """Watchdog timer function.

        Recreates sources which have not generated events in 10*interval if
        they have watchdog set to true in their configuration
        """
        for i, source in enumerate(self.sources):
            if not source.config.get('watchdog', False):
                continue
            sn = repr(source)
            last = self.lastEvents.get(source, None)
            if last:
                try:
                    if last < (time.time()-(source.inter*10)):
                        log.msg("Trying to restart stale source %s: %ss" % (
                            sn, int(time.time() - last)
                        ))

                        s = self.sources.pop(i)
                        try:
                            s.t.stop()
                        except Exception as e:
                            log.msg("Could not stop timer for %s: %s" % (
                                sn, e))

                        config = copy.deepcopy(s.config)

                        del self.lastEvents[source]
                        del s, source

                        source = self.createSource(config)

                        reactor.callLater(0, self._startSource, source)
                except Exception as e:
                    log.msg("Could not reset source %s: %s" % (
                        sn, e))


def _parse_format(self, format):
        """
        Converts the input format to a regular
        expression, as well as extracting fields

        Raises an exception if it couldn't compile
        the generated regex.
        """
        format = format.strip()
        format = re.sub('[ \t]+',' ',format)

        subpatterns = []

        findquotes = re.compile(r'^\\"')
        findreferreragent = re.compile('Referer|User-Agent')
        findpercent = re.compile('^%.*t$')
        lstripquotes = re.compile(r'^\\"')
        rstripquotes = re.compile(r'\\"$')
        header = re.compile(r'.*%\{([^\}]+)\}i')

        for element in format.split(' '):

            hasquotes = 0
            if findquotes.search(element): hasquotes = 1

            if hasquotes:
                element = lstripquotes.sub('', element)
                element = rstripquotes.sub('', element)

            head = header.match(element)
            if head:
                self._names.append(head.groups()[0].lower())
                self._types.append(str)
            else:
                self._names.append(self.alias(element))
                self._types.append(self.types.get(element, [None, str])[1])

            subpattern = '(\S*)'

            if hasquotes:
                if element == '%r' or findreferreragent.search(element):
                    subpattern = r'\"([^"\\]*(?:\\.[^"\\]*)*)\"'
                else:
                    subpattern = r'\"([^\"]*)\"'

            elif findpercent.search(element):
                subpattern = r'(\[[^\]]+\])'

            elif element == '%U':
                subpattern = '(.+?)'

            subpatterns.append(subpattern)

        self._pattern = '^' + ' '.join(subpatterns) + '$'
        try:
            self._regex = re.compile(self._pattern)
        except Exception as e:
            raise ApacheLogParserError(e)

def
_parse_format
(
self
,
format
)
:
format
=
format
.
strip
(
)
format
=
re
.
sub
(
'[ \t]+'
,
' '
,
format
)
subpatterns
=
[
]
findquotes
=
re
.
compile
(
r'^\\"'
)
findreferreragent
=
re
.
compile
(
'Referer|User-Agent'
)
findpercent
=
re
.
compile
(
'^%.*t$'
)
lstripquotes
=
re
.
compile
(
r'^\\"'
)
rstripquotes
=
re
.
compile
(
r'\\"$'
)
header
=
re
.
compile
(
r'.*%\{([^\}]+)\}i'
)
for
element
in
format
.
split
(
' '
)
:
hasquotes
=
0
if
findquotes
.
search
(
element
)
:
hasquotes
=
1
if
hasquotes
:
element
=
lstripquotes
.
sub
(
''
,
element
)
element
=
rstripquotes
.
sub
(
''
,
element
)
head
=
header
.
match
(
element
)
if
head
:
self
.
_names
.
append
(
head
.
groups
(
)
[
0
]
.
lower
(
)
)
self
.
_types
.
append
(
str
)
else
:
self
.
_names
.
append
(
self
.
alias
(
element
)
)
self
.
_types
.
append
(
self
.
types
.
get
(
element
,
[
,
str
]
)
[
1
]
)
subpattern
=
'(\S*)'
if
hasquotes
:
if
element
==
'%r'
or
findreferreragent
.
search
(
element
)
:
subpattern
=
r'\"([^"\\]*(?:\\.[^"\\]*)*)\"'
else
:
subpattern
=
r'\"([^\"]*)\"'
elif
findpercent
.
search
(
element
)
:
subpattern
=
r'(\[[^\]]+\])'
elif
element
==
'%U'
:
subpattern
=
'(.+?)'
subpatterns
.
append
(
subpattern
)
self
.
_pattern
=
'^'
+
' '
.
join
(
subpatterns
)
+
'$'
try
:
self
.
_regex
=
re
.
compile
(
self
.
_pattern
)
except
Exception
as
e
:
raise
ApacheLogParserError
(
e
)

Converts the input format to a regular expression as well as extracting fields


def parse(self, line):
        """
        Parses a single line from the log file and returns
        a dictionary of it's contents.

        Raises and exception if it couldn't parse the line
        """
        line = line.strip()
        match = self._regex.match(line)

        if match:
            data = {}
            for i, e in enumerate(match.groups()):
                if e == "-":
                    k, v = self._names[i], None
                else:
                    k, v = self._names[i], self._types[i](e)
                data[k] = v
            return data

        raise ApacheLogParserError("Unable to parse: %s" % line)

def
parse
(
self
,
line
)
:
line
=
line
.
strip
(
)
match
=
self
.
_regex
.
match
(
line
)
if
match
:
data
=
{
}
for
i
,
e
in
enumerate
(
match
.
groups
(
)
)
:
if
e
==
"-"
:
k
,
v
=
self
.
_names
[
i
]
,
else
:
k
,
v
=
self
.
_names
[
i
]
,
self
.
_types
[
i
]
(
e
)
data
[
k
]
=
v
return
data
raise
ApacheLogParserError
(
"Unable to parse: %s"
%
line
)

Parses a single line from the log file and returns a dictionary of it s contents .


def validate_expires_at(form, field):
    """Validate that date is in the future."""
    if form.accept.data:
        if not field.data or datetime.utcnow().date() >= field.data:
            raise validators.StopValidation(_(
                "Please provide a future date."
            ))
        if not field.data or \
                datetime.utcnow().date() + timedelta(days=365) < field.data:
            raise validators.StopValidation(_(
                "Please provide a date no more than 1 year into the future."
            ))


def validate_message(form, field):
        """Validate message."""
        if form.reject.data and not field.data.strip():
            raise validators.ValidationError(
                _("You are required to provide message to the requester when"
                  " you reject a request.")
            )


def verify_token():
    """Verify token and save in session if it's valid."""
    try:
        from .models import SecretLink
        token = request.args['token']
        # if the token is valid
        if token and SecretLink.validate_token(token, {}):
            # then save in session the token
            session['accessrequests-secret-token'] = token
    except KeyError:
        pass

def
verify_token
(
)
:
try
:
from
.
models
import
SecretLink
token
=
request
.
args
[
'token'
]
# if the token is valid
if
token
and
SecretLink
.
validate_token
(
token
,
{
}
)
:
# then save in session the token
session
[
'accessrequests-secret-token'
]
=
token
except
KeyError
:
pass

Verify token and save in session if it s valid .


def name(self):
        """ Return a basic meaningful name based on device type """
        if (
            self.device_type and
            self.device_type.code in (DeviceType.MOBILE, DeviceType.TABLET)
        ):
            return self.device
        else:
            return self.browser

def name ( self ) : if ( self . device_type and self . device_type . code in ( DeviceType . MOBILE , DeviceType . TABLET ) ) : return self . device else : return self . browser


def _warn_node(self, msg, *args, **kwargs):
    """Do not warn on external images."""
    if not msg.startswith('nonlocal image URI found:'):
        _warn_node_old(self, msg, *args, **kwargs)

def
_warn_node
(
self
,
msg
,
*
args
,
*
*
kwargs
)
:
if
not
msg
.
startswith
(
'nonlocal image URI found:'
)
:
_warn_node_old
(
self
,
msg
,
*
args
,
*
*
kwargs
)


def connect_receivers():
    """Connect receivers to signals."""
    request_created.connect(send_email_validation)
    request_confirmed.connect(send_confirmed_notifications)
    request_rejected.connect(send_reject_notification)
    # Order is important:
    request_accepted.connect(create_secret_link)
    request_accepted.connect(send_accept_notification)

def
connect_receivers
(
)
:
request_created
.
connect
(
send_email_validation
)
request_confirmed
.
connect
(
send_confirmed_notifications
)
request_rejected
.
connect
(
send_reject_notification
)
# Order is important:
request_accepted
.
connect
(
create_secret_link
)
request_accepted
.
connect
(
send_accept_notification
)


def create_secret_link(request, message=None, expires_at=None):
    """Receiver for request-accepted signal."""
    pid, record = get_record(request.recid)
    if not record:
        raise RecordNotFound(request.recid)

    description = render_template(
        "zenodo_accessrequests/link_description.tpl",
        request=request,
        record=record,
        pid=pid,
        expires_at=expires_at,
        message=message,
    )

    request.create_secret_link(
        record["title"],
        description=description,
        expires_at=expires_at
    )

Receiver for request - accepted signal .


def send_accept_notification(request, message=None, expires_at=None):
    """Receiver for request-accepted signal to send email notification."""
    pid, record = get_record(request.recid)
    _send_notification(
        request.sender_email,
        _("Access request accepted"),
        "zenodo_accessrequests/emails/accepted.tpl",
        request=request,
        record=record,
        pid=pid,
        record_link=request.link.get_absolute_url('invenio_records_ui.recid'),
        message=message,
        expires_at=expires_at,
    )

Receiver for request - accepted signal to send email notification .


def send_confirmed_notifications(request):
    """Receiver for request-confirmed signal to send email notification."""
    pid, record = get_record(request.recid)
    if record is None:
        current_app.logger.error("Cannot retrieve record %s. Emails not sent"
                                 % request.recid)
        return
    title = _("Access request: %(record)s", record=record["title"])

    _send_notification(
        request.receiver.email,
        title,
        "zenodo_accessrequests/emails/new_request.tpl",
        request=request,
        record=record,
        pid=pid,
    )

    _send_notification(
        request.sender_email,
        title,
        "zenodo_accessrequests/emails/confirmation.tpl",
        request=request,
        record=record,
        pid=pid,
    )

Receiver for request - confirmed signal to send email notification .


def send_email_validation(request):
    """Receiver for request-created signal to send email notification."""
    token = EmailConfirmationSerializer().create_token(
        request.id, dict(email=request.sender_email)
    )
    pid, record = get_record(request.recid)

    _send_notification(
        request.sender_email,
        _("Access request verification"),
        "zenodo_accessrequests/emails/validate_email.tpl",
        request=request,
        record=record,
        pid=pid,
        days=timedelta(
            seconds=current_app.config["ACCESSREQUESTS_CONFIRMLINK_EXPIRES_IN"]
        ).days,
        confirm_link=url_for(
            "invenio_records_ui.recid_access_request_email_confirm",
            pid_value=request.recid,
            token=token,
            _external=True,
        )
    )

Receiver for request - created signal to send email notification .


def send_reject_notification(request, message=None):
    """Receiver for request-rejected signal to send email notification."""
    pid, record = get_record(request.recid)
    _send_notification(
        request.sender_email,
        _("Access request rejected"),
        "zenodo_accessrequests/emails/rejected.tpl",
        request=request,
        record=record,
        pid=pid,
        message=message,
    )

Receiver for request - rejected signal to send email notification .