def Rule(name, loc=None):
    """A proxy for a rule called ``name`` which may not be yet defined."""
    @llrule(loc, lambda parser: getattr(parser, name).expected(parser))
    def rule(parser):
        return getattr(parser, name)()
    return rule

def Rule ( name , loc = None ) : @ llrule ( loc , lambda parser : getattr ( parser , name ) . expected ( parser ) ) def rule ( parser ) : return getattr ( parser , name ) ( ) return rule

A proxy for a rule called name which may not be yet defined .


def Expect(inner_rule, loc=None):
    """A rule that executes ``inner_rule`` and emits a diagnostic error if it returns None."""
    @llrule(loc, inner_rule.expected)
    def rule(parser):
        result = inner_rule(parser)
        if result is unmatched:
            expected = reduce(list.__add__, [rule.expected(parser) for rule in parser._errrules])
            expected = list(sorted(set(expected)))

            if len(expected) > 1:
                expected = " or ".join([", ".join(expected[0:-1]), expected[-1]])
            elif len(expected) == 1:
                expected = expected[0]
            else:
                expected = "(impossible)"

            error_tok = parser._tokens[parser._errindex]
            error = diagnostic.Diagnostic(
                "fatal", "unexpected {actual}: expected {expected}",
                {"actual": error_tok.kind, "expected": expected},
                error_tok.loc)
            parser.diagnostic_engine.process(error)
        return result
    return rule

A rule that executes inner_rule and emits a diagnostic error if it returns None .


def Seq(first_rule, *rest_of_rules, **kwargs):
    """
    A rule that accepts a sequence of tokens satisfying ``rules`` and returns a tuple
    containing their return values, or None if the first rule was not satisfied.
    """
    @llrule(kwargs.get("loc", None), first_rule.expected)
    def rule(parser):
        result = first_rule(parser)
        if result is unmatched:
            return result

        results = [result]
        for rule in rest_of_rules:
            result = rule(parser)
            if result is unmatched:
                return result
            results.append(result)
        return tuple(results)
    return rule

def Seq ( first_rule , * rest_of_rules , * * kwargs ) : @ llrule ( kwargs . get ( "loc" , None ) , first_rule . expected ) def rule ( parser ) : result = first_rule ( parser ) if result is unmatched : return result results = [ result ] for rule in rest_of_rules : result = rule ( parser ) if result is unmatched : return result results . append ( result ) return tuple ( results ) return rule

A rule that accepts a sequence of tokens satisfying rules and returns a tuple containing their return values or None if the first rule was not satisfied .


def SeqN(n, *inner_rules, **kwargs):
    """
    A rule that accepts a sequence of tokens satisfying ``rules`` and returns
    the value returned by rule number ``n``, or None if the first rule was not satisfied.
    """
    @action(Seq(*inner_rules), loc=kwargs.get("loc", None))
    def rule(parser, *values):
        return values[n]
    return rule

def SeqN ( n , * inner_rules , * * kwargs ) : @ action ( Seq ( * inner_rules ) , loc = kwargs . get ( "loc" , None ) ) def rule ( parser , * values ) : return values [ n ] return rule

A rule that accepts a sequence of tokens satisfying rules and returns the value returned by rule number n or None if the first rule was not satisfied .


def Newline(loc=None):
    """A rule that accepts token of kind ``newline`` and returns an empty list."""
    @llrule(loc, lambda parser: ["newline"])
    def rule(parser):
        result = parser._accept("newline")
        if result is unmatched:
            return result
        return []
    return rule

def Newline ( loc = None ) : @ llrule ( loc , lambda parser : [ "newline" ] ) def rule ( parser ) : result = parser . _accept ( "newline" ) if result is unmatched : return result return [ ] return rule

A rule that accepts token of kind newline and returns an empty list .


def urljoin(base, url, allow_fragments=True):
    """Join a base URL and a possibly relative URL to form an absolute
    interpretation of the latter."""
    if not base:
        return url
    if not url:
        return base
    bscheme, bnetloc, bpath, bparams, bquery, bfragment = \
            urlparse(base, '', allow_fragments)
    scheme, netloc, path, params, query, fragment = \
            urlparse(url, bscheme, allow_fragments)
    if scheme != bscheme or scheme not in uses_relative:
        return url
    if scheme in uses_netloc:
        if netloc:
            return urlunparse((scheme, netloc, path,
                               params, query, fragment))
        netloc = bnetloc
    if path[:1] == '/':
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    if not path and not params:
        path = bpath
        params = bparams
        if not query:
            query = bquery
        return urlunparse((scheme, netloc, path,
                           params, query, fragment))
    segments = bpath.split('/')[:-1] + path.split('/')
    # XXX The stuff below is bogus in various ways...
    if segments[-1] == '.':
        segments[-1] = ''
    while '.' in segments:
        segments.remove('.')
    while 1:
        i = 1
        n = len(segments) - 1
        while i < n:
            if (segments[i] == '..'
                and segments[i-1] not in ('', '..')):
                del segments[i-1:i+1]
                break
            i = i+1
        else:
            break
    if segments == ['', '..']:
        segments[-1] = ''
    elif len(segments) >= 2 and segments[-1] == '..':
        segments[-2:] = ['']
    return urlunparse((scheme, netloc, '/'.join(segments),
                       params, query, fragment))

Join a base URL and a possibly relative URL to form an absolute interpretation of the latter .


def urldefrag(url):
    """Removes any existing fragment from URL.

    Returns a tuple of the defragmented URL and the fragment.  If
    the URL contained no fragments, the second element is the
    empty string.
    """
    if '#' in url:
        s, n, p, a, q, frag = urlparse(url)
        defrag = urlunparse((s, n, p, a, q, ''))
        return defrag, frag
    else:
        return url, ''

def urldefrag ( url ) : if '#' in url : s , n , p , a , q , frag = urlparse ( url ) defrag = urlunparse ( ( s , n , p , a , q , '' ) ) return defrag , frag else : return url , ''

Removes any existing fragment from URL .


def _replace(_self, **kwds):
        'Return a new SplitResult object replacing specified fields with new values'
        result = _self._make(map(kwds.pop, ('scheme', 'netloc', 'path', 'query', 'fragment'), _self))
        if kwds:
            raise ValueError('Got unexpected field names: %r' % kwds.keys())
        return result

def
_replace
(
_self
,
*
*
kwds
)
:
result
=
_self
.
_make
(
map
(
kwds
.
pop
,
(
'scheme'
,
'netloc'
,
'path'
,
'query'
,
'fragment'
)
,
_self
)
)
if
kwds
:
raise
ValueError
(
'Got unexpected field names: %r'
%
kwds
.
keys
(
)
)
return
result


def getlines(filename, module_globals=None):
    """Get the lines for a file from the cache.
    Update the cache if it doesn't contain an entry for this file already."""

    if filename in cache:
        return cache[filename][2]

    try:
        return updatecache(filename, module_globals)
    except MemoryError:
        clearcache()
        return []

def getlines ( filename , module_globals = None ) : if filename in cache : return cache [ filename ] [ 2 ] try : return updatecache ( filename , module_globals ) except MemoryError : clearcache ( ) return [ ]

Get the lines for a file from the cache . Update the cache if it doesn t contain an entry for this file already .


def updatecache(filename, module_globals=None):
    """Update a cache entry and return its list of lines.
    If something's wrong, print a message, discard the cache entry,
    and return an empty list."""

    if filename in cache:
        del cache[filename]
    if not filename or (filename.startswith('<') and filename.endswith('>')):
        return []

    fullname = filename
    try:
        stat = os.stat(fullname)
    except OSError:
        basename = filename

        # Try for a __loader__, if available
        if module_globals and '__loader__' in module_globals:
            name = module_globals.get('__name__')
            loader = module_globals['__loader__']
            get_source = getattr(loader, 'get_source', None)

            if name and get_source:
                try:
                    data = get_source(name)
                except (ImportError, IOError):
                    pass
                else:
                    if data is None:
                        # No luck, the PEP302 loader cannot find the source
                        # for this module.
                        return []
                    cache[filename] = (
                        len(data), None,
                        [line+'\n' for line in data.splitlines()], fullname
                    )
                    return cache[filename][2]

        # Try looking through the module search path, which is only useful
        # when handling a relative filename.
        if os.path.isabs(filename):
            return []

        for dirname in sys.path:
            # When using imputil, sys.path may contain things other than
            # strings; ignore them when it happens.
            try:
                fullname = os.path.join(dirname, basename)
            except (TypeError, AttributeError):
                # Not sufficiently string-like to do anything useful with.
                continue
            try:
                stat = os.stat(fullname)
                break
            except os.error:
                pass
        else:
            return []
    try:
        with open(fullname, 'rU') as fp:
            lines = fp.readlines()
    except IOError:
        return []
    if lines and not lines[-1].endswith('\n'):
        lines[-1] += '\n'
    size, mtime = stat.st_size, stat.st_mtime
    cache[filename] = size, mtime, lines, fullname
    return lines

Update a cache entry and return its list of lines . If something s wrong print a message discard the cache entry and return an empty list .


def isfile(path):
    """Test whether a path is a regular file"""
    try:
        st = os.stat(path)
    except os.error:
        return False
    return stat.S_ISREG(st.st_mode)

def isfile ( path ) : try : st = os . stat ( path ) except os . error : return False return stat . S_ISREG ( st . st_mode )


def isdir(s):
    """Return true if the pathname refers to an existing directory."""
    try:
        st = os.stat(s)
    except os.error:
        return False
    return stat.S_ISDIR(st.st_mode)

def isdir ( s ) : try : st = os . stat ( s ) except os . error : return False return stat . S_ISDIR ( st . st_mode )

Return true if the pathname refers to an existing directory .


def commonprefix(m):
    "Given a list of pathnames, returns the longest common leading component"
    if not m: return ''
    s1 = min(m)
    s2 = max(m)
    for i, c in enumerate(s1):
        if c != s2[i]:
            return s1[:i]
    return s1

def commonprefix ( m ) : if not m : return '' s1 = min ( m ) s2 = max ( m ) for i , c in enumerate ( s1 ) : if c != s2 [ i ] : return s1 [ : i ] return s1

Given a list of pathnames returns the longest common leading component


def _splitext(p, sep, altsep, extsep):
    """Split the extension from a pathname.

    Extension is everything from the last dot to the end, ignoring
    leading dots.  Returns "(root, ext)"; ext may be empty."""

    sepIndex = p.rfind(sep)
    if altsep:
        altsepIndex = p.rfind(altsep)
        sepIndex = max(sepIndex, altsepIndex)

    dotIndex = p.rfind(extsep)
    if dotIndex > sepIndex:
        # skip all leading dots
        filenameIndex = sepIndex + 1
        while filenameIndex < dotIndex:
            if p[filenameIndex] != extsep:
                return p[:dotIndex], p[dotIndex:]
            filenameIndex += 1

    return p, ''

def
_splitext
(
p
,
sep
,
altsep
,
extsep
)
:
sepIndex
=
p
.
rfind
(
sep
)
if
altsep
:
altsepIndex
=
p
.
rfind
(
altsep
)
sepIndex
=
max
(
sepIndex
,
altsepIndex
)
dotIndex
=
p
.
rfind
(
extsep
)
if
dotIndex
>
sepIndex
:
# skip all leading dots
filenameIndex
=
sepIndex
+
1
while
filenameIndex
<
dotIndex
:
if
p
[
filenameIndex
]
!=
extsep
:
return
p
[
:
dotIndex
]
,
p
[
dotIndex
:
]
filenameIndex
+=
1
return
p
,
''


def wrap(text, width=70, **kwargs):
    """Wrap a single paragraph of text, returning a list of wrapped lines.

    Reformat the single paragraph in 'text' so it fits in lines of no
    more than 'width' columns, and return a list of wrapped lines.  By
    default, tabs in 'text' are expanded with string.expandtabs(), and
    all other whitespace characters (including newline) are converted to
    space.  See TextWrapper class for available keyword args to customize
    wrapping behaviour.
    """
    w = TextWrapper(width=width, **kwargs)
    return w.wrap(text)

def wrap ( text , width = 70 , * * kwargs ) : w = TextWrapper ( width = width , * * kwargs ) return w . wrap ( text )

Wrap a single paragraph of text returning a list of wrapped lines .


def fill(text, width=70, **kwargs):
    """Fill a single paragraph of text, returning a new string.

    Reformat the single paragraph in 'text' to fit in lines of no more
    than 'width' columns, and return a new string containing the entire
    wrapped paragraph.  As with wrap(), tabs are expanded and other
    whitespace characters converted to space.  See TextWrapper class for
    available keyword args to customize wrapping behaviour.
    """
    w = TextWrapper(width=width, **kwargs)
    return w.fill(text)

def fill ( text , width = 70 , * * kwargs ) : w = TextWrapper ( width = width , * * kwargs ) return w . fill ( text )

Fill a single paragraph of text returning a new string .


def dedent(text):
    """Remove any common leading whitespace from every line in `text`.

    This can be used to make triple-quoted strings line up with the left
    edge of the display, while still presenting them in the source code
    in indented form.

    Note that tabs and spaces are both treated as whitespace, but they
    are not equal: the lines "  hello" and "\\thello" are
    considered to have no common leading whitespace.  (This behaviour is
    new in Python 2.5; older versions of this module incorrectly
    expanded tabs before searching for common leading whitespace.)
    """
    # Look for the longest leading string of spaces and tabs common to
    # all lines.
    margin = None
    text = _whitespace_only_re.sub('', text)
    indents = _leading_whitespace_re.findall(text)
    for indent in indents:
        if margin is None:
            margin = indent

        # Current line more deeply indented than previous winner:
        # no change (previous winner is still on top).
        elif indent.startswith(margin):
            pass

        # Current line consistent with and no deeper than previous winner:
        # it's the new winner.
        elif margin.startswith(indent):
            margin = indent

        # Find the largest common whitespace between current line and previous
        # winner.
        else:
            for i, (x, y) in enumerate(zip(margin, indent)):
                if x != y:
                    margin = margin[:i]
                    break
            else:
                margin = margin[:len(indent)]

    # sanity check (testing/debugging only)
    if 0 and margin:
        for line in text.split("\n"):
            assert not line or line.startswith(margin), \
                   "line = %r, margin = %r" % (line, margin)

    if margin:
        text = re.sub(r'(?m)^' + margin, '', text)
    return text

Remove any common leading whitespace from every line in text .


def _long2bytesBigEndian(n, blocksize=0):
    """Convert a long integer to a byte string.

    If optional blocksize is given and greater than zero, pad the front
    of the byte string with binary zeros so that the length is a multiple
    of blocksize.
    """

    # After much testing, this algorithm was deemed to be the fastest.
    s = b''
    pack = struct.pack
    while n > 0:
        s = pack('>I', n & 0xffffffff) + s
        n = n >> 32

    # Strip off leading zeros.
    for i in range(len(s)):
        if s[i] != '\000':
            break
    else:
        # Only happens when n == 0.
        s = '\000'
        i = 0

    s = s[i:]

    # Add back some pad bytes. This could be done more efficiently
    # w.r.t. the de-padding being done above, but sigh...
    if blocksize > 0 and len(s) % blocksize:
        s = (blocksize - len(s) % blocksize) * '\000' + s

    return s

Convert a long integer to a byte string .


def _bytelist2longBigEndian(list):
    "Transform a list of characters into a list of longs."

    imax = len(list) // 4
    hl = [0] * imax

    j = 0
    i = 0
    while i < imax:
        b0 = ord(list[j]) << 24
        b1 = ord(list[j+1]) << 16
        b2 = ord(list[j+2]) << 8
        b3 = ord(list[j+3])
        hl[i] = b0 | b1 | b2 | b3
        i = i+1
        j = j+4

    return hl

def _bytelist2longBigEndian ( list ) : imax = len ( list ) // 4 hl = [ 0 ] * imax j = 0 i = 0 while i < imax : b0 = ord ( list [ j ] ) << 24 b1 = ord ( list [ j + 1 ] ) << 16 b2 = ord ( list [ j + 2 ] ) << 8 b3 = ord ( list [ j + 3 ] ) hl [ i ] = b0 | b1 | b2 | b3 i = i + 1 j = j + 4 return hl

Transform a list of characters into a list of longs .


def init(self):
        "Initialize the message-digest and set all fields to zero."

        self.length = 0
        self.input = []

        # Initial 160 bit message digest (5 times 32 bit).
        self.H0 = 0x67452301
        self.H1 = 0xEFCDAB89
        self.H2 = 0x98BADCFE
        self.H3 = 0x10325476
        self.H4 = 0xC3D2E1F0

Initialize the message - digest and set all fields to zero .


def enterabs(self, time, priority, action, argument):
        """Enter a new event in the queue at an absolute time.
        Returns an ID for the event which can be used to remove it,
        if necessary.
        """
        event = Event(time, priority, action, argument)
        heapq.heappush(self._queue, event)
        return event

def enterabs ( self , time , priority , action , argument ) : event = Event ( time , priority , action , argument ) heapq . heappush ( self . _queue , event ) return event

Enter a new event in the queue at an absolute time . Returns an ID for the event which can be used to remove it if necessary .


def copy(x):
    """Shallow copy operation on arbitrary Python objects.

    See the module's __doc__ string for more info.
    """

    cls = type(x)

    copier = _copy_dispatch.get(cls)
    if copier:
        return copier(x)

    copier = getattr(cls, "__copy__", None)
    if copier:
        return copier(x)

    reductor = dispatch_table.get(cls)
    if reductor:
        rv = reductor(x)
    else:
        reductor = getattr(x, "__reduce_ex__", None)
        if reductor:
            rv = reductor(2)
        else:
            reductor = getattr(x, "__reduce__", None)
            if reductor:
                rv = reductor()
            else:
                raise Error("un(shallow)copyable object of type %s" % cls)

    return _reconstruct(x, rv, 0)

def
copy
(
x
)
:
cls
=
type
(
x
)
copier
=
_copy_dispatch
.
get
(
cls
)
if
copier
:
return
copier
(
x
)
copier
=
getattr
(
cls
,
"__copy__"
,
)
if
copier
:
return
copier
(
x
)
reductor
=
dispatch_table
.
get
(
cls
)
if
reductor
:
rv
=
reductor
(
x
)
else
:
reductor
=
getattr
(
x
,
"__reduce_ex__"
,
)
if
reductor
:
rv
=
reductor
(
2
)
else
:
reductor
=
getattr
(
x
,
"__reduce__"
,
)
if
reductor
:
rv
=
reductor
(
)
else
:
raise
Error
(
"un(shallow)copyable object of type %s"
%
cls
)
return
_reconstruct
(
x
,
rv
,
0
)

Shallow copy operation on arbitrary Python objects .


def deepcopy(x, memo=None, _nil=[]):
    """Deep copy operation on arbitrary Python objects.

    See the module's __doc__ string for more info.
    """

    if memo is None:
        memo = {}

    d = id(x)
    y = memo.get(d, _nil)
    if y is not _nil:
        return y

    cls = type(x)

    copier = _deepcopy_dispatch.get(cls)
    if copier:
        y = copier(x, memo)
    else:
        try:
            issc = issubclass(cls, type)
        except TypeError: # cls is not a class (old Boost; see SF #502085)
            issc = 0
        if issc:
            y = _deepcopy_atomic(x, memo)
        else:
            copier = getattr(x, "__deepcopy__", None)
            if copier:
                y = copier(memo)
            else:
                reductor = dispatch_table.get(cls)
                if reductor:
                    rv = reductor(x)
                else:
                    reductor = getattr(x, "__reduce_ex__", None)
                    if reductor:
                        rv = reductor(2)
                    else:
                        reductor = getattr(x, "__reduce__", None)
                        if reductor:
                            rv = reductor()
                        else:
                            raise Error(
                                "un(deep)copyable object of type %s" % cls)
                y = _reconstruct(x, rv, 1, memo)

    memo[d] = y
    _keep_alive(x, memo) # Make sure x lives at least as long as d
    return y

Deep copy operation on arbitrary Python objects .


def _keep_alive(x, memo):
    """Keeps a reference to the object x in the memo.

    Because we remember objects by their id, we have
    to assure that possibly temporary objects are kept
    alive by referencing them.
    We store a reference at the id of the memo, which should
    normally not be used unless someone tries to deepcopy
    the memo itself...
    """
    try:
        memo[id(memo)].append(x)
    except KeyError:
        # aha, this is the first one :-)
        memo[id(memo)]=[x]

def
_keep_alive
(
x
,
memo
)
:
try
:
memo
[
id
(
memo
)
]
.
append
(
x
)
except
KeyError
:
# aha, this is the first one :-)
memo
[
id
(
memo
)
]
=
[
x
]

Keeps a reference to the object x in the memo .


def warnpy3k(message, category=None, stacklevel=1):
    """Issue a deprecation warning for Python 3.x related changes.

    Warnings are omitted unless Python is started with the -3 option.
    """
    if sys.py3kwarning:
        if category is None:
            category = DeprecationWarning
        warn(message, category, stacklevel+1)

def warnpy3k ( message , category = None , stacklevel = 1 ) : if sys . py3kwarning : if category is None : category = DeprecationWarning warn ( message , category , stacklevel + 1 )

Issue a deprecation warning for Python 3 . x related changes .


def _show_warning(message, category, filename, lineno, file=None, line=None):
    """Hook to write a warning to a file; replace if you like."""
    if file is None:
        file = sys.stderr
        if file is None:
            # sys.stderr is None - warnings get lost
            return
    try:
        file.write(formatwarning(message, category, filename, lineno, line))
    except (IOError, UnicodeError):
        pass # the file (probably stderr) is invalid - this warning gets lost.

Hook to write a warning to a file ; replace if you like .


def formatwarning(message, category, filename, lineno, line=None):
    """Function to format a warning the standard way."""
    try:
        unicodetype = unicode
    except NameError:
        unicodetype = ()
    try:
        message = str(message)
    except UnicodeEncodeError:
        pass
    s =  "%s: %s: %s\n" % (lineno, category.__name__, message)
    line = linecache.getline(filename, lineno) if line is None else line
    if line:
        line = line.strip()
        if isinstance(s, unicodetype) and isinstance(line, str):
            line = unicode(line, 'latin1')
        s += "  %s\n" % line
    if isinstance(s, unicodetype) and isinstance(filename, str):
        enc = sys.getfilesystemencoding()
        if enc:
            try:
                filename = unicode(filename, enc)
            except UnicodeDecodeError:
                pass
    s = "%s:%s" % (filename, s)
    return s

def
formatwarning
(
message
,
category
,
filename
,
lineno
,
line
=
)
:
try
:
unicodetype
=
unicode
except
NameError
:
unicodetype
=
(
)
try
:
message
=
str
(
message
)
except
UnicodeEncodeError
:
pass
s
=
"%s: %s: %s\n"
%
(
lineno
,
category
.
__name__
,
message
)
line
=
linecache
.
getline
(
filename
,
lineno
)
if
line
is
else
line
if
line
:
line
=
line
.
strip
(
)
if
isinstance
(
s
,
unicodetype
)
and
isinstance
(
line
,
str
)
:
line
=
unicode
(
line
,
'latin1'
)
s
+=
"  %s\n"
%
line
if
isinstance
(
s
,
unicodetype
)
and
isinstance
(
filename
,
str
)
:
enc
=
sys
.
getfilesystemencoding
(
)
if
enc
:
try
:
filename
=
unicode
(
filename
,
enc
)
except
UnicodeDecodeError
:
pass
s
=
"%s:%s"
%
(
filename
,
s
)
return
s

Function to format a warning the standard way .


def warn(message, category=None, stacklevel=1):
    """Issue a warning, or maybe ignore it or raise an exception."""
    # Check if message is already a Warning object
    if isinstance(message, Warning):
        category = message.__class__
    # Check category argument
    if category is None:
        category = UserWarning
    assert issubclass(category, Warning)
    # Get context information
    try:
        caller = sys._getframe(stacklevel)
    except ValueError:
        globals = sys.__dict__
        lineno = 1
    else:
        globals = caller.f_globals
        lineno = caller.f_lineno
    if '__name__' in globals:
        module = globals['__name__']
    else:
        module = "<string>"
    filename = globals.get('__file__')
    if filename:
        fnl = filename.lower()
        if fnl.endswith((".pyc", ".pyo")):
            filename = filename[:-1]
    else:
        if module == "__main__":
            try:
                filename = sys.argv[0]
            except AttributeError:
                # embedded interpreters don't have sys.argv, see bug #839151
                filename = '__main__'
        if not filename:
            filename = module
    registry = globals.setdefault("__warningregistry__", {})
    warn_explicit(message, category, filename, lineno, module, registry,
                  globals)

Issue a warning or maybe ignore it or raise an exception .


def _hash(self):
        """Compute the hash value of a set.

        Note that we don't define __hash__: not all sets are hashable.
        But if you define a hashable set type, its __hash__ should
        call this function.

        This must be compatible __eq__.

        All sets ought to compare equal if they contain the same
        elements, regardless of how they are implemented, and
        regardless of the order of the elements; so there's not much
        freedom for __eq__ or __hash__.  We match the algorithm used
        by the built-in frozenset type.
        """
        MAX = sys.maxint
        MASK = 2 * MAX + 1
        n = len(self)
        h = 1927868237 * (n + 1)
        h &= MASK
        for x in self:
            hx = hash(x)
            h ^= (hx ^ (hx << 16) ^ 89869747)  * 3644798167
            h &= MASK
        h = h * 69069 + 907133923
        h &= MASK
        if h > MAX:
            h -= MASK + 1
        if h == -1:
            h = 590923713
        return h


def remove(self, value):
        """Remove an element. If not a member, raise a KeyError."""
        if value not in self:
            raise KeyError(value)
        self.discard(value)

def remove ( self , value ) : if value not in self : raise KeyError ( value ) self . discard ( value )

Remove an element . If not a member raise a KeyError .


def pop(self):
        """Return the popped value.  Raise KeyError if empty."""
        it = iter(self)
        try:
            value = next(it)
        except StopIteration:
            raise KeyError
        self.discard(value)
        return value

def pop ( self ) : it = iter ( self ) try : value = next ( it ) except StopIteration : raise KeyError self . discard ( value ) return value

Return the popped value . Raise KeyError if empty .


def go_str(value):
  """Returns value as a valid Go string literal."""
  io = StringIO.StringIO()
  io.write('"')
  for c in value:
    if c in _ESCAPES:
      io.write(_ESCAPES[c])
    elif c in _SIMPLE_CHARS:
      io.write(c)
    else:
      io.write(r'\x{:02x}'.format(ord(c)))
  io.write('"')
  return io.getvalue()

def go_str ( value ) : io = StringIO . StringIO ( ) io . write ( '"' ) for c in value : if c in _ESCAPES : io . write ( _ESCAPES [ c ] ) elif c in _SIMPLE_CHARS : io . write ( c ) else : io . write ( r'\x{:02x}' . format ( ord ( c ) ) ) io . write ( '"' ) return io . getvalue ( )

Returns value as a valid Go string literal .


def acquire(self, blocking=1):
        """Acquire a lock, blocking or non-blocking.

        When invoked without arguments: if this thread already owns the lock,
        increment the recursion level by one, and return immediately. Otherwise,
        if another thread owns the lock, block until the lock is unlocked. Once
        the lock is unlocked (not owned by any thread), then grab ownership, set
        the recursion level to one, and return. If more than one thread is
        blocked waiting until the lock is unlocked, only one at a time will be
        able to grab ownership of the lock. There is no return value in this
        case.

        When invoked with the blocking argument set to true, do the same thing
        as when called without arguments, and return true.

        When invoked with the blocking argument set to false, do not block. If a
        call without an argument would block, return false immediately;
        otherwise, do the same thing as when called without arguments, and
        return true.

        """
        me = _get_ident()
        if self.__owner == me:
            self.__count = self.__count + 1
            if __debug__:
                self._note("%s.acquire(%s): recursive success", self, blocking)
            return 1
        rc = self.__block.acquire(blocking)
        if rc:
            self.__owner = me
            self.__count = 1
            if __debug__:
                self._note("%s.acquire(%s): initial success", self, blocking)
        else:
            if __debug__:
                self._note("%s.acquire(%s): failure", self, blocking)
        return rc

def
acquire
(
self
,
blocking
=
1
)
:
me
=
_get_ident
(
)
if
self
.
__owner
==
me
:
self
.
__count
=
self
.
__count
+
1
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): recursive success"
,
self
,
blocking
)
return
1
rc
=
self
.
__block
.
acquire
(
blocking
)
if
rc
:
self
.
__owner
=
me
self
.
__count
=
1
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): initial success"
,
self
,
blocking
)
else
:
if
__debug__
:
self
.
_note
(
"%s.acquire(%s): failure"
,
self
,
blocking
)
return
rc

Acquire a lock blocking or non - blocking .


def release(self):
        """Release a lock, decrementing the recursion level.

        If after the decrement it is zero, reset the lock to unlocked (not owned
        by any thread), and if any other threads are blocked waiting for the
        lock to become unlocked, allow exactly one of them to proceed. If after
        the decrement the recursion level is still nonzero, the lock remains
        locked and owned by the calling thread.

        Only call this method when the calling thread owns the lock. A
        RuntimeError is raised if this method is called when the lock is
        unlocked.

        There is no return value.

        """
        if self.__owner != _get_ident():
            raise RuntimeError("cannot release un-acquired lock")
        self.__count = count = self.__count - 1
        if not count:
            self.__owner = None
            self.__block.release()
            if __debug__:
                self._note("%s.release(): final release", self)
        else:
            if __debug__:
                self._note("%s.release(): non-final release", self)

def
release
(
self
)
:
if
self
.
__owner
!=
_get_ident
(
)
:
raise
RuntimeError
(
"cannot release un-acquired lock"
)
self
.
__count
=
count
=
self
.
__count
1
if
not
count
:
self
.
__owner
=
self
.
__block
.
release
(
)
if
__debug__
:
self
.
_note
(
"%s.release(): final release"
,
self
)
else
:
if
__debug__
:
self
.
_note
(
"%s.release(): non-final release"
,
self
)

Release a lock decrementing the recursion level .


def wait(self, timeout=None):
        """Wait until notified or until a timeout occurs.

        If the calling thread has not acquired the lock when this method is
        called, a RuntimeError is raised.

        This method releases the underlying lock, and then blocks until it is
        awakened by a notify() or notifyAll() call for the same condition
        variable in another thread, or until the optional timeout occurs. Once
        awakened or timed out, it re-acquires the lock and returns.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof).

        When the underlying lock is an RLock, it is not released using its
        release() method, since this may not actually unlock the lock when it
        was acquired multiple times recursively. Instead, an internal interface
        of the RLock class is used, which really unlocks it even when it has
        been recursively acquired several times. Another internal interface is
        then used to restore the recursion level when the lock is reacquired.

        """
        if not self._is_owned():
            raise RuntimeError("cannot wait on un-acquired lock")
        waiter = _allocate_lock()
        waiter.acquire()
        self.__waiters.append(waiter)
        saved_state = self._release_save()
        try:    # restore state no matter what (e.g., KeyboardInterrupt)
            if timeout is None:
                waiter.acquire()
                if __debug__:
                    self._note("%s.wait(): got it", self)
            else:
                # Balancing act:  We can't afford a pure busy loop, so we
                # have to sleep; but if we sleep the whole timeout time,
                # we'll be unresponsive.  The scheme here sleeps very
                # little at first, longer as time goes on, but never longer
                # than 20 times per second (or the timeout time remaining).
                endtime = _time() + timeout
                delay = 0.0005 # 500 us -> initial delay of 1 ms
                while True:
                    gotit = waiter.acquire(0)
                    if gotit:
                        break
                    remaining = endtime - _time()
                    if remaining <= 0:
                        break
                    delay = min(delay * 2, remaining, .05)
                    _sleep(delay)
                if not gotit:
                    if __debug__:
                        self._note("%s.wait(%s): timed out", self, timeout)
                    try:
                        self.__waiters.remove(waiter)
                    except ValueError:
                        pass
                else:
                    if __debug__:
                        self._note("%s.wait(%s): got it", self, timeout)
        finally:
            self._acquire_restore(saved_state)

Wait until notified or until a timeout occurs .


def notify(self, n=1):
        """Wake up one or more threads waiting on this condition, if any.

        If the calling thread has not acquired the lock when this method is
        called, a RuntimeError is raised.

        This method wakes up at most n of the threads waiting for the condition
        variable; it is a no-op if no threads are waiting.

        """
        if not self._is_owned():
            raise RuntimeError("cannot notify on un-acquired lock")
        __waiters = self.__waiters
        waiters = __waiters[:n]
        if not waiters:
            if __debug__:
                self._note("%s.notify(): no waiters", self)
            return
        self._note("%s.notify(): notifying %d waiter%s", self, n,
                   n!=1 and "s" or "")
        for waiter in waiters:
            waiter.release()
            try:
                __waiters.remove(waiter)
            except ValueError:
                pass

def
notify
(
self
,
n
=
1
)
:
if
not
self
.
_is_owned
(
)
:
raise
RuntimeError
(
"cannot notify on un-acquired lock"
)
__waiters
=
self
.
__waiters
waiters
=
__waiters
[
:
n
]
if
not
waiters
:
if
__debug__
:
self
.
_note
(
"%s.notify(): no waiters"
,
self
)
return
self
.
_note
(
"%s.notify(): notifying %d waiter%s"
,
self
,
n
,
n
!=
1
and
"s"
or
""
)
for
waiter
in
waiters
:
waiter
.
release
(
)
try
:
__waiters
.
remove
(
waiter
)
except
ValueError
:
pass

Wake up one or more threads waiting on this condition if any .


def acquire(self, blocking=1):
        """Acquire a semaphore, decrementing the internal counter by one.

        When invoked without arguments: if the internal counter is larger than
        zero on entry, decrement it by one and return immediately. If it is zero
        on entry, block, waiting until some other thread has called release() to
        make it larger than zero. This is done with proper interlocking so that
        if multiple acquire() calls are blocked, release() will wake exactly one
        of them up. The implementation may pick one at random, so the order in
        which blocked threads are awakened should not be relied on. There is no
        return value in this case.

        When invoked with blocking set to true, do the same thing as when called
        without arguments, and return true.

        When invoked with blocking set to false, do not block. If a call without
        an argument would block, return false immediately; otherwise, do the
        same thing as when called without arguments, and return true.

        """
        rc = False
        with self.__cond:
            while self.__value == 0:
                if not blocking:
                    break
                if __debug__:
                    self._note("%s.acquire(%s): blocked waiting, value=%s",
                            self, blocking, self.__value)
                self.__cond.wait()
            else:
                self.__value = self.__value - 1
                if __debug__:
                    self._note("%s.acquire: success, value=%s",
                            self, self.__value)
                rc = True
        return rc

Acquire a semaphore decrementing the internal counter by one .


def set(self):
        """Set the internal flag to true.

        All threads waiting for the flag to become true are awakened. Threads
        that call wait() once the flag is true will not block at all.

        """
        with self.__cond:
            self.__flag = True
            self.__cond.notify_all()

def set ( self ) : with self . __cond : self . __flag = True self . __cond . notify_all ( )


def wait(self, timeout=None):
        """Block until the internal flag is true.

        If the internal flag is true on entry, return immediately. Otherwise,
        block until another thread calls set() to set the flag to true, or until
        the optional timeout occurs.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof).

        This method returns the internal flag on exit, so it will always return
        True except if a timeout is given and the operation times out.

        """
        with self.__cond:
            if not self.__flag:
                self.__cond.wait(timeout)
            return self.__flag

def wait ( self , timeout = None ) : with self . __cond : if not self . __flag : self . __cond . wait ( timeout ) return self . __flag


def start(self):
        """Start the thread's activity.

        It must be called at most once per thread object. It arranges for the
        object's run() method to be invoked in a separate thread of control.

        This method will raise a RuntimeError if called more than once on the
        same thread object.

        """
        if not self.__initialized:
            raise RuntimeError("thread.__init__() not called")
        if self.__started.is_set():
            raise RuntimeError("threads can only be started once")
        if __debug__:
            self._note("%s.start(): starting thread", self)
        with _active_limbo_lock:
            _limbo[self] = self
        try:
            _start_new_thread(self.__bootstrap, ())
        except Exception:
            with _active_limbo_lock:
                del _limbo[self]
            raise
        self.__started.wait()

def
start
(
self
)
:
if
not
self
.
__initialized
:
raise
RuntimeError
(
"thread.__init__() not called"
)
if
self
.
__started
.
is_set
(
)
:
raise
RuntimeError
(
"threads can only be started once"
)
if
__debug__
:
self
.
_note
(
"%s.start(): starting thread"
,
self
)
with
_active_limbo_lock
:
_limbo
[
self
]
=
self
try
:
_start_new_thread
(
self
.
__bootstrap
,
(
)
)
except
Exception
:
with
_active_limbo_lock
:
del
_limbo
[
self
]
raise
self
.
__started
.
wait
(
)


def run(self):
        """Method representing the thread's activity.

        You may override this method in a subclass. The standard run() method
        invokes the callable object passed to the object's constructor as the
        target argument, if any, with sequential and keyword arguments taken
        from the args and kwargs arguments, respectively.

        """
        try:
            if self.__target:
                self.__target(*self.__args, **self.__kwargs)
        finally:
            # Avoid a refcycle if the thread is running a function with
            # an argument that has a member that points to the thread.
            del self.__target, self.__args, self.__kwargs

Method representing the thread s activity .


def join(self, timeout=None):
        """Wait until the thread terminates.

        This blocks the calling thread until the thread whose join() method is
        called terminates -- either normally or through an unhandled exception
        or until the optional timeout occurs.

        When the timeout argument is present and not None, it should be a
        floating point number specifying a timeout for the operation in seconds
        (or fractions thereof). As join() always returns None, you must call
        isAlive() after join() to decide whether a timeout happened -- if the
        thread is still alive, the join() call timed out.

        When the timeout argument is not present or None, the operation will
        block until the thread terminates.

        A thread can be join()ed many times.

        join() raises a RuntimeError if an attempt is made to join the current
        thread as that would cause a deadlock. It is also an error to join() a
        thread before it has been started and attempts to do so raises the same
        exception.

        """
        if not self.__initialized:
            raise RuntimeError("Thread.__init__() not called")
        if not self.__started.is_set():
            raise RuntimeError("cannot join thread before it is started")
        if self is current_thread():
            raise RuntimeError("cannot join current thread")

        if __debug__:
            if not self.__stopped:
                self._note("%s.join(): waiting until thread stops", self)
        self.__block.acquire()
        try:
            if timeout is None:
                while not self.__stopped:
                    self.__block.wait()
                if __debug__:
                    self._note("%s.join(): thread stopped", self)
            else:
                deadline = _time() + timeout
                while not self.__stopped:
                    delay = deadline - _time()
                    if delay <= 0:
                        if __debug__:
                            self._note("%s.join(): timed out", self)
                        break
                    self.__block.wait(delay)
                else:
                    if __debug__:
                        self._note("%s.join(): thread stopped", self)
        finally:
            self.__block.release()


def _dump_registry(cls, file=None):
        """Debug helper to print the ABC registry."""
        print >> file, "Class: %s.%s" % (cls.__module__, cls.__name__)
        print >> file, "Inv.counter: %s" % ABCMeta._abc_invalidation_counter
        for name in sorted(cls.__dict__.keys()):
            if name.startswith("_abc_"):
                value = getattr(cls, name)
                print >> file, "%s: %r" % (name, value)

def
_dump_registry
(
cls
,
file
=
)
:
print
>>
file
,
"Class: %s.%s"
%
(
cls
.
__module__
,
cls
.
__name__
)
print
>>
file
,
"Inv.counter: %s"
%
ABCMeta
.
_abc_invalidation_counter
for
name
in
sorted
(
cls
.
__dict__
.
keys
(
)
)
:
if
name
.
startswith
(
"_abc_"
)
:
value
=
getattr
(
cls
,
name
)
print
>>
file
,
"%s: %r"
%
(
name
,
value
)

Debug helper to print the ABC registry .


def b2a_qp(data, quotetabs=False, istext=True, header=False):
    """quotetabs=True means that tab and space characters are always
       quoted.
       istext=False means that \r and \n are treated as regular characters
       header=True encodes space characters with '_' and requires
       real '_' characters to be quoted.
    """
    MAXLINESIZE = 76

    # See if this string is using CRLF line ends
    lf = data.find('\n')
    crlf = lf > 0 and data[lf-1] == '\r'

    inp = 0
    linelen = 0
    odata = []
    while inp < len(data):
        c = data[inp]
        if (c > '~' or
            c == '=' or
            (header and c == '_') or
            (c == '.' and linelen == 0 and (inp+1 == len(data) or
                                            data[inp+1] == '\n' or
                                            data[inp+1] == '\r')) or
            (not istext and (c == '\r' or c == '\n')) or
            ((c == '\t' or c == ' ') and (inp + 1 == len(data))) or
            (c <= ' ' and c != '\r' and c != '\n' and
             (quotetabs or (not quotetabs and (c != '\t' and c != ' '))))):
            linelen += 3
            if linelen >= MAXLINESIZE:
                odata.append('=')
                if crlf: odata.append('\r')
                odata.append('\n')
                linelen = 3
            odata.append('=' + two_hex_digits(ord(c)))
            inp += 1
        else:
            if (istext and
                (c == '\n' or (inp+1 < len(data) and c == '\r' and
                               data[inp+1] == '\n'))):
                linelen = 0
                # Protect against whitespace on end of line
                if (len(odata) > 0 and
                    (odata[-1] == ' ' or odata[-1] == '\t')):
                    ch = ord(odata[-1])
                    odata[-1] = '='
                    odata.append(two_hex_digits(ch))

                if crlf: odata.append('\r')
                odata.append('\n')
                if c == '\r':
                    inp += 2
                else:
                    inp += 1
            else:
                if (inp + 1 < len(data) and
                    data[inp+1] != '\n' and
                    (linelen + 1) >= MAXLINESIZE):
                    odata.append('=')
                    if crlf: odata.append('\r')
                    odata.append('\n')
                    linelen = 0

                linelen += 1
                if header and c == ' ':
                    c = '_'
                odata.append(c)
                inp += 1
    return ''.join(odata)

quotetabs = True means that tab and space characters are always quoted . istext = False means that \ r and \ n are treated as regular characters header = True encodes space characters with _ and requires real _ characters to be quoted .


def rlecode_hqx(s):
    """
    Run length encoding for binhex4.
    The CPython implementation does not do run length encoding
    of \x90 characters. This implementation does.
    """
    if not s:
        return ''
    result = []
    prev = s[0]
    count = 1
    # Add a dummy character to get the loop to go one extra round.
    # The dummy must be different from the last character of s.
    # In the same step we remove the first character, which has
    # already been stored in prev.
    if s[-1] == '!':
        s = s[1:] + '?'
    else:
        s = s[1:] + '!'

    for c in s:
        if c == prev and count < 255:
            count += 1
        else:
            if count == 1:
                if prev != '\x90':
                    result.append(prev)
                else:
                    result += ['\x90', '\x00']
            elif count < 4:
                if prev != '\x90':
                    result += [prev] * count
                else:
                    result += ['\x90', '\x00'] * count
            else:
                if prev != '\x90':
                    result += [prev, '\x90', chr(count)]
                else:
                    result += ['\x90', '\x00', '\x90', chr(count)]
            count = 1
            prev = c

    return ''.join(result)

Run length encoding for binhex4 . The CPython implementation does not do run length encoding of \ x90 characters . This implementation does .


def _format_text(self, text):
        """
        Format a paragraph of free-form text for inclusion in the
        help output at the current indentation level.
        """
        text_width = max(self.width - self.current_indent, 11)
        indent = " "*self.current_indent
        return textwrap.fill(text,
                             text_width,
                             initial_indent=indent,
                             subsequent_indent=indent)

def
_format_text
(
self
,
text
)
:
text_width
=
max
(
self
.
width
self
.
current_indent
,
11
)
indent
=
" "
*
self
.
current_indent
return
textwrap
.
fill
(
text
,
text_width
,
initial_indent
=
indent
,
subsequent_indent
=
indent
)

Format a paragraph of free - form text for inclusion in the help output at the current indentation level .


def format_option_strings(self, option):
        """Return a comma-separated list of option strings & metavariables."""
        if option.takes_value():
            metavar = option.metavar or option.dest.upper()
            short_opts = [self._short_opt_fmt % (sopt, metavar)
                          for sopt in option._short_opts]
            long_opts = [self._long_opt_fmt % (lopt, metavar)
                         for lopt in option._long_opts]
        else:
            short_opts = option._short_opts
            long_opts = option._long_opts

        if self.short_first:
            opts = short_opts + long_opts
        else:
            opts = long_opts + short_opts

        return ", ".join(opts)

def
format_option_strings
(
self
,
option
)
:
if
option
.
takes_value
(
)
:
metavar
=
option
.
metavar
or
option
.
dest
.
upper
(
)
short_opts
=
[
self
.
_short_opt_fmt
%
(
sopt
,
metavar
)
for
sopt
in
option
.
_short_opts
]
long_opts
=
[
self
.
_long_opt_fmt
%
(
lopt
,
metavar
)
for
lopt
in
option
.
_long_opts
]
else
:
short_opts
=
option
.
_short_opts
long_opts
=
option
.
_long_opts
if
self
.
short_first
:
opts
=
short_opts
+
long_opts
else
:
opts
=
long_opts
+
short_opts
return
", "
.
join
(
opts
)

Return a comma - separated list of option strings & metavariables .


def _update_careful(self, dict):
        """
        Update the option values from an arbitrary dictionary, but only
        use keys from dict that already have a corresponding attribute
        in self.  Any keys in dict without a corresponding attribute
        are silently ignored.
        """
        for attr in dir(self):
            if attr in dict:
                dval = dict[attr]
                if dval is not None:
                    setattr(self, attr, dval)

def _update_careful ( self , dict ) : for attr in dir ( self ) : if attr in dict : dval = dict [ attr ] if dval is not None : setattr ( self , attr , dval )

Update the option values from an arbitrary dictionary but only use keys from dict that already have a corresponding attribute in self . Any keys in dict without a corresponding attribute are silently ignored .


def insort_right(a, x, lo=0, hi=None):
    """Insert item x in list a, and keep it sorted assuming a is sorted.

    If x is already in a, insert it to the right of the rightmost x.

    Optional args lo (default 0) and hi (default len(a)) bound the
    slice of a to be searched.
    """

    if lo < 0:
        raise ValueError('lo must be non-negative')
    if hi is None:
        hi = len(a)
    while lo < hi:
        mid = (lo+hi)//2
        if x < a[mid]: hi = mid
        else: lo = mid+1
    a.insert(lo, x)

def
insort_right
(
a
,
x
,
lo
=
0
,
hi
=
)
:
if
lo
<
0
:
raise
ValueError
(
'lo must be non-negative'
)
if
hi
is
:
hi
=
len
(
a
)
while
lo
<
hi
:
mid
=
(
lo
+
hi
)
//
2
if
x
<
a
[
mid
]
:
hi
=
mid
else
:
lo
=
mid
+
1
a
.
insert
(
lo
,
x
)

Insert item x in list a and keep it sorted assuming a is sorted .


def lock(self, function, argument):
        """Lock a mutex, call the function with supplied argument
        when it is acquired.  If the mutex is already locked, place
        function and argument in the queue."""
        if self.testandset():
            function(argument)
        else:
            self.queue.append((function, argument))

def lock ( self , function , argument ) : if self . testandset ( ) : function ( argument ) else : self . queue . append ( ( function , argument ) )

Lock a mutex call the function with supplied argument when it is acquired . If the mutex is already locked place function and argument in the queue .


def unlock(self):
        """Unlock a mutex.  If the queue is not empty, call the next
        function with its argument."""
        if self.queue:
            function, argument = self.queue.popleft()
            function(argument)
        else:
            self.locked = False

def unlock ( self ) : if self . queue : function , argument = self . queue . popleft ( ) function ( argument ) else : self . locked = False

Unlock a mutex . If the queue is not empty call the next function with its argument .


def copy(self):
    """Return a clone object.

    Return a copy ('clone') of the md5 object. This can be used
    to efficiently compute the digests of strings that share
    a common initial substring.
    """
    if 0:  # set this to 1 to make the flow space crash
      return copy.deepcopy(self)
    clone = self.__class__()
    clone.length = self.length
    clone.count = [] + self.count[:]
    clone.input = [] + self.input
    clone.A = self.A
    clone.B = self.B
    clone.C = self.C
    clone.D = self.D
    return clone


def search(self, string, pos=0, endpos=sys.maxint):
        """Scan through string looking for a location where this regular
        expression produces a match, and return a corresponding MatchObject
        instance. Return None if no position in the string matches the
        pattern."""
        state = _State(string, pos, endpos, self.flags)
        if state.search(self._code):
            return SRE_Match(self, state)
        else:
            return None

def search ( self , string , pos = 0 , endpos = sys . maxint ) : state = _State ( string , pos , endpos , self . flags ) if state . search ( self . _code ) : return SRE_Match ( self , state ) else : return None

Scan through string looking for a location where this regular expression produces a match and return a corresponding MatchObject instance . Return None if no position in the string matches the pattern .


def sub(self, repl, string, count=0):
        """Return the string obtained by replacing the leftmost non-overlapping
        occurrences of pattern in string by the replacement repl."""
        return self._subx(repl, string, count, False)

def sub ( self , repl , string , count = 0 ) : return self . _subx ( repl , string , count , False )

Return the string obtained by replacing the leftmost non - overlapping occurrences of pattern in string by the replacement repl .


def split(self, string, maxsplit=0):
        """Split string by the occurrences of pattern."""
        splitlist = []
        state = _State(string, 0, sys.maxint, self.flags)
        n = 0
        last = state.start
        while not maxsplit or n < maxsplit:
            state.reset()
            state.string_position = state.start
            if not state.search(self._code):
                break
            if state.start == state.string_position: # zero-width match
                if last == state.end:                # or end of string
                    break
                state.start += 1
                continue
            splitlist.append(string[last:state.start])
            # add groups (if any)
            if self.groups:
                match = SRE_Match(self, state)
                # TODO: Use .extend once it is implemented.
                # splitlist.extend(list(match.groups(None)))
                splitlist += (list(match.groups(None)))
            n += 1
            last = state.start = state.string_position
        splitlist.append(string[last:state.end])
        return splitlist

Split string by the occurrences of pattern .


def _create_regs(self, state):
        """Creates a tuple of index pairs representing matched groups."""
        regs = [(state.start, state.string_position)]
        for group in range(self.re.groups):
            mark_index = 2 * group
            if mark_index + 1 < len(state.marks) \
                                    and state.marks[mark_index] is not None \
                                    and state.marks[mark_index + 1] is not None:
                regs.append((state.marks[mark_index], state.marks[mark_index + 1]))
            else:
                regs.append((-1, -1))
        return tuple(regs)

def _create_regs ( self , state ) : regs = [ ( state . start , state . string_position ) ] for group in range ( self . re . groups ) : mark_index = 2 * group if mark_index + 1 < len ( state . marks ) and state . marks [ mark_index ] is not None and state . marks [ mark_index + 1 ] is not None : regs . append ( ( state . marks [ mark_index ] , state . marks [ mark_index + 1 ] ) ) else : regs . append ( ( - 1 , - 1 ) ) return tuple ( regs )

Creates a tuple of index pairs representing matched groups .


def group(self, *args):
        """Returns one or more subgroups of the match. Each argument is either a
        group index or a group name."""
        if len(args) == 0:
            args = (0,)
        grouplist = []
        for group in args:
            grouplist.append(self._get_slice(self._get_index(group), None))
        if len(grouplist) == 1:
            return grouplist[0]
        else:
            return tuple(grouplist)

def group ( self , * args ) : if len ( args ) == 0 : args = ( 0 , ) grouplist = [ ] for group in args : grouplist . append ( self . _get_slice ( self . _get_index ( group ) , None ) ) if len ( grouplist ) == 1 : return grouplist [ 0 ] else : return tuple ( grouplist )

Returns one or more subgroups of the match . Each argument is either a group index or a group name .


def fast_search(self, pattern_codes):
        """Skips forward in a string as fast as possible using information from
        an optimization info block."""
        # pattern starts with a known prefix
        # <5=length> <6=skip> <7=prefix data> <overlap data>
        flags = pattern_codes[2]
        prefix_len = pattern_codes[5]
        prefix_skip = pattern_codes[6] # don't really know what this is good for
        prefix = pattern_codes[7:7 + prefix_len]
        overlap = pattern_codes[7 + prefix_len - 1:pattern_codes[1] + 1]
        pattern_codes = pattern_codes[pattern_codes[1] + 1:]
        i = 0
        string_position = self.string_position
        while string_position < self.end:
            while True:
                if ord(self.string[string_position]) != prefix[i]:
                    if i == 0:
                        break
                    else:
                        i = overlap[i]
                else:
                    i += 1
                    if i == prefix_len:
                        # found a potential match
                        self.start = string_position + 1 - prefix_len
                        self.string_position = string_position + 1 \
                                                     - prefix_len + prefix_skip
                        if flags & SRE_INFO_LITERAL:
                            return True # matched all of pure literal pattern
                        if self.match(pattern_codes[2 * prefix_skip:]):
                            return True
                        i = overlap[i]
                    break
            string_position += 1
        return False

Skips forward in a string as fast as possible using information from an optimization info block .


def push_new_context(self, pattern_offset):
        """Creates a new child context of this context and pushes it on the
        stack. pattern_offset is the offset off the current code position to
        start interpreting from."""
        child_context = _MatchContext(self.state,
            self.pattern_codes[self.code_position + pattern_offset:])
        self.state.context_stack.append(child_context)
        return child_context

def push_new_context ( self , pattern_offset ) : child_context = _MatchContext ( self . state , self . pattern_codes [ self . code_position + pattern_offset : ] ) self . state . context_stack . append ( child_context ) return child_context

Creates a new child context of this context and pushes it on the stack . pattern_offset is the offset off the current code position to start interpreting from .


def match(self, context):
        """Returns True if the current context matches, False if it doesn't and
        None if matching is not finished, ie must be resumed after child
        contexts have been matched."""
        while context.remaining_codes() > 0 and context.has_matched is None:
            opcode = context.peek_code()
            if not self.dispatch(opcode, context):
                return None
        if context.has_matched is None:
            context.has_matched = False
        return context.has_matched

def match ( self , context ) : while context . remaining_codes ( ) > 0 and context . has_matched is None : opcode = context . peek_code ( ) if not self . dispatch ( opcode , context ) : return None if context . has_matched is None : context . has_matched = False return context . has_matched

Returns True if the current context matches False if it doesn t and None if matching is not finished ie must be resumed after child contexts have been matched .


def dispatch(self, opcode, context):
        """Dispatches a context on a given opcode. Returns True if the context
        is done matching, False if it must be resumed when next encountered."""
        if id(context) in self.executing_contexts:
            generator = self.executing_contexts[id(context)]
            del self.executing_contexts[id(context)]
            has_finished = generator.next()
        else:
            method = self.DISPATCH_TABLE.get(opcode, _OpcodeDispatcher.unknown)
            has_finished = method(self, context)
            if hasattr(has_finished, "next"): # avoid using the types module
                generator = has_finished
                has_finished = generator.next()
        if not has_finished:
            self.executing_contexts[id(context)] = generator
        return has_finished

def
dispatch
(
self
,
opcode
,
context
)
:
if
id
(
context
)
in
self
.
executing_contexts
:
generator
=
self
.
executing_contexts
[
id
(
context
)
]
del
self
.
executing_contexts
[
id
(
context
)
]
has_finished
=
generator
.
next
(
)
else
:
method
=
self
.
DISPATCH_TABLE
.
get
(
opcode
,
_OpcodeDispatcher
.
unknown
)
has_finished
=
method
(
self
,
context
)
if
hasattr
(
has_finished
,
"next"
)
:
# avoid using the types module
generator
=
has_finished
has_finished
=
generator
.
next
(
)
if
not
has_finished
:
self
.
executing_contexts
[
id
(
context
)
]
=
generator
return
has_finished

Dispatches a context on a given opcode . Returns True if the context is done matching False if it must be resumed when next encountered .


def check_charset(self, ctx, char):
        """Checks whether a character matches set of arbitrary length. Assumes
        the code pointer is at the first member of the set."""
        self.set_dispatcher.reset(char)
        save_position = ctx.code_position
        result = None
        while result is None:
            result = self.set_dispatcher.dispatch(ctx.peek_code(), ctx)
        ctx.code_position = save_position
        return result

def check_charset ( self , ctx , char ) : self . set_dispatcher . reset ( char ) save_position = ctx . code_position result = None while result is None : result = self . set_dispatcher . dispatch ( ctx . peek_code ( ) , ctx ) ctx . code_position = save_position return result

Checks whether a character matches set of arbitrary length . Assumes the code pointer is at the first member of the set .


def unexpo(intpart, fraction, expo):
    """Remove the exponent by changing intpart and fraction."""
    if expo > 0: # Move the point left
        f = len(fraction)
        intpart, fraction = intpart + fraction[:expo], fraction[expo:]
        if expo > f:
            intpart = intpart + '0'*(expo-f)
    elif expo < 0: # Move the point right
        i = len(intpart)
        intpart, fraction = intpart[:expo], intpart[expo:] + fraction
        if expo < -i:
            fraction = '0'*(-expo-i) + fraction
    return intpart, fraction

def
unexpo
(
intpart
,
fraction
,
expo
)
:
if
expo
>
0
:
# Move the point left
f
=
len
(
fraction
)
intpart
,
fraction
=
intpart
+
fraction
[
:
expo
]
,
fraction
[
expo
:
]
if
expo
>
f
:
intpart
=
intpart
+
'0'
*
(
expo
f
)
elif
expo
<
0
:
# Move the point right
i
=
len
(
intpart
)
intpart
,
fraction
=
intpart
[
:
expo
]
,
intpart
[
expo
:
]
+
fraction
if
expo
<
i
:
fraction
=
'0'
*
(
expo
i
)
+
fraction
return
intpart
,
fraction

Remove the exponent by changing intpart and fraction .


def roundfrac(intpart, fraction, digs):
    """Round or extend the fraction to size digs."""
    f = len(fraction)
    if f <= digs:
        return intpart, fraction + '0'*(digs-f)
    i = len(intpart)
    if i+digs < 0:
        return '0'*-digs, ''
    total = intpart + fraction
    nextdigit = total[i+digs]
    if nextdigit >= '5': # Hard case: increment last digit, may have carry!
        n = i + digs - 1
        while n >= 0:
            if total[n] != '9': break
            n = n-1
        else:
            total = '0' + total
            i = i+1
            n = 0
        total = total[:n] + chr(ord(total[n]) + 1) + '0'*(len(total)-n-1)
        intpart, fraction = total[:i], total[i:]
    if digs >= 0:
        return intpart, fraction[:digs]
    else:
        return intpart[:digs] + '0'*-digs, ''

Round or extend the fraction to size digs .


def _randbelow(self, n):
    """Return a random int in the range [0,n)."""
    # TODO
    # change once int.bit_length is implemented.
    # k = n.bit_length()
    k = _int_bit_length(n)
    r = self.getrandbits(k)
    while r >= n:
      r = self.getrandbits(k)
    return r

Return a random int in the range [ 0 n ) .


def filter(names, pat):
    """Return the subset of the list NAMES that match PAT"""
    import os
    # import posixpath
    result=[]
    # pat=os.path.normcase(pat)
    try:
        re_pat = _cache[pat]
    except KeyError:
        res = translate(pat)
        if len(_cache) >= _MAXCACHE:
            # _cache.clear()
            globals()['_cache'] = {}
        _cache[pat] = re_pat = re.compile(res)
    match = re_pat.match
    # if os.path is posixpath:
    if 1:
        # normcase on posix is NOP. Optimize it away from the loop.
        for name in names:
            if match(name):
                result.append(name)
    else:
        for name in names:
            if match(os.path.normcase(name)):
                result.append(name)
    return result


def fnmatchcase(name, pat):
    """Test whether FILENAME matches PATTERN, including case.

    This is a version of fnmatch() which doesn't case-normalize
    its arguments.
    """

    try:
        re_pat = _cache[pat]
    except KeyError:
        res = translate(pat)
        if len(_cache) >= _MAXCACHE:
            # _cache.clear()
            globals()['_cache'] = {}
        _cache[pat] = re_pat = re.compile(res)
    return re_pat.match(name) is not None

def
fnmatchcase
(
name
,
pat
)
:
try
:
re_pat
=
_cache
[
pat
]
except
KeyError
:
res
=
translate
(
pat
)
if
len
(
_cache
)
>=
_MAXCACHE
:
# _cache.clear()
globals
(
)
[
'_cache'
]
=
{
}
_cache
[
pat
]
=
re_pat
=
re
.
compile
(
res
)
return
re_pat
.
match
(
name
)
is
not

Test whether FILENAME matches PATTERN including case .


def translate(pat):
    """Translate a shell PATTERN to a regular expression.

    There is no way to quote meta-characters.
    """

    i, n = 0, len(pat)
    res = ''
    while i < n:
        c = pat[i]
        i = i+1
        if c == '*':
            res = res + '.*'
        elif c == '?':
            res = res + '.'
        elif c == '[':
            j = i
            if j < n and pat[j] == '!':
                j = j+1
            if j < n and pat[j] == ']':
                j = j+1
            while j < n and pat[j] != ']':
                j = j+1
            if j >= n:
                res = res + '\\['
            else:
                stuff = pat[i:j].replace('\\','\\\\')
                i = j+1
                if stuff[0] == '!':
                    stuff = '^' + stuff[1:]
                elif stuff[0] == '^':
                    stuff = '\\' + stuff
                res = '%s[%s]' % (res, stuff)
        else:
            res = res + re.escape(c)
    return res + '\Z(?ms)'

def translate ( pat ) : i , n = 0 , len ( pat ) res = '' while i < n : c = pat [ i ] i = i + 1 if c == '*' : res = res + '.*' elif c == '?' : res = res + '.' elif c == '[' : j = i if j < n and pat [ j ] == '!' : j = j + 1 if j < n and pat [ j ] == ']' : j = j + 1 while j < n and pat [ j ] != ']' : j = j + 1 if j >= n : res = res + '\\[' else : stuff = pat [ i : j ] . replace ( '\\' , '\\\\' ) i = j + 1 if stuff [ 0 ] == '!' : stuff = '^' + stuff [ 1 : ] elif stuff [ 0 ] == '^' : stuff = '\\' + stuff res = '%s[%s]' % ( res , stuff ) else : res = res + re . escape ( c ) return res + '\Z(?ms)'

Translate a shell PATTERN to a regular expression .


def task_done(self):
        """Indicate that a formerly enqueued task is complete.

        Used by Queue consumer threads.  For each get() used to fetch a task,
        a subsequent call to task_done() tells the queue that the processing
        on the task is complete.

        If a join() is currently blocking, it will resume when all items
        have been processed (meaning that a task_done() call was received
        for every item that had been put() into the queue).

        Raises a ValueError if called more times than there were items
        placed in the queue.
        """
        self.all_tasks_done.acquire()
        try:
            unfinished = self.unfinished_tasks - 1
            if unfinished <= 0:
                if unfinished < 0:
                    raise ValueError('task_done() called too many times')
                self.all_tasks_done.notify_all()
            self.unfinished_tasks = unfinished
        finally:
            self.all_tasks_done.release()

def
task_done
(
self
)
:
self
.
all_tasks_done
.
acquire
(
)
try
:
unfinished
=
self
.
unfinished_tasks
1
if
unfinished
<=
0
:
if
unfinished
<
0
:
raise
ValueError
(
'task_done() called too many times'
)
self
.
all_tasks_done
.
notify_all
(
)
self
.
unfinished_tasks
=
unfinished
finally
:
self
.
all_tasks_done
.
release
(
)

Indicate that a formerly enqueued task is complete .


def put(self, item, block=True, timeout=None):
        """Put an item into the queue.

        If optional args 'block' is true and 'timeout' is None (the default),
        block if necessary until a free slot is available. If 'timeout' is
        a non-negative number, it blocks at most 'timeout' seconds and raises
        the Full exception if no free slot was available within that time.
        Otherwise ('block' is false), put an item on the queue if a free slot
        is immediately available, else raise the Full exception ('timeout'
        is ignored in that case).
        """
        self.not_full.acquire()
        try:
            if self.maxsize > 0:
                if not block:
                    if self._qsize() == self.maxsize:
                        raise Full
                elif timeout is None:
                    while self._qsize() == self.maxsize:
                        self.not_full.wait()
                elif timeout < 0:
                    raise ValueError("'timeout' must be a non-negative number")
                else:
                    endtime = _time() + timeout
                    while self._qsize() == self.maxsize:
                        remaining = endtime - _time()
                        if remaining <= 0.0:
                            raise Full
                        self.not_full.wait(remaining)
            self._put(item)
            self.unfinished_tasks += 1
            self.not_empty.notify()
        finally:
            self.not_full.release()


def calculate_transitive_deps(modname, script, gopath):
  """Determines all modules that script transitively depends upon."""
  deps = set()
  def calc(modname, script):
    if modname in deps:
      return
    deps.add(modname)
    for imp in collect_imports(modname, script, gopath):
      if imp.is_native:
        deps.add(imp.name)
        continue
      parts = imp.name.split('.')
      calc(imp.name, imp.script)
      if len(parts) == 1:
        continue
      # For submodules, the parent packages are also deps.
      package_dir, filename = os.path.split(imp.script)
      if filename == '__init__.py':
        package_dir = os.path.dirname(package_dir)
      for i in xrange(len(parts) - 1, 0, -1):
        modname = '.'.join(parts[:i])
        script = os.path.join(package_dir, '__init__.py')
        calc(modname, script)
        package_dir = os.path.dirname(package_dir)
  calc(modname, script)
  deps.remove(modname)
  return deps

Determines all modules that script transitively depends upon .


def _make_future_features(node):
  """Processes a future import statement, returning set of flags it defines."""
  assert isinstance(node, ast.ImportFrom)
  assert node.module == '__future__'
  features = FutureFeatures()
  for alias in node.names:
    name = alias.name
    if name in _FUTURE_FEATURES:
      if name not in _IMPLEMENTED_FUTURE_FEATURES:
        msg = 'future feature {} not yet implemented by grumpy'.format(name)
        raise util.ParseError(node, msg)
      setattr(features, name, True)
    elif name == 'braces':
      raise util.ParseError(node, 'not a chance')
    elif name not in _REDUNDANT_FUTURE_FEATURES:
      msg = 'future feature {} is not defined'.format(name)
      raise util.ParseError(node, msg)
  return features

Processes a future import statement returning set of flags it defines .


def nested(*managers):
    """Combine multiple context managers into a single nested context manager.

   This function has been deprecated in favour of the multiple manager form
   of the with statement.

   The one advantage of this function over the multiple manager form of the
   with statement is that argument unpacking allows it to be
   used with a variable number of context managers as follows:

      with nested(*managers):
          do_something()

    """
    warn("With-statements now directly support multiple context managers",
         DeprecationWarning, 3)
    exits = []
    vars = []
    exc = (None, None, None)
    try:
        for mgr in managers:
            exit = mgr.__exit__
            enter = mgr.__enter__
            vars.append(enter())
            exits.append(exit)
        yield vars
    except:
        exc = sys.exc_info()
    finally:
        while exits:
            exit = exits.pop()
            try:
                if exit(*exc):
                    exc = (None, None, None)
            except:
                exc = sys.exc_info()
        if exc != (None, None, None):
            # Don't rely on sys.exc_info() still containing
            # the right information. Another exception may
            # have been raised and caught by an exit method
            raise exc[0], exc[1], exc[2]

Combine multiple context managers into a single nested context manager .


def tf_loss(self, states, internals, reward, update, reference=None):
        """
        Creates the TensorFlow operations for calculating the L2 loss between predicted
        state values and actual rewards.

        Args:
            states: Dict of state tensors.
            internals: List of prior internal state tensors.
            reward: Reward tensor.
            update: Boolean tensor indicating whether this call happens during an update.
            reference: Optional reference tensor(s), in case of a comparative loss.

        Returns:
            Loss tensor
        """
        prediction = self.predict(states=states, internals=internals, update=update)
        return tf.nn.l2_loss(t=(prediction - reward))

def tf_loss ( self , states , internals , reward , update , reference = None ) : prediction = self . predict ( states = states , internals = internals , update = update ) return tf . nn . l2_loss ( t = ( prediction - reward ) )

Creates the TensorFlow operations for calculating the L2 loss between predicted state values and actual rewards .


def get_variables(self, include_nontrainable=False):
        """
        Returns the TensorFlow variables used by the baseline.

        Returns:
            List of variables
        """
        if include_nontrainable:
            return [self.all_variables[key] for key in sorted(self.all_variables)]
        else:
            return [self.variables[key] for key in sorted(self.variables)]

def get_variables ( self , include_nontrainable = False ) : if include_nontrainable : return [ self . all_variables [ key ] for key in sorted ( self . all_variables ) ] else : return [ self . variables [ key ] for key in sorted ( self . variables ) ]

Returns the TensorFlow variables used by the baseline .


def from_spec(spec, kwargs=None):
        """
        Creates a baseline from a specification dict.
        """
        baseline = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.baselines.baselines,
            kwargs=kwargs
        )
        assert isinstance(baseline, Baseline)
        return baseline

def from_spec ( spec , kwargs = None ) : baseline = util . get_object ( obj = spec , predefined_objects = tensorforce . core . baselines . baselines , kwargs = kwargs ) assert isinstance ( baseline , Baseline ) return baseline

Creates a baseline from a specification dict .


def reset(self):
        """
        Resets the environment to its initialization state. This method needs to be called to start a
        new episode after the last episode ended.

        :return: initial state
        """
        self.level.reset()  # optional: episode=-1, seed=None
        return self.level.observations()[self.state_attribute]

def
reset
(
self
)
:
self
.
level
.
reset
(
)
# optional: episode=-1, seed=None
return
self
.
level
.
observations
(
)
[
self
.
state_attribute
]

Resets the environment to its initialization state . This method needs to be called to start a new episode after the last episode ended .


def execute(self, action):
        """
        Pass action to universe environment, return reward, next step, terminal state and
        additional info.

        :param action: action to execute as numpy array, should have dtype np.intc and should adhere to
            the specification given in DeepMindLabEnvironment.action_spec(level_id)
        :return: dict containing the next state, the reward, and a boolean indicating if the
            next state is a terminal state
        """
        adjusted_action = list()
        for action_spec in self.level.action_spec():
            if action_spec['min'] == -1 and action_spec['max'] == 1:
                adjusted_action.append(action[action_spec['name']] - 1)
            else:
                adjusted_action.append(action[action_spec['name']])  # clip?
        action = np.array(adjusted_action, dtype=np.intc)

        reward = self.level.step(action=action, num_steps=self.repeat_action)
        state = self.level.observations()['RGB_INTERLACED']
        terminal = not self.level.is_running()
        return state, terminal, reward

def
execute
(
self
,
action
)
:
adjusted_action
=
list
(
)
for
action_spec
in
self
.
level
.
action_spec
(
)
:
if
action_spec
[
'min'
]
==
1
and
action_spec
[
'max'
]
==
1
:
adjusted_action
.
append
(
action
[
action_spec
[
'name'
]
]
1
)
else
:
adjusted_action
.
append
(
action
[
action_spec
[
'name'
]
]
)
# clip?
action
=
np
.
array
(
adjusted_action
,
dtype
=
np
.
intc
)
reward
=
self
.
level
.
step
(
action
=
action
,
num_steps
=
self
.
repeat_action
)
state
=
self
.
level
.
observations
(
)
[
'RGB_INTERLACED'
]
terminal
=
not
self
.
level
.
is_running
(
)
return
state
,
terminal
,
reward

Pass action to universe environment return reward next step terminal state and additional info .


def tf_step(self, x, iteration, conjugate, residual, squared_residual):
        """
        Iteration loop body of the conjugate gradient algorithm.

        Args:
            x: Current solution estimate $x_t$.
            iteration: Current iteration counter $t$.
            conjugate: Current conjugate $c_t$.
            residual: Current residual $r_t$.
            squared_residual: Current squared residual $r_t^2$.

        Returns:
            Updated arguments for next iteration.
        """
        x, next_iteration, conjugate, residual, squared_residual = super(ConjugateGradient, self).tf_step(
            x, iteration, conjugate, residual, squared_residual
        )

        # Ac := A * c_t
        A_conjugate = self.fn_x(conjugate)

        # TODO: reference?
        if self.damping > 0.0:
            A_conjugate = [A_conj + self.damping * conj for A_conj, conj in zip(A_conjugate, conjugate)]

        # cAc := c_t^T * Ac
        conjugate_A_conjugate = tf.add_n(
            inputs=[tf.reduce_sum(input_tensor=(conj * A_conj)) for conj, A_conj in zip(conjugate, A_conjugate)]
        )

        # \alpha := r_t^2 / cAc
        alpha = squared_residual / tf.maximum(x=conjugate_A_conjugate, y=util.epsilon)

        # x_{t+1} := x_t + \alpha * c_t
        next_x = [t + alpha * conj for t, conj in zip(x, conjugate)]

        # r_{t+1} := r_t - \alpha * Ac
        next_residual = [res - alpha * A_conj for res, A_conj in zip(residual, A_conjugate)]

        # r_{t+1}^2 := r_{t+1}^T * r_{t+1}
        next_squared_residual = tf.add_n(inputs=[tf.reduce_sum(input_tensor=(res * res)) for res in next_residual])

        # \beta = r_{t+1}^2 / r_t^2
        beta = next_squared_residual / tf.maximum(x=squared_residual, y=util.epsilon)

        # c_{t+1} := r_{t+1} + \beta * c_t
        next_conjugate = [res + beta * conj for res, conj in zip(next_residual, conjugate)]

        return next_x, next_iteration, next_conjugate, next_residual, next_squared_residual

def
tf_step
(
self
,
x
,
iteration
,
conjugate
,
residual
,
squared_residual
)
:
x
,
next_iteration
,
conjugate
,
residual
,
squared_residual
=
super
(
ConjugateGradient
,
self
)
.
tf_step
(
x
,
iteration
,
conjugate
,
residual
,
squared_residual
)
# Ac := A * c_t
A_conjugate
=
self
.
fn_x
(
conjugate
)
# TODO: reference?
if
self
.
damping
>
0.0
:
A_conjugate
=
[
A_conj
+
self
.
damping
*
conj
for
A_conj
,
conj
in
zip
(
A_conjugate
,
conjugate
)
]
# cAc := c_t^T * Ac
conjugate_A_conjugate
=
tf
.
add_n
(
inputs
=
[
tf
.
reduce_sum
(
input_tensor
=
(
conj
*
A_conj
)
)
for
conj
,
A_conj
in
zip
(
conjugate
,
A_conjugate
)
]
)
# \alpha := r_t^2 / cAc
alpha
=
squared_residual
/
tf
.
maximum
(
x
=
conjugate_A_conjugate
,
y
=
util
.
epsilon
)
# x_{t+1} := x_t + \alpha * c_t
next_x
=
[
t
+
alpha
*
conj
for
t
,
conj
in
zip
(
x
,
conjugate
)
]
# r_{t+1} := r_t - \alpha * Ac
next_residual
=
[
res
alpha
*
A_conj
for
res
,
A_conj
in
zip
(
residual
,
A_conjugate
)
]
# r_{t+1}^2 := r_{t+1}^T * r_{t+1}
next_squared_residual
=
tf
.
add_n
(
inputs
=
[
tf
.
reduce_sum
(
input_tensor
=
(
res
*
res
)
)
for
res
in
next_residual
]
)
# \beta = r_{t+1}^2 / r_t^2
beta
=
next_squared_residual
/
tf
.
maximum
(
x
=
squared_residual
,
y
=
util
.
epsilon
)
# c_{t+1} := r_{t+1} + \beta * c_t
next_conjugate
=
[
res
+
beta
*
conj
for
res
,
conj
in
zip
(
next_residual
,
conjugate
)
]
return
next_x
,
next_iteration
,
next_conjugate
,
next_residual
,
next_squared_residual

Iteration loop body of the conjugate gradient algorithm .


def from_spec(spec, kwargs=None):
        """
        Creates a layer from a specification dict.
        """
        layer = util.get_object(
            obj=spec,
            predefined_objects=tensorforce.core.networks.layers,
            kwargs=kwargs
        )
        assert isinstance(layer, Layer)
        return layer

def from_spec ( spec , kwargs = None ) : layer = util . get_object ( obj = spec , predefined_objects = tensorforce . core . networks . layers , kwargs = kwargs ) assert isinstance ( layer , Layer ) return layer

Creates a layer from a specification dict .


def target_optimizer_arguments(self):
        """
        Returns the target optimizer arguments including the time, the list of variables to
        optimize, and various functions which the optimizer might require to perform an update
        step.

        Returns:
            Target optimizer arguments as dict.
        """
        variables = self.target_network.get_variables() + [
            variable for name in sorted(self.target_distributions)
            for variable in self.target_distributions[name].get_variables()
        ]
        source_variables = self.network.get_variables() + [
            variable for name in sorted(self.distributions)
            for variable in self.distributions[name].get_variables()
        ]
        arguments = dict(
            time=self.global_timestep,
            variables=variables,
            source_variables=source_variables
        )
        if self.global_model is not None:
            arguments['global_variables'] = self.global_model.target_network.get_variables() + [
                variable for name in sorted(self.global_model.target_distributions)
                for variable in self.global_model.target_distributions[name].get_variables()
            ]
        return arguments

def target_optimizer_arguments ( self ) : variables = self . target_network . get_variables ( ) + [ variable for name in sorted ( self . target_distributions ) for variable in self . target_distributions [ name ] . get_variables ( ) ] source_variables = self . network . get_variables ( ) + [ variable for name in sorted ( self . distributions ) for variable in self . distributions [ name ] . get_variables ( ) ] arguments = dict ( time = self . global_timestep , variables = variables , source_variables = source_variables ) if self . global_model is not None : arguments [ 'global_variables' ] = self . global_model . target_network . get_variables ( ) + [ variable for name in sorted ( self . global_model . target_distributions ) for variable in self . global_model . target_distributions [ name ] . get_variables ( ) ] return arguments

Returns the target optimizer arguments including the time the list of variables to optimize and various functions which the optimizer might require to perform an update step .


def from_spec(spec, kwargs):
        """
        Creates an environment from a specification dict.
        """
        env = tensorforce.util.get_object(
            obj=spec,
            predefined_objects=tensorforce.environments.environments,
            kwargs=kwargs
        )
        assert isinstance(env, Environment)
        return env

def from_spec ( spec , kwargs ) : env = tensorforce . util . get_object ( obj = spec , predefined_objects = tensorforce . environments . environments , kwargs = kwargs ) assert isinstance ( env , Environment ) return env

Creates an environment from a specification dict .


def setup(app):
    """When used for spinx extension."""
    global _is_sphinx
    _is_sphinx = True
    app.add_config_value('no_underscore_emphasis', False, 'env')
    app.add_source_parser('.md', M2RParser)
    app.add_directive('mdinclude', MdInclude)

def setup ( app ) : global _is_sphinx _is_sphinx = True app . add_config_value ( 'no_underscore_emphasis' , False , 'env' ) app . add_source_parser ( '.md' , M2RParser ) app . add_directive ( 'mdinclude' , MdInclude )


def output_image_link(self, m):
        """Pass through rest role."""
        return self.renderer.image_link(
            m.group('url'), m.group('target'), m.group('alt'))

def output_image_link ( self , m ) : return self . renderer . image_link ( m . group ( 'url' ) , m . group ( 'target' ) , m . group ( 'alt' ) )


def output_eol_literal_marker(self, m):
        """Pass through rest link."""
        marker = ':' if m.group(1) is None else ''
        return self.renderer.eol_literal_marker(marker)

def output_eol_literal_marker ( self , m ) : marker = ':' if m . group ( 1 ) is None else '' return self . renderer . eol_literal_marker ( marker )


def table(self, header, body):
        """Rendering table element. Wrap header and body in it.

        :param header: header part of the table.
        :param body: body part of the table.
        """
        table = '\n.. list-table::\n'
        if header and not header.isspace():
            table = (table + self.indent + ':header-rows: 1\n\n' +
                     self._indent_block(header) + '\n')
        else:
            table = table + '\n'
        table = table + self._indent_block(body) + '\n\n'
        return table

def
table
(
self
,
header
,
body
)
:
table
=
'\n.. list-table::\n'
if
header
and
not
header
.
isspace
(
)
:
table
=
(
table
+
self
.
indent
+
':header-rows: 1\n\n'
+
self
.
_indent_block
(
header
)
+
'\n'
)
else
:
table
=
table
+
'\n'
table
=
table
+
self
.
_indent_block
(
body
)
+
'\n\n'
return
table

Rendering table element . Wrap header and body in it .


def run(self):
        """Most of this method is from ``docutils.parser.rst.Directive``.

        docutils version: 0.12
        """
        if not self.state.document.settings.file_insertion_enabled:
            raise self.warning('"%s" directive disabled.' % self.name)
        source = self.state_machine.input_lines.source(
            self.lineno - self.state_machine.input_offset - 1)
        source_dir = os.path.dirname(os.path.abspath(source))
        path = rst.directives.path(self.arguments[0])
        path = os.path.normpath(os.path.join(source_dir, path))
        path = utils.relative_path(None, path)
        path = nodes.reprunicode(path)

        # get options (currently not use directive-specific options)
        encoding = self.options.get(
            'encoding', self.state.document.settings.input_encoding)
        e_handler = self.state.document.settings.input_encoding_error_handler
        tab_width = self.options.get(
            'tab-width', self.state.document.settings.tab_width)

        # open the inclding file
        try:
            self.state.document.settings.record_dependencies.add(path)
            include_file = io.FileInput(source_path=path,
                                        encoding=encoding,
                                        error_handler=e_handler)
        except UnicodeEncodeError as error:
            raise self.severe('Problems with "%s" directive path:\n'
                              'Cannot encode input file path "%s" '
                              '(wrong locale?).' %
                              (self.name, SafeString(path)))
        except IOError as error:
            raise self.severe('Problems with "%s" directive path:\n%s.' %
                              (self.name, ErrorString(error)))

        # read from the file
        try:
            rawtext = include_file.read()
        except UnicodeError as error:
            raise self.severe('Problem with "%s" directive:\n%s' %
                              (self.name, ErrorString(error)))

        config = self.state.document.settings.env.config
        converter = M2R(no_underscore_emphasis=config.no_underscore_emphasis)
        include_lines = statemachine.string2lines(converter(rawtext),
                                                  tab_width,
                                                  convert_whitespace=True)
        self.state_machine.insert_input(include_lines, path)
        return []

Most of this method is from docutils . parser . rst . Directive .


def WorkerAgentGenerator(agent_class):
    """
    Worker Agent generator, receives an Agent class and creates a Worker Agent class that inherits from that Agent.
    """

    # Support special case where class is given as type-string (AgentsDictionary) or class-name-string.
    if isinstance(agent_class, str):
        agent_class = AgentsDictionary.get(agent_class)
        # Last resort: Class name given as string?
        if not agent_class and agent_class.find('.') != -1:
            module_name, function_name = agent_class.rsplit('.', 1)
            module = importlib.import_module(module_name)
            agent_class = getattr(module, function_name)

    class WorkerAgent(agent_class):
        """
        Worker agent receiving a shared model to avoid creating multiple models.
        """

        def __init__(self, model=None, **kwargs):
            # Set our model externally.
            self.model = model
            # Be robust against `network` coming in from kwargs even though this agent doesn't have one
            if not issubclass(agent_class, LearningAgent):
                kwargs.pop("network")
            # Call super c'tor (which will call initialize_model and assign self.model to the return value).
            super(WorkerAgent, self).__init__(**kwargs)

        def initialize_model(self):
            # Return our model (already given and initialized).
            return self.model

    return WorkerAgent

Worker Agent generator receives an Agent class and creates a Worker Agent class that inherits from that Agent .


def _run_single(self, thread_id, agent, environment, deterministic=False,
                    max_episode_timesteps=-1, episode_finished=None, testing=False, sleep=None):
        """
        The target function for a thread, runs an agent and environment until signaled to stop.
        Adds rewards to shared episode rewards list.

        Args:
            thread_id (int): The ID of the thread that's running this target function.
            agent (Agent): The Agent object that this particular thread uses.
            environment (Environment): The Environment object that this particular thread uses.
            max_episode_timesteps (int): Max. number of timesteps per episode. Use -1 or 0 for non-limited episodes.
            episode_finished (callable): Function called after each episode that takes an episode summary spec and
                returns False, if this single run should terminate after this episode.
                Can be used e.g. to set a particular mean reward threshold.
        """

        # figure out whether we are using the deprecated way of "episode_finished" reporting
        old_episode_finished = False
        if episode_finished is not None and len(getargspec(episode_finished).args) == 1:
            old_episode_finished = True

        episode = 0
        # Run this single worker (episode loop) as long as global count thresholds have not been reached.
        while not self.should_stop:
            state = environment.reset()
            agent.reset()
            self.global_timestep, self.global_episode = agent.timestep, agent.episode
            episode_reward = 0

            # Time step (within episode) loop
            time_step = 0
            time_start = time.time()
            while True:
                action, internals, states = agent.act(states=state, deterministic=deterministic, buffered=False)
                reward = 0
                for repeat in xrange(self.repeat_actions):
                    state, terminal, step_reward = environment.execute(action=action)
                    reward += step_reward
                    if terminal:
                        break

                if not testing:
                    # agent.observe(reward=reward, terminal=terminal)
                    # Insert everything at once.
                    agent.atomic_observe(
                        states=state,
                        actions=action,
                        internals=internals,
                        reward=reward,
                        terminal=terminal
                    )

                if sleep is not None:
                    time.sleep(sleep)

                time_step += 1
                episode_reward += reward

                if terminal or time_step == max_episode_timesteps:
                    break

                # Abort the episode (discard its results) when global says so.
                if self.should_stop:
                    return

            self.global_timestep += time_step

            # Avoid race condition where order in episode_rewards won't match order in episode_timesteps.
            self.episode_list_lock.acquire()
            self.episode_rewards.append(episode_reward)
            self.episode_timesteps.append(time_step)
            self.episode_times.append(time.time() - time_start)
            self.episode_list_lock.release()

            if episode_finished is not None:
                # old way of calling episode_finished
                if old_episode_finished:
                    summary_data = {
                        "thread_id": thread_id,
                        "episode": episode,
                        "timestep": time_step,
                        "episode_reward": episode_reward
                    }
                    if not episode_finished(summary_data):
                        return
                # New way with BasicRunner (self) and thread-id.
                elif not episode_finished(self, thread_id):
                    return

            episode += 1

The target function for a thread runs an agent and environment until signaled to stop . Adds rewards to shared episode rewards list .


def _int_to_pos(self, flat_position):
        """Returns x, y from flat_position integer.

        Args:
            flat_position: flattened position integer

        Returns: x, y

        """
        return flat_position % self.env.action_space.screen_shape[0],\
            flat_position % self.env.action_space.screen_shape[1]

def _int_to_pos ( self , flat_position ) : return flat_position % self . env . action_space . screen_shape [ 0 ] , flat_position % self . env . action_space . screen_shape [ 1 ]

Returns x y from flat_position integer .