def directives(self):
    """The diretives parsed from the comments."""
    if self._directives is None:
      self._directives = []
      for comment in self.comments:
        self._directives.extend(self.directives_from_comment(comment))
    return self._directives

def directives ( self ) : if self . _directives is None : self . _directives = [ ] for comment in self . comments : self . _directives . extend ( self . directives_from_comment ( comment ) ) return self . _directives

The diretives parsed from the comments .


def output_source(self):
    """The python source of the parser generated from the input source."""
    if self._output_source is None:
      self._output_source = self._compile()
    return self._output_source

def output_source ( self ) : if self . _output_source is None : self . _output_source = self . _compile ( ) return self . _output_source

The python source of the parser generated from the input source .


def _compile(self):
    """Returns the python source code for the generated parser."""
    fmt = """\"\"\"This parser was generated by pyebnf on {date}.\"\"\"
    from enum import Enum

    from pyebnf import parser_base as PB
    from pyebnf.primitive import alternation, concatenation, exclusion, one_or_more
    from pyebnf.primitive import option, repeated, repetition, terminal, zero_or_more
    {imports}

    {token_type_enum}

    {class_definition}
    """

    fmt = self._clean_fmt(fmt)

    return fmt.format(date=datetime.utcnow().isoformat(),
                      imports=self._get_imports(),
                      token_type_enum=self._get_token_type_enum(),
                      class_definition=self._get_class_definition())

Returns the python source code for the generated parser .


def _get_imports(self):
    """Reads the directives and generates source code for custom imports."""
    import_directives = [d for d in self.directives if d.name == "import"]
    if import_directives:
      return "\n" + "\n".join(d.args["value"] for d in import_directives)
    else:
      return ""

def _get_imports ( self ) : import_directives = [ d for d in self . directives if d . name == "import" ] if import_directives : return "\n" + "\n" . join ( d . args [ "value" ] for d in import_directives ) else : return ""

Reads the directives and generates source code for custom imports .


def _get_token_type_enum(self):
    """Builds the python source code for the Parser TokenType enum."""
    fmt = "class TokenType(Enum):\n" \
          "{indent}\"\"\"The token types for parse nodes generated by the Parser.\"\"\"\n" \
          "{indent}" + \
          "\n{indent}".join("{1} = {0}".format(num + 1, r.name) for num, r in enumerate(self.rules))
    return fmt.format(indent=self.indent)

Builds the python source code for the Parser TokenType enum .


def _get_class_definition(self):
    """Builds the class definition of the parser."""
    fmt = """class Parser({parser_base}):
             {indent}\"\"\"This class contains methods for reading source code and generating a parse tree.\"\"\"
             {indent}entry_point = "{entry_point}"

             {rule_definitions}
             """
    fmt = self._clean_fmt(fmt)
    return fmt.format(parser_base=self._get_parser_base(),
                      indent=self.indent,
                      entry_point=self._get_entry_point(),
                      rule_definitions="\n".join(self._get_rule_definitions()))

Builds the class definition of the parser .


def _get_entry_point(self):
    """Gets the entry_point value for the parser."""
    ep = self._find_directive("entry_point")
    if ep:
      return ep.args["value"]
    else:
      return self.rules[0].name

def _get_entry_point ( self ) : ep = self . _find_directive ( "entry_point" ) if ep : return ep . args [ "value" ] else : return self . rules [ 0 ] . name

Gets the entry_point value for the parser .


def _get_rule_definition(self, rule):
    """Generates the source code for a rule."""
    fmt = """def {rule_fxn_name}(self, text):
             {indent}\"\"\"{rule_source}\"\"\"
             {indent}self._attempting(text)
             {indent}return {rule_definition}(text){transform}
          """
    fmt = self._clean_fmt(fmt)
    source = self._indent(self._ast_to_code(rule.expression), skip_first_line=True)

    # All the primitives will accept a string x in place of terminal(x). This is terminal shorthand.
    # However, if a rule is only a wrapper around a single terminal, we have to actually make a
    # terminal call. This handles that situation.
    if self.use_terminal_shorthand and len(source) == 1 and source[0].startswith(("'", '"')):
      source = ["terminal({})".format(source[0])]

    rule_source = fmt.format(rule_fxn_name=self._get_rule_fxn_name(rule.name),
                             indent=self.indent,
                             rule_source=self._get_rule_source(rule),
                             rule_definition="\n".join(source),
                             transform=self._get_rule_transform(rule))
    return self._indent(rule_source, 1)


def _get_rule_source(self, rule):
    """Gets the variable part of the source code for a rule."""
    p = len(self.input_source) + rule.position
    source = self.input_source[p:p + rule.consumed].rstrip()
    return self._indent(source, depth=self.indent + "   ", skip_first_line=True)

def
_get_rule_source
(
self
,
rule
)
:
p
=
len
(
self
.
input_source
)
+
rule
.
position
source
=
self
.
input_source
[
p
:
p
+
rule
.
consumed
]
.
rstrip
(
)
return
self
.
_indent
(
source
,
depth
=
self
.
indent
+
"   "
,
skip_first_line
=
True
)

Gets the variable part of the source code for a rule .


def _get_rule_transform(self, rule):
    """The return value for each rule can be either retyped, compressed or left alone. This method
    determines that and returns the source code text for accomplishing it.
    """
    rd = self._find_directive(lambda d: d.name == "rule" and d.args.get("name") == rule.name)

    if rd:
      args = rd.args
    else:
      args = {}

    transform = args.get("transform", "retype")

    if transform == "retype":
      new_name = args.get("to_type", "TokenType.{0}".format(rule.name))
      return ".retyped({0})".format(new_name)
    elif transform == "compress":
      new_name = args.get("to_type", "TokenType.{0}".format(rule.name))
      if new_name == "identity":
        return ".compressed()"
      else:
        return ".compressed({0})".format(new_name)
    elif transform == "identity":
      return ""

def _get_rule_transform ( self , rule ) : rd = self . _find_directive ( lambda d : d . name == "rule" and d . args . get ( "name" ) == rule . name ) if rd : args = rd . args else : args = { } transform = args . get ( "transform" , "retype" ) if transform == "retype" : new_name = args . get ( "to_type" , "TokenType.{0}" . format ( rule . name ) ) return ".retyped({0})" . format ( new_name ) elif transform == "compress" : new_name = args . get ( "to_type" , "TokenType.{0}" . format ( rule . name ) ) if new_name == "identity" : return ".compressed()" else : return ".compressed({0})" . format ( new_name ) elif transform == "identity" : return ""

The return value for each rule can be either retyped compressed or left alone . This method determines that and returns the source code text for accomplishing it .


def _expression_to_asn(self, expression):
    """Convert an expression to an Abstract Syntax Tree Node."""
    new_children = [self._node_to_asn(c) for c in expression.children]
    return self._remove_grouping_groups(infix_to_optree(new_children))

def _expression_to_asn ( self , expression ) : new_children = [ self . _node_to_asn ( c ) for c in expression . children ] return self . _remove_grouping_groups ( infix_to_optree ( new_children ) )

Convert an expression to an Abstract Syntax Tree Node .


def _node_to_asn(self, node):
    """Convert a parse tree node into an absract syntax tree node."""
    if node.is_type(TokenType.identifier):
      return Identifier(node.svalue)

    elif node.is_type(TokenType.terminal):
      return Terminal(node.svalue)

    elif node.is_type(TokenType.option_group):
      expr = node.children[0]
      return OptionGroup(self._expression_to_asn(expr))

    elif node.is_type(TokenType.repetition_group):
      expr = node.children[0]
      return RepetitionGroup(self._expression_to_asn(expr))

    elif node.is_type(TokenType.grouping_group):
      expr = node.children[0]
      return GroupingGroup(self._expression_to_asn(expr))

    elif node.is_type(TokenType.special_handling):
      ident = node.children[0]
      return SpecialHandling(ident)

    elif node.is_type(TokenType.number):
      return Number(node.svalue)

    elif node.is_type((TokenType.operator, TokenType.op_mult, TokenType.op_add)):
      return OperatorNode(OPERATOR_INDEX[node.svalue], node.position)

    else:
      raise Exception("Unhandled parse tree node: {0}".format(node))

def
_node_to_asn
(
self
,
node
)
:
if
node
.
is_type
(
TokenType
.
identifier
)
:
return
Identifier
(
node
.
svalue
)
elif
node
.
is_type
(
TokenType
.
terminal
)
:
return
Terminal
(
node
.
svalue
)
elif
node
.
is_type
(
TokenType
.
option_group
)
:
expr
=
node
.
children
[
0
]
return
OptionGroup
(
self
.
_expression_to_asn
(
expr
)
)
elif
node
.
is_type
(
TokenType
.
repetition_group
)
:
expr
=
node
.
children
[
0
]
return
RepetitionGroup
(
self
.
_expression_to_asn
(
expr
)
)
elif
node
.
is_type
(
TokenType
.
grouping_group
)
:
expr
=
node
.
children
[
0
]
return
GroupingGroup
(
self
.
_expression_to_asn
(
expr
)
)
elif
node
.
is_type
(
TokenType
.
special_handling
)
:
ident
=
node
.
children
[
0
]
return
SpecialHandling
(
ident
)
elif
node
.
is_type
(
TokenType
.
number
)
:
return
Number
(
node
.
svalue
)
elif
node
.
is_type
(
(
TokenType
.
operator
,
TokenType
.
op_mult
,
TokenType
.
op_add
)
)
:
return
OperatorNode
(
OPERATOR_INDEX
[
node
.
svalue
]
,
node
.
position
)
else
:
raise
Exception
(
"Unhandled parse tree node: {0}"
.
format
(
node
)
)

Convert a parse tree node into an absract syntax tree node .


def _hoist_operands(self, operands, pred):
    """Flattens a list of optree operands based on a pred.

    This is used to convert concatenation([x, concatenation[y, ...]]) (or alternation) to
    concatenation([x, y, ...]).
    """
    hopper = list(operands)
    new_operands = []
    while hopper:
      target = hopper.pop(0)
      if pred(target):
        hopper = list(target.operands) + hopper
      else:
        new_operands.append(target)
    return new_operands

def _hoist_operands ( self , operands , pred ) : hopper = list ( operands ) new_operands = [ ] while hopper : target = hopper . pop ( 0 ) if pred ( target ) : hopper = list ( target . operands ) + hopper else : new_operands . append ( target ) return new_operands

Flattens a list of optree operands based on a pred .


def _remove_grouping_groups(self, optree):
    """Grouping groups are implied by optrees, this function hoists grouping group expressions up
    to their parent node.
    """
    new_operands = []
    for operand in optree.operands:
      if isinstance(operand, OptreeNode):
        new_operands.append(self._remove_grouping_groups(operand))
      elif isinstance(operand, GroupingGroup):
        new_operands.append(operand.expression)
      else:
        new_operands.append(operand)

    return OptreeNode(optree.opnode, new_operands)

def _remove_grouping_groups ( self , optree ) : new_operands = [ ] for operand in optree . operands : if isinstance ( operand , OptreeNode ) : new_operands . append ( self . _remove_grouping_groups ( operand ) ) elif isinstance ( operand , GroupingGroup ) : new_operands . append ( operand . expression ) else : new_operands . append ( operand ) return OptreeNode ( optree . opnode , new_operands )

Grouping groups are implied by optrees this function hoists grouping group expressions up to their parent node .


def _ast_to_code(self, node, **kwargs):
    """Convert an abstract syntax tree to python source code."""
    if isinstance(node, OptreeNode):
      return self._ast_optree_node_to_code(node, **kwargs)
    elif isinstance(node, Identifier):
      return self._ast_identifier_to_code(node, **kwargs)
    elif isinstance(node, Terminal):
      return self._ast_terminal_to_code(node, **kwargs)
    elif isinstance(node, OptionGroup):
      return self._ast_option_group_to_code(node, **kwargs)
    elif isinstance(node, RepetitionGroup):
      return self._ast_repetition_group_to_code(node, **kwargs)
    elif isinstance(node, SpecialHandling):
      return self._ast_special_handling_to_code(node, **kwargs)
    elif isinstance(node, Number):
      return self._ast_number_to_code(node, **kwargs)
    else:
      raise Exception("Unhandled ast node: {0}".format(node))

def
_ast_to_code
(
self
,
node
,
*
*
kwargs
)
:
if
isinstance
(
node
,
OptreeNode
)
:
return
self
.
_ast_optree_node_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
Identifier
)
:
return
self
.
_ast_identifier_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
Terminal
)
:
return
self
.
_ast_terminal_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
OptionGroup
)
:
return
self
.
_ast_option_group_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
RepetitionGroup
)
:
return
self
.
_ast_repetition_group_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
SpecialHandling
)
:
return
self
.
_ast_special_handling_to_code
(
node
,
*
*
kwargs
)
elif
isinstance
(
node
,
Number
)
:
return
self
.
_ast_number_to_code
(
node
,
*
*
kwargs
)
else
:
raise
Exception
(
"Unhandled ast node: {0}"
.
format
(
node
)
)

Convert an abstract syntax tree to python source code .


def _ast_optree_node_to_code(self, node, **kwargs):
    """Convert an abstract syntax operator tree to python source code."""
    opnode = node.opnode
    if opnode is None:
      return self._ast_to_code(node.operands[0])
    else:
      operator = opnode.operator
      if operator is OP_ALTERNATE:
        return self._ast_op_alternate_to_code(node, **kwargs)
      elif operator is OP_WS_CONCAT:
        kwargs["ignore_whitespace"] = False
        return self._ast_op_concat_to_code(node, **kwargs)
      elif operator is OP_CONCAT:
        kwargs["ignore_whitespace"] = True
        return self._ast_op_concat_to_code(node, **kwargs)
      elif operator is OP_EXCLUDE:
        return self._ast_op_exclude_to_code(node, **kwargs)
      elif operator is OP_MULTIPLY:
        return self._ast_op_multiply_to_code(node, **kwargs)
      elif operator is OP_REPEAT:
        return self._ast_op_repeat_to_code(node, **kwargs)
      else:
        raise Exception("Unhandled optree node: {0}".format(node))

def
_ast_optree_node_to_code
(
self
,
node
,
*
*
kwargs
)
:
opnode
=
node
.
opnode
if
opnode
is
:
return
self
.
_ast_to_code
(
node
.
operands
[
0
]
)
else
:
operator
=
opnode
.
operator
if
operator
is
OP_ALTERNATE
:
return
self
.
_ast_op_alternate_to_code
(
node
,
*
*
kwargs
)
elif
operator
is
OP_WS_CONCAT
:
kwargs
[
"ignore_whitespace"
]
=
False
return
self
.
_ast_op_concat_to_code
(
node
,
*
*
kwargs
)
elif
operator
is
OP_CONCAT
:
kwargs
[
"ignore_whitespace"
]
=
True
return
self
.
_ast_op_concat_to_code
(
node
,
*
*
kwargs
)
elif
operator
is
OP_EXCLUDE
:
return
self
.
_ast_op_exclude_to_code
(
node
,
*
*
kwargs
)
elif
operator
is
OP_MULTIPLY
:
return
self
.
_ast_op_multiply_to_code
(
node
,
*
*
kwargs
)
elif
operator
is
OP_REPEAT
:
return
self
.
_ast_op_repeat_to_code
(
node
,
*
*
kwargs
)
else
:
raise
Exception
(
"Unhandled optree node: {0}"
.
format
(
node
)
)

Convert an abstract syntax operator tree to python source code .


def _ast_terminal_to_code(self, terminal, **kwargs):
    """Convert an AST terminal to python source code."""
    value = _replace(terminal.value)
    if self.use_terminal_shorthand:
      return [value]
    else:
      return ["terminal({})".format(value)]

def _ast_terminal_to_code ( self , terminal , * * kwargs ) : value = _replace ( terminal . value ) if self . use_terminal_shorthand : return [ value ] else : return [ "terminal({})" . format ( value ) ]

Convert an AST terminal to python source code .


def _ast_option_group_to_code(self, option_group, **kwargs):
    """Convert an AST option group to python source code."""
    lines = ["option("]
    lines.extend(self._indent(self._ast_to_code(option_group.expression)))
    lines.append(")")
    return lines

def _ast_option_group_to_code ( self , option_group , * * kwargs ) : lines = [ "option(" ] lines . extend ( self . _indent ( self . _ast_to_code ( option_group . expression ) ) ) lines . append ( ")" ) return lines

Convert an AST option group to python source code .


def _ast_repetition_group_to_code(self, repetition_group, ignore_whitespace=False, **kwargs):
    """Convert an AST repetition group to python source code."""
    lines = ["zero_or_more("]
    lines.extend(self._indent(self._ast_to_code(repetition_group.expression)))
    lines[-1] += ","
    lines.append(self._indent("ignore_whitespace={}".format(bool(ignore_whitespace))))
    lines.append(")")
    return lines

def _ast_repetition_group_to_code ( self , repetition_group , ignore_whitespace = False , * * kwargs ) : lines = [ "zero_or_more(" ] lines . extend ( self . _indent ( self . _ast_to_code ( repetition_group . expression ) ) ) lines [ - 1 ] += "," lines . append ( self . _indent ( "ignore_whitespace={}" . format ( bool ( ignore_whitespace ) ) ) ) lines . append ( ")" ) return lines

Convert an AST repetition group to python source code .


def _ast_special_handling_to_code(self, special_handling, **kwargs):
    """Convert an AST sepcial handling to python source code."""
    ident = special_handling.value.svalue
    if ident in PB_SPECIAL_HANDLING:
      return ["PB.{0}".format(ident)]
    else:
      return ["self.{0}".format(ident)]

def _ast_special_handling_to_code ( self , special_handling , * * kwargs ) : ident = special_handling . value . svalue if ident in PB_SPECIAL_HANDLING : return [ "PB.{0}" . format ( ident ) ] else : return [ "self.{0}" . format ( ident ) ]

Convert an AST sepcial handling to python source code .


def _ast_op_alternate_to_code(self, opr, **kwargs):
    """Convert an AST alternate op to python source code."""
    hoist_target = OP_ALTERNATE
    operands = self._hoist_operands(opr.operands, lambda t: isinstance(t, OptreeNode) and t.opnode.operator is hoist_target)

    lines = ["alternation(["]
    for op in operands:
      lines.extend(self._indent(self._ast_to_code(op)))
      lines[-1] += ","
    lines.append("])")
    return lines

def _ast_op_alternate_to_code ( self , opr , * * kwargs ) : hoist_target = OP_ALTERNATE operands = self . _hoist_operands ( opr . operands , lambda t : isinstance ( t , OptreeNode ) and t . opnode . operator is hoist_target ) lines = [ "alternation([" ] for op in operands : lines . extend ( self . _indent ( self . _ast_to_code ( op ) ) ) lines [ - 1 ] += "," lines . append ( "])" ) return lines

Convert an AST alternate op to python source code .


def _ast_op_concat_to_code(self, opr, *, ignore_whitespace, **kwargs):
    """Convert an AST concatenate op to python source code."""
    hoist_target = OP_CONCAT if ignore_whitespace else OP_WS_CONCAT
    operands = self._hoist_operands(opr.operands, lambda t: isinstance(t, OptreeNode) and t.opnode.operator is hoist_target)

    lines = ["concatenation(["]
    for op in operands:
      lines.extend(self._indent(self._ast_to_code(op, ignore_whitespace=ignore_whitespace)))
      lines[-1] += ","
    lines.append("], ignore_whitespace={})".format(bool(ignore_whitespace)))
    return lines

def
_ast_op_concat_to_code
(
self
,
opr
,
*
,
ignore_whitespace
,
*
*
kwargs
)
:
hoist_target
=
OP_CONCAT
if
ignore_whitespace
else
OP_WS_CONCAT
operands
=
self
.
_hoist_operands
(
opr
.
operands
,
lambda
t
:
isinstance
(
t
,
OptreeNode
)
and
t
.
opnode
.
operator
is
hoist_target
)
lines
=
[
"concatenation(["
]
for
op
in
operands
:
lines
.
extend
(
self
.
_indent
(
self
.
_ast_to_code
(
op
,
ignore_whitespace
=
ignore_whitespace
)
)
)
lines
[
1
]
+=
","
lines
.
append
(
"], ignore_whitespace={})"
.
format
(
bool
(
ignore_whitespace
)
)
)
return
lines

Convert an AST concatenate op to python source code .


def _ast_op_exclude_to_code(self, opr, **kwargs):
    """Convert an AST exclude op to python source code."""
    opl, opr = opr.operands

    lines = ["exclusion("]
    lines.extend(self._indent(self._ast_to_code(opl)))
    lines[-1] += ","
    lines.extend(self._indent(self._ast_to_code(opr)))
    lines.append(")")
    return lines

def _ast_op_exclude_to_code ( self , opr , * * kwargs ) : opl , opr = opr . operands lines = [ "exclusion(" ] lines . extend ( self . _indent ( self . _ast_to_code ( opl ) ) ) lines [ - 1 ] += "," lines . extend ( self . _indent ( self . _ast_to_code ( opr ) ) ) lines . append ( ")" ) return lines

Convert an AST exclude op to python source code .


def _ast_op_multiply_to_code(self, opr, ignore_whitespace=False, **kwargs):
    """Convert an AST multiply op to python source code."""
    opl, opr = opr.operands

    if isinstance(opl, Number):
      times = opl.value
      subject = self._ast_to_code(opr)
    else:
      times = opr.value
      subject = self._ast_to_code(opl)

    lines = ["repeated("]
    lines.extend(self._indent(subject))
    lines[-1] += ","
    lines.append("{0}times={1},".format(self.indent, times))
    lines.append("{0}ignore_whitespace={1}".format(self.indent, bool(ignore_whitespace)))
    lines.append(")")
    return lines

def _ast_op_multiply_to_code ( self , opr , ignore_whitespace = False , * * kwargs ) : opl , opr = opr . operands if isinstance ( opl , Number ) : times = opl . value subject = self . _ast_to_code ( opr ) else : times = opr . value subject = self . _ast_to_code ( opl ) lines = [ "repeated(" ] lines . extend ( self . _indent ( subject ) ) lines [ - 1 ] += "," lines . append ( "{0}times={1}," . format ( self . indent , times ) ) lines . append ( "{0}ignore_whitespace={1}" . format ( self . indent , bool ( ignore_whitespace ) ) ) lines . append ( ")" ) return lines

Convert an AST multiply op to python source code .


def _ast_op_repeat_to_code(self, opr, ignore_whitespace=False, **kwargs):
    """Convert an AST repeat op to python source code."""
    lines = ["one_or_more("]
    lines.extend(self._indent(self._ast_to_code(opr.operands[0])))
    lines[-1] += ","
    lines.append(self._indent("ignore_whitespace={}".format(bool(ignore_whitespace))))
    lines.append(")")
    return lines

def _ast_op_repeat_to_code ( self , opr , ignore_whitespace = False , * * kwargs ) : lines = [ "one_or_more(" ] lines . extend ( self . _indent ( self . _ast_to_code ( opr . operands [ 0 ] ) ) ) lines [ - 1 ] += "," lines . append ( self . _indent ( "ignore_whitespace={}" . format ( bool ( ignore_whitespace ) ) ) ) lines . append ( ")" ) return lines

Convert an AST repeat op to python source code .


def _find_directives(self, pred):
    """Finds all directives with a certain name, or that passes a predicate."""
    if isinstance(pred, str):
      return [d for d in self.directives if d.name == pred]
    else:
      return [d for d in self.directives if pred(d)]

def _find_directives ( self , pred ) : if isinstance ( pred , str ) : return [ d for d in self . directives if d . name == pred ] else : return [ d for d in self . directives if pred ( d ) ]

Finds all directives with a certain name or that passes a predicate .


def _flatten(child, parent):
    """Custom flattening method for the parse tree."""
    return parent.is_type(TokenType.expression) and child.node_type == parent.node_type

def _flatten ( child , parent ) : return parent . is_type ( TokenType . expression ) and child . node_type == parent . node_type

Custom flattening method for the parse tree .


def directives_from_comment(cls, comment):
    """A directive is a line in a comment that begins with '!'."""
    comment_contents = comment.value[2:-2].strip()
    comment_lines = (l.strip() for l in comment_contents.split("\n"))
    directives = (l[1:].strip() for l in comment_lines if l.startswith("!"))

    for directive_def in directives:
      yield cls.parse_directive_def(directive_def)

def directives_from_comment ( cls , comment ) : comment_contents = comment . value [ 2 : - 2 ] . strip ( ) comment_lines = ( l . strip ( ) for l in comment_contents . split ( "\n" ) ) directives = ( l [ 1 : ] . strip ( ) for l in comment_lines if l . startswith ( "!" ) ) for directive_def in directives : yield cls . parse_directive_def ( directive_def )

A directive is a line in a comment that begins with ! .


def _handle_results(self):
        """
        Handle the results of the API call
        """

        # Only process if we get HTTP return code other 200.
        if self._api_result.status_code != requests.codes.ok:
            print(self.colorize_json(self._api_result.text))


def get_id(id):
 """Get a new id if the provided one is None."""
 if id == None:
  id = wx.NewId()
  logger.debug('Generated new ID %s.', id)
 else:
  logger.debug('Using provided id %s.', id)
 return id

def
get_id
(
id
)
:
if
id
==
:
id
=
wx
.
NewId
(
)
logger
.
debug
(
'Generated new ID %s.'
,
id
)
else
:
logger
.
debug
(
'Using provided id %s.'
,
id
)
return
id

Get a new id if the provided one is None .


def remove_hotkey(control, key):
 """
 Remove a global hotkey.

 control - The control to affect
 key - The key to remove.
 """
 l = _hotkeys.get(control, [])
 for a in l:
  key_str, id = a
  if key_str == key:
   control.Unbind(wx.EVT_HOTKEY, id = id)
   control.UnregisterHotKey(id)
   l.remove(a)
   if l:
    _hotkeys[control] = l
   else:
    del _hotkeys[control]

def remove_hotkey ( control , key ) : l = _hotkeys . get ( control , [ ] ) for a in l : key_str , id = a if key_str == key : control . Unbind ( wx . EVT_HOTKEY , id = id ) control . UnregisterHotKey ( id ) l . remove ( a ) if l : _hotkeys [ control ] = l else : del _hotkeys [ control ]

Remove a global hotkey . control - The control to affect key - The key to remove .


def add_arguments(self):
        """
        Configure handling of command line arguments.
        """
        self.add_logging_argument()
        self.parser.add_argument('-a', '--api-host', dest='api_host', action='store', metavar="api_host",
                                 help='{0} API host endpoint'.format(self.product_name))
        self.parser.add_argument('-e', '--email', dest='email', action='store', metavar="e_mail",
                                 help='e-mail that has access to the {0} account'.format(self.product_name))
        self.parser.add_argument('-t', '--api-token', dest='api_token', required=False, action='store',
                                 metavar="api_token",
                                 help='API token for given e-mail that has access to the {0} account'.format(
                                     self.product_name))
        self.parser.add_argument('-z', '--curl', dest='curl', required=False, action='store_true', default=False,
                                 help='Output the corresponding curl command line and exit')

Configure handling of command line arguments .


def _configure_logging(self):
        """
        Configure logging based on command line options
        """
        if self.args.logLevel is not None:
            logging.basicConfig(level=self.levels[self.args.logLevel])
        logging.info("Set logging level to {0}".format(self.args.logLevel))

def
_configure_logging
(
self
)
:
if
self
.
args
.
logLevel
is
not
:
logging
.
basicConfig
(
level
=
self
.
levels
[
self
.
args
.
logLevel
]
)
logging
.
info
(
"Set logging level to {0}"
.
format
(
self
.
args
.
logLevel
)
)


def _validate_arguments(self):
        """
        Validates the command line arguments passed to the CLI
        Derived classes that override need to call this method before
        validating their arguments
        """
        if self._email is None:
            self.set_error_message("E-mail for the account not provided")
            return False
        if self._api_token is None:
            self.set_error_message("API Token for the account not provided")
            return False
        return True


def infix_to_postfix(nodes, *, recurse_types=None):
  """Convert a list of nodes in infix order to a list of nodes in postfix order.

  E.G. with normal algebraic precedence, 3 + 4 * 5 -> 3 4 5 * +
  """
  output = []
  operators = []

  for node in nodes:
    if isinstance(node, OperatorNode):
      # Drain out all operators whose precedence is gte the node's...
      cmp_operator = node.operator
      while operators:
        current_operator = operators[-1].operator
        if current_operator.precedence > cmp_operator.precedence or \
           current_operator.precedence == cmp_operator.precedence and current_operator.association == Association.left:
          output.append(operators.pop())
        else:
          break
      operators.append(node)
    else:
      if recurse_types is not None and node.node_type in recurse_types:
        output.extend(infix_to_postfix(node.children, recurse_types=recurse_types))
      else:
        output.append(node)

  return output + list(reversed(operators))

Convert a list of nodes in infix order to a list of nodes in postfix order .


def postfix_to_optree(nodes):
  """Convert a list of nodes in postfix order to an Optree."""
  while len(nodes) > 1:
    nodes = _reduce(nodes)

  if len(nodes) == 0:
    raise OperatorError("Empty node list")

  node = nodes[0]

  if isinstance(node, OperatorNode):
    raise OperatorError("Operator without operands")

  if isinstance(node, OptreeNode):
    return node

  return OptreeNode(None, (node, ))

def
postfix_to_optree
(
nodes
)
:
while
len
(
nodes
)
>
1
:
nodes
=
_reduce
(
nodes
)
if
len
(
nodes
)
==
0
:
raise
OperatorError
(
"Empty node list"
)
node
=
nodes
[
0
]
if
isinstance
(
node
,
OperatorNode
)
:
raise
OperatorError
(
"Operator without operands"
)
if
isinstance
(
node
,
OptreeNode
)
:
return
node
return
OptreeNode
(
,
(
node
,
)
)

Convert a list of nodes in postfix order to an Optree .


def _reduce(nodes):
  """Finds the first operator in the list, converts it and its operands to a OptreeNode, then
  returns a new list with the operator and operands replaced by the new OptreeNode.
  """
  i = 0
  while i < len(nodes):
    if isinstance(nodes[i], OperatorNode):
      break
    else:
      i += 1

  if i == len(nodes):
    raise OperatorError("No operator found")

  operator_node = nodes[i]
  operator = operator_node.operator
  operands_lbound = i - operator.cardinality

  if operands_lbound < 0:
    raise OperatorError("Insufficient operands for operator {0}".format(operator.symbol))

  return nodes[:operands_lbound] + \
         [OptreeNode(operator_node, tuple(nodes[operands_lbound:i]))] + \
         nodes[i+1:]

Finds the first operator in the list converts it and its operands to a OptreeNode then returns a new list with the operator and operands replaced by the new OptreeNode .


def add_arguments(self):
        """
        Add the specific arguments of this CLI
        """
        MetricCommon.add_arguments(self)
        self.parser.add_argument('-n', '--metric-name', dest='metricName', action='store',
                                 required=True, metavar='metric_name', help='Metric identifier')
        self.parser.add_argument('-d', '--display-name', dest='displayName', action='store',
                                 required=True, metavar='display_name', help='Metric display name')
        self.parser.add_argument('-s', '--display-name-short', dest='displayNameShort', action='store',
                                 required=True, metavar='display_short_name', help='Metric short display name')
        self.parser.add_argument('-i', '--description', dest='description', action='store',
                                 required=not self.update, metavar='description', help='Metric description')
        self.parser.add_argument('-g', '--aggregate', dest='aggregate', action='store',
                                 required=True, choices=['avg', 'max', 'min', 'sum'],
                                 help='Metric default aggregate')
        self.parser.add_argument('-u', '--unit', dest='unit', action='store',
                                 required=False, choices=['percent', 'number', 'bytecount', 'duration'],
                                 help='Metric unit')
        self.parser.add_argument('-r', '--resolution', dest='resolution', action='store', metavar='resolution',
                                 required=False, help='Metric default resolution')
        self.parser.add_argument('-y', '--type', dest='type', action='store', default=None,
                                 required=False, metavar='type', help='Sets the type metadata field')
        self.parser.add_argument('-x', '--is-disabled', dest='isDisabled', action='store', default=None,
                                 required=False,
                                 choices=['true', 'false'], help='Enable or disable the metric definition')


def load(self):
        """
        Read the file and parse JSON into dictionary
        """
        manifest = PluginManifest(self.file_path)
        manifest.get()
        self.manifest = manifest.get_manifest()

def load ( self ) : manifest = PluginManifest ( self . file_path ) manifest . get ( ) self . manifest = manifest . get_manifest ( )


def getMetricDefinition(self, name):
        """
        Looks up the metric definition from the definitions from the API call
        """
        metric = None
        for m in self.metric_definitions:
            if m['name'] == name:
                metric = m
                break
        return metric

def getMetricDefinition ( self , name ) : metric = None for m in self . metric_definitions : if m [ 'name' ] == name : metric = m break return metric


def getFieldsColumnLengths(self):
        """
        Gets the maximum length of each column in the field table
        """
        nameLen = 0
        descLen = 0
        for f in self.fields:
            nameLen = max(nameLen, len(f['title']))
            descLen = max(descLen, len(f['description']))
        return (nameLen, descLen)

def getFieldsColumnLengths ( self ) : nameLen = 0 descLen = 0 for f in self . fields : nameLen = max ( nameLen , len ( f [ 'title' ] ) ) descLen = max ( descLen , len ( f [ 'description' ] ) ) return ( nameLen , descLen )


def getMetricsColumnLengths(self):
        """
        Gets the maximum length of each column
        """
        displayLen = 0
        descLen = 0
        for m in self.metrics:
            displayLen = max(displayLen, len(m['displayName']))
            descLen = max(descLen, len(m['description']))
        return (displayLen, descLen)

def getMetricsColumnLengths ( self ) : displayLen = 0 descLen = 0 for m in self . metrics : displayLen = max ( displayLen , len ( m [ 'displayName' ] ) ) descLen = max ( descLen , len ( m [ 'description' ] ) ) return ( displayLen , descLen )


def escapeUnderscores(self):
        """
        Escape underscores so that the markdown is correct
        """
        new_metrics = []
        for m in self.metrics:
            m['name'] = m['name'].replace("_", "\_")
            new_metrics.append(m)
        self.metrics = new_metrics

def escapeUnderscores ( self ) : new_metrics = [ ] for m in self . metrics : m [ 'name' ] = m [ 'name' ] . replace ( "_" , "\_" ) new_metrics . append ( m ) self . metrics = new_metrics


def outputFieldMarkdown(self):
        """
        Sends the field definitions ot standard out
        """
        f, d = self.getFieldsColumnLengths()

        fc, dc = self.printFieldsHeader(f, d)
        f = max(fc, f)
        d = max(dc, d)
        self.printFields(f, d)

def outputFieldMarkdown ( self ) : f , d = self . getFieldsColumnLengths ( ) fc , dc = self . printFieldsHeader ( f , d ) f = max ( fc , f ) d = max ( dc , d ) self . printFields ( f , d )


def outputMetricMarkdown(self):
        """
        Sends the markdown of the metric definitions to standard out
        """
        self.escapeUnderscores()
        m, d = self.getMetricsColumnLengths()

        self.printMetricsHeader(m, d)
        self.printMetrics(m, d)

def outputMetricMarkdown ( self ) : self . escapeUnderscores ( ) m , d = self . getMetricsColumnLengths ( ) self . printMetricsHeader ( m , d ) self . printMetrics ( m , d )


def generateMarkdown(self):
        """
        Look up each of the metrics and then output in Markdown
        """
        self.generateMetricDefinitions()
        self.generateFieldDefinitions()
        self.generateDashboardDefinitions()
        self.outputMarkdown()

def generateMarkdown ( self ) : self . generateMetricDefinitions ( ) self . generateFieldDefinitions ( ) self . generateDashboardDefinitions ( ) self . outputMarkdown ( )


def parse(self, text):
    """Attempt to parse source code."""
    self.original_text = text

    try:
      return getattr(self, self.entry_point)(text)
    except (DeadEnd) as exc:
      raise ParserError(self.most_consumed, "Failed to parse input") from exc
    return tree

def
parse
(
self
,
text
)
:
self
.
original_text
=
text
try
:
return
getattr
(
self
,
self
.
entry_point
)
(
text
)
except
(
DeadEnd
)
as
exc
:
raise
ParserError
(
self
.
most_consumed
,
"Failed to parse input"
)
from
exc
return
tree


def _attempting(self, text):
    """Keeps track of the furthest point in the source code the parser has reached to this point."""
    consumed = len(self.original_text) - len(text)
    self.most_consumed = max(consumed, self.most_consumed)

def _attempting ( self , text ) : consumed = len ( self . original_text ) - len ( text ) self . most_consumed = max ( consumed , self . most_consumed )

Keeps track of the furthest point in the source code the parser has reached to this point .


def add_arguments(self):
        """
        Add specific command line arguments for this command
        """

        # Call our parent to add the default arguments
        ApiCli.add_arguments(self)

        # Command specific arguments
        self.parser.add_argument('-f', '--format', dest='format', action='store', required=False,
                                 choices=['csv', 'json', 'raw', 'xml'], help='Output format. Default is raw')
        self.parser.add_argument('-n', '--name', dest='metric_name', action='store', required=True,
                                 metavar="metric_name", help='Metric identifier')
        self.parser.add_argument('-g', '--aggregate', dest='aggregate', action='store', required=False,
                                 choices=['sum', 'avg', 'max', 'min'], help='Metric default aggregate')
        self.parser.add_argument('-r', '--sample', dest='sample', action='store', type=int, metavar="sample",
                                 help='Down sample rate sample in seconds')
        self.parser.add_argument('-s', '--source', dest='source', action='store', metavar="source", required=True,
                                 help='Source of measurement')
        self.parser.add_argument('-b', '--start', dest='start', action='store', required=True, metavar="start",
                                 help='Start of time range as ISO 8601 string or epoch seconds')
        self.parser.add_argument('-d', '--end', dest='end', action='store', metavar="end", required=False,
                                 help='End of time range as ISO 8601 string or epoch seconds')

        self.parser.add_argument('-o', '--date-format', dest='date_format', action='store', metavar="format",
                                 required=False,
                                 help='For CSV, JSON, and XML output formats dates (see Python date.strftime). ' +
                                      'Default format is %%s')


def parse_time_date(self, s):
        """
        Attempt to parse the passed in string into a valid datetime.
        If we get a parse error then assume the string is an epoch time
        and convert to a datetime.
        """
        try:
            ret = parser.parse(str(s))
        except ValueError:
            try:
                ret = datetime.fromtimestamp(int(s))
            except TypeError:
                ret = None
        return ret

def parse_time_date ( self , s ) : try : ret = parser . parse ( str ( s ) ) except ValueError : try : ret = datetime . fromtimestamp ( int ( s ) ) except TypeError : ret = None return ret

Attempt to parse the passed in string into a valid datetime . If we get a parse error then assume the string is an epoch time and convert to a datetime .


def output_csv(self, text):
        """
        Output results in CSV format
        """
        payload = json.loads(text)
        # Print CSV header
        print("{0},{1},{2},{3},{4}".format('timestamp', 'metric', 'aggregate', 'source', 'value'))
        metric_name = self._metric_name
        # Loop through the aggregates one row per timestamp, and 1 or more source/value pairs
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            # timestamp = string.strip(timestamp, ' ')
            # timestamp = string.strip(timestamp, "'")
            for s in r[1]:
                print('{0},"{1}","{2}","{3}",{4}'.format(timestamp, metric_name, self.aggregate, s[0], s[1]))


def output_json(self, text):
        """
        Output results in structured JSON format
        """
        payload = json.loads(text)
        data = []
        metric_name = self._metric_name
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            for s in r[1]:
                data.append({
                    "timestamp": timestamp,
                    "metric": metric_name,
                    "aggregate": self.aggregate,
                    "source": s[0],
                    "value": s[1],
                })
        payload = {"data": data}
        out = json.dumps(payload, indent=self._indent, separators=(',', ': '))
        print(self.colorize_json(out))

def
output_json
(
self
,
text
)
:
payload
=
json
.
loads
(
text
)
data
=
[
]
metric_name
=
self
.
_metric_name
for
r
in
payload
[
'result'
]
[
'aggregates'
]
[
'key'
]
:
timestamp
=
self
.
_format_timestamp
(
r
[
0
]
[
0
]
)
for
s
in
r
[
1
]
:
data
.
append
(
{
"timestamp"
:
timestamp
,
"metric"
:
metric_name
,
"aggregate"
:
self
.
aggregate
,
"source"
:
s
[
0
]
,
"value"
:
s
[
1
]
,
}
)
payload
=
{
"data"
:
data
}
out
=
json
.
dumps
(
payload
,
indent
=
self
.
_indent
,
separators
=
(
','
,
': '
)
)
print
(
self
.
colorize_json
(
out
)
)


def output_raw(self, text):
        """
        Output results in raw JSON format
        """
        payload = json.loads(text)
        out = json.dumps(payload, sort_keys=True, indent=self._indent, separators=(',', ': '))
        print(self.colorize_json(out))

def
output_raw
(
self
,
text
)
:
payload
=
json
.
loads
(
text
)
out
=
json
.
dumps
(
payload
,
sort_keys
=
True
,
indent
=
self
.
_indent
,
separators
=
(
','
,
': '
)
)
print
(
self
.
colorize_json
(
out
)
)


def output_xml(self, text):
        """
        Output results in JSON format
        """

        # Create the main document nodes
        document = Element('results')
        comment = Comment('Generated by TrueSight Pulse measurement-get CLI')
        document.append(comment)
        aggregates = SubElement(document, 'aggregates')
        aggregate = SubElement(aggregates, 'aggregate')
        measurements = SubElement(aggregate, 'measurements')

        # Parse the JSON result so we can translate to XML
        payload = json.loads(text)

        # Current only support a single metric, if we move to the batch API then
        # we can handle multiple
        metric_name = self._metric_name

        # Loop through the aggregates one row per timestamp, and 1 or more source/value pairs
        for r in payload['result']['aggregates']['key']:
            timestamp = self._format_timestamp(r[0][0])
            for s in r[1]:
                # Each timestamp, metric, source, values is placed in a measure tag
                measure_node = SubElement(measurements, 'measure')
                source = s[0]
                value = str(s[1])
                ts_node = SubElement(measure_node, 'timestamp')
                ts_node.text = str(timestamp)
                metric_node = SubElement(measure_node, 'metric')
                metric_node.text = metric_name
                metric_node = SubElement(measure_node, 'aggregate')
                metric_node.text = self.aggregate
                source_node = SubElement(measure_node, 'source')
                source_node.text = source
                value_node = SubElement(measure_node, 'value')
                value_node.text = value

        rough_string = ElementTree.tostring(document, 'utf-8')
        reparse = minidom.parseString(rough_string)
        output = reparse.toprettyxml(indent=" ")
        print(self.colorize_xml(output))


def trimmed_pred_default(node, parent):
  """The default predicate used in Node.trimmed."""
  return isinstance(node, ParseNode) and (node.is_empty or node.is_type(ParseNodeType.terminal))

def trimmed_pred_default ( node , parent ) : return isinstance ( node , ParseNode ) and ( node . is_empty or node . is_type ( ParseNodeType . terminal ) )

The default predicate used in Node . trimmed .


def pprint(root, depth=0, space_unit="    ", *, source_len=0, file=None):
  """Pretting print a parse tree."""
  spacing = space_unit * depth

  if isinstance(root, str):
    print("{0}terminal@(?): {1}".format(spacing, root), file=file)
  else:
    if root.position is None:
      position = -1
    elif root.position < 0:
      position = source_len + root.position
    else:
      position = root.position

    if root.is_value:
      print("{0}{1}@({2}:{3}):\t{4}".format(spacing, root.node_type, position, root.consumed, root.svalue), file=file)
    else:
      print("{0}{1}@({2}:{3}):".format(spacing, root.node_type, position, root.consumed), file=file)
      for child in root.children:
        pprint(child, depth + 1, source_len=source_len, file=file)

def
pprint
(
root
,
depth
=
0
,
space_unit
=
"    "
,
*
,
source_len
=
0
,
file
=
)
:
spacing
=
space_unit
*
depth
if
isinstance
(
root
,
str
)
:
print
(
"{0}terminal@(?): {1}"
.
format
(
spacing
,
root
)
,
file
=
file
)
else
:
if
root
.
position
is
:
position
=
1
elif
root
.
position
<
0
:
position
=
source_len
+
root
.
position
else
:
position
=
root
.
position
if
root
.
is_value
:
print
(
"{0}{1}@({2}:{3}):\t{4}"
.
format
(
spacing
,
root
.
node_type
,
position
,
root
.
consumed
,
root
.
svalue
)
,
file
=
file
)
else
:
print
(
"{0}{1}@({2}:{3}):"
.
format
(
spacing
,
root
.
node_type
,
position
,
root
.
consumed
)
,
file
=
file
)
for
child
in
root
.
children
:
pprint
(
child
,
depth
+
1
,
source_len
=
source_len
,
file
=
file
)


def repetition(extractor, bounds, *, ignore_whitespace=False):
  """Returns a partial of _get_repetition that accepts only a text argument."""
  return partial(_get_repetition, extractor, bounds=bounds, ignore_whitespace=ignore_whitespace)

def repetition ( extractor , bounds , * , ignore_whitespace = False ) : return partial ( _get_repetition , extractor , bounds = bounds , ignore_whitespace = ignore_whitespace )

Returns a partial of _get_repetition that accepts only a text argument .


def _get_terminal(value, text):
  """Checks the beginning of text for a value. If it is found, a terminal ParseNode is returned
  filled out appropriately for the value it found. DeadEnd is raised if the value does not match.
  """
  if text and text.startswith(value):
    return ParseNode(ParseNodeType.terminal,
                      children=[value],
                      consumed=len(value),
                      position=-len(text))
  else:
    raise DeadEnd()

def _get_terminal ( value , text ) : if text and text . startswith ( value ) : return ParseNode ( ParseNodeType . terminal , children = [ value ] , consumed = len ( value ) , position = - len ( text ) ) else : raise DeadEnd ( )

Checks the beginning of text for a value . If it is found a terminal ParseNode is returned filled out appropriately for the value it found . DeadEnd is raised if the value does not match .


def _get_repetition(extractor, text, *, bounds=(0, None), ignore_whitespace=False):
  """Tries to pull text with extractor repeatedly.

  Bounds is a 2-tuple of (lbound, ubound) where lbound is a number and ubound is a number or None.
  If the ubound is None, this method will execute extractor on text until extrator raises DeadEnd.
  Otherwise, extractor will be called until it raises DeadEnd, or it has extracted ubound times.

  If the number of children extracted is >= lbound, then a ParseNode with type repetition is
  returned. Otherwise, DeadEnd is raised.

  Bounds are interpreted as (lbound, ubound]

  This method is used to implement:
  - option (0, 1)
  - zero_or_more (0, None)
  - one_or_more (1, None)
  - exact_repeat (n, n)
  """
  minr, maxr = bounds
  children = []

  while maxr is None or len(children) <= maxr:
    ignored_ws, use_text = _split_ignored(text, ignore_whitespace)
    try:
      child = _call_extractor(extractor, use_text)
      child.add_ignored(ignored_ws)
    except DeadEnd:
      break

    if child.is_empty:
      break

    children.append(child)
    text = text[child.consumed:]

  if len(children) >= minr:
    return ParseNode(ParseNodeType.repetition,
                      children=children)
  else:
    raise DeadEnd()

def _get_repetition ( extractor , text , * , bounds = ( 0 , None ) , ignore_whitespace = False ) : minr , maxr = bounds children = [ ] while maxr is None or len ( children ) <= maxr : ignored_ws , use_text = _split_ignored ( text , ignore_whitespace ) try : child = _call_extractor ( extractor , use_text ) child . add_ignored ( ignored_ws ) except DeadEnd : break if child . is_empty : break children . append ( child ) text = text [ child . consumed : ] if len ( children ) >= minr : return ParseNode ( ParseNodeType . repetition , children = children ) else : raise DeadEnd ( )

Tries to pull text with extractor repeatedly .


def _get_exclusion(extractor, exclusion, text):
  """Returns extractor's result if exclusion does not match.

  If exclusion raises DeadEnd (meaning it did not match) then the result of extractor(text) is
  returned. Otherwise, if exclusion does not raise DeadEnd it means it did match, and we then
  raise DeadEnd.
  """
  try:
    _call_extractor(exclusion, text)
    exclusion_matches = True
  except DeadEnd:
    exclusion_matches = False

  if exclusion_matches:
    raise DeadEnd()
  else:
    return _call_extractor(extractor, text)

def _get_exclusion ( extractor , exclusion , text ) : try : _call_extractor ( exclusion , text ) exclusion_matches = True except DeadEnd : exclusion_matches = False if exclusion_matches : raise DeadEnd ( ) else : return _call_extractor ( extractor , text )

Returns extractor s result if exclusion does not match .


def _count_leading_whitespace(text):
  """Returns the number of characters at the beginning of text that are whitespace."""
  idx = 0
  for idx, char in enumerate(text):
    if not char.isspace():
      return idx
  return idx + 1

def _count_leading_whitespace ( text ) : idx = 0 for idx , char in enumerate ( text ) : if not char . isspace ( ) : return idx return idx + 1

Returns the number of characters at the beginning of text that are whitespace .


def _call_extractor(extractor, text):
  """This method calls an extractor on some text.

  If extractor is just a string, it is passed as the first value to _get_terminal. Otherwise it is
  treated as a callable and text is passed directly to it.

  This makes it so you can have a shorthand of terminal(val) <-> val.
  """
  if isinstance(extractor, str):
    return _get_terminal(extractor, text)
  else:
    return extractor(text)

def _call_extractor ( extractor , text ) : if isinstance ( extractor , str ) : return _get_terminal ( extractor , text ) else : return extractor ( text )

This method calls an extractor on some text .


def position(self):
    """Gets the position of the text the ParseNode processed. If the ParseNode does not have its
    own position, it looks to its first child for its position.

    'Value Nodes' (terminals) must have their own position, otherwise this method will throw an
    exception when it tries to get the position property of the string child.
    """
    pos = self._position
    if pos is None and self.children:
      ch1 = self.children[0]
      if isinstance(ch1, ParseNode):
        pos = ch1.position
    return pos

def position ( self ) : pos = self . _position if pos is None and self . children : ch1 = self . children [ 0 ] if isinstance ( ch1 , ParseNode ) : pos = ch1 . position return pos

Gets the position of the text the ParseNode processed . If the ParseNode does not have its own position it looks to its first child for its position .


def is_empty(self):
    """Returns True if this node has no children, or if all of its children are ParseNode instances
    and are empty.
    """
    return all(isinstance(c, ParseNode) and c.is_empty for c in self.children)

def is_empty ( self ) : return all ( isinstance ( c , ParseNode ) and c . is_empty for c in self . children )

Returns True if this node has no children or if all of its children are ParseNode instances and are empty .


def add_ignored(self, ignored):
    """Add ignored text to the node. This will add the length of the ignored text to the node's
    consumed property.
    """
    if ignored:
      if self.ignored:
        self.ignored = ignored + self.ignored
      else:
        self.ignored = ignored

    self.consumed += len(ignored)

def add_ignored ( self , ignored ) : if ignored : if self . ignored : self . ignored = ignored + self . ignored else : self . ignored = ignored self . consumed += len ( ignored )

Add ignored text to the node . This will add the length of the ignored text to the node s consumed property .


def is_type(self, value):
    """Returns True if node_type == value.

    If value is a tuple, node_type is checked against each member and True is returned if any of
    them match.
    """
    if isinstance(value, tuple):
      for opt in value:
        if self.node_type == opt:
          return True
      return False
    else:
      return self.node_type == value

def is_type ( self , value ) : if isinstance ( value , tuple ) : for opt in value : if self . node_type == opt : return True return False else : return self . node_type == value


def flattened(self, pred=flattened_pred_default):
    """Flattens nodes by hoisting children up to ancestor nodes.

    A node is hoisted if pred(node) returns True.
    """
    if self.is_value:
      return self

    new_children = []

    for child in self.children:
      if child.is_empty:
        continue

      new_child = child.flattened(pred)

      if pred(new_child, self):
        new_children.extend(new_child.children)
      else:
        new_children.append(new_child)

    return ParseNode(self.node_type,
                     children=new_children,
                     consumed=self.consumed,
                     position=self.position,
                     ignored=self.ignored)

def flattened ( self , pred = flattened_pred_default ) : if self . is_value : return self new_children = [ ] for child in self . children : if child . is_empty : continue new_child = child . flattened ( pred ) if pred ( new_child , self ) : new_children . extend ( new_child . children ) else : new_children . append ( new_child ) return ParseNode ( self . node_type , children = new_children , consumed = self . consumed , position = self . position , ignored = self . ignored )

Flattens nodes by hoisting children up to ancestor nodes .


def trimmed(self, pred=trimmed_pred_default):
    """Trim a ParseTree.

    A node is trimmed if pred(node) returns True.
    """
    new_children = []

    for child in self.children:
      if isinstance(child, ParseNode):
        new_child = child.trimmed(pred)
      else:
        new_child = child

      if not pred(new_child, self):
        new_children.append(new_child)

    return ParseNode(self.node_type,
                     children=new_children,
                     consumed=self.consumed,
                     position=self.position,
                     ignored=self.ignored)

def trimmed ( self , pred = trimmed_pred_default ) : new_children = [ ] for child in self . children : if isinstance ( child , ParseNode ) : new_child = child . trimmed ( pred ) else : new_child = child if not pred ( new_child , self ) : new_children . append ( new_child ) return ParseNode ( self . node_type , children = new_children , consumed = self . consumed , position = self . position , ignored = self . ignored )


def merged(self, other):
    """Returns a new ParseNode whose type is this node's type, and whose children are all the
    children from this node and the other whose length is not 0.
    """
    children = [c for c in itertools.chain(self.children, other.children) if len(c) > 0]
    # NOTE: Only terminals should have ignored text attached to them, and terminals shouldn't be
    #       merged (probably) so it shouldn't be necessary to copy of ignored -- it should always
    #       be None. But, we'll go ahead and copy it over anyway, recognizing that other's
    #       ignored text will be lost.
    return ParseNode(self.node_type,
                      children=children,
                      consumed=self.consumed + other.consumed,
                      ignored=self.ignored)

Returns a new ParseNode whose type is this node s type and whose children are all the children from this node and the other whose length is not 0 .


def retyped(self, new_type):
    """Returns a new node with the same contents as self, but with a new node_type."""
    return ParseNode(new_type,
                      children=list(self.children),
                      consumed=self.consumed,
                      position=self.position,
                      ignored=self.ignored)

def retyped ( self , new_type ) : return ParseNode ( new_type , children = list ( self . children ) , consumed = self . consumed , position = self . position , ignored = self . ignored )

Returns a new node with the same contents as self but with a new node_type .


def compressed(self, new_type=None, *, include_ignored=False):
    """Turns the node into a value node, whose single string child is the concatenation of all its
    children.
    """
    values = []
    consumed = 0
    ignored = None

    for i, child in enumerate(self.children):
      consumed += child.consumed
      if i == 0 and not include_ignored:
        ignored = child.ignored
      if child.is_value:
        if include_ignored:
          values.append("{0}{1}".format(child.ignored or "", child.value))
        else:
          values.append(child.value)
      else:
        values.append(child.compressed(include_ignored=include_ignored).value)

    return ParseNode(new_type or self.node_type,
                      children=["".join(values)],
                      consumed=consumed,
                      ignored=ignored,
                      position=self.position)

def compressed ( self , new_type = None , * , include_ignored = False ) : values = [ ] consumed = 0 ignored = None for i , child in enumerate ( self . children ) : consumed += child . consumed if i == 0 and not include_ignored : ignored = child . ignored if child . is_value : if include_ignored : values . append ( "{0}{1}" . format ( child . ignored or "" , child . value ) ) else : values . append ( child . value ) else : values . append ( child . compressed ( include_ignored = include_ignored ) . value ) return ParseNode ( new_type or self . node_type , children = [ "" . join ( values ) ] , consumed = consumed , ignored = ignored , position = self . position )

Turns the node into a value node whose single string child is the concatenation of all its children .


def position(self) -> Position:
        """The current position of the cursor."""
        return Position(self._index, self._lineno, self._col_offset)

def position ( self ) -> Position : return Position ( self . _index , self . _lineno , self . _col_offset )


def max_readed_position(self) -> Position:
        """The index of the deepest character readed."""
        return Position(self._maxindex, self._maxline, self._maxcol)

def max_readed_position ( self ) -> Position : return Position ( self . _maxindex , self . _maxline , self . _maxcol )

The index of the deepest character readed .


def step_next_char(self):
        """Puts the cursor on the next character."""
        self._index += 1
        self._col_offset += 1
        if self._index > self._maxindex:
            self._maxindex = self._index
            self._maxcol = self._col_offset
            self._maxline = self._lineno

def step_next_char ( self ) : self . _index += 1 self . _col_offset += 1 if self . _index > self . _maxindex : self . _maxindex = self . _index self . _maxcol = self . _col_offset self . _maxline = self . _lineno


def step_next_line(self):
        """Sets cursor as beginning of next line."""
        self._eol.append(self.position)
        self._lineno += 1
        self._col_offset = 0

def step_next_line ( self ) : self . _eol . append ( self . position ) self . _lineno += 1 self . _col_offset = 0


def step_prev_line(self):
        """Sets cursor as end of previous line."""
        #TODO(bps): raise explicit error for unregistered eol
        #assert self._eol[-1].index == self._index
        if len(self._eol) > 0:
            self.position = self._eol.pop()


def last_readed_line(self) -> str:
        """Usefull string to compute error message."""
        mpos = self._cursor.max_readed_position
        mindex = mpos.index
        # search last \n
        prevline = mindex - 1 if mindex == self.eos_index else mindex
        while prevline >= 0 and self._content[prevline] != '\n':
            prevline -= 1
        # search next \n
        nextline = mindex
        while nextline < self.eos_index and self._content[nextline] != '\n':
            nextline += 1
        last_line = self._content[prevline + 1:nextline]
        return last_line

def
last_readed_line
(
self
)
->
str
:
mpos
=
self
.
_cursor
.
max_readed_position
mindex
=
mpos
.
index
# search last \n
prevline
=
mindex
1
if
mindex
==
self
.
eos_index
else
mindex
while
prevline
>=
0
and
self
.
_content
[
prevline
]
!=
'\n'
:
prevline
-=
1
# search next \n
nextline
=
mindex
while
nextline
<
self
.
eos_index
and
self
.
_content
[
nextline
]
!=
'\n'
:
nextline
+=
1
last_line
=
self
.
_content
[
prevline
+
1
:
nextline
]
return
last_line

Usefull string to compute error message .


def incpos(self, length: int=1) -> int:
        """Increment the cursor to the next character."""
        if length < 0:
            raise ValueError("length must be positive")
        i = 0
        while (i < length):
            if self._cursor.index < self._len:
                if self.peek_char == '\n':
                    self._cursor.step_next_line()
                self._cursor.step_next_char()
            i += 1
        return self._cursor.index

def
incpos
(
self
,
length
:
int
=
1
)
->
int
:
if
length
<
0
:
raise
ValueError
(
"length must be positive"
)
i
=
0
while
(
i
<
length
)
:
if
self
.
_cursor
.
index
<
self
.
_len
:
if
self
.
peek_char
==
'\n'
:
self
.
_cursor
.
step_next_line
(
)
self
.
_cursor
.
step_next_char
(
)
i
+=
1
return
self
.
_cursor
.
index

Increment the cursor to the next character .


def save_context(self) -> bool:
        """Save current position."""
        self._contexts.append(self._cursor.position)
        return True

def save_context ( self ) -> bool : self . _contexts . append ( self . _cursor . position ) return True


def restore_context(self) -> bool:
        """Rollback to previous saved position."""
        self._cursor.position = self._contexts.pop()
        return False

def restore_context ( self ) -> bool : self . _cursor . position = self . _contexts . pop ( ) return False


def to_fmt(self, with_from=False) -> fmt.indentable:
    """
    Return a Fmt representation of Translator for pretty-printing
    """
    txt = fmt.sep("\n", [
        fmt.sep(
            " ",
            [
                self._type_source,
                "to",
                self._type_target,
                '=',
                self._fun.to_fmt()
            ]
        ),
        self._notify.get_content(with_from)
    ])
    return txt

def
to_fmt
(
self
,
with_from
=
False
)
->
fmt
.
indentable
:
txt
=
fmt
.
sep
(
"\n"
,
[
fmt
.
sep
(
" "
,
[
self
.
_type_source
,
"to"
,
self
.
_type_target
,
'='
,
self
.
_fun
.
to_fmt
(
)
]
)
,
self
.
_notify
.
get_content
(
with_from
)
]
)
return
txt

Return a Fmt representation of Translator for pretty - printing


def set_name(self, name: str):
        """ You could set the name after construction """
        self.name = name
        # update internal names
        lsig = self._hsig.values()
        self._hsig = {}
        for s in lsig:
            self._hsig[s.internal_name()] = s

def
set_name
(
self
,
name
:
str
)
:
self
.
name
=
name
# update internal names
lsig
=
self
.
_hsig
.
values
(
)
self
.
_hsig
=
{
}
for
s
in
lsig
:
self
.
_hsig
[
s
.
internal_name
(
)
]
=
s


def count_vars(self) -> int:
        """ Count var define by this scope """
        n = 0
        for s in self._hsig.values():
            if hasattr(s, 'is_var') and s.is_var:
                n += 1
        return n

def count_vars ( self ) -> int : n = 0 for s in self . _hsig . values ( ) : if hasattr ( s , 'is_var' ) and s . is_var : n += 1 return n


def count_funs(self) -> int:
        """ Count function define by this scope """
        n = 0
        for s in self._hsig.values():
            if hasattr(s, 'is_fun') and s.is_fun:
                n += 1
        return n

def count_funs ( self ) -> int : n = 0 for s in self . _hsig . values ( ) : if hasattr ( s , 'is_fun' ) and s . is_fun : n += 1 return n


def __update_count(self):
        """ Update internal counters """
        self._ntypes = self.count_types()
        self._nvars = self.count_vars()
        self._nfuns = self.count_funs()

def __update_count ( self ) : self . _ntypes = self . count_types ( ) self . _nvars = self . count_vars ( ) self . _nfuns = self . count_funs ( )


def update(self, sig: list or Scope) -> Scope:
        """ Update the Set with values of another Set """
        values = sig
        if hasattr(sig, 'values'):
            values = sig.values()
        for s in values:
            if self.is_namespace:
                s.set_parent(self)
            if isinstance(s, Scope):
                s.state = StateScope.EMBEDDED
            self._hsig[s.internal_name()] = s
        self.__update_count()
        return self

def update ( self , sig : list or Scope ) -> Scope : values = sig if hasattr ( sig , 'values' ) : values = sig . values ( ) for s in values : if self . is_namespace : s . set_parent ( self ) if isinstance ( s , Scope ) : s . state = StateScope . EMBEDDED self . _hsig [ s . internal_name ( ) ] = s self . __update_count ( ) return self


def union(self, sig: Scope) -> Scope:
        """ Create a new Set produce by the union of 2 Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new |= sig
        return new

def union ( self , sig : Scope ) -> Scope : new = Scope ( sig = self . _hsig . values ( ) , state = self . state ) new |= sig return new


def intersection_update(self, oset: Scope) -> Scope:
        """ Update Set with common values of another Set """
        keys = list(self._hsig.keys())
        for k in keys:
            if k not in oset:
                del self._hsig[k]
            else:
                self._hsig[k] = oset.get(k)
        return self

def intersection_update ( self , oset : Scope ) -> Scope : keys = list ( self . _hsig . keys ( ) ) for k in keys : if k not in oset : del self . _hsig [ k ] else : self . _hsig [ k ] = oset . get ( k ) return self


def intersection(self, sig: Scope) -> Scope:
        """ Create a new Set produce by the intersection of 2 Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new &= sig
        return new

def intersection ( self , sig : Scope ) -> Scope : new = Scope ( sig = self . _hsig . values ( ) , state = self . state ) new &= sig return new


def difference_update(self, oset: Scope) -> Scope:
        """ Remove values common with another Set """
        keys = list(self._hsig.keys())
        for k in keys:
            if k in oset:
                del self._hsig[k]
        return self

def difference_update ( self , oset : Scope ) -> Scope : keys = list ( self . _hsig . keys ( ) ) for k in keys : if k in oset : del self . _hsig [ k ] return self


def difference(self, sig: Scope) -> Scope:
        """ Create a new Set produce by a Set subtracted by another Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new -= sig
        return new

def difference ( self , sig : Scope ) -> Scope : new = Scope ( sig = self . _hsig . values ( ) , state = self . state ) new -= sig return new


def symmetric_difference_update(self, oset: Scope) -> Scope:
        """ Remove common values
            and Update specific values from another Set
        """
        skey = set()
        keys = list(self._hsig.keys())
        for k in keys:
            if k in oset:
                skey.add(k)
        for k in oset._hsig.keys():
            if k not in skey:
                self._hsig[k] = oset.get(k)
        for k in skey:
            del self._hsig[k]
        return self

def symmetric_difference_update ( self , oset : Scope ) -> Scope : skey = set ( ) keys = list ( self . _hsig . keys ( ) ) for k in keys : if k in oset : skey . add ( k ) for k in oset . _hsig . keys ( ) : if k not in skey : self . _hsig [ k ] = oset . get ( k ) for k in skey : del self . _hsig [ k ] return self


def symmetric_difference(self, sig: Scope) -> Scope:
        """ Create a new Set with values present in only one Set """
        new = Scope(sig=self._hsig.values(), state=self.state)
        new ^= sig
        return new

def symmetric_difference ( self , sig : Scope ) -> Scope : new = Scope ( sig = self . _hsig . values ( ) , state = self . state ) new ^= sig return new


def add(self, it: Signature) -> bool:
        """ Add it to the Set """
        if isinstance(it, Scope):
            it.state = StateScope.EMBEDDED
        txt = it.internal_name()
        it.set_parent(self)
        if self.is_namespace:
            txt = it.internal_name()
        if txt == "":
            txt = '_' + str(len(self._hsig))
        if txt in self._hsig:
            raise KeyError("Already exists %s" % txt)
        self._hsig[txt] = it
        self.__update_count()
        return True

def
add
(
self
,
it
:
Signature
)
->
bool
:
if
isinstance
(
it
,
Scope
)
:
it
.
state
=
StateScope
.
EMBEDDED
txt
=
it
.
internal_name
(
)
it
.
set_parent
(
self
)
if
self
.
is_namespace
:
txt
=
it
.
internal_name
(
)
if
txt
==
""
:
txt
=
'_'
+
str
(
len
(
self
.
_hsig
)
)
if
txt
in
self
.
_hsig
:
raise
KeyError
(
"Already exists %s"
%
txt
)
self
.
_hsig
[
txt
]
=
it
self
.
__update_count
(
)
return
True


def remove(self, it: Signature) -> bool:
        """ Remove it but raise KeyError if not found """
        txt = it.internal_name()
        if txt not in self._hsig:
            raise KeyError(it.show_name() + ' not in Set')
        sig = self._hsig[txt]
        if isinstance(sig, Scope):
            sig.state = StateScope.LINKED
        del self._hsig[txt]
        return True

def
remove
(
self
,
it
:
Signature
)
->
bool
:
txt
=
it
.
internal_name
(
)
if
txt
not
in
self
.
_hsig
:
raise
KeyError
(
it
.
show_name
(
)
+
' not in Set'
)
sig
=
self
.
_hsig
[
txt
]
if
isinstance
(
sig
,
Scope
)
:
sig
.
state
=
StateScope
.
LINKED
del
self
.
_hsig
[
txt
]
return
True


def discard(self, it: Signature) -> bool:
        """ Remove it only if present """
        txt = it.internal_name()
        if txt in self._hsig:
            sig = self._hsig[txt]
            if isinstance(sig, Scope):
                sig.state = StateScope.LINKED
            del self._hsig[txt]
            return True
        return False

def discard ( self , it : Signature ) -> bool : txt = it . internal_name ( ) if txt in self . _hsig : sig = self . _hsig [ txt ] if isinstance ( sig , Scope ) : sig . state = StateScope . LINKED del self . _hsig [ txt ] return True return False


def values(self) -> [Signature]:
        """ Retrieve all values """
        if self.state == StateScope.EMBEDDED and self.parent is not None:
            return list(self._hsig.values()) + list(self.parent().values())
        else:
            return self._hsig.values()

def values ( self ) -> [ Signature ] : if self . state == StateScope . EMBEDDED and self . parent is not None : return list ( self . _hsig . values ( ) ) + list ( self . parent ( ) . values ( ) ) else : return self . _hsig . values ( )


def first(self) -> Signature:
        """ Retrieve the first Signature ordered by mangling descendant """
        k = sorted(self._hsig.keys())
        return self._hsig[k[0]]

def first ( self ) -> Signature : k = sorted ( self . _hsig . keys ( ) ) return self . _hsig [ k [ 0 ] ]


def last(self) -> Signature:
        """ Retrieve the last Signature ordered by mangling descendant """
        k = sorted(self._hsig.keys())
        return self._hsig[k[-1]]

def last ( self ) -> Signature : k = sorted ( self . _hsig . keys ( ) ) return self . _hsig [ k [ - 1 ] ]


def get(self, key: str, default=None) -> Signature:
        """ Get a signature instance by its internal_name """
        item = default
        if key in self._hsig:
            item = self._hsig[key]
        return item

def get ( self , key : str , default = None ) -> Signature : item = default if key in self . _hsig : item = self . _hsig [ key ] return item


def get_by_symbol_name(self, name: str) -> Scope:
        """ Retrieve a Set of all signature by symbol name """
        lst = []
        for s in self.values():
            if s.name == name:
                # create an EvalCtx only when necessary
                lst.append(EvalCtx.from_sig(s))
        # include parent
        # TODO: see all case of local redefinition for
        #       global overloads
        # possible algos... take all with different internal_name
        if len(lst) == 0:
            p = self.get_parent()
            if p is not None:
                return p.get_by_symbol_name(name)
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return rscope


def getsig_by_symbol_name(self, name: str) -> Signature:
        """ Retrieve the unique Signature of a symbol.
        Fail if the Signature is not unique
        """
        subscope = self.get_by_symbol_name(name)
        if len(subscope) != 1:
            raise KeyError("%s have multiple candidates in scope" % name)
        v = list(subscope.values())
        return v[0]

def
getsig_by_symbol_name
(
self
,
name
:
str
)
->
Signature
:
subscope
=
self
.
get_by_symbol_name
(
name
)
if
len
(
subscope
)
!=
1
:
raise
KeyError
(
"%s have multiple candidates in scope"
%
name
)
v
=
list
(
subscope
.
values
(
)
)
return
v
[
0
]

Retrieve the unique Signature of a symbol . Fail if the Signature is not unique


def get_all_polymorphic_return(self) -> bool:
        """ For now, polymorphic return type are handle by symbol artefact.

        --> possible multi-polymorphic but with different constraint attached!
        """
        lst = []
        for s in self.values():
            if hasattr(s, 'tret') and s.tret.is_polymorphic:
                # encapsulate s into a EvalCtx for meta-var resolution
                lst.append(EvalCtx.from_sig(s))
        rscope = Scope(sig=lst, state=StateScope.LINKED, is_namespace=False)
        # inherit type/translation from parent
        rscope.set_parent(self)
        return rscope

For now polymorphic return type are handle by symbol artefact .


def callInjector(self, old: Node, trans: Translator) -> Node:
        """ If don't have injector call from parent """
        if self.astTranslatorInjector is None:
            if self.parent is not None:
                # TODO: think if we forward for all StateScope
                # forward to parent scope
                return self.parent().callInjector(old, trans)
            else:
                raise TypeError("Must define an Translator Injector")
        return self.astTranslatorInjector(old, trans)


def normalize(ast: Node) -> Node:
    """
    Normalize an AST nodes.

    all builtins containers are replace by referencable subclasses
    """
    res = ast
    typemap = {DictNode, ListNode, TupleNode}
    if type(ast) is dict:
        res = DictNode(ast)
    elif type(ast) is list:
        res = ListNode(ast)
    elif type(ast) is tuple:
        res = TupleNode(ast)
    # in-depth change
    if hasattr(res, 'items'):
        for k, v in res.items():
            res[k] = normalize(v)
    elif hasattr(res, '__getitem__'):
        for idx, v in zip(range(len(res)), res):
            res[idx] = normalize(v)
    if type(res) not in typemap and hasattr(res, '__dict__'):
        subattr = vars(res)
        for k, v in subattr.items():
            setattr(res, k, normalize(v))
    return res

def
normalize
(
ast
:
Node
)
->
Node
:
res
=
ast
typemap
=
{
DictNode
,
ListNode
,
TupleNode
}
if
type
(
ast
)
is
dict
:
res
=
DictNode
(
ast
)
elif
type
(
ast
)
is
list
:
res
=
ListNode
(
ast
)
elif
type
(
ast
)
is
tuple
:
res
=
TupleNode
(
ast
)
# in-depth change
if
hasattr
(
res
,
'items'
)
:
for
k
,
v
in
res
.
items
(
)
:
res
[
k
]
=
normalize
(
v
)
elif
hasattr
(
res
,
'__getitem__'
)
:
for
idx
,
v
in
zip
(
range
(
len
(
res
)
)
,
res
)
:
res
[
idx
]
=
normalize
(
v
)
if
type
(
res
)
not
in
typemap
and
hasattr
(
res
,
'__dict__'
)
:
subattr
=
vars
(
res
)
for
k
,
v
in
subattr
.
items
(
)
:
setattr
(
res
,
k
,
normalize
(
v
)
)
return
res


def set(self, othernode):
        """allow to completly mutate the node into any subclasses of Node"""
        self.__class__ = othernode.__class__
        self.clean()
        if len(othernode) > 0:
            for k, v in othernode.items():
                self[k] = v
        for k, v in vars(othernode).items():
            setattr(self, k, v)

def set ( self , othernode ) : self . __class__ = othernode . __class__ self . clean ( ) if len ( othernode ) > 0 : for k , v in othernode . items ( ) : self [ k ] = v for k , v in vars ( othernode ) . items ( ) : setattr ( self , k , v )


def rvalues(self):
        """
        in reversed order
        """
        tmp = self
        while tmp is not None:
            yield tmp.data
            tmp = tmp.prev

def rvalues ( self ) : tmp = self while tmp is not None : yield tmp . data tmp = tmp . prev


def _hit_ok(hit, min_hit_charge, max_hit_charge):
    ''' Check if given hit is withing the limits.
    '''
    # Omit hits with charge < min_hit_charge
    if hit['charge'] < min_hit_charge:
        return False

    # Omit hits with charge > max_hit_charge
    if max_hit_charge != 0 and hit['charge'] > max_hit_charge:
        return False

    return True

Check if given hit is withing the limits .


def get_compute_sig(self) -> Signature:
        """
        Compute a signature Using resolution!!!

        TODO: discuss of relevance of a final generation for a signature
        """
        tret = []
        tparams = []
        for t in self.tret.components:
            if t in self.resolution and self.resolution[t] is not None:
                tret.append(self.resolution[t]().show_name())
            else:
                tret.append(t)
        if hasattr(self, 'tparams'):
            for p in self.tparams:
                tp = []
                for t in p.components:
                    if t in self.resolution and self.resolution[t] is not None:
                        tp.append(self.resolution[t]().show_name())
                    else:
                        tp.append(t)
                tparams.append(" ".join(tp))
            if self.variadic:
                if self._variadic_types is None:
                    raise ValueError("Can't compute the sig "
                                     + "with unresolved variadic argument"
                                     )
                for p in self._variadic_types:
                    tp = []
                    for t in p.components:
                        if (t in self.resolution
                            and self.resolution[t] is not None
                        ):
                            tp.append(self.resolution[t]().show_name())
                        else:
                            tp.append(t)
                    tparams.append(" ".join(tp))
        ret = Fun(self.name, " ".join(tret), tparams)
        # transform as-is into our internal Signature (Val, Var, whatever)
        ret.__class__ = self._sig.__class__
        return ret


def resolve(self):
        """
        Process the signature and find definition for type.
        """
        # collect types for resolution
        t2resolv = []
        if hasattr(self._sig, 'tret'):
            t2resolv.append(self._sig.tret)
        if hasattr(self._sig, 'tparams') and self._sig.tparams is not None:
            for p in self._sig.tparams:
                t2resolv.append(p)
        if self._translate_to is not None:
            t2resolv.append(self._translate_to.target)
        if self._variadic_types is not None:
            for t in self._variadic_types:
                t2resolv.append(t)
        for t in t2resolv:
            for c in t.components:
                if c not in self.resolution or self.resolution[c] is None:
                    # try to find what is c
                    parent = self.get_parent()
                    if parent is not None:
                        sc = parent.get_by_symbol_name(c)
                        if len(sc) == 1:
                            sc = list(sc.values())[0]
                            # unwrap EvalCtx around Type
                            if isinstance(sc, EvalCtx):
                                sc = sc._sig
                            rtyp = weakref.ref(sc)
                            self.resolution[c] = rtyp
                            continue
                    # unresolved
                    self.resolution[c] = None

def
resolve
(
self
)
:
# collect types for resolution
t2resolv
=
[
]
if
hasattr
(
self
.
_sig
,
'tret'
)
:
t2resolv
.
append
(
self
.
_sig
.
tret
)
if
hasattr
(
self
.
_sig
,
'tparams'
)
and
self
.
_sig
.
tparams
is
not
:
for
p
in
self
.
_sig
.
tparams
:
t2resolv
.
append
(
p
)
if
self
.
_translate_to
is
not
:
t2resolv
.
append
(
self
.
_translate_to
.
target
)
if
self
.
_variadic_types
is
not
:
for
t
in
self
.
_variadic_types
:
t2resolv
.
append
(
t
)
for
t
in
t2resolv
:
for
c
in
t
.
components
:
if
c
not
in
self
.
resolution
or
self
.
resolution
[
c
]
is
:
# try to find what is c
parent
=
self
.
get_parent
(
)
if
parent
is
not
:
sc
=
parent
.
get_by_symbol_name
(
c
)
if
len
(
sc
)
==
1
:
sc
=
list
(
sc
.
values
(
)
)
[
0
]
# unwrap EvalCtx around Type
if
isinstance
(
sc
,
EvalCtx
)
:
sc
=
sc
.
_sig
rtyp
=
weakref
.
ref
(
sc
)
self
.
resolution
[
c
]
=
rtyp
continue
# unresolved
self
.
resolution
[
c
]
=

Process the signature and find definition for type .


def get_resolved_names(self, type_name: TypeName) -> list:
        """
        Use self.resolution to subsitute type_name.
        Allow to instanciate polymorphic type ?1, ?toto
        """
        if not isinstance(type_name, TypeName):
            raise Exception("Take a TypeName as parameter not a %s"
                            % type(type_name))
        rnames = []
        for name in type_name.components:
            if name not in self.resolution:
                raise Exception("Unknown type %s in a EvalCtx" % name)
            rname = self.resolution[name]
            if rname is not None:
                rname = rname().show_name()
            else:
                rname = name
            rnames.append(rname)
        return rnames

def
get_resolved_names
(
self
,
type_name
:
TypeName
)
->
list
:
if
not
isinstance
(
type_name
,
TypeName
)
:
raise
Exception
(
"Take a TypeName as parameter not a %s"
%
type
(
type_name
)
)
rnames
=
[
]
for
name
in
type_name
.
components
:
if
name
not
in
self
.
resolution
:
raise
Exception
(
"Unknown type %s in a EvalCtx"
%
name
)
rname
=
self
.
resolution
[
name
]
if
rname
is
not
:
rname
=
rname
(
)
.
show_name
(
)
else
:
rname
=
name
rnames
.
append
(
rname
)
return
rnames

Use self . resolution to subsitute type_name . Allow to instanciate polymorphic type ?1 ?toto


def set_resolved_name(self, ref: dict, type_name2solve: TypeName,
                          type_name_ref: TypeName):
        """
        Warning!!! Need to rethink it when global poly type
        """
        if self.resolution[type_name2solve.value] is None:
            self.resolution[type_name2solve.value] = ref[type_name_ref.value]

def set_resolved_name ( self , ref : dict , type_name2solve : TypeName , type_name_ref : TypeName ) : if self . resolution [ type_name2solve . value ] is None : self . resolution [ type_name2solve . value ] = ref [ type_name_ref . value ]


def _delete_local(self, filename):
        """Deletes the specified file from the local filesystem."""

        if os.path.exists(filename):
            os.remove(filename)

def _delete_local ( self , filename ) : if os . path . exists ( filename ) : os . remove ( filename )

Deletes the specified file from the local filesystem .


def _delete_s3(self, filename, bucket_name):
        """Deletes the specified file from the given S3 bucket."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(bucket_name)

        if type(filename).__name__ == 'Key':
            filename = '/' + filename.name

        path = self._get_s3_path(filename)
        k = Key(bucket)
        k.key = path

        try:
            bucket.delete_key(k)
        except S3ResponseError:
            pass

def _delete_s3 ( self , filename , bucket_name ) : conn = S3Connection ( self . access_key_id , self . access_key_secret ) bucket = conn . get_bucket ( bucket_name ) if type ( filename ) . __name__ == 'Key' : filename = '/' + filename . name path = self . _get_s3_path ( filename ) k = Key ( bucket ) k . key = path try : bucket . delete_key ( k ) except S3ResponseError : pass

Deletes the specified file from the given S3 bucket .


def delete(self, filename, storage_type=None, bucket_name=None):
        """Deletes the specified file, either locally or from S3, depending on the file's storage type."""

        if not (storage_type and bucket_name):
            self._delete_local(filename)
        else:
            if storage_type != 's3':
                raise ValueError('Storage type "%s" is invalid, the only supported storage type (apart from default local storage) is s3.' % storage_type)

            self._delete_s3(filename, bucket_name)

Deletes the specified file either locally or from S3 depending on the file s storage type .


def _save_local(self, temp_file, filename, obj):
        """Saves the specified file to the local file system."""

        path = self._get_path(filename)
        if not os.path.exists(os.path.dirname(path)):
            os.makedirs(os.path.dirname(path), self.permission | 0o111)

        fd = open(path, 'wb')

        # Thanks to:
        # http://stackoverflow.com/a/3253276/2066849
        temp_file.seek(0)
        t = temp_file.read(1048576)
        while t:
            fd.write(t)
            t = temp_file.read(1048576)

        fd.close()

        if self.filesize_field:
            setattr(obj, self.filesize_field, os.path.getsize(path))

        return filename

Saves the specified file to the local file system .


def _save_s3(self, temp_file, filename, obj):
        """Saves the specified file to the configured S3 bucket."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(self.bucket_name)

        path = self._get_s3_path(filename)
        k = bucket.new_key(path)
        k.set_contents_from_string(temp_file.getvalue())
        k.set_acl(self.acl)

        if self.filesize_field:
            setattr(obj, self.filesize_field, k.size)

        return filename

def _save_s3 ( self , temp_file , filename , obj ) : conn = S3Connection ( self . access_key_id , self . access_key_secret ) bucket = conn . get_bucket ( self . bucket_name ) path = self . _get_s3_path ( filename ) k = bucket . new_key ( path ) k . set_contents_from_string ( temp_file . getvalue ( ) ) k . set_acl ( self . acl ) if self . filesize_field : setattr ( obj , self . filesize_field , k . size ) return filename

Saves the specified file to the configured S3 bucket .


def save(self, temp_file, filename, obj):
        """Saves the specified file to either S3 or the local filesystem, depending on the currently enabled storage type."""

        if not (self.storage_type and self.bucket_name):
            ret = self._save_local(temp_file, filename, obj)
        else:
            if self.storage_type != 's3':
                raise ValueError('Storage type "%s" is invalid, the only supported storage type (apart from default local storage) is s3.' % self.storage_type)

            ret = self._save_s3(temp_file, filename, obj)

        if self.field_name:
            setattr(obj, self.field_name, ret)

        if self.storage_type == 's3':
            if self.storage_type_field:
                setattr(obj, self.storage_type_field, self.storage_type)
            if self.bucket_name_field:
                setattr(obj, self.bucket_name_field, self.bucket_name)
        else:
            if self.storage_type_field:
                setattr(obj, self.storage_type_field, '')
            if self.bucket_name_field:
                setattr(obj, self.bucket_name_field, '')

        return ret

Saves the specified file to either S3 or the local filesystem depending on the currently enabled storage type .


def _find_by_path_s3(self, path, bucket_name):
        """Finds files by licking an S3 bucket's contents by prefix."""

        conn = S3Connection(self.access_key_id, self.access_key_secret)
        bucket = conn.get_bucket(bucket_name)

        s3_path = self._get_s3_path(path)

        return bucket.list(prefix=s3_path)

def _find_by_path_s3 ( self , path , bucket_name ) : conn = S3Connection ( self . access_key_id , self . access_key_secret ) bucket = conn . get_bucket ( bucket_name ) s3_path = self . _get_s3_path ( path ) return bucket . list ( prefix = s3_path )

Finds files by licking an S3 bucket s contents by prefix .


def enum(*sequential, **named):
    """
    Build an enum statement
    """
    #: build enums from parameter
    enums = dict(zip(sequential, range(len(sequential))), **named)
    enums['map'] = copy.copy(enums)
    #: build reverse mapping
    enums['rmap'] = {}
    for key, value in enums.items():
        if type(value) is int:
            enums['rmap'][value] = key
    return type('Enum', (), enums)

def
enum
(
*
sequential
,
*
*
named
)
:
#: build enums from parameter
enums
=
dict
(
zip
(
sequential
,
range
(
len
(
sequential
)
)
)
,
*
*
named
)
enums
[
'map'
]
=
copy
.
copy
(
enums
)
#: build reverse mapping
enums
[
'rmap'
]
=
{
}
for
key
,
value
in
enums
.
items
(
)
:
if
type
(
value
)
is
int
:
enums
[
'rmap'
]
[
value
]
=
key
return
type
(
'Enum'
,
(
)
,
enums
)


def checktypes(func):
    """Decorator to verify arguments and return types."""
    sig = inspect.signature(func)

    types = {}
    for param in sig.parameters.values():
        # Iterate through function's parameters and build the list of
        # arguments types
        param_type = param.annotation
        if param_type is param.empty or not inspect.isclass(param_type):
            # Missing annotation or not a type, skip it
            continue

        types[param.name] = param_type

        # If the argument has a type specified, let's check that its
        # default value (if present) conforms with the type.
        if (param.default is not param.empty and
                not isinstance(param.default, param_type)):
            raise ValueError(
                "{func}: wrong type of a default value for {arg!r}".format(
                    func=func.__qualname__, arg=param.name)
            )

    def check_type(sig, arg_name, arg_type, arg_value):
        # Internal function that encapsulates arguments type checking
        if not isinstance(arg_value, arg_type):
            raise ValueError("{func}: wrong type of {arg!r} argument, "
                             "{exp!r} expected, got {got!r}".
                             format(func=func.__qualname__, arg=arg_name,
                                    exp=arg_type.__name__,
                                    got=type(arg_value).__name__))

    @functools.wraps(func)
    def wrapper(*args, **kwargs):
        # Let's bind the arguments
        ba = sig.bind(*args, **kwargs)
        for arg_name, arg in ba.arguments.items():
            # And iterate through the bound arguments
            try:
                type_ = types[arg_name]
            except KeyError:
                continue
            else:
                # OK, we have a type for the argument, lets get the
                # corresponding parameter description from the signature object
                param = sig.parameters[arg_name]
                if param.kind == param.VAR_POSITIONAL:
                    # If this parameter is a variable-argument parameter,
                    # then we need to check each of its values
                    for value in arg:
                        check_type(sig, arg_name, type_, value)
                elif param.kind == param.VAR_KEYWORD:
                    # If this parameter is a variable-keyword-argument
                    # parameter:
                    for subname, value in arg.items():
                        check_type(sig, arg_name + ':' + subname, type_, value)
                else:
                    # And, finally, if this parameter a regular one:
                    check_type(sig, arg_name, type_, arg)

        result = func(*ba.args, **ba.kwargs)

        # The last bit - let's check that the result is correct
        return_type = sig.return_annotation
        if (return_type is not sig.empty and
                isinstance(return_type, type) and
                not isinstance(result, return_type)):

            raise ValueError(
                '{func}: wrong return type, {exp} expected, got {got}'.format(
                    func=func.__qualname__, exp=return_type.__name__,
                    got=type(result).__name__)
            )
        return result

    return wrapper

Decorator to verify arguments and return types .


def set_one(chainmap, thing_name, callobject):
    """ Add a mapping with key thing_name for callobject in chainmap with
        namespace handling.
    """
    namespaces = reversed(thing_name.split("."))
    lstname = []
    for name in namespaces:
        lstname.insert(0, name)
        strname = '.'.join(lstname)
        chainmap[strname] = callobject

def set_one ( chainmap , thing_name , callobject ) : namespaces = reversed ( thing_name . split ( "." ) ) lstname = [ ] for name in namespaces : lstname . insert ( 0 , name ) strname = '.' . join ( lstname ) chainmap [ strname ] = callobject

Add a mapping with key thing_name for callobject in chainmap with namespace handling .


def add_method(cls):
    """Attach a method to a class."""
    def wrapper(f):
        #if hasattr(cls, f.__name__):
        #    raise AttributeError("{} already has a '{}' attribute".format(
        #        cls.__name__, f.__name__))
        setattr(cls, f.__name__, f)
        return f
    return wrapper


def hook(cls, hookname=None, erase=False):
    """Attach a method to a parsing class and register it as a parser hook.

       The method is registered with its name unless hookname is provided.
    """
    if not hasattr(cls, '_hooks'):
        raise TypeError(
            "%s didn't seems to be a BasicParser subsclasse" % cls.__name__)
    class_hook_list = cls._hooks
    class_rule_list = cls._rules

    def wrapper(f):
        nonlocal hookname
        add_method(cls)(f)
        if hookname is None:
            hookname = f.__name__
        if not erase and (hookname in class_hook_list or hookname in class_rule_list):
            raise TypeError("%s is already define has rule or hook" % hookname)
        if '.' not in hookname:
            hookname = '.'.join([cls.__module__, cls.__name__, hookname])
        set_one(class_hook_list, hookname, f)
        return f
    return wrapper

Attach a method to a parsing class and register it as a parser hook .


def rule(cls, rulename=None, erase=False):
    """Attach a method to a parsing class and register it as a parser rule.

       The method is registered with its name unless rulename is provided.
    """
    if not hasattr(cls, '_rules'):
        raise TypeError(
            "%s didn't seems to be a BasicParser subsclasse" % cls.__name__)
    class_hook_list = cls._hooks
    class_rule_list = cls._rules

    def wrapper(f):
        nonlocal rulename
        add_method(cls)(f)
        if rulename is None:
            rulename = f.__name__
        if not erase and (rulename in class_hook_list or rulename in class_rule_list):
            raise TypeError("%s is already define has rule or hook" % rulename)
        if '.' not in rulename:
            rulename = cls.__module__ + '.' + cls.__name__ + '.' + rulename
        set_one(class_rule_list, rulename, f)
        return f
    return wrapper

Attach a method to a parsing class and register it as a parser rule .


def directive(directname=None):
    """Attach a class to a parsing class and register it as a parser directive.

        The class is registered with its name unless directname is provided.
    """
    global _directives
    class_dir_list = _directives

    def wrapper(f):
        nonlocal directname
        if directname is None:
            directname = f.__name__
        f.ns_name = directname
        set_one(class_dir_list, directname, f)
        return f
    return wrapper

def directive ( directname = None ) : global _directives class_dir_list = _directives def wrapper ( f ) : nonlocal directname if directname is None : directname = f . __name__ f . ns_name = directname set_one ( class_dir_list , directname , f ) return f return wrapper

Attach a class to a parsing class and register it as a parser directive .