"""
Define general engineering functions.

[[[cog
import os, sys
if sys.hexversion < 0x03000000:
    import __builtin__
else:
    import builtins as __builtin__
sys.path.append(os.environ['TRACER_DIR'])
import trace_ex_eng_functions
exobj_eng = trace_ex_eng_functions.trace_module(no_print=True)
]]]
[[[end]]]
"""
# functions.py
# Copyright (c) 2013-2019 Pablo Acosta-Serafini
# See LICENSE for details
# pylint: disable=C0103,C0111,C0413,E0611,R0914,R0915,R1710,W0105,W0611,W0631

# Standard library imports
import collections
import copy
import math
import re
import textwrap
import decimal
from decimal import Decimal
import sys

if sys.hexversion < 0x03000000:  # pragma: no cover
    from itertools import izip_longest as zip_longest
else:  # pragma: no cover
    from itertools import zip_longest
import warnings

# PyPI imports
with warnings.catch_warnings():
    warnings.filterwarnings("ignore", category=RuntimeWarning)
    import numpy as np
import pexdoc.exh
from pexdoc.ptypes import non_negative_integer
import pyparsing

# Intra-package imports imports
from .ptypes import engineering_notation_number, engineering_notation_suffix

pyparsing.ParserElement.enablePackrat()

###
# Global variables
###
_OP_PREC = [
    "|",
    "^",
    "&",
    ["<<", ">>"],
    ["+", "-"],
    ["*", "/", "//", "%"],
    ["+", "-", "~"],
    "**",
]

_POWER_TO_SUFFIX_DICT = dict(
    (exp, prf) for exp, prf in zip(range(-24, 27, 3), "yzafpnum kMGTPEZY")
)
_SUFFIX_TO_POWER_DICT = dict(
    (value, key) for key, value in _POWER_TO_SUFFIX_DICT.items()
)
_SUFFIX_POWER_DICT = dict(
    (key, float(10 ** value)) for key, value in _SUFFIX_TO_POWER_DICT.items()
)

EngPower = collections.namedtuple("EngPower", ["suffix", "exp"])
"""Constructor for engineering notation suffix."""

NumComp = collections.namedtuple("NumComp", ["mant", "exp"])
"""Constructor for number components representation."""

###
# Functions
###
def _build_expr(tokens, higher_oplevel=-1, ldelim="(", rdelim=")"):
    """Build mathematical expression from hierarchical list."""
    # Numbers
    if isinstance(tokens, str):
        return tokens
    # Unary operators
    if len(tokens) == 2:
        return "".join(tokens)
    # Multi-term operators
    oplevel = _get_op_level(tokens[1])
    stoken = ""
    for num, item in enumerate(tokens):
        if num % 2 == 0:
            stoken += _build_expr(item, oplevel, ldelim=ldelim, rdelim=rdelim)
        else:
            stoken += item
    if (oplevel < higher_oplevel) or (
        (oplevel == higher_oplevel) and (oplevel in _OP_PREC_PAR)
    ):
        stoken = ldelim + stoken + rdelim
    return stoken

def _get_op_level(operator, nops=2):
    # There is no error checking that operator is not in the
    # operator precedence group, it is assumed that only valid
    # operators are in mathematical expression at higher function
    # levels and/or at API entry point
    for num, item in enumerate(_OP_PREC):  # pragma: no cover
        if operator in item:
            if operator not in ["+", "-"]:
                return num
            if (nops == 2) and item == ["+", "-"]:
                return num
            if (nops == 1) and item == ["+", "-", "~"]:
                return num

# Operators groups that need parenthesis around expression if lower in parse
# tree and next level up has the same operator precedence level
_OP_PREC_PAR = [_get_op_level("<<"), _get_op_level("*")]

def _next_rdelim(items, pos):
    """Return position of next matching closing delimiter."""
    for num, item in enumerate(items):
        if item > pos:
            break
    else:
        raise RuntimeError("Mismatched delimiters")
    del items[num]
    return item

def _get_functions(expr, ldelim="(", rdelim=")"):
    """Parse function calls."""
    tpars = _pair_delims(expr, ldelim=ldelim, rdelim=rdelim)
    alphas = "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
    fchars = "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "_"
    tfuncs = []
    for lnum, rnum in tpars:
        if lnum and expr[lnum - 1] in fchars:
            for cnum, char in enumerate(reversed(expr[:lnum])):
                if char not in fchars:
                    break
            else:
                cnum = lnum
            tfuncs.append(
                {
                    "fname": expr[lnum - cnum : lnum],
                    "expr": expr[lnum + 1 : rnum],
                    "start": lnum - cnum,
                    "stop": rnum,
                }
            )
            if expr[lnum - cnum] not in alphas:
                raise RuntimeError(
                    "Function name `{0}` is not valid".format(expr[lnum - cnum : lnum])
                )
    return tfuncs

def _pair_delims(expr, ldelim="(", rdelim=")"):
    """Pair delimiters."""
    # Find where remaining delimiters are
    lindex = reversed([num for num, item in enumerate(expr) if item == ldelim])
    rindex = [num for num, item in enumerate(expr) if item == rdelim]
    # Pair remaining delimiters
    return [(lpos, _next_rdelim(rindex, lpos)) for lpos in lindex][::-1]

def _parse_expr(text, ldelim="(", rdelim=")"):
    """Parse mathematical expression using PyParsing."""
    var = pyparsing.Word(pyparsing.alphas + "_", pyparsing.alphanums + "_")
    point = pyparsing.Literal(".")
    exp = pyparsing.CaselessLiteral("E")
    number = pyparsing.Combine(
        pyparsing.Word("+-" + pyparsing.nums, pyparsing.nums)
        + pyparsing.Optional(point + pyparsing.Optional(pyparsing.Word(pyparsing.nums)))
        + pyparsing.Optional(
            exp + pyparsing.Word("+-" + pyparsing.nums, pyparsing.nums)
        )
    )
    atom = var | number
    oplist = [
        (pyparsing.Literal("**"), 2, pyparsing.opAssoc.RIGHT),
        (pyparsing.oneOf("+ - ~"), 1, pyparsing.opAssoc.RIGHT),
        (pyparsing.oneOf("* / // %"), 2, pyparsing.opAssoc.LEFT),
        (pyparsing.oneOf("+ -"), 2, pyparsing.opAssoc.LEFT),
        (pyparsing.oneOf("<< >>"), 2, pyparsing.opAssoc.LEFT),
        (pyparsing.Literal("&"), 2, pyparsing.opAssoc.LEFT),
        (pyparsing.Literal("^"), 2, pyparsing.opAssoc.LEFT),
        (pyparsing.Literal("|"), 2, pyparsing.opAssoc.LEFT),
    ]
    # Get functions
    expr = pyparsing.infixNotation(
        atom, oplist, lpar=pyparsing.Suppress(ldelim), rpar=pyparsing.Suppress(rdelim)
    )
    return expr.parseString(text)[0]

def _remove_consecutive_delims(expr, ldelim="(", rdelim=")"):
    """Remove consecutive delimiters."""
    tpars = _pair_delims(expr, ldelim=ldelim, rdelim=rdelim)
    # Flag superfluous delimiters
    ddelim = []
    for ctuple, ntuple in zip(tpars, tpars[1:]):
        if ctuple == (ntuple[0] - 1, ntuple[1] + 1):
            ddelim.extend(ntuple)
    ddelim.sort()
    # Actually remove delimiters from expression
    for num, item in enumerate(ddelim):
        expr = expr[: item - num] + expr[item - num + 1 :]
    # Get functions
    return expr

def _remove_extra_delims(expr, ldelim="(", rdelim=")", fcount=None):
    """
    Remove unnecessary delimiters (parenthesis, brackets, etc.).

    Internal function that can be recursed
    """
    if not expr.strip():
        return ""
    fcount = [0] if fcount is None else fcount
    tfuncs = _get_functions(expr, ldelim=ldelim, rdelim=rdelim)
    # Replace function call with tokens
    for fdict in reversed(tfuncs):
        fcount[0] += 1
        fdict["token"] = "__" + str(fcount[0])
        expr = expr[: fdict["start"]] + fdict["token"] + expr[fdict["stop"] + 1 :]
        fdict["expr"] = _remove_extra_delims(
            fdict["expr"], ldelim=ldelim, rdelim=rdelim, fcount=fcount
        )
    # Parse expression with function calls removed
    expr = _build_expr(
        _parse_expr(expr, ldelim=ldelim, rdelim=rdelim), ldelim=ldelim, rdelim=rdelim
    )
    # Insert cleaned-up function calls
    for fdict in tfuncs:
        expr = expr.replace(
            fdict["token"], fdict["fname"] + ldelim + fdict["expr"] + rdelim
        )
    return expr

def _split_every(text, sep, count, lstrip=False, rstrip=False):
    """
    Return list of the words in the string, using count of a separator as delimiter.

    :param text: String to split
    :type  text: string

    :param sep: Separator
    :type  sep: string

    :param count: Number of separators to use as delimiter
    :type  count: integer

    :param lstrip: Flag that indicates whether whitespace is removed
                   from the beginning of each list item (True) or not
                   (False)
    :type  lstrip: boolean

    :param rstrip: Flag that indicates whether whitespace is removed
                   from the end of each list item (True) or not (False)
    :type  rstrip: boolean

    :rtype: tuple
    """
    ltr = "_rl "[2 * lstrip + rstrip].strip()
    func = lambda x: getattr(x, ltr + "strip")() if ltr != "_" else x
    items = text.split(sep)
    groups = zip_longest(*[iter(items)] * count, fillvalue="")
    joints = (sep.join(group).rstrip(sep) for group in groups)
    return tuple(func(joint) for joint in joints)

def _to_eng_tuple(number):
    """
    Return tuple with mantissa and exponent of number formatted in engineering notation.

    :param number: Number
    :type  number: integer or float

    :rtype: tuple
    """
    # pylint: disable=W0141
    # Helper function: split integer and fractional part of mantissa
    #  + ljust ensures that integer part in engineering notation has
    #    at most 3 digits (say if number given is 1E4)
    #  + rstrip ensures that there is no empty fractional part
    split = lambda x, p: (x.ljust(3 + neg, "0")[:p], x[p:].rstrip("0"))
    # Convert number to scientific notation, a "constant" format
    mant, exp = to_scientific_tuple(number)
    mant, neg = mant.replace(".", ""), mant.startswith("-")
    # New values
    new_mant = ".".join(filter(None, split(mant, 1 + (exp % 3) + neg)))
    new_exp = int(3 * math.floor(exp / 3))
    return NumComp(new_mant, new_exp)

@pexdoc.pcontracts.contract(number="number")
def no_exp(number):
    r"""
    Convert number to string guaranteeing result is not in scientific notation.

    :param number: Number to convert
    :type  number: integer or float

    :rtype: string

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for peng.functions.no_exp

    :raises: RuntimeError (Argument \`number\` is not valid)

    .. [[[end]]]
    """
    mant, exp = to_scientific_tuple(number)
    if not exp:
        return str(number)
    floating_mant = "." in mant
    mant = mant.replace(".", "")
    if exp < 0:
        return "0." + "0" * (-exp - 1) + mant
    if not floating_mant:
        return mant + "0" * exp + (".0" if isinstance(number, float) else "")
    lfpart = len(mant) - 1
    if lfpart < exp:
        return (mant + "0" * (exp - lfpart)).rstrip(".")
    return mant

@pexdoc.pcontracts.contract(
    number="int|float", frac_length="non_negative_integer", rjust=bool
)
def peng(number, frac_length, rjust=True):
    r"""
    Convert a number to engineering notation.

    The absolute value of the number (if it is not exactly zero) is bounded to
    the interval [1E-24, 1E+24)

    :param number: Number to convert
    :type  number: integer or float

    :param frac_length: Number of digits of fractional part
    :type  frac_length: `NonNegativeInteger
                        <https://pexdoc.readthedocs.io/en/stable/
                        ptypes.html#nonnegativeinteger>`_

    :param rjust: Flag that indicates whether the number is
                  right-justified (True) or not (False)
    :type  rjust: boolean

    :rtype: string

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for peng.functions.peng

    :raises:
     * RuntimeError (Argument \`frac_length\` is not valid)

     * RuntimeError (Argument \`number\` is not valid)

     * RuntimeError (Argument \`rjust\` is not valid)

    .. [[[end]]]

    The supported engineering suffixes are:

    +----------+-------+--------+
    | Exponent | Name  | Suffix |
    +==========+=======+========+
    | 1E-24    | yocto | y      |
    +----------+-------+--------+
    | 1E-21    | zepto | z      |
    +----------+-------+--------+
    | 1E-18    | atto  | a      |
    +----------+-------+--------+
    | 1E-15    | femto | f      |
    +----------+-------+--------+
    | 1E-12    | pico  | p      |
    +----------+-------+--------+
    | 1E-9     | nano  | n      |
    +----------+-------+--------+
    | 1E-6     | micro | u      |
    +----------+-------+--------+
    | 1E-3     | milli | m      |
    +----------+-------+--------+
    | 1E+0     |       |        |
    +----------+-------+--------+
    | 1E+3     | kilo  | k      |
    +----------+-------+--------+
    | 1E+6     | mega  | M      |
    +----------+-------+--------+
    | 1E+9     | giga  | G      |
    +----------+-------+--------+
    | 1E+12    | tera  | T      |
    +----------+-------+--------+
    | 1E+15    | peta  | P      |
    +----------+-------+--------+
    | 1E+18    | exa   | E      |
    +----------+-------+--------+
    | 1E+21    | zetta | Z      |
    +----------+-------+--------+
    | 1E+24    | yotta | Y      |
    +----------+-------+--------+

    For example:

        >>> import peng
        >>> peng.peng(1235.6789E3, 3, False)
        '1.236M'
    """
    # The decimal module has a to_eng_string() function, but it does not seem
    # to work well in all cases. For example:
    # >>> decimal.Decimal('34.5712233E8').to_eng_string()
    # '3.45712233E+9'
    # >>> decimal.Decimal('34.57122334E8').to_eng_string()
    # '3457122334'
    # It seems that the conversion function does not work in all cases
    #
    # Return formatted zero if number is zero, easier to not deal with this
    # special case through the rest of the algorithm
    if number == 0:
        number = "0.{zrs}".format(zrs="0" * frac_length) if frac_length else "0"
        # Engineering notation numbers can have a sign, a 3-digit integer part,
        # a period, and a fractional part of length frac_length, so the
        # length of the number to the left of, and including, the period is 5
        return "{0} ".format(number.rjust(5 + frac_length)) if rjust else number
    # Low-bound number
    sign = +1 if number >= 0 else -1
    ssign = "-" if sign == -1 else ""
    anumber = abs(number)
    if anumber < 1e-24:
        anumber = 1e-24
        number = sign * 1e-24
    # Round fractional part if requested frac_length is less than length
    # of fractional part. Rounding method is to add a '5' at the decimal
    # position just after the end of frac_length digits
    exp = 3.0 * math.floor(math.floor(math.log10(anumber)) / 3.0)
    mant = number / 10 ** exp
    # Because exponent is a float, mantissa is a float and its string
    # representation always includes a period
    smant = str(mant)
    ppos = smant.find(".")
    if len(smant) - ppos - 1 > frac_length:
        mant += sign * 5 * 10 ** (-frac_length - 1)
        if abs(mant) >= 1000:
            exp += 3
            mant = mant / 1e3
        smant = str(mant)
        ppos = smant.find(".")
    # Make fractional part have frac_length digits
    bfrac_length = bool(frac_length)
    flength = ppos - (not bfrac_length) + frac_length + 1
    new_mant = smant[:flength].ljust(flength, "0")
    # Upper-bound number
    if exp > 24:
        new_mant, exp = (
            "{sign}999.{frac}".format(sign=ssign, frac="9" * frac_length),
            24,
        )
    # Right-justify number, engineering notation numbers can have a sign,
    # a 3-digit integer part and a period, and a fractional part of length
    # frac_length, so the length of the number to the left of the
    # period is 4
    new_mant = new_mant.rjust(rjust * (4 + bfrac_length + frac_length))
    # Format number
    num = "{mant}{suffix}".format(
        mant=new_mant, suffix=_POWER_TO_SUFFIX_DICT[exp] if exp else " " * bool(rjust)
    )
    return num

@pexdoc.pcontracts.contract(snum="engineering_notation_number")
def peng_float(snum):
    r"""
    Return floating point equivalent of a number represented in engineering notation.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: string

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_float

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_float(peng.peng(1235.6789E3, 3, False))
        1236000.0
    """
    # This can be coded as peng_mant(snum)*(peng_power(snum)[1]), but the
    # "function unrolling" is about 4x faster
    snum = snum.rstrip()
    power = _SUFFIX_POWER_DICT[" " if snum[-1].isdigit() else snum[-1]]
    return float(snum if snum[-1].isdigit() else snum[:-1]) * power

@pexdoc.pcontracts.contract(snum="engineering_notation_number")
def peng_frac(snum):
    r"""
    Return the fractional part of a number represented in engineering notation.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: integer

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_frac

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_frac(peng.peng(1235.6789E3, 3, False))
        236
    """
    snum = snum.rstrip()
    pindex = snum.find(".")
    if pindex == -1:
        return 0
    return int(snum[pindex + 1 :] if snum[-1].isdigit() else snum[pindex + 1 : -1])

def peng_int(snum):
    r"""
    Return the integer part of a number represented in engineering notation.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: integer

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for peng.functions.peng_int

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_int(peng.peng(1235.6789E3, 3, False))
        1
    """
    return int(peng_mant(snum))

@pexdoc.pcontracts.contract(snum="engineering_notation_number")
def peng_mant(snum):
    r"""
    Return the mantissa of a number represented in engineering notation.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: float

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_mant

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_mant(peng.peng(1235.6789E3, 3, False))
        1.236
    """
    snum = snum.rstrip()
    return float(snum if snum[-1].isdigit() else snum[:-1])

@pexdoc.pcontracts.contract(snum="engineering_notation_number")
def peng_power(snum):
    r"""
    Return engineering suffix and its floating point equivalent of a number.

    :py:func:`peng.peng` lists the correspondence between suffix and floating
    point exponent.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: named tuple in which the first item is the engineering suffix and
            the second item is the floating point equivalent of the suffix
            when the number is represented in engineering notation.

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_power

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_power(peng.peng(1235.6789E3, 3, False))
        EngPower(suffix='M', exp=1000000.0)
    """
    suffix = " " if snum[-1].isdigit() else snum[-1]
    return EngPower(suffix, _SUFFIX_POWER_DICT[suffix])

@pexdoc.pcontracts.contract(snum="engineering_notation_number")
def peng_suffix(snum):
    r"""
    Return the suffix of a number represented in engineering notation.

    :param snum: Number
    :type  snum: :ref:`EngineeringNotationNumber`

    :rtype: string

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_suffix

    :raises: RuntimeError (Argument \`snum\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_suffix(peng.peng(1235.6789E3, 3, False))
        'M'
    """
    snum = snum.rstrip()
    return " " if snum[-1].isdigit() else snum[-1]

@pexdoc.pcontracts.contract(suffix="engineering_notation_suffix", offset=int)
def peng_suffix_math(suffix, offset):
    r"""
    Return engineering suffix from a starting suffix and an number of suffixes offset.

    :param suffix: Engineering suffix
    :type  suffix: :ref:`EngineeringNotationSuffix`

    :param offset: Engineering suffix offset
    :type  offset: integer

    :rtype: string

    .. [[[cog cog.out(exobj_eng.get_sphinx_autodoc()) ]]]
    .. Auto-generated exceptions documentation for
    .. peng.functions.peng_suffix_math

    :raises:
     * RuntimeError (Argument \`offset\` is not valid)

     * RuntimeError (Argument \`suffix\` is not valid)

     * ValueError (Argument \`offset\` is not valid)

    .. [[[end]]]

    For example:

        >>> import peng
        >>> peng.peng_suffix_math('u', 6)
        'T'
    """
    # pylint: disable=W0212
    eobj = pexdoc.exh.addex(ValueError, "Argument `offset` is not valid")
    try:
        return _POWER_TO_SUFFIX_DICT[_SUFFIX_TO_POWER_DICT[suffix] + 3 * offset]
    except KeyError:
        eobj(True)

def remove_extra_delims(expr, ldelim="(", rdelim=")"):
    r"""
    Remove unnecessary delimiters in mathematical expressions.

    Delimiters (parenthesis, brackets, etc.) may be removed either because
    there are multiple consecutive delimiters enclosing a single expressions or
    because the delimiters are implied by operator precedence rules. Function
    names must start with a letter and then can contain alphanumeric characters
    and a maximum of one underscore

    :param expr: Mathematical expression
    :type  expr: string

    :param ldelim: Single character left delimiter
    :type  ldelim: string

    :param rdelim: Single character right delimiter
    :type  rdelim: string

    :rtype: string

    :raises:
     * RuntimeError (Argument \`expr\` is not valid)

     * RuntimeError (Argument \`ldelim\` is not valid)

     * RuntimeError (Argument \`rdelim\` is not valid)

     * RuntimeError (Function name `*[function_name]*` is not valid)

     * RuntimeError (Mismatched delimiters)
    """
    op_group = ""
    for item1 in _OP_PREC:
        if isinstance(item1, list):
            for item2 in item1:
                op_group += item2
        else:
            op_group += item1
    iobj = zip([expr, ldelim, rdelim], ["expr", "ldelim", "rdelim"])
    for item, desc in iobj:
        if not isinstance(item, str):
            raise RuntimeError("Argument `{0}` is not valid".format(desc))
    if (len(ldelim) != 1) or ((len(ldelim) == 1) and (ldelim in op_group)):
        raise RuntimeError("Argument `ldelim` is not valid")
    if (len(rdelim) != 1) or ((len(rdelim) == 1) and (rdelim in op_group)):
        raise RuntimeError("Argument `rdelim` is not valid")
    if expr.count(ldelim) != expr.count(rdelim):
        raise RuntimeError("Mismatched delimiters")
    if not expr:
        return expr
    vchars = (
        "abcdefghijklmnopqrstuvwxyz"
        "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
        ".0123456789"
        r"_()[]\{\}" + rdelim + ldelim + op_group
    )
    if any([item not in vchars for item in expr]) or ("__" in expr):
        raise RuntimeError("Argument `expr` is not valid")
    expr = _remove_consecutive_delims(expr, ldelim=ldelim, rdelim=rdelim)
    expr = expr.replace(ldelim + rdelim, "")
    return _remove_extra_delims(expr, ldelim=ldelim, rdelim=rdelim)

def round_mantissa(arg, decimals=0):
    """
    Round floating point number(s) mantissa to given number of digits.

    Integers are not altered. The mantissa used is that of the floating point
    number(s) when expressed in `normalized scientific notation
    <https://en.wikipedia.org/wiki/Scientific_notation#Normalized_notation>`_

    :param arg: Input data
    :type  arg: integer, float, Numpy vector of integers or floats, or None

    :param decimals: Number of digits to round the fractional part of the
                     mantissa to.
    :type  decimals: integer

    :rtype: same as **arg**

    For example::

        >>> import peng
        >>> peng.round_mantissa(012345678E-6, 3)
        12.35
        >>> peng.round_mantissa(5, 3)
        5
    """
    if arg is None:
        return arg
    if isinstance(arg, np.ndarray):
        foi = [isinstance(item, int) for item in arg]
        return np.array(
            [
                item if isint else float(to_scientific_string(item, decimals))
                for isint, item in zip(foi, arg)
            ]
        )
    if isinstance(arg, int):
        return arg
    return float(to_scientific_string(arg, decimals))

def pprint_vector(vector, limit=False, width=None, indent=0, eng=False, frac_length=3):
    r"""
    Format a list of numbers (vector) or a Numpy vector for printing.

    If the argument **vector** is :code:`None` the string :code:`'None'` is
    returned

    :param vector: Vector to pretty print or None
    :type  vector: list of integers or floats, Numpy vector or None

    :param limit: Flag that indicates whether at most 6 vector items are
                  printed (all vector items if its length is equal or less
                  than 6, first and last 3 vector items if it is not) (True),
                  or the entire vector is printed (False)
    :type  limit: boolean

    :param width: Number of available characters per line. If None the vector
                  is printed in one line
    :type  width: integer or None

    :param indent: Flag that indicates whether all subsequent lines after the
                   first one are indented (True) or not (False). Only relevant
                   if **width** is not None
    :type  indent: boolean

    :param eng: Flag that indicates whether engineering notation is used
                (True) or not (False)
    :type  eng: boolean

    :param frac_length: Number of digits of fractional part (only applicable
                        if **eng** is True)
    :type  frac_length: integer

    :raises: ValueError (Argument \`width\` is too small)

    :rtype: string

    For example:

        >>> from __future__ import print_function
        >>> import peng
        >>> header = 'Vector: '
        >>> data = [1e-3, 20e-6, 300e+6, 4e-12, 5.25e3, -6e-9, 700, 8, 9]
        >>> print(
        ...     header+peng.pprint_vector(
        ...         data,
        ...         width=30,
        ...         eng=True,
        ...         frac_length=1,
        ...         limit=True,
        ...         indent=len(header)
        ...     )
        ... )
        Vector: [    1.0m,   20.0u,  300.0M,
                             ...
                   700.0 ,    8.0 ,    9.0  ]
        >>> print(
        ...     header+peng.pprint_vector(
        ...         data,
        ...         width=30,
        ...         eng=True,
        ...         frac_length=0,
        ...         indent=len(header)
        ...     )
        ... )
        Vector: [    1m,   20u,  300M,    4p,
                     5k,   -6n,  700 ,    8 ,
                     9  ]
        >>> print(peng.pprint_vector(data, eng=True, frac_length=0))
        [    1m,   20u,  300M,    4p,    5k,   -6n,  700 ,    8 ,    9  ]
        >>> print(peng.pprint_vector(data, limit=True))
        [ 0.001, 2e-05, 300000000.0, ..., 700, 8, 9 ]
    """
    # pylint: disable=R0912,R0913
    num_digits = 12
    approx = lambda x: float(x) if "." not in x else round(float(x), num_digits)

    def limstr(value):
        str1 = str(value)
        iscomplex = isinstance(value, complex)
        str1 = str1.lstrip("(").rstrip(")")
        if "." not in str1:
            return str1
        if iscomplex:
            sign = "+" if value.imag >= 0 else "-"
            regexp = re.compile(
                r"(.*(?:[Ee][\+-]\d+)?)"
                + (r"\+" if sign == "+" else "-")
                + r"(.*(?:[Ee][\+-]\d+)?j)"
            )
            rvalue, ivalue = regexp.match(str1).groups()
            return (
                str(complex(approx(rvalue), approx(sign + ivalue.strip("j"))))
                .lstrip("(")
                .rstrip(")")
            )
        str2 = str(round(value, num_digits))
        return str2 if len(str1) > len(str2) else str1

    def _str(*args):
        """
        Convert numbers to string, optionally represented in engineering notation.

        Numbers may be integers, float or complex
        """
        ret = [
            (limstr(element) if not eng else peng(element, frac_length, True))
            if not isinstance(element, complex)
            else (
                limstr(element)
                if not eng
                else "{real}{sign}{imag}j".format(
                    real=peng(element.real, frac_length, True),
                    imag=peng(abs(element.imag), frac_length, True),
                    sign="+" if element.imag >= 0 else "-",
                )
            )
            for element in args
        ]
        return ret[0] if len(ret) == 1 else ret

    if vector is None:
        return "None"
    lvector = len(vector)
    if (not limit) or (limit and (lvector < 7)):
        items = _str(*vector)
        uret = "[ {0} ]".format(", ".join(items))
    else:
        items = _str(*(vector[:3] + vector[-3:]))
        uret = "[ {0}, ..., {1} ]".format(", ".join(items[:3]), ", ".join(items[-3:]))
    if (width is None) or (len(uret) < width):
        return uret
    # -4 comes from the fact that an opening '[ ' and a closing ' ]'
    # are added to the multi-line vector string
    if any([len(item) > width - 4 for item in items]):
        raise ValueError("Argument `width` is too small")
    # Text needs to be wrapped in multiple lines
    # Figure out how long the first line needs to be
    wobj = textwrap.TextWrapper(initial_indent="[ ", width=width)
    # uret[2:] -> do not include initial '[ ' as this is specified as
    # the initial indent to the text wrapper
    rlist = wobj.wrap(uret[2:])
    first_line = rlist[0]
    first_line_elements = first_line.count(",")
    # Reconstruct string representation of vector excluding first line
    # Remove ... from text to be wrapped because it is placed in a single
    # line centered with the content
    uret_left = (",".join(uret.split(",")[first_line_elements:])).replace("...,", "")
    wobj = textwrap.TextWrapper(width=width - 2)
    wrapped_text = wobj.wrap(uret_left.lstrip())
    # Construct candidate wrapped and indented list of vector elements
    rlist = [first_line] + [
        (" " * (indent + 2)) + item.rstrip() for item in wrapped_text
    ]
    last_line = rlist[-1]
    last_line_elements = last_line.count(",") + 1
    # "Manually" format limit output so that it is either 3 lines, first and
    # last line with 3 elements and the middle with '...' or 7 lines, each with
    # 1 element and the middle with '...'
    # If numbers are not to be aligned at commas (variable width) then use the
    # existing results of the wrap() function
    if limit and (lvector > 6):
        if (first_line_elements < 3) or (
            (first_line_elements == 3) and (last_line_elements < 3)
        ):
            rlist = [
                "[ {0},".format(_str(vector[0])),
                _str(vector[1]),
                _str(vector[2]),
                "...",
                _str(vector[-3]),
                _str(vector[-2]),
                "{0} ]".format(_str(vector[-1])),
            ]
            first_line_elements = 1
        else:
            rlist = [
                "[ {0},".format(", ".join(_str(*vector[:3]))),
                "...",
                "{0} ]".format(", ".join(_str(*vector[-3:]))),
            ]
        first_line = rlist[0]
    elif limit:
        rlist = [item.lstrip() for item in rlist]
    first_comma_index = first_line.find(",")
    actual_width = len(first_line) - 2
    if not eng:
        if not limit:
            return "\n".join(rlist)
        num_elements = len(rlist)
        return "\n".join(
            [
                "{spaces}{line}{comma}".format(
                    spaces=(" " * (indent + 2)) if num > 0 else "",
                    line=(
                        line.center(actual_width).rstrip()
                        if line.strip() == "..."
                        else line
                    ),
                    comma=(
                        ","
                        if (
                            (num < num_elements - 1)
                            and (not line.endswith(","))
                            and (line.strip() != "...")
                        )
                        else ""
                    ),
                )
                if num > 0
                else line
                for num, line in enumerate(rlist)
            ]
        )
    # Align elements across multiple lines
    if limit:
        remainder_list = [line.lstrip() for line in rlist[1:]]
    else:
        remainder_list = _split_every(
            text=uret[len(first_line) :],
            sep=",",
            count=first_line_elements,
            lstrip=True,
        )
    new_wrapped_lines_list = [first_line]
    for line in remainder_list[:-1]:
        new_wrapped_lines_list.append(
            "{0},".format(line).rjust(actual_width)
            if line != "..."
            else line.center(actual_width).rstrip()
        )
    # Align last line on fist comma (if it exists) or
    # on length of field if does not
    if remainder_list[-1].find(",") == -1:
        marker = len(remainder_list[-1]) - 2
    else:
        marker = remainder_list[-1].find(",")
    new_wrapped_lines_list.append(
        "{0}{1}".format((first_comma_index - marker - 2) * " ", remainder_list[-1])
    )
    return "\n".join(
        [
            "{spaces}{line}".format(spaces=" " * (indent + 2), line=line)
            if num > 0
            else line
            for num, line in enumerate(new_wrapped_lines_list)
        ]
    )

def to_scientific_string(number, frac_length=None, exp_length=None, sign_always=False):
    """
    Convert number or number string to a number string in scientific notation.

    Full precision is maintained if the number is represented as a string

    :param number: Number to convert
    :type  number: number or string

    :param frac_length: Number of digits of fractional part, None indicates
                        that the fractional part of the number should not be
                        limited
    :type  frac_length: integer or None

    :param exp_length: Number of digits of the exponent; the actual length of
                       the exponent takes precedence if it is longer
    :type  exp_length: integer or None

    :param sign_always: Flag that indicates whether the sign always
                        precedes the number for both non-negative and negative
                        numbers (True) or only for negative numbers (False)
    :type  sign_always: boolean

    :rtype: string

    For example:

        >>> import peng
        >>> peng.to_scientific_string(333)
        '3.33E+2'
        >>> peng.to_scientific_string(0.00101)
        '1.01E-3'
        >>> peng.to_scientific_string(99.999, 1, 2, True)
        '+1.0E+02'
    """
    # pylint: disable=W0702
    try:
        number = -1e20 if np.isneginf(number) else number
    except:
        pass
    try:
        number = +1e20 if np.isposinf(number) else number
    except:
        pass
    exp_length = 0 if not exp_length else exp_length
    mant, exp = to_scientific_tuple(number)
    fmant = float(mant)
    if (not frac_length) or (fmant == int(fmant)):
        return "{sign}{mant}{period}{zeros}E{exp_sign}{exp}".format(
            sign="+" if sign_always and (fmant >= 0) else "",
            mant=mant,
            period="." if frac_length else "",
            zeros="0" * frac_length if frac_length else "",
            exp_sign="-" if exp < 0 else "+",
            exp=str(abs(exp)).rjust(exp_length, "0"),
        )
    rounded_mant = round(fmant, frac_length)
    # Avoid infinite recursion when rounded mantissa is _exactly_ 10
    if abs(rounded_mant) == 10:
        rounded_mant = fmant = -1.0 if number < 0 else 1.0
        frac_length = 1
        exp = exp + 1
    zeros = 2 + (1 if (fmant < 0) else 0) + frac_length - len(str(rounded_mant))
    return "{sign}{mant}{zeros}E{exp_sign}{exp}".format(
        sign="+" if sign_always and (fmant >= 0) else "",
        mant=rounded_mant,
        zeros="0" * zeros,
        exp_sign="-" if exp < 0 else "+",
        exp=str(abs(exp)).rjust(exp_length, "0"),
    )

def to_scientific_tuple(number):
    """
    Return mantissa and exponent of a number in scientific notation.

    Full precision is maintained if the number is represented as a string

    :param number: Number
    :type  number: integer, float or string

    :rtype: named tuple in which the first item is the mantissa (*string*)
            and the second item is the exponent (*integer*) of the number
            when expressed in scientific notation

    For example:

        >>> import peng
        >>> peng.to_scientific_tuple('135.56E-8')
        NumComp(mant='1.3556', exp=-6)
        >>> peng.to_scientific_tuple(0.0000013556)
        NumComp(mant='1.3556', exp=-6)
    """
    # pylint: disable=W0632
    convert = not isinstance(number, str)
    # Detect zero and return, simplifies subsequent algorithm
    if (convert and (number == 0)) or (
        (not convert) and (not number.strip("0").strip("."))
    ):
        return ("0", 0)
    # Break down number into its components, use Decimal type to
    # preserve resolution:
    # sign  : 0 -> +, 1 -> -
    # digits: tuple with digits of number
    # exp   : exponent that gives null fractional part
    sign, digits, exp = Decimal(str(number) if convert else number).as_tuple()
    mant = (
        "{sign}{itg}{frac}".format(
            sign="-" if sign else "",
            itg=digits[0],
            frac=(
                ".{frac}".format(frac="".join([str(num) for num in digits[1:]]))
                if len(digits) > 1
                else ""
            ),
        )
        .rstrip("0")
        .rstrip(".")
    )
    exp += len(digits) - 1
    return NumComp(mant, exp)


from __future__ import unicode_literals

import hashlib
import io
import json
import logging
import os
import posixpath
import subprocess

from django.conf import settings
from django.contrib.staticfiles.storage import staticfiles_storage
from django.utils.encoding import force_text

import semantic_version

from .jspm import locate_package_json

logger = logging.getLogger(__name__)

JSPM_LOG_VERSION = semantic_version.Version('0.16.3')

SOURCEMAPPING_URL_COMMENT = b'//# sourceMappingURL='

NODE_ENV_VAR = 'NODE_PATH'

class BundleError(OSError):
    pass

class System(object):

    def __init__(self, **opts):
        self.opts = opts
        self.stdout = self.stdin = self.stderr = subprocess.PIPE
        self.cwd = None
        self.version = None  # JSPM version

    def _has_jspm_log(self):
        return self.jspm_version and self.jspm_version >= JSPM_LOG_VERSION

    def get_jspm_version(self, opts):
        jspm_bin = opts['jspm']
        cmd = '{} --version'.format(jspm_bin)
        proc = subprocess.Popen(
            cmd, shell=True, cwd=self.cwd, stdout=self.stdout,
            stdin=self.stdin, stderr=self.stderr)
        result, err = proc.communicate()  # block until it's done
        if err:
            raise BundleError("Could not determine JSPM version, error: %s", err)
        version_string = result.decode().split()[0]
        return semantic_version.Version(version_string, partial=True)

    @property
    def jspm_version(self):
        if not self.version:
            options = self.opts.copy()
            options.setdefault('jspm', settings.SYSTEMJS_JSPM_EXECUTABLE)
            self.version = self.get_jspm_version(options)
        return self.version

    def bundle(self, app):
        bundle = SystemBundle(self, app, **self.opts)
        return bundle.bundle()

    @staticmethod
    def get_bundle_path(app):
        """
        Returns the path relative to STATIC_URL for the bundle for app.
        """
        bundle = SystemBundle(None, app)
        return bundle.get_paths()[1]

class SystemBundle(object):
    """
    Represents a single app to be bundled.
    """

    def __init__(self, system, app, **options):
        """
        Initialize a SystemBundle object.

        :param system: a System instance that holds the non-bundle specific
        meta information (such as jspm version, configuration)

        :param app: string, the name of the JS package to bundle. This may be
        missing the '.js' extension.

        :param options: dict containing the bundle-specific options. Possible
        options:
            `jspm`: `jspm` executable (if it's not on $PATH, for example)
            `log`: logging mode for jspm, can be ok|warn|err. Only available
                   for jspm >= 0.16.3
            `minify`: boolean, whether go generate minified bundles or not
            `sfx`: boolean, generate a self-executing bundle or not
            `skip-source-maps: boolean, whether to generate source maps or not
        """
        self.system = system
        self.app = app

        # set the bundle options
        options.setdefault('jspm', settings.SYSTEMJS_JSPM_EXECUTABLE)
        self.opts = options

        bundle_cmd = self.get_bundle_sfx_cmd() if self.opts.get('sfx') else 'bundle'
        self.command = '{jspm} ' + bundle_cmd + ' {app} {outfile}'

        self.stdout = self.stdin = self.stderr = subprocess.PIPE

    def get_bundle_sfx_cmd(self):
        spec = semantic_version.Spec('>=0.17.0')
        return 'build' if self.system.jspm_version in spec else 'bundle-sfx'

    def get_outfile(self):
        js_file = '{app}{ext}'.format(app=self.app, ext='.js' if self.needs_ext() else '')
        outfile = os.path.join(settings.STATIC_ROOT, settings.SYSTEMJS_OUTPUT_DIR, js_file)
        return outfile

    def get_paths(self):
        """
        Return a tuple with the absolute path and relative path (relative to STATIC_URL)
        """
        outfile = self.get_outfile()
        rel_path = os.path.relpath(outfile, settings.STATIC_ROOT)
        return outfile, rel_path

    def needs_ext(self):
        """
        Check whether `self.app` is missing the '.js' extension and if it needs it.
        """
        if settings.SYSTEMJS_DEFAULT_JS_EXTENSIONS:
            name, ext = posixpath.splitext(self.app)
            if not ext:
                return True
        return False

    def bundle(self):
        """
        Bundle the app and return the static url to the bundle.
        """
        outfile, rel_path = self.get_paths()

        options = self.opts
        if self.system._has_jspm_log():
            self.command += ' --log {log}'
            options.setdefault('log', 'err')

        if options.get('minify'):
            self.command += ' --minify'

        if options.get('skip_source_maps'):
            self.command += ' --skip-source-maps'

        try:
            cmd = self.command.format(app=self.app, outfile=outfile, **options)
            proc = subprocess.Popen(
                cmd, shell=True, cwd=self.system.cwd, stdout=self.stdout,
                stdin=self.stdin, stderr=self.stderr)

            result, err = proc.communicate()  # block until it's done
            if err and self.system._has_jspm_log():
                fmt = 'Could not bundle \'%s\': \n%s'
                logger.warn(fmt, self.app, err)
                raise BundleError(fmt % (self.app, err))
            if result.strip():
                logger.info(result)
        except (IOError, OSError) as e:
            if isinstance(e, BundleError):
                raise
            raise BundleError('Unable to apply %s (%r): %s' % (
                              self.__class__.__name__, cmd, e))
        else:
            if not options.get('sfx'):
                # add the import statement, which is missing for non-sfx bundles
                sourcemap = find_sourcemap_comment(outfile)
                with open(outfile, 'a') as of:
                    of.write("\nSystem.import('{app}{ext}');\n{sourcemap}".format(
                        app=self.app,
                        ext='.js' if self.needs_ext() else '',
                        sourcemap=sourcemap if sourcemap else '',
                    ))
        return rel_path

class TraceError(Exception):
    pass

class SystemTracer(object):

    def __init__(self, node_path=None):
        node_env = os.environ.copy()
        if node_path and NODE_ENV_VAR not in node_env:
            node_env[NODE_ENV_VAR] = node_path
        self.env = node_env
        self.name = 'deps.json'
        self.storage = staticfiles_storage
        self._trace_cache = {}
        self._package_json_dir = os.path.dirname(locate_package_json())

    @property
    def cache_file_path(self):
        if not os.path.exists(settings.SYSTEMJS_CACHE_DIR):
            os.makedirs(settings.SYSTEMJS_CACHE_DIR)
        return os.path.join(settings.SYSTEMJS_CACHE_DIR, self.name)

    def trace(self, app):
        """
        Trace the dependencies for app.

        A tracer-instance is shortlived, and re-tracing the same app should
        yield the same results. Since tracing is an expensive process, cache
        the result on the tracer instance.
        """
        if app not in self._trace_cache:
            process = subprocess.Popen(
                "trace-deps.js {}".format(app), shell=True,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                env=self.env, universal_newlines=True, cwd=self._package_json_dir
            )
            out, err = process.communicate()
            if err:
                raise TraceError(err)
            self._trace_cache[app] = json.loads(out)
        return self._trace_cache[app]

    def get_hash(self, path):
        md5 = hashlib.md5()
        with self.storage.open(path) as infile:
            for chunk in infile.chunks():
                md5.update(chunk)
        return md5.hexdigest()

    def write_depcache(self, app_deps, bundle_options):  # TODO: use storage
        all_deps = {
            'version': 1,
            'packages': app_deps,
            'hashes': {},
            'options': bundle_options,
        }

        for pkg_deptree in app_deps.values():
            for module, info in pkg_deptree.items():
                # issue #13 - external resources are not included in the bundle,
                # so don't include them in the depcache either
                if info.get('skip', False):
                    continue
                path = info['path']
                if path not in all_deps['hashes']:
                    all_deps['hashes'][path] = self.get_hash(path)

        with open(self.cache_file_path, 'w') as outfile:
            json.dump(all_deps, outfile)

    @property
    def cached_deps(self):
        if not hasattr(self, '_depcache'):
            with open(self.cache_file_path, 'r') as infile:
                self._depcache = json.load(infile)
        return self._depcache

    def get_depcache(self, app):
        # cache in memory for faster lookup
        if self.cached_deps.get('version') == 1:
            return self.cached_deps['packages'].get(app)
        else:
            raise NotImplementedError  # noqa

    def get_hashes(self):
        if self.cached_deps.get('version') == 1:
            return self.cached_deps['hashes']
        else:
            raise NotImplementedError  # noqa

    def get_bundle_options(self):
        if self.cached_deps.get('version') == 1:
            return self.cached_deps.get('options')
        else:
            raise NotImplementedError  # noqa

    def hashes_match(self, dep_tree):
        """
        Compares the app deptree file hashes with the hashes stored in the
        cache.
        """
        hashes = self.get_hashes()
        for module, info in dep_tree.items():
            md5 = self.get_hash(info['path'])
            if md5 != hashes[info['path']]:
                return False
        return True

    def check_needs_update(self, app):
        cached_deps = self.get_depcache(app)
        deps = self.trace(app)
        # no re-bundle needed if the trees, mtimes and file hashes match
        if deps == cached_deps and self.hashes_match(deps):
            return False
        return True

def find_sourcemap_comment(filepath, block_size=100):
    """
    Seeks and removes the sourcemap comment. If found, the sourcemap line is
    returned.

    Bundled output files can have massive amounts of lines, and the sourceMap
    comment is always at the end. So, to extract it efficiently, we read out the
    lines of the file starting from the end. We look back at most 2 lines.

    :param:filepath: path to output bundle file containing the sourcemap comment
    :param:blocksize: integer saying how many bytes to read at once
    :return:string with the sourcemap comment or None
    """

    MAX_TRACKBACK = 2  # look back at most 2 lines, catching potential blank line at the end

    block_number = -1
    # blocks of size block_size, in reverse order starting from the end of the file
    blocks = []
    sourcemap = None

    try:
        # open file in binary read+write mode, so we can seek with negative offsets
        of = io.open(filepath, 'br+')
        # figure out what's the end byte
        of.seek(0, os.SEEK_END)
        block_end_byte = of.tell()

        # track back for maximum MAX_TRACKBACK lines and while we can track back
        while block_end_byte > 0 and MAX_TRACKBACK > 0:
            if (block_end_byte - block_size > 0):
                # read the last block we haven't yet read
                of.seek(block_number*block_size, os.SEEK_END)
                blocks.append(of.read(block_size))
            else:
                # file too small, start from begining
                of.seek(0, os.SEEK_SET)
                # only read what was not read
                blocks = [of.read(block_end_byte)]

            # update variables that control while loop
            content = b''.join(reversed(blocks))
            lines_found = content.count(b'\n')
            MAX_TRACKBACK -= lines_found
            block_end_byte -= block_size
            block_number -= 1

            # early check and bail out if we found the sourcemap comment
            if SOURCEMAPPING_URL_COMMENT in content:
                offset = 0
                # splitlines eats the last \n if its followed by a blank line
                lines = content.split(b'\n')
                for i, line in enumerate(lines):
                    if line.startswith(SOURCEMAPPING_URL_COMMENT):
                        offset = len(line)
                        sourcemap = line
                        break
                while i+1 < len(lines):
                    offset += 1  # for the newline char
                    offset += len(lines[i+1])
                    i += 1
                # track back until the start of the comment, and truncate the comment
                if sourcemap:
                    offset += 1  # for the newline before the sourcemap comment
                    of.seek(-offset, os.SEEK_END)
                    of.truncate()
                return force_text(sourcemap)
    finally:
        of.close()
    return sourcemap


# This file is copyright Arch Systems, Inc.
# Except as otherwise provided in the relevant LICENSE file, all rights are reserved.

# pylint: disable=unused-argument,missing-docstring

#bytes.py
#Simple bytearray type

import sys
from binascii import unhexlify, hexlify

def convert(arg, **kwargs):
    if isinstance(arg, bytearray):
        return arg
    if isinstance(arg, str):
        if len(arg) > 2 and arg.startswith("0x"):
            data = unhexlify(arg[2:])
        else:
            data = arg

        return bytearray(data)
    if isinstance(arg, bytes):
        return arg

    raise TypeError("You must create a bytes object from bytes, bytearray or a hex string")

def convert_binary(arg, **kwargs):
    return bytearray(arg)

def default_formatter(arg, **kwargs):
    return str(arg)

def format_repr(arg):
    return repr(arg)

def format_hex(arg):
    """Convert the bytes object to a hex string."""

    return hexlify(arg).decode('utf-8')

def format_hexdump(arg):
    """Convert the bytes object to a hexdump.

    The output format will be:

    <offset, 4-byte>  <16-bytes of output separated by 1 space>  <16 ascii characters>
    """

    line = ''

    for i in range(0, len(arg), 16):
        if i > 0:
            line += '\n'
        chunk = arg[i:i + 16]
        hex_chunk = hexlify(chunk).decode('utf-8')
        hex_line = ' '.join(hex_chunk[j:j + 2] for j in range(0, len(hex_chunk), 2))

        if len(hex_line) < (3 * 16) - 1:
            hex_line += ' ' * (((3 * 16) - 1) - len(hex_line))

        ascii_line = ''.join(_convert_to_ascii(x) for x in chunk)
        offset_line = '%08x' % i

        line += "%s  %s  %s" % (offset_line, hex_line, ascii_line)

    return line

def _convert_to_ascii(byte):
    if byte < 32 or byte > 126:
        return '.'

    return chr(byte)


"""Routines for extracting parameter and return information from a docstring."""

import inspect
from .doc_parser import parse_param, parse_return

def parse_docstring(doc):
    """Parse a docstring into ParameterInfo and ReturnInfo objects."""

    doc = inspect.cleandoc(doc)
    lines = doc.split('\n')
    section = None
    section_indent = None

    params = {}
    returns = None

    for line in lines:
        line = line.rstrip()

        if len(line) == 0:
            continue
        elif str(line) == 'Args:':
            section = 'args'
            section_indent = None
            continue
        elif str(line) == 'Returns:':
            section = 'return'
            section_indent = None
            continue

        if section is not None:
            stripped = line.lstrip()
            margin = len(line) - len(stripped)

            if section_indent is None:
                section_indent = margin

            if margin != section_indent:
                continue

            # These are all the param lines in the docstring that are
            # not continuations of the previous line
            if section == 'args':
                param_name, type_info = parse_param(stripped)
                params[param_name] = type_info
            elif section == 'return':
                returns = parse_return(stripped)

    return params, returns


"""Hierarchical Shell provides as REPL like environment for executing commmands."""

# This file is adapted from python code released by WellDone International
# under the terms of the LGPLv3.  WellDone International's contact information is
# info@welldone.org
# http://welldone.org
#
# Modifications to this file from the original created at WellDone International
# are copyright Arch Systems Inc.

#shell.py
#Given a command line string, attempt to map it to a function and fill in
#the parameters based on that function's annotated type information.

import inspect
import shlex
import platform
import importlib
from typedargs.exceptions import ArgumentError, NotFoundError, ValidationError
from typedargs import annotate, utils
from typedargs import iprint
from typedargs.typeinfo import type_system

@annotate.context("root")
class InitialContext(dict):
    """A basic context for holding the root callable functions for a shell."""
    pass

class HierarchicalShell:
    """A hierarchical shell for navigating through python package API functions."""

    def __init__(self, name):
        self.name = name
        self.init_commands = {}

        self.root = InitialContext()
        self.contexts = [self.root]

        # Keep track of whether we are on windows because shlex does not dequote
        # strings the same on Windows as on other platforms
        self.posix_lex = platform.system() != 'Windows'

        #Initialize the root context if required
        self._check_initialize_context()

        # Initialize our builtin functions
        self.builtins = {}
        self.add_builtin('back', self._builtin_back)
        self.add_builtin('help', self._builtin_help)
        self.add_builtin('quit', self._builtin_quit)

    def add_builtin(self, name, callable):
        """Add a builtin function callable from all contexts.

        Callable should be an annotated function like any other that you
        want to call from a HierarchicalShell

        Args:
            name (str): The name of the function
            callable (callable): The annotated function that we should call
        """

        self.builtins[name] = callable

    def root_update(self, dict_like):
        """Add entries to root from a dict_line object."""
        self.root.update(dict_like)

    def root_add(self, name, value):
        """Add a single function to the root context.

        Args:
            name (str): The name of the callable to add
            value (str or callable): The callable function or a string to
                lazily resolve to the callable later.
        """

        self.root[name] = value

    def context_name(self):
        """Get the string name of the current context."""
        return utils.context_name(self.contexts[-1])

    def finished(self):
        """Check if we have finalized all contexts.

        Returns:
            bool: True if there are no nested contexts left, False otherwise
        """

        return len(self.contexts) == 0

    def valid_identifiers(self):
        """Get a list of all valid identifiers for the current context.

        Returns:
            list(str): A list of all of the valid identifiers for this context
        """

        funcs = list(utils.find_all(self.contexts[-1])) + list(self.builtins)
        return funcs

    @classmethod
    def _deferred_add(cls, add_action):
        """Lazily load a callable.

        Perform a lazy import of a context so that we don't have a huge initial startup time
        loading all of the modules that someone might want even though they probably only
        will use a few of them.
        """

        module, _, obj = add_action.partition(',')

        mod = importlib.import_module(module)
        if obj == "":
            _, con = annotate.context_from_module(mod)
            return con

        if hasattr(mod, obj):
            return getattr(mod, obj)

        raise ArgumentError("Attempted to import nonexistent object from module", module=module, object=obj)

    @classmethod
    def _remove_quotes(cls, word):
        if len(word) > 0 and word.startswith(("'", '"')) and word[0] == word[-1]:
            return word[1:-1]

        return word

    def _split_line(self, line):
        """Split a line into arguments using shlex and a dequoting routine."""

        parts = shlex.split(line, posix=self.posix_lex)
        if not self.posix_lex:
            parts = [self._remove_quotes(x) for x in parts]

        return parts

    def _check_initialize_context(self):
        """
        Check if our context matches something that we have initialization commands
        for.  If so, run them to initialize the context before proceeding with other
        commands.
        """

        path = ".".join([annotate.context_name(x) for x in self.contexts])

        #Make sure we don't clutter up the output with return values from
        #initialization functions
        old_interactive = type_system.interactive
        type_system.interactive = False

        for key, cmds in self.init_commands.items():
            if path.endswith(key):
                for cmd in cmds:
                    line = self._split_line(cmd)
                    self.invoke(line)

        type_system.interactive = old_interactive

    @annotate.finalizer
    def _builtin_back(self):
        """Pop the current context and return to its parent."""
        pass  # Popping the context is handled by invoke since this is marked as a finalizer

    @annotate.takes_cmdline
    def _builtin_quit(self, _cmdline):
        """Quit this hierarchical shell."""
        del self.contexts[:]

    @annotate.takes_cmdline
    @annotate.stringable
    def _builtin_help(self, args):
        """Return help information for a context or function."""

        if len(args) == 0:
            return self.list_dir(self.contexts[-1])
        if len(args) == 1:
            func = self.find_function(self.contexts[-1], args[0])
            return annotate.get_help(func)

        help_text = "Too many arguments: " + str(args) + "\n"
        help_text += "Usage: help [function]"
        return help_text

    def find_function(self, context, funname):
        """Find a function in the given context by name.

        This function will first search the list of builtins and if the
        desired function is not a builtin, it will continue to search
        the given context.

        Args:
            context (object): A dict or class that is a typedargs context
            funname (str): The name of the function to find

        Returns:
            callable: The found function.
        """

        if funname in self.builtins:
            return self.builtins[funname]

        func = None
        if isinstance(context, dict):
            if funname in context:
                func = context[funname]

                #Allowed lazy loading of functions
                if isinstance(func, str):
                    func = self._deferred_add(func)
                    context[funname] = func
        elif hasattr(context, funname):
            func = getattr(context, funname)

        if func is None:
            raise NotFoundError("Function not found", function=funname)

        return func

    def list_dir(self, context):
        """Return a listing of all of the functions in this context including builtins.

        Args:
            context (object): The context to print a directory for.

        Returns:
            str
        """

        doc = inspect.getdoc(context)

        listing = ""
        listing += "\n"

        listing += annotate.context_name(context) + "\n"

        if doc is not None:
            doc = inspect.cleandoc(doc)
            listing += doc + "\n"

        listing += "\nDefined Functions:\n"
        is_dict = False

        if isinstance(context, dict):
            funs = context.keys()
            is_dict = True
        else:
            funs = utils.find_all(context)

        for fun in sorted(funs):
            override_name = None
            if is_dict:
                override_name = fun

            fun = self.find_function(context, fun)

            if isinstance(fun, dict):
                if is_dict:
                    listing += " - " + override_name + '\n'
                else:
                    listing += " - " + fun.metadata.name + '\n'
            else:
                listing += " - " + fun.metadata.signature(name=override_name) + '\n'

            if annotate.short_description(fun) != "":
                listing += "   " + annotate.short_description(fun) + '\n'

        listing += "\nBuiltin Functions\n"
        for bif in sorted(self.builtins.keys()):
            listing += ' - ' + bif + '\n'

        listing += '\n'
        return listing

    @classmethod
    def _is_flag(cls, arg):
        """Check if an argument is a flag.

        A flag starts with - or -- and the next character must be a letter
        followed by letters, numbers, - or _.  Currently we only check the
        alpha'ness of the first non-dash character to make sure we're not just
        looking at a negative number.

        Returns:
            bool: Whether the argument is a flag.
        """

        if arg == '--':
            return False

        if not arg.startswith('-'):
            return False

        if arg.startswith('--'):
            first_char = arg[2]
        else:
            first_char = arg[1]

        if not first_char.isalpha():
            return False

        return True

    def process_arguments(self, func, args):
        """Process arguments from the command line into positional and kw args.

        Arguments are consumed until the argument spec for the function is filled
        or a -- is found or there are no more arguments.  Keyword arguments can be
        specified using --field=value, -f value or --field value.  Positional
        arguments are specified just on the command line itself.

        If a keyword argument (`field`) is a boolean, it can be set to True by just passing
        --field or -f without needing to explicitly pass True unless this would cause
        ambiguity in parsing since the next expected positional argument is also a boolean
        or a string.

        Args:
            func (callable): A function previously annotated with type information
            args (list): A list of all of the potential arguments to this function.

        Returns:
            (args, kw_args, unused args): A tuple with a list of args, a dict of
                keyword args and a list of any unused args that were not processed.
        """

        pos_args = []
        kw_args = {}

        while len(args) > 0:
            if func.metadata.spec_filled(pos_args, kw_args) and not self._is_flag(args[0]):
                break

            arg = args.pop(0)

            if arg == '--':
                break
            elif self._is_flag(arg):
                arg_value = None
                arg_name = None

                if len(arg) == 2:
                    arg_name = func.metadata.match_shortname(arg[1:], filled_args=pos_args)
                else:
                    if not arg.startswith('--'):
                        raise ArgumentError("Invalid method of specifying keyword argument that did not start with --", argument=arg)

                    # Skip the --
                    arg = arg[2:]

                    # Check if the value is embedded in the parameter
                    # Make sure we allow the value after the equals sign to also
                    # contain an equals sign.
                    if '=' in arg:
                        arg, arg_value = arg.split('=', 1)

                    arg_name = func.metadata.match_shortname(arg, filled_args=pos_args)

                arg_type = func.metadata.param_type(arg_name)
                if arg_type is None:
                    raise ArgumentError("Attempting to set a parameter from command line that does not have type information", argument=arg_name)

                # If we don't have a value yet, attempt to get one from the next parameter on the command line
                if arg_value is None:
                    arg_value = self._extract_arg_value(arg_name, arg_type, args)

                kw_args[arg_name] = arg_value
            else:
                pos_args.append(arg)

        # Always check if there is a trailing '--' and chomp so that we always
        # start on a function name.  This can happen if there is a gratuitous
        # -- for a 0 arg function or after an implicit boolean flag like -f --
        if len(args) > 0 and args[0] == '--':
            args.pop(0)

        return pos_args, kw_args, args

    @classmethod
    def _extract_arg_value(cls, arg_name, arg_type, remaining):
        """Try to find the value for a keyword argument."""

        next_arg = None
        should_consume = False
        if len(remaining) > 0:
            next_arg = remaining[0]
            should_consume = True

            if next_arg == '--':
                next_arg = None

        # Generally we just return the next argument, however if the type
        # is bool we allow not specifying anything to mean true if there
        # is no ambiguity
        if arg_type == "bool":
            if next_arg is None or next_arg.startswith('-'):
                next_arg = True
                should_consume = False
        else:
            if next_arg is None:
                raise ArgumentError("Could not find value for keyword argument", argument=arg_name)

        if should_consume:
            remaining.pop(0)

        return next_arg

    def invoke_one(self, line):
        """Invoke a function given a list of arguments with the function listed first.

        The function is searched for using the current context on the context stack
        and its annotated type information is used to convert all of the string parameters
        passed in line to appropriate python types.

        Args:
            line (list): The list of command line arguments.

        Returns:
            (object, list, bool): A tuple containing the return value of the function, if any,
                a boolean specifying if the function created a new context (False if a new context
                was created) and a list with the remainder of the command line if this function
                did not consume all arguments.
        """

        funname = line.pop(0)

        context = self.contexts[-1]
        func = self.find_function(context, funname)

        #If this is a context derived from a module or package, just jump to it
        #since there is no initialization function
        if isinstance(func, dict):
            self.contexts.append(func)
            self._check_initialize_context()
            return None, line, False

        # If the function wants arguments directly, do not parse them, otherwise turn them
        # into positional and kw arguments
        if func.takes_cmdline is True:
            val = func(line)
            line = []
        else:
            posargs, kwargs, line = self.process_arguments(func, line)

            #We need to check for not enough args for classes before calling or the call won't make it all the way to __init__
            if inspect.isclass(func) and not func.metadata.spec_filled(posargs, kwargs):
                raise ValidationError("Not enough parameters specified to call function", function=func.metadata.name, signature=func.metadata.signature())

            val = func(*posargs, **kwargs)

        # Update our current context if this function destroyed it or returned a new one.
        finished = True

        if func.finalizer is True:
            self.contexts.pop()
        elif val is not None:
            if func.metadata.returns_data():
                val = func.metadata.format_returnvalue(val)
            else:
                self.contexts.append(val)
                self._check_initialize_context()
                finished = False
                val = None

        return val, line, finished

    def invoke(self, line):
        """Invoke a one or more function given a list of arguments.

        The functions are searched for using the current context on the context stack
        and its annotated type information is used to convert all of the string parameters
        passed in line to appropriate python types.

        Args:
            line (list): The list of command line arguments.

        Returns:
            bool: A boolean specifying if the last function created a new context
                (False if a new context was created) and a list with the remainder of the
                command line if this function did not consume all arguments.)
        """

        finished = True

        while len(line) > 0:
            val, line, finished = self.invoke_one(line)
            if val is not None:
                iprint(val)

        return finished

    def invoke_string(self, line):
        """Parse and invoke a string line.

        Args:
            line (str): The line that we want to parse and invoke.

        Returns:
            bool: A boolean specifying if the last function created a new context
                (False if a new context was created) and a list with the remainder of the
                command line if this function did not consume all arguments.)
        """

        # Make sure line is a unicode string on all python versions
        line = str(line)

        # Ignore empty lines and comments
        if len(line) == 0:
            return True

        if line[0] == u'#':
            return True

        args = self._split_line(line)
        return self.invoke(args)


"""Experimental module for better google docstring parsing."""

import inspect
from io import StringIO
from collections import namedtuple
from textwrap import fill, dedent
from .basic_structures import ParameterInfo, ReturnInfo
from .exceptions import ValidationError
from .terminal import get_terminal_size

BlankLine = namedtuple("BlankLine", ['contents'])
SectionHeader = namedtuple("SectionHeader", ['name'])
ContinuationLine = namedtuple("ContinuationLine", ['contents'])
Line = namedtuple("Line", ['contents'])
ListItem = namedtuple("ListItem", ['marker', 'contents'])

# pylint: disable=too-few-public-methods;Experimental class
class ParsedDocstring:
    """A parser for google docstrings.

    The parser will collect all of the sections and parse them into
    paragraphs that are separated by blank lines or indented sections
    depending on what kind of section it is.
    """

    MAIN_SECTION = 0
    ARGS_SECTION = 1
    RETURN_SECTION = 2

    def __init__(self, doc):
        self._sections = {ParsedDocstring.MAIN_SECTION: [], ParsedDocstring.ARGS_SECTION: [], ParsedDocstring.RETURN_SECTION: []}

        sections = self._collect_sections(doc)

        for order, section in sorted(sections, key=lambda x: x[0]):
            lines = sections[(order, section)]
            sec_type = self._classify_section(section)
            if sec_type is None:
                lines = [BlankLine(""), Line(section + ":"), BlankLine("")] + lines + [BlankLine("")]
                sec_type = ParsedDocstring.MAIN_SECTION

            self._sections[sec_type].extend(lines)

        for section, lines in self._sections.items():
            lines = self._merge_blank_lines(lines)
            self._sections[section] = lines

        self.param_info = {parse_param(arg.contents, True)[0]: parse_param(arg.contents, True)[1] for arg in self._sections.get(self.ARGS_SECTION, []) if isinstance(arg, Line)}
        self.return_info = None
        if len(self._sections.get(ParsedDocstring.RETURN_SECTION, [])) > 0:
            self.return_info = parse_return(self._sections[self.RETURN_SECTION][0].contents, True)

        self.maindoc = self._merge_adjacent_lines(self._sections[ParsedDocstring.MAIN_SECTION])

    @property
    def short_desc(self):
        """One line description of this parsed docstring.

        Returns:
            str
        """

        if len(self.maindoc) == 0:
            return ""

        return self.maindoc[0].contents

    @classmethod
    def _merge_adjacent_lines(cls, lines):
        out_lines = []

        for line in lines:
            if not isinstance(line, Line) or len(out_lines) == 0 or not isinstance(out_lines[-1], Line):
                out_lines.append(line)
            else:
                out_lines[-1] = Line(out_lines[-1].contents + " " + line.contents)

        return out_lines

    @classmethod
    def _merge_blank_lines(cls, lines):
        out_lines = []

        for line in lines:
            if not isinstance(line, BlankLine) or len(out_lines) == 0 or not isinstance(out_lines[-1], BlankLine):
                out_lines.append(line)

        if len(out_lines) > 0 and isinstance(out_lines[-1], BlankLine):
            out_lines = out_lines[:-1]

        if len(out_lines) > 0 and isinstance(out_lines[0], BlankLine):
            out_lines = out_lines[1:]

        return out_lines

    @classmethod
    def _merge_continuations(cls, lines):
        if len(lines) == 0:
            return []

        out_lines = []
        curr_line = lines[0]

        for line in lines[1:]:
            if isinstance(line, ContinuationLine):
                if curr_line is None:
                    out_lines.append(Line(line.contents))
                elif isinstance(curr_line, Line):
                    curr_line = Line(curr_line.contents + ' ' + line.contents)
                elif isinstance(curr_line, ListItem):
                    curr_line = ListItem(curr_line.marker, curr_line.contents + ' ' + line.contents)
            else:
                if curr_line is not None:
                    out_lines.append(curr_line)

                curr_line = None

                if isinstance(line, BlankLine):
                    out_lines.append(line)
                else:
                    curr_line = line

        if curr_line is not None:
            out_lines.append(curr_line)

        return out_lines

    @classmethod
    def _classify_section(cls, section):
        """Attempt to find the canonical name of this section."""

        name = section.lower()

        if name in frozenset(['args', 'arguments', "params", "parameters"]):
            return cls.ARGS_SECTION

        if name in frozenset(['returns', 'return']):
            return cls.RETURN_SECTION

        if name in frozenset(['main']):
            return cls.MAIN_SECTION

        return None

    @classmethod
    def _classify_line(cls, line):
        """Classify a line into a type of object."""

        line = line.rstrip()

        if len(line) == 0:
            return BlankLine('')

        if ' ' not in line and line.endswith(':'):
            name = line[:-1]
            return SectionHeader(name)

        if line.startswith('  '):
            return ContinuationLine(line.lstrip())

        if line.startswith(' - '):
            return ListItem('-', line[3:].lstrip())

        if line.startswith('- '):
            return ListItem('-', line[2:].lstrip())

        return Line(line)

    @classmethod
    def _parse_sections(cls, lines):
        curr_section = "Main"
        curr_lines = []
        sections = {}

        section_counter = 0

        for line in lines:
            line_obj = cls._classify_line(line)
            if isinstance(line_obj, SectionHeader):
                sections[(section_counter, curr_section)] = curr_lines
                curr_lines = []
                curr_section = line_obj.name
                section_counter += 1
            else:
                curr_lines.append(line)

        sections[(section_counter, curr_section)] = curr_lines
        return sections

    @classmethod
    def _dedent_and_parse(cls, lines):
        paragraph = '\n'.join(lines)
        dedented = dedent(paragraph)
        lines = dedented.splitlines()
        return [cls._classify_line(x) for x in lines]

    #pylint:disable=too-many-branches
    @classmethod
    def _collect_sections(cls, doc):
        doc = inspect.cleandoc(doc)
        lines = doc.splitlines()

        sections = cls._parse_sections(lines)

        for section in sections:
            reclassified = cls._dedent_and_parse(sections[section])
            sections[section] = cls._merge_continuations(reclassified)

        return sections

    @classmethod
    def _join_paragraph(cls, lines, leading_blanks, trailing_blanks):
        if leading_blanks is False:
            remove_count = 0
            for line in lines:
                if len(line) == 0:
                    remove_count += 1
                else:
                    break

            if remove_count > 0:
                lines = lines[remove_count:]

        if trailing_blanks is False:
            remove_count = 0
            for line in reversed(lines):
                if len(line) == 0:
                    remove_count += 1
                else:
                    break

            if remove_count > 0:
                lines = lines[:-remove_count]

        return " ".join(lines)

    @classmethod
    def _join_paragraphs(cls, lines, use_indent=False, leading_blanks=False, trailing_blanks=False):
        """Join adjacent lines together into paragraphs using either a blank line or indent as separator."""

        curr_para = []
        paragraphs = []

        for line in lines:
            if use_indent:
                if line.startswith(' '):
                    curr_para.append(line.lstrip())
                    continue
                elif line == '':
                    continue
                else:
                    if len(curr_para) > 0:
                        paragraphs.append(cls._join_paragraph(curr_para, leading_blanks, trailing_blanks))

                    curr_para = [line.lstrip()]
            else:
                if len(line) != 0:
                    curr_para.append(line)
                else:
                    paragraphs.append(cls._join_paragraph(curr_para, leading_blanks, trailing_blanks))
                    curr_para = []

        # Finish the last paragraph if there is one
        if len(curr_para) > 0:
            paragraphs.append(cls._join_paragraph(curr_para, leading_blanks, trailing_blanks))

        return paragraphs

    def wrap_and_format(self, width=None, include_params=False, include_return=False, excluded_params=None):
        """Wrap, format and print this docstring for a specific width.

        Args:
            width (int): The number of characters per line.  If set to None
                this will be inferred from the terminal width and default
                to 80 if not passed or if passed as None and the terminal
                width cannot be determined.
            include_return (bool): Include the return information section
                in the output.
            include_params (bool): Include a parameter information section
                in the output.
            excluded_params (list): An optional list of parameter names to exclude.
                Options for excluding things are, for example, 'self' or 'cls'.
        """

        if excluded_params is None:
            excluded_params = []

        out = StringIO()
        if width is None:
            width, _height = get_terminal_size()

        for line in self.maindoc:
            if isinstance(line, Line):
                out.write(fill(line.contents, width=width))
                out.write('\n')
            elif isinstance(line, BlankLine):
                out.write('\n')
            elif isinstance(line, ListItem):
                out.write(fill(line.contents, initial_indent=" %s " % line.marker[0], subsequent_indent="   ", width=width))
                out.write('\n')

        if include_params:
            included_params = set(self.param_info) - set(excluded_params)
            if len(included_params) > 0:
                out.write("\nParameters:\n")

                for param in included_params:
                    info = self.param_info[param]
                    out.write(" - %s (%s):\n" % (param, info.type_name))
                    out.write(fill(info.desc, initial_indent="   ", subsequent_indent="   ", width=width))
                    out.write('\n')

        if include_return:
            print("Returns:")
            print("    " + self.return_info.type_name)
            #pylint:disable=fixme; Issue tracked in #32
            # TODO: Also include description information here

        return out.getvalue()

def parse_param(param, include_desc=False):
    """Parse a single typed parameter statement."""

    param_def, _colon, desc = param.partition(':')
    if not include_desc:
        desc = None
    else:
        desc = desc.lstrip()

    if _colon == "":
        raise ValidationError("Invalid parameter declaration in docstring, missing colon", declaration=param)

    param_name, _space, param_type = param_def.partition(' ')
    if len(param_type) < 2 or param_type[0] != '(' or param_type[-1] != ')':
        raise ValidationError("Invalid parameter type string not enclosed in ( ) characters", param_string=param_def, type_string=param_type)

    param_type = param_type[1:-1]
    return param_name, ParameterInfo(param_type, [], desc)

def parse_return(return_line, include_desc=False):
    """Parse a single return statement declaration.

    The valid types of return declarion are a Returns: section heading
    followed a line that looks like:
    type [format-as formatter]: description

    OR

    type [show-as (string | context)]: description sentence
    """

    ret_def, _colon, desc = return_line.partition(':')
    if _colon == "":
        raise ValidationError("Invalid return declaration in docstring, missing colon", declaration=ret_def)

    if not include_desc:
        desc = None

    if 'show-as' in ret_def:
        ret_type, _showas, show_type = ret_def.partition('show-as')
        ret_type = ret_type.strip()
        show_type = show_type.strip()

        if show_type not in ('string', 'context'):
            raise ValidationError("Unkown show-as formatting specifier", found=show_type, expected=['string', 'context'])

        if show_type == 'string':
            return ReturnInfo(None, str, True, desc)

        return ReturnInfo(None, None, False, desc)

    if 'format-as' in ret_def:
        ret_type, _showas, formatter = ret_def.partition('format-as')
        ret_type = ret_type.strip()
        formatter = formatter.strip()

        return ReturnInfo(ret_type, formatter, True, desc)

    return ReturnInfo(ret_def, None, True, desc)