# -*- coding: utf-8 -*-
#
# The MIT License (MIT)
#
# Copyright (c) 2018 Philippe Faist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
#

r"""
The ``latexwalker`` module provides a simple API for parsing LaTeX snippets,
and representing the contents using a data structure based on nodes classes.

LatexWalker will understand the syntax of most common macros.  However,
``latexwalker`` is NOT a replacement for a full LaTeX engine.  (Originally,
``latexwalker`` was desigend to extract useful text for indexing for text
database searches of LaTeX content.)

You can also use `latexwalker` directly in command-line, producing JSON or a
human-readable node tree::

    $ echo '\textit{italic} text' | python -m pylatexenc.latexwalker  \
                                    --output-format=json --json-compact
    [{"nodetype": "LatexMacroNode", "macroname": "textit", "nodeoptarg": null,
    "nodeargs": [{"nodetype": "LatexGroupNode", "nodelist": [{"nodetype":
    "LatexCharsNode", "chars": "italic"}]}], "macro_post_space": ""},
    {"nodetype": "LatexCharsNode", "chars": " text"}]

    $ python -m pylatexenc.latexwalker --help
    [...]
"""

from __future__ import print_function

import re
from collections import namedtuple
import sys
import logging
import json

if sys.version_info.major > 2:
    # Py3
    def unicode(string): return string
    _basestring = str
    _str_from_unicode = lambda x: x
    _unicode_from_str = lambda x: x
else:
    # Py2
    _basestring = basestring
    _str_from_unicode = lambda x: unicode(x).encode('utf-8')
    _unicode_from_str = lambda x: x.decode('utf-8')

logger = logging.getLogger(__name__)

class LatexWalkerError(Exception):
    """
    Generic exception class raised by this module.
    """
    pass

class LatexWalkerParseError(LatexWalkerError):
    """
    Parse error.  The following attributes are available: `msg` (the error message), `s`
    (the parsed string), `pos` (the position of the error in the string, 0-based index).
    """
    def __init__(self, msg, s=None, pos=None):
        self.msg = msg
        self.s = s
        self.pos = pos
        disp = '...'+s[max(pos-25,0):pos]
        disp = '\n%s\n'%(disp)  +  (' '*len(disp)) + s[pos:pos+25]+'...'
        LatexWalkerError.__init__(self, msg + (
            " @ %d:\n%s" %(pos, disp)
            ))

class LatexWalkerEndOfStream(LatexWalkerError):
    """
    Reached end of input stream (e.g., end of file).
    """
    pass

MacrosDef = namedtuple('MacrosDef', ['macname', 'optarg', 'numargs'])
"""
Class which stores a Macro syntax.

- `macname` stores the name of the macro, without the leading backslash.

- `optarg` may be one of `True`, `False`, or `None`.

  + if `True`, the macro expects as first argument an optional argument in square
    brackets. Then, `numargs` specifies the number of additional mandatory arguments to
    the command, given in usual curly braces (or simply as one TeX token)

  + if `False`, the macro only expects a number of mandatory arguments given by
    `numargs`. The mandatory arguments are given in usual curly braces (or simply as one
    TeX token)

  + if `None`, then `numargs` is a string of either characters "{" or "[", in which each
    curly brace specifies a mandatory argument and each square bracket specifies an
    optional argument in square brackets.  For example, "{{[{" expects two mandatory
    arguments, then an optional argument in square brackets, and then another mandatory
    argument.
"""

_default_macro_list = (
    MacrosDef('documentclass', True, 1),
    MacrosDef('usepackage', True, 1),
    MacrosDef('selectlanguage', True, 1),
    MacrosDef('setlength', True, 2),
    MacrosDef('addlength', True, 2),
    MacrosDef('setcounter', True, 2),
    MacrosDef('addcounter', True, 2),
    MacrosDef('newcommand', None, "{[{"),
    MacrosDef('renewcommand', None, "{[{"),
    MacrosDef('DeclareMathOperator', False, 2),
    MacrosDef('input', False, 1),

    MacrosDef('hspace', False, 1),
    MacrosDef('vspace', False, 1),

    MacrosDef('\\', True, 0), # (Note: single backslash) end of line with optional spacing, e.g.  \\[2mm]
    MacrosDef('item', True, 0),

    # \input{someotherfile}
    MacrosDef('input', False, 1),
    MacrosDef('include', False, 1),

    MacrosDef('includegraphics', True, 1),

    MacrosDef('emph', False, 1),
    MacrosDef('textit', False, 1),
    MacrosDef('textbf', False, 1),
    MacrosDef('textsc', False, 1),
    MacrosDef('textsl', False, 1),
    MacrosDef('text', False, 1),
    MacrosDef('mathrm', False, 1),

    MacrosDef('label', False, 1),
    MacrosDef('ref', False, 1),
    MacrosDef('eqref', False, 1),
    MacrosDef('url', False, 1),
    MacrosDef('hypersetup', False, 1),
    MacrosDef('footnote', True, 1),

    MacrosDef('keywords', False, 1),

    MacrosDef('hphantom', True, 1),
    MacrosDef('vphantom', True, 1),

    MacrosDef("'", False, 1),
    MacrosDef("`", False, 1),
    MacrosDef('"', False, 1),
    MacrosDef("c", False, 1),
    MacrosDef("^", False, 1),
    MacrosDef("~", False, 1),
    MacrosDef("H", False, 1),
    MacrosDef("k", False, 1),
    MacrosDef("=", False, 1),
    MacrosDef("b", False, 1),
    MacrosDef(".", False, 1),
    MacrosDef("d", False, 1),
    MacrosDef("r", False, 1),
    MacrosDef("u", False, 1),
    MacrosDef("v", False, 1),

    MacrosDef("vec", False, 1),
    MacrosDef("dot", False, 1),
    MacrosDef("hat", False, 1),
    MacrosDef("check", False, 1),
    MacrosDef("breve", False, 1),
    MacrosDef("acute", False, 1),
    MacrosDef("grave", False, 1),
    MacrosDef("tilde", False, 1),
    MacrosDef("bar", False, 1),
    MacrosDef("ddot", False, 1),

    MacrosDef('frac', False, 2),
    MacrosDef('nicefrac', False, 2),

    MacrosDef('sqrt', True, 1),

    MacrosDef('ket', False, 1),
    MacrosDef('bra', False, 1),
    MacrosDef('braket', False, 2),
    MacrosDef('ketbra', False, 2),

    MacrosDef('texorpdfstring', False, 2),

    # ethuebung
    MacrosDef('UebungLoesungFont', False, 1),
    MacrosDef('UebungHinweisFont', False, 1),
    MacrosDef('UebungExTitleFont', False, 1),
    MacrosDef('UebungSubExTitleFont', False, 1),
    MacrosDef('UebungTipsFont', False, 1),
    MacrosDef('UebungLabel', False, 1),
    MacrosDef('UebungSubLabel', False, 1),
    MacrosDef('UebungLabelEnum', False, 1),
    MacrosDef('UebungLabelEnumSub', False, 1),
    MacrosDef('UebungSolLabel', False, 1),
    MacrosDef('UebungHinweisLabel', False, 1),
    MacrosDef('UebungHinweiseLabel', False, 1),
    MacrosDef('UebungSolEquationLabel', False, 1),
    MacrosDef('UebungTipsLabel', False, 1),
    MacrosDef('UebungTipsEquationLabel', False, 1),
    MacrosDef('UebungsblattTitleSeries', False, 1),
    MacrosDef('UebungsblattTitleSolutions', False, 1),
    MacrosDef('UebungsblattTitleTips', False, 1),
    MacrosDef('UebungsblattNumber', False, 1),
    MacrosDef('UebungsblattTitleFont', False, 1),
    MacrosDef('UebungTitleCenterVSpacing', False, 1),
    MacrosDef('UebungAttachedSolutionTitleTop', False, 1),
    MacrosDef('UebungAttachedSolutionTitleFont', False, 1),
    MacrosDef('UebungAttachedSolutionTitle', False, 1),
    MacrosDef('UebungTextAttachedSolution', False, 1),
    MacrosDef('UebungDueByLabel', False, 1),
    MacrosDef('UebungDueBy', False, 1),
    MacrosDef('UebungLecture', False, 1),
    MacrosDef('UebungProf', False, 1),
    MacrosDef('UebungLecturer', False, 1),
    MacrosDef('UebungSemester', False, 1),
    MacrosDef('UebungLogoFile', False, 1),
    MacrosDef('UebungLanguage', False, 1),
    MacrosDef('UebungStyle', False, 1),
    #
    MacrosDef('uebung', False, '{['),
    MacrosDef('exercise', False, '{['),
    MacrosDef('keywords', False, 1),
    MacrosDef('subuebung', False, 1),
    MacrosDef('subexercise', False, 1),
    MacrosDef('pdfloesung', True, 1),
    MacrosDef('pdfsolution', True, 1),
    MacrosDef('exenumfulllabel', False, 1),
    MacrosDef('hint', False, 1),
    MacrosDef('hints', False, 1),
    MacrosDef('hinweis', False, 1),
    MacrosDef('hinweise', False, 1),

    )

default_macro_dict = dict([(m.macname, m) for m in _default_macro_list])
"""
The default context dictionary of known LaTeX macros.  The keys are the macro names
(:py:class:`MacrosDef.macname <MacrosDef>`) and the values are :py:class:`MacrosDef`
instances.
"""

# ------------------------------------------------

class LatexToken(object):
    r"""
    Represents a token read from the LaTeX input.

    This is used internally by :py:class:`LatexWalker`'s methods.  You probably
    don't need to worry about individual tokens.  Rather, you should use the
    high-level functions provided by :py:class:`LatexWalker` (e.g.,
    :py:meth:`~latexwalker.LatexWalker.get_latex_nodes()`).  So most likely, you
    can ignore this class entirely.

    This is not the same thing as a LaTeX token, it's just a part of the input
    which we treat in the same way (e.g. a bunch of content characters, a
    comment, a macro, etc.)

    Information about the object is stored into the fields `tok` and `arg`. The
    `tok` field is a string which identifies the type of the token. The `arg`
    depends on what `tok` is, and describes the actual input.

    Additionally, this class stores information about the position of the token
    in the input stream in the field `pos`.  This `pos` is an integer which
    corresponds to the index in the input string.  The field `len` stores the
    length of the token in the input string.  This means that this token spans
    in the input string from `pos` to `pos+len`.

    Leading whitespace before the token is not returned as a separate
    'char'-type token, but it is given in the `pre_space` field of the token
    which follows.  Pre-space may contain a newline, but not two consecutive
    newlines.

    The `post_space` is only used for 'macro' and 'comment' tokens, and it
    stores any spaces encountered after a macro, or the newline with any
    following spaces that terminates a LaTeX comment.

    The `tok` field may be one of:

      - 'char': raw characters which have no special LaTeX meaning; they are
        part of the text content.

        The `arg` field contains the characters themselves.

      - 'macro': a macro invokation, but not '\begin' or '\end'

        The `arg` field contains the name of the macro, without the leading
        backslash.

      - 'begin_environment': an invokation of '\begin{environment}'.

        The `arg` field contains the name of the environment inside the braces.

      - 'end_environment': an invokation of '\end{environment}'.

        The `arg` field contains the name of the environment inside the braces.

      - 'comment': a LaTeX comment delimited by a percent sign up to the end of
        the line.

        The `arg` field contains the text in the comment line, not including the
        percent sign nor the newline.

      - 'brace_open': an opening brace.  This is usually a curly brace, and
        sometimes also a square bracket.  What is parsed as a brace depends on
        the arguments to :py:func:`get_token()`.

        The `arg` is a string which contains the relevant brace character.

      - 'brace_close': a closing brace.  This is usually a curly brace, and
        sometimes also a square bracket.  What is parsed as a brace depends on
        the arguments to :py:func:`get_token()`.

        The `arg` is a string which contains the relevant brace character.

      - 'mathmode_inline': a delimiter which starts inline math.  This is (e.g.)
        a single '$' character which is not part of a double '$$' display
        environment delimiter.

        The `arg` is the string value of the delimiter in question ('$')
    """
    def __init__(self, tok, arg, pos, len, pre_space, post_space=''):
        self.tok = tok
        self.arg = arg
        self.pos = pos
        self.len = len
        self.pre_space = pre_space
        self.post_space = post_space
        self._fields = ['tok', 'arg', 'pos', 'len', 'pre_space']
        if self.tok in ('macro', 'comment'):
            self._fields.append('post_space')
        super(LatexToken, self).__init__()

    def __unicode__(self):
        return _unicode_from_str(self.__str__())

    def __repr__(self):
        return (
            "LatexToken(" +
            ", ".join([ "%s=%r"%(k,getattr(self,k))
                        for k in self._fields ]) +
            ")"
            )

    def __str__(self):
        return self.__repr__()

    def __eq__(self, other):
        return all( ( getattr(self, f) == getattr(other, f)  for f in self._fields ) )

    def __ne__(self, other): return NotImplemented
    __hash__ = None

# ------------------------------------------------

class LatexNode(object):
    """
    Represents an abstract 'node' of the latex document.

    Use :py:meth:`nodeType()` to figure out what type of node this is, and
    :py:meth:`isNodeType()` to test whether it is of a given type.
    """
    def __init__(self, **kwargs):
        """
        Important: subclasses must set `self._fields` to a list (or tuple or iterable) of the
        fields stored in this object.
        """
        super(LatexNode, self).__init__(**kwargs)

    def nodeType(self):
        """
        Returns the class which corresponds to the type of this node.  This is a Python class
        object, that is one of :py:class:`~pylatexenc.latexwalker.LatexCharsNode`,
        :py:class:`~pylatexenc.latexwalker.LatexGroupNode`, etc.
        """
        return LatexNode

    def isNodeType(self, t):
        """
        Returns `True` if the current node is of the given type.  The argument `t` must be a
        Python class such as, e.g. :py:class:`~pylatexenc.latexwalker.LatexGroupNode`.
        """
        return isinstance(self, t)

    def __eq__(self, other):
        return other is not None  and  self.nodeType() == other.nodeType()  and  all(
            ( getattr(self, f) == getattr(other, f)  for f in self._fields )
        )
    def __ne__(self, other): return NotImplemented
    __hash__ = None

    def __unicode__(self):
        return _unicode_from_str(self.__str__())
    def __str__(self):
        return self.__repr__()
    def __repr__(self):
        return (
            self.nodeType().__name__ + "(" +
            ", ".join([ "%s=%r"%(k,getattr(self,k))  for k in self._fields ]) +
            ")"
            )

class LatexCharsNode(LatexNode):
    """
    A string of characters in the LaTeX document, without any special LaTeX code.

    .. py:attribute:: chars

       The string of characters represented by this node.

    """
    def __init__(self, chars, **kwargs):
        r"""
        Constructor arguments:

            - `chars`: the actual characters.

        """
        super(LatexCharsNode, self).__init__(**kwargs)
        self._fields = ('chars',)
        self.chars = chars

    def nodeType(self):
        return LatexCharsNode

class LatexGroupNode(LatexNode):
    r"""
    A LaTeX group delimited by braces, ``{like this}``.

    .. py:attribute:: nodelist

       A list of nodes describing the contents of the LaTeX braced group.  Each
       item of the list is a :py:class:`LatexNode`.

    """
    def __init__(self, nodelist, **kwargs):
        super(LatexGroupNode, self).__init__(**kwargs)
        self._fields = ('nodelist',)
        self.nodelist = nodelist

    def nodeType(self):
        return LatexGroupNode

class LatexCommentNode(LatexNode):
    r"""
    A LaTeX comment, delimited by a percent sign until the end of line.

    .. py:attribute:: comment

       The comment string, not including the '%' sign nor the following newline

    .. py:attribute:: comment_post_space

       The newline that terminated the comment possibly followed by spaces
       (e.g., indentation spaces of the next line)

    """
    def __init__(self, comment, comment_post_space='', **kwargs):
        super(LatexCommentNode, self).__init__(**kwargs)
        self._fields = ('comment',)
        self.comment = comment
        self.comment_post_space = comment_post_space

    def nodeType(self):
        return LatexCommentNode

class LatexMacroNode(LatexNode):
    r"""
    Represents a 'macro' type node, e.g. '\textbf'

    .. py:attribute:: macroname

       The name of the macro (string), *without* the leading backslash.

    .. py:attribute:: nodeoptarg

       If non-`None`, this corresponds to the optional argument of the macro.

    .. py:attribute:: nodeargs

       A list of arguments to the macro. Each item in the list is a
       :py:class:`LatexNode`.

    .. py:attribute:: macro_post_space

       Any spaces that were encountered immediately after the macro.

    """
    def __init__(self, macroname, nodeoptarg=None, nodeargs=[], macro_post_space='',
                 **kwargs):
        super(LatexMacroNode, self).__init__(**kwargs)
        self._fields = ('macroname','nodeoptarg','nodeargs','macro_post_space')
        self.macroname = macroname
        self.nodeoptarg = nodeoptarg
        self.nodeargs = nodeargs
        self.macro_post_space = macro_post_space

    def nodeType(self):
        return LatexMacroNode

class LatexEnvironmentNode(LatexNode):
    r"""
    A LaTeX Environment Node, i.e. ``\begin{something} ... \end{something}``.

    .. py:attribute:: envname

       The name of the environment ('itemize', 'equation', ...)

    .. py:attribute:: nodelist

       A list of :py:class:`LatexNode`'s that represent all the contents between
       the ``\begin{...}`` instruction and the ``\end{...}`` instruction.

    .. py:attribute:: optargs

       Any possible optional argument passed to the ``\begin{...}`` instruction,
       for example in ``\begin{enumerate}[label=\roman*)]`` (Currently, only a
       single optional argument is parsed, but this attribute is still a list of
       :py:class:`LatexNode`'s.

    .. py:attribute:: args

       Any possible regular arguments passed to the ``\begin{...}`` instruction,
       for example in ``\begin{tabular}{clr}``. Currently, at most a single
       regular argument is parsed, but this is anyway a list of
       :py:class:`LatexNode`'s

    """

    def __init__(self, envname, nodelist, optargs=[], args=[], **kwargs):
        super(LatexEnvironmentNode, self).__init__(**kwargs)
        self._fields = ('envname','nodelist','optargs','args',)
        self.envname = envname
        self.nodelist = nodelist
        self.optargs = optargs
        self.args = args

    def nodeType(self):
        return LatexEnvironmentNode

class LatexMathNode(LatexNode):
    r"""
    A Math node type.

    Note that currently only 'inline' math environments are detected.

    .. py:attribute:: displaytype

       Either 'inline' or 'display', to indicate an inline math block or a
       display math block. (Note that math environments such as
       `\begin{equation}...\end{equation}`, are reported as
       :py:class:`LatexEnvironmentNode`'s, and not as
       :py:class:`LatexMathNode`'s.

    .. note::

       Currently, the 'display' type is never used.  Display blocks delimited
       e.g. by ``$$ .. $$`` or ``\[ ... \]`` are always reported as regular text
       with :py:class:`LatexCharsNode`.  This might change in the future.

    .. py:attribute:: nodelist

       The contents of the environment, given as a list of
       :py:class:`LatexNode`'s.
    """
    def __init__(self, displaytype, nodelist=[], **kwargs):
        super(LatexMathNode, self).__init__(**kwargs)
        self._fields = ('displaytype','nodelist',)
        self.displaytype = displaytype
        self.nodelist = nodelist

    def nodeType(self):
        return LatexMathNode

# ------------------------------------------------------------------------------

class _PushPropOverride(object):
    def __init__(self, obj, propname, new_value):
        super(_PushPropOverride, self).__init__()
        self.obj = obj
        self.propname = propname
        self.new_value = new_value

    def __enter__(self):
        if self.new_value is not None:
            self.initval = getattr(self.obj, self.propname)
            setattr(self.obj, self.propname, self.new_value)
        return self

    def __exit__(self, type, value, traceback):
        # clean-up
        if self.new_value is not None:
            setattr(self.obj, self.propname, self.initval)

# ------------------------------------------------------------------------------

class LatexWalker(object):
    r"""
    A parser which walks through an input stream, parsing it as LaTeX markup.

    Arguments:

      - `s`: the string to parse as LaTeX code

      - `macro_dict`: a context dictionary of known LaTeX macros.  By default, the default
        global macro dictionary `default_macro_dict` is used.  This should be a dictionary
        where the keys are macro names (see :py:class:`MacrosDef.macname <MacrosDef>`) and
        values are :py:class:`MacrosDef` instances.

    Additional keyword arguments are flags which influence the parsing.  Accepted flags are:

      - `keep_inline_math=True|False` If this option is set to `True`, then inline math is
        parsed and stored using :py:class:`LatexMathNode` instances.  Otherwise, inline
        math is not treated differently and is simply kept as text.

      - `tolerant_parsing=True|False` If set to `True`, then the parser generally ignores
        syntax errors rather than raising an exception.

      - `strict_braces=True|False` This option refers specifically to reading a
        encountering a closing brace when an expression is needed.  You generally won't
        need to specify this flag, use `tolerant_parsing` instead.

    The methods provided in this class perform various parsing of the given
    string `s`.  These methods typically accept a `pos` parameter, which must be
    an integer, which defines the position in the string `s` to start parsing.

    These methods, unless otherwise documented, return a tuple `(node, pos,
    len)`, where node is a :py:class:`LatexNode` describing the parsed content,
    `pos` is the position at which the LaTeX element of iterest was encountered,
    and `len` is the length of the string that is considered to be part of the
    `node`.  That is, the position in the string that is immediately after the
    node is `pos+len`.
    """

    def __init__(self, s, macro_dict=None, **flags):
        self.s = s
        if macro_dict is not None:
            self.macro_dict = macro_dict
        else:
            self.macro_dict = default_macro_dict
        #
        # now parsing flags:
        #
        self.keep_inline_math = flags.pop('keep_inline_math', False)
        self.tolerant_parsing = flags.pop('tolerant_parsing', True)
        self.strict_braces = flags.pop('strict_braces', False)
        if flags:
            # any flags left which we haven't recognized
            logger.warning("LatexWalker(): Unknown flag(s) encountered: %r", flags.keys())

    def parse_flags(self):
        """
        The parse flags currently set on this object.  Returns a dictionary with keys
        'keep_inline_math', 'tolerant_parsing' and 'strict_braces'.
        """
        return {
            'keep_inline_math': self.keep_inline_math,
            'tolerant_parsing': self.tolerant_parsing,
            'strict_braces': self.strict_braces,
        }

    def get_token(self, pos, brackets_are_chars=True, environments=True, keep_inline_math=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single "token", as defined by
        :py:class:`LatexToken`.

        Parse the token in the stream pointed to at position `pos`.

        Returns a :py:class:`LatexToken`. Raises :py:exc:`LatexWalkerEndOfStream` if end
        of stream reached.

        If `brackets_are_chars=False`, then square bracket characters count as
        'brace_open' and 'brace_close' token types (see :py:class:`LatexToken`); otherwise
        (the default) they are considered just like other normal characters.

        If `environments=False`, then '\\begin' and '\\end' tokens count as regular
        'macro' tokens (see :py:class:`LatexToken`); otherwise (the default) they are
        considered as the token types 'begin_environment' and 'end_environment'.

        If `keep_inline_math` is not `None`, then that value overrides that of
        `self.keep_inline_math` for the duration of this method call.
        """

        s = self.s # shorthand

        with _PushPropOverride(self, 'keep_inline_math', keep_inline_math):

            space = ''
            while (pos < len(s) and s[pos].isspace()):
                space += s[pos]
                pos += 1
                if (space.endswith('\n\n')):  # two \n's indicate new paragraph.
                    # Adding pre-space is overkill here I think.
                    return LatexToken(tok='char', arg='\n\n', pos=pos-2, len=2, pre_space='')

            if (pos >= len(s)):
                raise LatexWalkerEndOfStream()

            if (s[pos] == '\\'):
                # escape sequence
                i = 2
                macro = s[pos+1] # next char is necessarily part of macro
                # following chars part of macro only if all are alphabetical
                isalphamacro = False
                if (s[pos+1].isalpha()):
                    isalphamacro = True
                    while pos+i<len(s) and s[pos+i].isalpha():
                        macro += s[pos+i]
                        i += 1
                # possibly followed by a star
                if (pos+i<len(s) and s[pos+i] == '*'):
                    macro += '*'
                    i += 1

                # see if we have a begin/end environment
                if (environments and (macro == 'begin' or macro == 'end')):
                    # \begin{environment} or \end{environment}
                    envmatch = re.match(r'^\s*\{([\w*]+)\}', s[pos+i:])
                    if (envmatch is None):
                        raise LatexWalkerParseError(
                            s=s,
                            pos=pos,
                            msg="Bad \\%s macro: expected {environment}" %(macro)
                        )

                    return LatexToken(
                        tok=('begin_environment' if macro == 'begin' else  'end_environment'),
                        arg=envmatch.group(1),
                        pos=pos,
                        len=i+envmatch.end(), # !!envmatch.end() counts from pos+i
                        pre_space=space
                        )

                # get the following whitespace, and store it in the macro's post_space
                post_space = ''
                if isalphamacro:
                    # important, LaTeX does not consume space after non-alpha macros, like \&
                    while pos+i<len(s) and s[pos+i].isspace():
                        post_space += s[pos+i]
                        i += 1

                return LatexToken(tok='macro', arg=macro, pos=pos, len=i,
                                  pre_space=space, post_space=post_space)

            if (s[pos] == '%'):
                # latex comment
                m = re.search(r'(\n|\r|\n\r)\s*', s[pos:])
                mlen = None
                if m is not None:
                    arglen = m.start() # relative to pos already
                    mlen = m.end() # relative to pos already
                    mspace = m.group()
                else:
                    arglen = len(s)-pos# [  ==len(s[pos:])  ]
                    mlen = arglen
                    mspace = ''
                return LatexToken(tok='comment', arg=s[pos+1:pos+arglen], pos=pos, len=mlen,
                                  pre_space=space, post_space=mspace)

            openbracechars = '{'
            closebracechars = '}'
            if not brackets_are_chars:
                openbracechars += '['
                closebracechars += ']'

            if s[pos] in openbracechars:
                return LatexToken(tok='brace_open', arg=s[pos], pos=pos, len=1, pre_space=space)

            if s[pos] in closebracechars:
                return LatexToken(tok='brace_close', arg=s[pos], pos=pos, len=1, pre_space=space)

            # check if it is an inline math char, if we care about inline math.
            if (s[pos] == '$' and self.keep_inline_math):
                # check that we don't have double-$$, which would be a display environment.
                if not (pos+1 < len(s) and s[pos+1] == '$'):
                    return LatexToken(tok='mathmode_inline', arg=s[pos], pos=pos, len=1, pre_space=space)
                # otherwise, proceed to 'char' type.

            return LatexToken(tok='char', arg=s[pos], pos=pos, len=1, pre_space=space)

    def get_latex_expression(self, pos, strict_braces=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to parse a single LaTeX expression.

        Reads a latex expression, e.g. macro argument. This may be a single char, an escape
        sequence, or a expression placed in braces.  This is what TeX calls a "token" (and
        not what we call a token... anyway).

        Returns a tuple `(node, pos, len)`, where `pos` is the position of the
        first char of the expression and `len` the length of the expression.
        """

        with _PushPropOverride(self, 'strict_braces', strict_braces):

            tok = self.get_token(pos, environments=False, keep_inline_math=False)

            if (tok.tok == 'macro'):
                if (tok.arg == 'end'):
                    if not self.tolerant_parsing:
                        # error, this should be an \end{environment}, not an argument in itself
                        raise LatexWalkerParseError("Expected expression, got \end", self.s, pos)
                    else:
                        return (LatexCharsNode(chars=''), tok.pos, 0)
                return (LatexMacroNode(macroname=tok.arg, nodeoptarg=None, nodeargs=[],
                                       macro_post_space=tok.post_space),
                        tok.pos, tok.len)
            if (tok.tok == 'comment'):
                return self.get_latex_expression(pos+tok.len)
            if (tok.tok == 'brace_open'):
                return self.get_latex_braced_group(tok.pos)
            if (tok.tok == 'brace_close'):
                if (self.strict_braces and not self.tolerant_parsing):
                    raise LatexWalkerParseError("Expected expression, got closing brace!", self.s, pos)
                return (LatexCharsNode(chars=''), tok.pos, 0)
            if (tok.tok == 'char'):
                return (LatexCharsNode(chars=tok.arg), tok.pos, tok.len)

            raise LatexWalkerParseError("Unknown token type: %s" %(tok.tok), self.s, pos)

    def get_latex_maybe_optional_arg(self, pos):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to attempt to parse an optional argument.

        Attempts to parse an optional argument. If this is successful, we return
        a tuple `(node, pos, len)` if success where `node` is a
        :py:class:`LatexGroupNode`.  Otherwise, this method returns None.
        """

        tok = self.get_token(pos, brackets_are_chars=False, environments=False)
        if (tok.tok == 'brace_open' and tok.arg == '['):
            return self.get_latex_braced_group(pos, brace_type='[')

        return None

    def get_latex_braced_group(self, pos, brace_type='{'):
        """
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to read a latex group delimited by braces.

        Reads a latex expression enclosed in braces ``{ ... }``. The first token of
        `s[pos:]` must be an opening brace.

        Returns a tuple `(node, pos, len)`, where `node` is a
        :py:class:`LatexGroupNode` instance, `pos` is the position of the first
        char of the expression (which has to be an opening brace), and `len` is
        the length of the group, including the closing brace (relative to the
        starting position).
        """

        closing_brace = None
        if (brace_type == '{'):
            closing_brace = '}'
        elif (brace_type == '['):
            closing_brace = ']'
        else:
            raise LatexWalkerParseError(s=self.s, pos=pos, msg="Uknown brace type: %s" %(brace_type))
        brackets_are_chars = (brace_type != '[')

        firsttok = self.get_token(pos, brackets_are_chars=brackets_are_chars)
        if (firsttok.tok != 'brace_open'  or  firsttok.arg != brace_type):
            raise LatexWalkerParseError(
                s=self.s,
                pos=pos,
                msg='get_latex_braced_group: not an opening brace/bracket: %s' %(self.s[pos])
            )

        #pos = firsttok.pos + firsttok.len

        (nodelist, npos, nlen) = self.get_latex_nodes(firsttok.pos + firsttok.len,
                                                      stop_upon_closing_brace=closing_brace)

        return (LatexGroupNode(nodelist=nodelist), firsttok.pos, npos + nlen - firsttok.pos)

    def get_latex_environment(self, pos, environmentname=None):
        r"""
        Parses the latex content given to the constructor (and stored in `self.s`),
        starting at position `pos`, to read a latex environment.

        Reads a latex expression enclosed in a
        ``\begin{environment}...\end{environment}``.  The first token in the
        stream must be the ``\begin{environment}``.

        If `environmentname` is given and nonempty, then additionally a
        :py:exc:`LatexWalkerParseError` is raised if the environment in the
        input stream does not match the provided name.

        This function will attempt to heuristically parse an optional argument,
        and possibly a mandatory argument given to the environment.  No space is
        allowed between ``\begin{environment}`` and an opening square bracket or
        opening brace.

        Returns a tuple (node, pos, len) with node being a
        :py:class:`LatexEnvironmentNode`.
        """

        startpos = pos

        firsttok = self.get_token(pos)
        if (firsttok.tok != 'begin_environment'  or
            (environmentname is not None and firsttok.arg != environmentname)):
            raise LatexWalkerParseError(s=self.s, pos=pos,
                                        msg=r'get_latex_environment: expected \begin{%s}: %s' %(
                environmentname if environmentname is not None else '<environment name>',
                firsttok.arg
                ))
        if (environmentname is None):
            environmentname = firsttok.arg

        pos = firsttok.pos + firsttok.len

        optargs = []
        args = []

        # see if the \begin{environment} is immediately followed by some
        # options.  Important: Don't eat the brace of a commutator!! Don't allow
        # any space between the environment and the open bracket.
        optargtuple = None
        if (self.s[pos] == '['):
            optargtuple = self.get_latex_maybe_optional_arg(pos)

        if (optargtuple is not None):
            optargs.append(optargtuple[0])
            pos = optargtuple[1]+optargtuple[2]
        else:
            # Try to see if we have a mandatory argument.  Don't use get_token
            # as we don't want to skip any space.
            if self.s[pos] == '{':
                (argnode, apos, alen) = self.get_latex_braced_group(pos)
                args.append(argnode)
                pos = apos+alen

        (nodelist, npos, nlen) = self.get_latex_nodes(pos, stop_upon_end_environment=environmentname)

        return (LatexEnvironmentNode(envname=environmentname,
                                     nodelist=nodelist,
                                     optargs=optargs,
                                     args=args),
                startpos, npos+nlen-startpos)

    def get_latex_nodes(self, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None,
                        stop_upon_closing_mathmode=None):
        """
        Parses the latex content given to the constructor (and stored in `self.s`)
        into a list of nodes.

        Returns a tuple `(nodelist, pos, len)` where nodelist is a list of
        :py:class:`LatexNode`\ 's.

        If `stop_upon_closing_brace` is given and set to a character, then parsing stops
        once the given closing brace is encountered (but not inside a subgroup).  The
        brace is given as a character, ']' or '}'.  The returned `len` includes the
        closing brace, but the closing brace is not included in any of the nodes in the
        `nodelist`.

        If `stop_upon_end_environment` is provided, then parsing stops once the given
        environment was closed.  If there is an environment mismatch, then a
        `LatexWalkerParseError` is raised except in tolerant parsing mode (see
        py:meth:`parse_flags()`).  Again, the closing environment is included in the
        length count but not the nodes.

        If `stop_upon_closing_mathmode` is specified, then the parsing stops
        once the corresponding math mode (assumed already open) is closed.
        Currently, only inline math modes delimited by ``$`` are supported.
        I.e., currently, if set, only the value
        ``stop_upon_closing_mathmode='$'`` is valid.
        """

        nodelist = []

        brackets_are_chars = True
        if (stop_upon_closing_brace == ']'):
            brackets_are_chars = False

        origpos = pos

        class PosPointer:
            def __init__(self, pos=0, lastchars=''):
                self.pos = pos
                self.lastchars = lastchars

        p = PosPointer(pos)

        def do_read(nodelist, p):
            """
            Read a single token and process it, recursing into brace blocks and environments etc if
            needed, and appending stuff to nodelist.

            Return True whenever we should stop trying to read more. (e.g. upon reaching the a matched
            stop_upon_end_environment etc.)
            """

            try:
                tok = self.get_token(p.pos, brackets_are_chars=brackets_are_chars)
            except LatexWalkerEndOfStream:
                if self.tolerant_parsing:
                    return True
                raise # re-raise

            p.pos = tok.pos + tok.len

            # if it's a char, just append it to the stream of last characters.
            if (tok.tok == 'char'):
                p.lastchars += tok.pre_space + tok.arg
                return False

            # if it's not a char, push the last `p.lastchars` into the node list before anything else
            if len(p.lastchars):
                strnode = LatexCharsNode(chars=p.lastchars+tok.pre_space)
                nodelist.append(strnode)
                p.lastchars = ''
            elif len(tok.pre_space):
                # If we have pre_space, add a separate chars node that contains
                # the spaces.  We do this seperately, so that latex2text can
                # ignore these groups by default to avoid too much space on the
                # output.  This allows latex2text to implement the
                # `strict_latex_spaces=True` flag correctly.
                spacestrnode = LatexCharsNode(chars=tok.pre_space)
                nodelist.append(spacestrnode)

            # and see what the token is.

            if (tok.tok == 'brace_close'):
                # we've reached the end of the group. stop the parsing.
                if (tok.arg != stop_upon_closing_brace):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing brace: `%s\'' %(tok.arg)
                        )
                    return False
                return True

            if (tok.tok == 'end_environment'):
                # we've reached the end of an environment.
                if (tok.arg != stop_upon_end_environment):
                    if (not self.tolerant_parsing):
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg=('Unexpected mismatching closing environment: `%s\', '
                                 'expecting `%s\'' %(tok.arg, stop_upon_end_environment))
                        )
                    return False
                return True

            if (tok.tok == 'mathmode_inline'):
                # if we care about keeping math mode inlines verbatim, gulp all of the expression.
                if stop_upon_closing_mathmode is not None:
                    if stop_upon_closing_mathmode != '$':
                        raise LatexWalkerParseError(
                            s=self.s,
                            pos=tok.pos,
                            msg='Unexpected mismatching closing math mode: `$\''
                        )
                    return True

                # we have encountered a new math inline, so gulp all of the math expression
                (mathinline_nodelist, mpos, mlen) = self.get_latex_nodes(p.pos, stop_upon_closing_mathmode='$')
                p.pos = mpos + mlen

                nodelist.append(LatexMathNode(displaytype='inline', nodelist=mathinline_nodelist))
                return

            if (tok.tok == 'comment'):
                commentnode = LatexCommentNode(comment=tok.arg, comment_post_space=tok.post_space)
                nodelist.append(commentnode)
                return

            if (tok.tok == 'brace_open'):
                # another braced group to read.
                (groupnode, bpos, blen) = self.get_latex_braced_group(tok.pos)
                p.pos = bpos + blen
                nodelist.append(groupnode)
                return

            if (tok.tok == 'begin_environment'):
                # an environment to read.
                (envnode, epos, elen) = self.get_latex_environment(tok.pos, environmentname=tok.arg)
                p.pos = epos + elen
                # add node and continue.
                nodelist.append(envnode)
                return

            if (tok.tok == 'macro'):
                # read a macro. see if it has arguments.
                nodeoptarg = None
                nodeargs = []
                macname = tok.arg.rstrip('*') # for lookup in macro_dict
                if macname in self.macro_dict:
                    mac = self.macro_dict[macname]

                    def getoptarg(pos):
                        """
                        Gets a possibly optional argument. returns (argnode, new-pos) where argnode
                        might be `None` if the argument was not specified.
                        """
                        optarginfotuple = self.get_latex_maybe_optional_arg(pos)
                        if optarginfotuple is not None:
                            (nodeoptarg, optargpos, optarglen) = optarginfotuple
                            return (nodeoptarg, optargpos+optarglen)
                        return (None, pos)

                    def getarg(pos):
                        """
                        Gets a mandatory argument. returns (argnode, new-pos)
                        """
                        (nodearg, npos, nlen) = self.get_latex_expression(pos, strict_braces=False)
                        return (nodearg, npos + nlen)

                    if mac.optarg:
                        (nodeoptarg, p.pos) = getoptarg(p.pos)

                    if isinstance(mac.numargs, _basestring):
                        # specific argument specification
                        for arg in mac.numargs:
                            if arg == '{':
                                (node, p.pos) = getarg(p.pos)
                                nodeargs.append(node)
                            elif arg == '[':
                                (node, p.pos) = getoptarg(p.pos)
                                nodeargs.append(node)
                            else:
                                raise LatexWalkerError(
                                    "Unknown macro argument kind for macro %s: %s"
                                    % (mac.macroname, arg)
                                )
                    else:
                        for n in range(mac.numargs):
                            (nodearg, p.pos) = getarg(p.pos)
                            nodeargs.append(nodearg)

                nodelist.append(LatexMacroNode(macroname=tok.arg,
                                               nodeoptarg=nodeoptarg,
                                               nodeargs=nodeargs,
                                               macro_post_space=tok.post_space))
                return None

            raise LatexWalkerParseError(s=self.s, pos=p.pos, msg="Unknown token: %r" %(tok))

        while True:
            try:
                r_endnow = do_read(nodelist, p)
            except LatexWalkerEndOfStream:
                if stop_upon_closing_brace or stop_upon_end_environment:
                    # unexpected eof
                    if (not self.tolerant_parsing):
                        raise LatexWalkerError("Unexpected end of stream!")
                    else:
                        r_endnow = False
                else:
                    r_endnow = True

            if (r_endnow):
                # add last chars
                if (p.lastchars):
                    strnode = LatexCharsNode(chars=p.lastchars)
                    nodelist.append(strnode)
                return (nodelist, origpos, p.pos - origpos)

        raise LatexWalkerError(                # lgtm [py/unreachable-statement]
            "CONGRATULATIONS !! "
            "You are the first human to telepathically break an infinite loop !!!!!!!"
        )

# ------------------------------------------------------------------------------

def get_token(s, pos, brackets_are_chars=True, environments=True, **parse_flags):
    """
    Parse the next token in the stream.

    Returns a `LatexToken`. Raises `LatexWalkerEndOfStream` if end of stream reached.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_token()` instead.
    """
    return LatexWalker(s, **parse_flags).get_token(pos=pos,
                                                   brackets_are_chars=brackets_are_chars,
                                                   environments=environments)

def get_latex_expression(s, pos, **parse_flags):
    """
    Reads a latex expression, e.g. macro argument. This may be a single char, an escape
    sequence, or a expression placed in braces.

    Returns a tuple `(<LatexNode instance>, pos, len)`. `pos` is the first char of the
    expression, and `len` is its length.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_latex_expression()` instead.
    """

    return LatexWalker(s, **parse_flags).get_latex_expression(pos=pos)

def get_latex_maybe_optional_arg(s, pos, **parse_flags):
    """
    Attempts to parse an optional argument. Returns a tuple `(groupnode, pos, len)` if
    success, otherwise returns None.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_latex_maybe_optional_arg()` instead.
    """

    return LatexWalker(s, **parse_flags).get_latex_maybe_optional_arg(pos=pos)

def get_latex_braced_group(s, pos, brace_type='{', **parse_flags):
    """
    Reads a latex expression enclosed in braces {...}. The first token of `s[pos:]` must
    be an opening brace.

    Returns a tuple `(node, pos, len)`. `pos` is the first char of the
    expression (which has to be an opening brace), and `len` is its length,
    including the closing brace.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_latex_braced_group()` instead.
    """

    return LatexWalker(s, **parse_flags).get_latex_braced_group(pos=pos, brace_type=brace_type)

def get_latex_environment(s, pos, environmentname=None, **parse_flags):
    """
    Reads a latex expression enclosed in a \\begin{environment}...\\end{environment}. The first
    token in the stream must be the \\begin{environment}.

    Returns a tuple (node, pos, len) with node being a :py:class:`LatexEnvironmentNode`.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_latex_environment()` instead.
    """

    return LatexWalker(s, **parse_flags).get_latex_environment(pos=pos, environmentname=environmentname)

def get_latex_nodes(s, pos=0, stop_upon_closing_brace=None, stop_upon_end_environment=None,
                    stop_upon_closing_mathmode=None, **parse_flags):
    """
    Parses latex content `s`.

    Returns a tuple `(nodelist, pos, len)` where nodelist is a list of `LatexNode` 's.

    If `stop_upon_closing_brace` is given, then `len` includes the closing brace, but the
    closing brace is not included in any of the nodes in the `nodelist`.

    .. deprecated:: 1.0
       Please use :py:meth:`LatexWalker.get_latex_nodes()` instead.
    """

    return LatexWalker(s, **parse_flags).get_latex_nodes(stop_upon_closing_brace=stop_upon_closing_brace,
                                                         stop_upon_end_environment=stop_upon_end_environment,
                                                         stop_upon_closing_mathmode=stop_upon_closing_mathmode)

# ------------------------------------------------------------------------------

#
# small utilities for displaying & debugging
#

def put_in_braces(brace_char, thestring):
    if (brace_char == '{'):
        return '{%s}' %(thestring)
    if (brace_char == '['):
        return '[%s]' %(thestring)
    if (brace_char == '('):
        return '(%s)' %(thestring)
    if (brace_char == '<'):
        return '<%s>' %(thestring)

    return brace_char + thestring + brace_char

def nodelist_to_latex(nodelist):
    latex = ''
    for n in nodelist:
        if n is None:
            continue
        if n.isNodeType(LatexCharsNode):
            latex += n.chars
            continue
        if n.isNodeType(LatexMacroNode):
            latex += r'\%s%s' %(n.macroname, n.macro_post_space)

            mac = None
            if (n.macroname in default_macro_dict):
                mac = default_macro_dict[n.macroname]

            if (n.nodeoptarg is not None):
                latex += '[%s]' %(nodelist_to_latex([n.nodeoptarg]))

            if mac is not None:
                macbraces = (mac.numargs if isinstance(mac.numargs, _basestring) else '{'*mac.numargs)
            else:
                macbraces = '{'*len(n.nodeargs)

            if (len(n.nodeargs) != len(macbraces)):
                raise LatexWalkerError("Error: number of arguments (%d) provided to macro `\\%s' does not "
                                       "match its specification of `%s'"
                                       %(len(n.nodeargs), n.macroname, macbraces))
            for i in range(len(n.nodeargs)):
                nodearg = n.nodeargs[i]
                if nodearg is not None:
                    latex += put_in_braces(macbraces[i], nodelist_to_latex([nodearg]))

            continue

        if n.isNodeType(LatexCommentNode):
            latex += '%'+n.comment+n.comment_post_space
            continue

        if n.isNodeType(LatexGroupNode):
            latex += put_in_braces('{', nodelist_to_latex(n.nodelist))
            continue

        if n.isNodeType(LatexEnvironmentNode):
            latex += r'\begin{%s}' %(n.envname)
            for optarg in n.optargs:
                latex += put_in_braces('[', nodelist_to_latex([optarg]))
            for arg in n.args:
                latex += put_in_braces('{', nodelist_to_latex([arg]))
            latex += nodelist_to_latex(n.nodelist)
            latex += r'\end{%s}' %(n.envname)
            continue

        latex += '<[UNKNOWN LATEX NODE: `%s\']>' %(n.nodeType().__name__)

    return latex

def math_node_to_latex(node):

    if (not node.isNodeType(LatexMathNode)):
        raise LatexWalkerError("Expected math node, got `%s'" %(node.nodeType().__name__))

    if (node.displaytype == 'inline'):
        return '$%s$' %(nodelist_to_latex(node.nodelist))
    if (node.displaytype == 'display'):
        return r'\[%s\]' %(nodelist_to_latex(node.nodelist))

    raise LatexWalkerError("Unkonwn displaytype: `%s'" %(node.displaytype))

def disp_node(n, indent=0, context='* ', skip_group=False):
    title = ''
    comment = ''
    iterchildren = []
    if n is None:
        title = '<None>'
    elif n.isNodeType(LatexCharsNode):
        title = "'%s'" %(n.chars) #.strip())
    elif n.isNodeType(LatexMacroNode):
        title = '\\'+n.macroname
        #comment = 'opt arg?: %d; %d args' % (n.arg.nodeoptarg is not None, len(n.arg.nodeargs))
        if (n.nodeoptarg):
            iterchildren.append(('[...]: ', [n.nodeoptarg], False))
        if (len(n.nodeargs)):
            iterchildren.append(('{...}: ', n.nodeargs, False))
    elif n.isNodeType(LatexCommentNode):
        title = '%' + n.comment.strip()
    elif n.isNodeType(LatexGroupNode):
        if (skip_group):
            for nn in n.arg:
                disp_node(nn, indent=indent, context=context)
            return
        title = 'Group: '
        iterchildren.append(('* ', n.nodelist, False))
    elif n.isNodeType(LatexEnvironmentNode):
        title = '\\begin{%s}' %(n.envname)
        iterchildren.append(('* ', n.nodelist, False))
    elif n.isNodeType(LatexMathNode):
        title = '$inline math$'
        iterchildren.append(('* ', n.nodelist, False))
    else:
        print("UNKNOWN NODE TYPE: %s"%(n.nodeType().__name__))

    print(' '*indent + context + title + '  '+comment)

    for context, nodelist, skip in iterchildren:
        for nn in nodelist:
            disp_node(nn, indent=indent+4, context=context, skip_group=skip)

class LatexNodesJSONEncoder(json.JSONEncoder):
    """
    A :py:class:`json.JSONEncoder` that can encode :py:class:`LatexNode` objects
    (and subclasses).
    """
    def default(self, obj):
        if not isinstance(obj, LatexNode):
            return super(LatexNodesJSONEncoder, self).default(obj)

        # Prepare a dictionary with the correct keys and values.
        n = obj
        d = {
            'nodetype': n.__class__.__name__,
        }
        for fld in n._fields:
            d[fld] = n.__dict__[fld]
        return d

def main(argv):
    import fileinput
    import argparse

    parser = argparse.ArgumentParser()

    parser.add_argument('--output-format', metavar="FORMAT", dest="output_format",
                        choices=["human", "json"], default='human',
                        help='Requested output format for the node tree')
    parser.add_argument('--json-indent', metavar="NUMSPACES", dest="json_indent",
                        type=int, default=2,
                        help='Indentation in JSON output (specify number of spaces per indentation level)')
    parser.add_argument('--json-compact', dest="json_indent", default=2,
                        action='store_const', const=None,
                        help='Output compact JSON')

    parser.add_argument('--keep-inline-math', action='store_const', const=True,
                        dest='keep_inline_math', default=True)
    parser.add_argument('--no-keep-inline-math', action='store_const', const=False,
                        dest='keep_inline_math',
                        help="Keep inline math between $ signs as a special math node (default yes)")

    parser.add_argument('--tolerant-parsing', action='store_const', const=True,
                        dest='tolerant_parsing', default=True)
    parser.add_argument('--no-tolerant-parsing', action='store_const', const=False,
                        dest='tolerant_parsing',
                        help="Tolerate syntax errors when parsing, and attempt to continue (default yes)")

    parser.add_argument('--strict-braces', action='store_const', const=True,
                        dest='strict_braces', default=False)
    parser.add_argument('--no-strict-braces', action='store_const', const=False,
                        dest='strict_braces',
                        help="Report errors for mismatching LaTeX braces (default no)")

    parser.add_argument('files', metavar="FILE", nargs='*',
                        help='Input files (if none specified, read from stdandard input)')

    args = parser.parse_args(argv)

    latex = ''
    for line in fileinput.input(files=args.files):
        latex += line

    latexwalker = LatexWalker(latex,
                              keep_inline_math=args.keep_inline_math,
                              tolerant_parsing=args.tolerant_parsing,
                              strict_braces=args.strict_braces)

    (nodelist, pos, len_) = latexwalker.get_latex_nodes()

    if args.output_format == 'human':
        print('\n--- NODES ---\n')
        for n in nodelist:
            disp_node(n)
        print('\n-------------\n')
        return

    if args.output_format == 'json':
        print(json.dumps(nodelist, cls=LatexNodesJSONEncoder,
                         indent=args.json_indent) + "\n")
        return

    raise ValueError("Invalid output format: "+args.output_format)

if __name__ == '__main__':

    try:

        main(sys.argv[1:])

    except SystemExit:
        raise
    except: # lgtm [py/catch-base-exception]
        import pdb
        import traceback
        traceback.print_exc()
        pdb.post_mortem()