expr_tokenizer yields the components ("tokens") forming the expression.

    Tokens are split by whitespace which is never considered a token in its
    own right. operator_tokens should likely include "(" and ")" and strictly
    the expression. This means that the word 'test' will be split into ['t',
    'e', 'st'] if 'e' is an operator.

    :param expr: The expression to break into tokens.
    :param operator_tokens: A list of operators to extract as tokens.

def expr_tokenizer(expr, operator_tokens):
    """expr_tokenizer yields the components ("tokens") forming the expression.

    Tokens are split by whitespace which is never considered a token in its
    own right. operator_tokens should likely include "(" and ")" and strictly
    the expression. This means that the word 'test' will be split into ['t',
    'e', 'st'] if 'e' is an operator.

    :param expr: The expression to break into tokens.
    :param operator_tokens: A list of operators to extract as tokens.
    """
    operator_tokens.sort(key=len, reverse=True)
    for m in re.finditer(
        r"""(\s+) |            # Whitespace
            ({0}) |            # Operators
            (.+?)(?={0}|\s|$)  # Patterns
            """.format("|".join(re.escape(op) for op in operator_tokens)),
        expr, re.X
    ):
        token = m.group(2) or m.group(3)
        if token:
            yield token