The main collector for all AST functions.

    This function is called recursively to find both variable use and function
    calls and returns a GenericBody with both those variables and function
    calls hanging off of it. The caller can figure out what to do with both of
    those

def _ast_worker(tokens, tokens_len, index, term):
    """The main collector for all AST functions.

    This function is called recursively to find both variable use and function
    calls and returns a GenericBody with both those variables and function
    calls hanging off of it. The caller can figure out what to do with both of
    those
    """
    statements = []
    arguments = []

    while index < tokens_len:
        if term:
            if term(index, tokens):
                break

        # Function call
        if tokens[index].type == TokenType.Word and \
           index + 1 < tokens_len and \
           tokens[index + 1].type == TokenType.LeftParen:
            index, statement = _handle_function_call(tokens,
                                                     tokens_len,
                                                     index)
            statements.append(statement)
        # Argument
        elif _is_word_type(tokens[index].type):
            arguments.append(Word(type=_word_type(tokens[index].type),
                                  contents=tokens[index].content,
                                  line=tokens[index].line,
                                  col=tokens[index].col,
                                  index=index))

        index = index + 1

    return (index, GenericBody(statements=statements,
                               arguments=arguments))