Consume a token.

        Returns tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.

def consume_token(self, tokens, index, tokens_len):
        """Consume a token.

        Returns tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.
        """
        del tokens_len

        consumption_ended = False

        q_type = self.quote_type

        begin_literal_type = getattr(TokenType,
                                     "Begin{0}QuotedLiteral".format(q_type))
        end_literal_type = getattr(TokenType,
                                   "End{0}QuotedLiteral".format(q_type))

        if (index != self.begin and
                tokens[index].type == begin_literal_type):
            # This is an edge case where a quote begins a line and matched
            # as a quoted region beginning and a quoted region ending.
            # Split the token before and after the quote, mark the
            # quote character itself as an ending and insert both
            # tokens back in, handling the ending afterwards.
            assert _RE_QUOTE_TYPE.match(tokens[index].content[0])

            # Mini-tokenize everything after the first token
            line_tokens = _scan_for_tokens(tokens[index].content[1:])
            end_type = getattr(TokenType,
                               "End{0}QuotedLiteral".format(q_type))
            replacement = [Token(type=end_type,
                                 content=tokens[index].content[0],
                                 line=tokens[index].line,
                                 col=tokens[index].col)]

            for after in line_tokens:
                replacement.append(Token(type=after.type,
                                         content=after.content,
                                         line=(tokens[index].line +
                                               after.line - 1),
                                         col=(tokens[index].col +
                                              after.col - 1)))

            tokens = _replace_token_range(tokens,
                                          index,
                                          index + 1,
                                          replacement)
            consumption_ended = True

        if tokens[index].type == end_literal_type:
            consumption_ended = True

        if consumption_ended:
            end = index + 1
            pasted = ""
            for i in range(self.begin, end):
                pasted += tokens[i].content

            tokens = _replace_token_range(tokens, self.begin, end,
                                          [Token(type=TokenType.QuotedLiteral,
                                                 content=pasted,
                                                 line=tokens[self.begin].line,
                                                 col=tokens[self.begin].col)])

            return (self.begin, len(tokens), tokens)