Consume a token.

        Returns a tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.

def consume_token(self, tokens, index, tokens_len):
        """Consume a token.

        Returns a tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.
        """
        finished = False

        if tokens[index].line > self.line:
            finished = True
            end = index
        elif index == tokens_len - 1:
            finished = True
            end = index + 1

        if finished:
            pasted_together_contents = ""
            for i in range(self.begin, end):
                pasted_together_contents += tokens[i].content

            replacement = [Token(type=TokenType.Comment,
                                 content=pasted_together_contents,
                                 line=tokens[self.begin].line,
                                 col=tokens[self.begin].col)]

            tokens = _replace_token_range(tokens,
                                          self.begin,
                                          end,
                                          replacement)

            return (self.begin, len(tokens), tokens)