Consume a token.

        Returns a tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.

def consume_token(self, tokens, index, tokens_len):
        """Consume a token.

        Returns a tuple of (tokens, tokens_len, index) when consumption is
        completed and tokens have been merged together.
        """
        if _is_really_comment(tokens, index):
            self.last_line_with_comment = tokens[index].line

        finished = False

        if (not _is_in_comment_type(tokens[index].type) and
                self.last_line_with_comment != tokens[index].line):
            finished = True
            end = index
        elif index == (tokens_len - 1):
            finished = True
            end = index + 1

        if finished:
            return _paste_tokens_line_by_line(tokens,
                                              TokenType.RST,
                                              self.begin,
                                              end)