returns a list of (token node ID, token) which represent the tokens
        of the input document (in the order they occur).

        Parameters
        ----------
        token_attrib : str
            name of the node attribute that contains the token string as its
            value (default: token).

        Yields
        -------
        result : generator of (str, unicode) or generator unicode
            a generator of (token node ID, token string) tuples if
            token_strings_only==False, a generator of token strings otherwise

def get_tokens(self, token_attrib='token', token_strings_only=False):
        """
        returns a list of (token node ID, token) which represent the tokens
        of the input document (in the order they occur).

        Parameters
        ----------
        token_attrib : str
            name of the node attribute that contains the token string as its
            value (default: token).

        Yields
        -------
        result : generator of (str, unicode) or generator unicode
            a generator of (token node ID, token string) tuples if
            token_strings_only==False, a generator of token strings otherwise
        """
        if token_strings_only:
            for token_id in self.tokens:
                yield self.get_token(token_id, token_attrib)
        else:
            for token_id in self.tokens:
                yield (token_id, self.get_token(token_id, token_attrib))