>>> tk = Tokenizer(ngrams=1, lc=True, collapse_urls=True, collapse_mentions=True, collapse_hashtags=False, retain_punc_toks=True)
        >>> tk.do_tokenize('http://www.foo.com fast-forward hi there :) how?? U.S.A. @you whaaaaaaat? #awesome.')
        ['THIS_IS_A_URL', 'fast-forward', 'hi', 'there', ':)', 'how', '??', 'u.s.a', '.', 'THIS_IS_A_MENTION', 'what', '?', 'HASHTAG_awesome', '.']
        >>> tk = Tokenizer(rt_prefix=True)
        >>> tk.do_tokenize('RT hi there')
        ['RT_hi', 'RT_there']
        >>> tk.do_tokenize('hi there')
        ['hi', 'there']

def do_tokenize(self, text):
        """
        >>> tk = Tokenizer(ngrams=1, lc=True, collapse_urls=True, collapse_mentions=True, collapse_hashtags=False, retain_punc_toks=True)
        >>> tk.do_tokenize('http://www.foo.com fast-forward hi there :) how?? U.S.A. @you whaaaaaaat? #awesome.')
        ['THIS_IS_A_URL', 'fast-forward', 'hi', 'there', ':)', 'how', '??', 'u.s.a', '.', 'THIS_IS_A_MENTION', 'what', '?', 'HASHTAG_awesome', '.']
        >>> tk = Tokenizer(rt_prefix=True)
        >>> tk.do_tokenize('RT hi there')
        ['RT_hi', 'RT_there']
        >>> tk.do_tokenize('hi there')
        ['hi', 'there']
        """
        text = text.lower() if self.lc else text
        if self.collapse_hashtags:
            text = re.sub('#\S+', 'THIS_IS_A_HASHTAG', text)
        else:
            text = re.sub('#(\S+)', r'HASHTAG_\1', text)
        if self.collapse_mentions:
            text = re.sub('@\S+', 'THIS_IS_A_MENTION', text)
        if self.collapse_urls:
            text = re.sub('http\S+', 'THIS_IS_A_URL', text)
        if self.limit_repeats:
            text = re.sub(r'(.)\1\1\1+', r'\1', text)
        if self.collapse_digits:
            text = re.sub(r'[0-9]+', '9', text)
        toks = []
        for tok in text.split():
            tok = re.sub(r'^(' + punc_re + '+)', r'\1 ', tok)
            tok = re.sub(r'(' + punc_re + '+)$', r' \1', tok)
            for subtok in tok.split():
                if self.retain_punc_toks or re.search('\w', subtok):
                    toks.append(subtok)
        if self.rt_prefix:
            rt_text = 'rt' if self.lc else 'RT'
            if rt_text in toks:
                toks.remove(rt_text)
                toks = ['RT_' + t for t in toks]
        return toks