>>> preprocess('#hi there http://www.foo.com @you isn"t RT &lt;&gt;')
    'hashtaghi hashtaghi there isn"t'

def preprocess(s):
    """
    >>> preprocess('#hi there http://www.foo.com @you isn"t RT &lt;&gt;')
    'hashtaghi hashtaghi there isn"t'
    """
    # s = re.sub('@\S+', 'thisisamention', s)  # map all mentions to thisisamention
    s = re.sub(r'@\S+', ' ', s)  # map all mentions to thisisamention
    # s = re.sub('http\S+', 'http', s)  # keep only http from urls
    s = re.sub(r'http\S+', ' ', s)  # keep only http from urls
    s = re.sub(r'#(\S+)', r'hashtag\1 hashtag\1', s)  # #foo -> hashtagfoo hashtagfoo (for retaining hashtags even using bigrams)
    # s = re.sub(r'[0-9]+', '9', s)  # 1234 -> 9
    s = re.sub(r'\bRT\b', ' ', s, re.IGNORECASE)
    s = re.sub(r'&[a-z]+;', ' ', s, re.IGNORECASE)
    s = re.sub(r'\s+', ' ', s).strip()
    return s.lower()