Returns list of LexicalTokens found in tweet. The list contains all the words in original tweet, but are
    optimally grouped up to form largest matching n-grams from lexicon. If no match is found, token is added as
    singleton.

    @param tweet      Tweet to lexically parse
    @param phrase_tree Token tree that contains all the lexical n-grams
    @return List of LexicalTokens

def lexically_parse_tweet(tweet, phrase_tree):
    """
    Returns list of LexicalTokens found in tweet. The list contains all the words in original tweet, but are
    optimally grouped up to form largest matching n-grams from lexicon. If no match is found, token is added as
    singleton.

    @param tweet      Tweet to lexically parse
    @param phrase_tree Token tree that contains all the lexical n-grams
    @return List of LexicalTokens
    """
    lexical_tokens = []

    prev = 0

    while True:
        match = RegexFilters.SENTENCE_END_PUNCTUATION.search(tweet[prev:])
        if match is None:
            break
        span = match.span()
        sentence = tweet[prev:prev + span[0]]
        punctuation = match.group(0)
        prev += span[1]
        lexical_tokens.extend(parse_sentence(sentence, punctuation, phrase_tree))

    lexical_tokens.extend(parse_sentence(tweet[prev:], None, phrase_tree))

    return lexical_tokens