def find_optimal_allocation(self, tokens):
        """
        Finds longest, non-overlapping word-ranges of phrases in tokens stored in TokenTrie

        :param tokens: tokens tokenize
        :type tokens: list of str
        :return: Optimal allocation of tokens to phrases
        :rtype: list of TokenTrie.Token
        """
        token_ranges = self.find_tracked_words(tokens)
        token_ranges.sort()

        for offset in range(1, len(token_ranges)):
            to_be_removed = []
            for candidate in token_ranges[offset:]:
                for i in range(offset):
                    if token_ranges[i].overlaps_with(candidate):
                        to_be_removed.append(candidate)
                        break

            token_ranges = [token for token in token_ranges if token not in to_be_removed]

        token_ranges.sort(key=lambda token: token.get_start_index())
        return token_ranges
def find_optimal_allocation ( self , tokens ) : token_ranges = self . find_tracked_words ( tokens ) token_ranges . sort ( ) for offset in range ( 1 , len ( token_ranges ) ) : to_be_removed = [ ] for candidate in token_ranges [ offset : ] : for i in range ( offset ) : if token_ranges [ i ] . overlaps_with ( candidate ) : to_be_removed . append ( candidate ) break token_ranges = [ token for token in token_ranges if token not in to_be_removed ] token_ranges . sort ( key = lambda token : token . get_start_index ( ) ) return token_ranges
Finds longest, non-overlapping word-ranges of phrases in tokens stored in TokenTrie

        :param tokens: tokens tokenize
        :type tokens: list of str
        :return: Optimal allocation of tokens to phrases
        :rtype: list of TokenTrie.Token