Similar to {@link #findOptimalAllocation(String[])}, but also includes the words not matching any longer n-gram
        in TokenTrie as singletons.

        :param tokens: tokens to tokenize
        :return: Optimal allocation of tokens to phrases, with non matching tokens as singletons.

def find_optimal_tokenization(self, tokens):
        """
        Similar to {@link #findOptimalAllocation(String[])}, but also includes the words not matching any longer n-gram
        in TokenTrie as singletons.

        :param tokens: tokens to tokenize
        :return: Optimal allocation of tokens to phrases, with non matching tokens as singletons.
        """
        token_ranges = self.find_optimal_allocation(tokens)
        tokenized_sentence = []

        set_index = 0
        for token in token_ranges:
            while set_index < token.get_start_index():
                tokenized_sentence.append(tokens[set_index])
                set_index += 1
            tokenized_sentence.append(" ".join(token.get_token_sequence()))
            set_index = token.get_end_index() + 1

        while set_index < len(tokens):
            tokenized_sentence.append(tokens[set_index])
            set_index += 1

        return tokenized_sentence