def get_similarity_task_tokens(args):
    """Returns a set of all tokens occurring the evaluation datasets."""
    tokens = set()
    for _, _, dataset in iterate_similarity_datasets(args):
        tokens.update(
            itertools.chain.from_iterable((d[0], d[1]) for d in dataset))
    return tokens
def get_similarity_task_tokens ( args ) : tokens = set ( ) for _ , _ , dataset in iterate_similarity_datasets ( args ) : tokens . update ( itertools . chain . from_iterable ( ( d [ 0 ] , d [ 1 ] ) for d in dataset ) ) return tokens
Returns a set of all tokens occurring the evaluation datasets.