def crypto_bloom_filter(record,      # type: Sequence[Text]
                        tokenizers,  # type: List[Callable[[Text, Optional[Text]], Iterable[Text]]]
                        schema,      # type: Schema
                        keys         # type: Sequence[Sequence[bytes]]
                        ):
    # type: (...) -> Tuple[bitarray, Text, int]
    """ Computes the composite Bloom filter encoding of a record.

        Using the method from
        http://www.record-linkage.de/-download=wp-grlc-2011-02.pdf

        :param record: plaintext record tuple. E.g. (index, name, dob, gender)
        :param tokenizers: A list of tokenizers. A tokenizer is a function that
            returns tokens from a string.
        :param schema: Schema
        :param keys: Keys for the hash functions as a tuple of lists of bytes.

        :return: 3-tuple:
                - bloom filter for record as a bitarray
                - first element of record (usually an index)
                - number of bits set in the bloomfilter
    """
    hash_l = schema.l * 2 ** schema.xor_folds

    bloomfilter = bitarray(hash_l)
    bloomfilter.setall(False)

    for (entry, tokenize, field, key) \
            in zip(record, tokenizers, schema.fields, keys):
        fhp = field.hashing_properties
        if fhp:
            ngrams = list(tokenize(field.format_value(entry)))
            hash_function = hashing_function_from_properties(fhp)
            bloomfilter |= hash_function(ngrams, key,
                                         fhp.ks(len(ngrams)),
                                         hash_l, fhp.encoding)

    c1 = bloomfilter.count()
    bloomfilter = fold_xor(bloomfilter, schema.xor_folds)
    c2 = bloomfilter.count()
    return bloomfilter, record[0], bloomfilter.count()
def crypto_bloom_filter ( record , tokenizers , schema , keys ) : hash_l = schema . l * 2 ** schema . xor_folds bloomfilter = bitarray ( hash_l ) bloomfilter . setall ( False ) for ( entry , tokenize , field , key ) in zip ( record , tokenizers , schema . fields , keys ) : fhp = field . hashing_properties if fhp : ngrams = list ( tokenize ( field . format_value ( entry ) ) ) hash_function = hashing_function_from_properties ( fhp ) bloomfilter |= hash_function ( ngrams , key , fhp . ks ( len ( ngrams ) ) , hash_l , fhp . encoding ) c1 = bloomfilter . count ( ) bloomfilter = fold_xor ( bloomfilter , schema . xor_folds ) c2 = bloomfilter . count ( ) return bloomfilter , record [ 0 ] , bloomfilter . count ( )
Computes the composite Bloom filter encoding of a record.

        Using the method from
        http://www.record-linkage.de/-download=wp-grlc-2011-02.pdf

        :param record: plaintext record tuple. E.g. (index, name, dob, gender)
        :param tokenizers: A list of tokenizers. A tokenizer is a function that
            returns tokens from a string.
        :param schema: Schema
        :param keys: Keys for the hash functions as a tuple of lists of bytes.

        :return: 3-tuple:
                - bloom filter for record as a bitarray
                - first element of record (usually an index)
                - number of bits set in the bloomfilter