streamcorpus_pipeline/_kvlayer_keyword_search.py
def make_hash_kw(self, tok):
        '''Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash

        '''
        if isinstance(tok, unicode):
            tok = tok.encode('utf-8')
        h = mmh3.hash(tok)
        if h == DOCUMENT_HASH_KEY:
            h = DOCUMENT_HASH_KEY_REPLACEMENT
        return (tok, h)
def make_hash_kw(self, tok):
        '''Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash

        '''
        if isinstance(tok, unicode):
            tok = tok.encode('utf-8')
        h = mmh3.hash(tok)
        if h == DOCUMENT_HASH_KEY:
            h = DOCUMENT_HASH_KEY_REPLACEMENT
        return (tok, h)
def make_hash_kw ( self , tok ) : if isinstance ( tok , unicode ) : tok = tok . encode ( 'utf-8' ) h = mmh3 . hash ( tok ) if h == DOCUMENT_HASH_KEY : h = DOCUMENT_HASH_KEY_REPLACEMENT return ( tok , h )
Get a Murmur hash and a normalized token.

        `tok` may be a :class:`unicode` string or a UTF-8-encoded
        byte string.  :data:`DOCUMENT_HASH_KEY`, hash value 0, is
        reserved for the document count, and this function remaps
        that value.

        :param tok: token to hash
        :return: pair of normalized `tok` and its hash
Get a Murmur hash and a normalized token .