streamcorpus_pipeline/_kvlayer_keyword_search.py
def index(self, si):
        '''Record index records for a single document.

        Which indexes this creates depends on the parameters to the
        constructor.  This records all of the requested indexes for
        a single document.

        '''
        if not si.body.clean_visible:
            logger.warn('stream item %s has no clean_visible part, '
                        'skipping keyword indexing', si.stream_id)
            return

        # Count tokens in si.clean_visible
        # We will recycle hash==0 for "# of documents"
        hash_counts = defaultdict(int)
        hash_counts[DOCUMENT_HASH_KEY] = 1
        hash_kw = defaultdict(int)
        words = self.collect_words(si)
        for tok, count in words.iteritems():
            (tok, tok_hash) = self.make_hash_kw(tok)
            hash_counts[tok_hash] += count
            hash_kw[tok] = tok_hash

        # Convert this and write it out
        if self.hash_docs:
            (k1, k2) = key_for_stream_item(si)
            kvps = [((h, k1, k2), n) for (h, n) in hash_counts.iteritems()
                    if h != DOCUMENT_HASH_KEY]
            self.client.put(HASH_TF_INDEX_TABLE, *kvps)

        if self.hash_frequencies:
            kvps = [((h,), 1) for h in hash_counts.iterkeys()]
            self.client.increment(HASH_FREQUENCY_TABLE, *kvps)

        if self.hash_keywords:
            kvps = [((h, t), 1) for (t, h) in hash_kw.iteritems()]
            self.client.increment(HASH_KEYWORD_INDEX_TABLE, *kvps)
def index(self, si):
        '''Record index records for a single document.

        Which indexes this creates depends on the parameters to the
        constructor.  This records all of the requested indexes for
        a single document.

        '''
        if not si.body.clean_visible:
            logger.warn('stream item %s has no clean_visible part, '
                        'skipping keyword indexing', si.stream_id)
            return

        # Count tokens in si.clean_visible
        # We will recycle hash==0 for "# of documents"
        hash_counts = defaultdict(int)
        hash_counts[DOCUMENT_HASH_KEY] = 1
        hash_kw = defaultdict(int)
        words = self.collect_words(si)
        for tok, count in words.iteritems():
            (tok, tok_hash) = self.make_hash_kw(tok)
            hash_counts[tok_hash] += count
            hash_kw[tok] = tok_hash

        # Convert this and write it out
        if self.hash_docs:
            (k1, k2) = key_for_stream_item(si)
            kvps = [((h, k1, k2), n) for (h, n) in hash_counts.iteritems()
                    if h != DOCUMENT_HASH_KEY]
            self.client.put(HASH_TF_INDEX_TABLE, *kvps)

        if self.hash_frequencies:
            kvps = [((h,), 1) for h in hash_counts.iterkeys()]
            self.client.increment(HASH_FREQUENCY_TABLE, *kvps)

        if self.hash_keywords:
            kvps = [((h, t), 1) for (t, h) in hash_kw.iteritems()]
            self.client.increment(HASH_KEYWORD_INDEX_TABLE, *kvps)
def index ( self , si ) : if not si . body . clean_visible : logger . warn ( 'stream item %s has no clean_visible part, ' 'skipping keyword indexing' , si . stream_id ) return # Count tokens in si.clean_visible # We will recycle hash==0 for "# of documents" hash_counts = defaultdict ( int ) hash_counts [ DOCUMENT_HASH_KEY ] = 1 hash_kw = defaultdict ( int ) words = self . collect_words ( si ) for tok , count in words . iteritems ( ) : ( tok , tok_hash ) = self . make_hash_kw ( tok ) hash_counts [ tok_hash ] += count hash_kw [ tok ] = tok_hash # Convert this and write it out if self . hash_docs : ( k1 , k2 ) = key_for_stream_item ( si ) kvps = [ ( ( h , k1 , k2 ) , n ) for ( h , n ) in hash_counts . iteritems ( ) if h != DOCUMENT_HASH_KEY ] self . client . put ( HASH_TF_INDEX_TABLE , * kvps ) if self . hash_frequencies : kvps = [ ( ( h , ) , 1 ) for h in hash_counts . iterkeys ( ) ] self . client . increment ( HASH_FREQUENCY_TABLE , * kvps ) if self . hash_keywords : kvps = [ ( ( h , t ) , 1 ) for ( t , h ) in hash_kw . iteritems ( ) ] self . client . increment ( HASH_KEYWORD_INDEX_TABLE , * kvps )
Record index records for a single document.

        Which indexes this creates depends on the parameters to the
        constructor.  This records all of the requested indexes for
        a single document.
Record index records for a single document .