datasketch/lshensemble.py
def index(self, entries):
        '''
        Index all sets given their keys, MinHashes, and sizes.
        It can be called only once after the index is created.

        Args:
            entries (`iterable` of `tuple`): An iterable of tuples, each must be
                in the form of `(key, minhash, size)`, where `key` is the unique
                identifier of a set, `minhash` is the MinHash of the set,
                and `size` is the size or number of unique items in the set.

        Note:
            `size` must be positive.
        '''
        if not self.is_empty():
            raise ValueError("Cannot call index again on a non-empty index")
        if not isinstance(entries, list):
            queue = deque([])
            for key, minhash, size in entries:
                if size <= 0:
                    raise ValueError("Set size must be positive")
                queue.append((key, minhash, size))
            entries = list(queue)
        if len(entries) == 0:
            raise ValueError("entries is empty")
        # Create optimal partitions.
        sizes, counts = np.array(sorted(
            Counter(e[2] for e in entries).most_common())).T
        partitions = optimal_partitions(sizes, counts, len(self.indexes))
        for i, (lower, upper) in enumerate(partitions):
            self.lowers[i], self.uppers[i] = lower, upper
        # Insert into partitions.
        entries.sort(key=lambda e : e[2])
        curr_part = 0
        for key, minhash, size in entries:
            if size > self.uppers[curr_part]:
                curr_part += 1
            for r in self.indexes[curr_part]:
                self.indexes[curr_part][r].insert(key, minhash)
def index(self, entries):
        '''
        Index all sets given their keys, MinHashes, and sizes.
        It can be called only once after the index is created.

        Args:
            entries (`iterable` of `tuple`): An iterable of tuples, each must be
                in the form of `(key, minhash, size)`, where `key` is the unique
                identifier of a set, `minhash` is the MinHash of the set,
                and `size` is the size or number of unique items in the set.

        Note:
            `size` must be positive.
        '''
        if not self.is_empty():
            raise ValueError("Cannot call index again on a non-empty index")
        if not isinstance(entries, list):
            queue = deque([])
            for key, minhash, size in entries:
                if size <= 0:
                    raise ValueError("Set size must be positive")
                queue.append((key, minhash, size))
            entries = list(queue)
        if len(entries) == 0:
            raise ValueError("entries is empty")
        # Create optimal partitions.
        sizes, counts = np.array(sorted(
            Counter(e[2] for e in entries).most_common())).T
        partitions = optimal_partitions(sizes, counts, len(self.indexes))
        for i, (lower, upper) in enumerate(partitions):
            self.lowers[i], self.uppers[i] = lower, upper
        # Insert into partitions.
        entries.sort(key=lambda e : e[2])
        curr_part = 0
        for key, minhash, size in entries:
            if size > self.uppers[curr_part]:
                curr_part += 1
            for r in self.indexes[curr_part]:
                self.indexes[curr_part][r].insert(key, minhash)
def index ( self , entries ) : if not self . is_empty ( ) : raise ValueError ( "Cannot call index again on a non-empty index" ) if not isinstance ( entries , list ) : queue = deque ( [ ] ) for key , minhash , size in entries : if size <= 0 : raise ValueError ( "Set size must be positive" ) queue . append ( ( key , minhash , size ) ) entries = list ( queue ) if len ( entries ) == 0 : raise ValueError ( "entries is empty" ) # Create optimal partitions. sizes , counts = np . array ( sorted ( Counter ( e [ 2 ] for e in entries ) . most_common ( ) ) ) . T partitions = optimal_partitions ( sizes , counts , len ( self . indexes ) ) for i , ( lower , upper ) in enumerate ( partitions ) : self . lowers [ i ] , self . uppers [ i ] = lower , upper # Insert into partitions. entries . sort ( key = lambda e : e [ 2 ] ) curr_part = 0 for key , minhash , size in entries : if size > self . uppers [ curr_part ] : curr_part += 1 for r in self . indexes [ curr_part ] : self . indexes [ curr_part ] [ r ] . insert ( key , minhash )
Index all sets given their keys, MinHashes, and sizes.
        It can be called only once after the index is created.

        Args:
            entries (`iterable` of `tuple`): An iterable of tuples, each must be
                in the form of `(key, minhash, size)`, where `key` is the unique
                identifier of a set, `minhash` is the MinHash of the set,
                and `size` is the size or number of unique items in the set.

        Note:
            `size` must be positive.
Index all sets given their keys MinHashes and sizes . It can be called only once after the index is created .