public static IndexSummary downsample(IndexSummary existing, int newSamplingLevel, int minIndexInterval, IPartitioner partitioner)
    {
        // To downsample the old index summary, we'll go through (potentially) several rounds of downsampling.
        // Conceptually, each round starts at position X and then removes every Nth item.  The value of X follows
        // a particular pattern to evenly space out the items that we remove.  The value of N decreases by one each
        // round.

        int currentSamplingLevel = existing.getSamplingLevel();
        assert currentSamplingLevel > newSamplingLevel;
        assert minIndexInterval == existing.getMinIndexInterval();

        // calculate starting indexes for downsampling rounds
        int[] startPoints = Downsampling.getStartPoints(currentSamplingLevel, newSamplingLevel);

        // calculate new off-heap size
        int newKeyCount = existing.size();
        long newEntriesLength = existing.getEntriesLength();
        for (int start : startPoints)
        {
            for (int j = start; j < existing.size(); j += currentSamplingLevel)
            {
                newKeyCount--;
                long length = existing.getEndInSummary(j) - existing.getPositionInSummary(j);
                newEntriesLength -= length;
            }
        }

        Memory oldEntries = existing.getEntries();
        Memory newOffsets = Memory.allocate(newKeyCount * 4);
        Memory newEntries = Memory.allocate(newEntriesLength);

        // Copy old entries to our new Memory.
        int i = 0;
        int newEntriesOffset = 0;
        outer:
        for (int oldSummaryIndex = 0; oldSummaryIndex < existing.size(); oldSummaryIndex++)
        {
            // to determine if we can skip this entry, go through the starting points for our downsampling rounds
            // and see if the entry's index is covered by that round
            for (int start : startPoints)
            {
                if ((oldSummaryIndex - start) % currentSamplingLevel == 0)
                    continue outer;
            }

            // write the position of the actual entry in the index summary (4 bytes)
            newOffsets.setInt(i * 4, newEntriesOffset);
            i++;
            long start = existing.getPositionInSummary(oldSummaryIndex);
            long length = existing.getEndInSummary(oldSummaryIndex) - start;
            newEntries.put(newEntriesOffset, oldEntries, start, length);
            newEntriesOffset += length;
        }
        assert newEntriesOffset == newEntriesLength;
        return new IndexSummary(partitioner, newOffsets, newKeyCount, newEntries, newEntriesLength,
                                existing.getMaxNumberOfEntries(), minIndexInterval, newSamplingLevel);
    }
public static IndexSummary downsample ( IndexSummary existing , int newSamplingLevel , int minIndexInterval , IPartitioner partitioner ) { int currentSamplingLevel = existing . getSamplingLevel ( ) ; assert currentSamplingLevel > newSamplingLevel ; assert minIndexInterval == existing . getMinIndexInterval ( ) ; int [ ] startPoints = Downsampling . getStartPoints ( currentSamplingLevel , newSamplingLevel ) ; int newKeyCount = existing . size ( ) ; long newEntriesLength = existing . getEntriesLength ( ) ; for ( int start : startPoints ) { for ( int j = start ; j < existing . size ( ) ; j += currentSamplingLevel ) { newKeyCount -- ; long length = existing . getEndInSummary ( j ) - existing . getPositionInSummary ( j ) ; newEntriesLength -= length ; } } Memory oldEntries = existing . getEntries ( ) ; Memory newOffsets = Memory . allocate ( newKeyCount * 4 ) ; Memory newEntries = Memory . allocate ( newEntriesLength ) ; int i = 0 ; int newEntriesOffset = 0 ; outer : for ( int oldSummaryIndex = 0 ; oldSummaryIndex < existing . size ( ) ; oldSummaryIndex ++ ) { for ( int start : startPoints ) { if ( ( oldSummaryIndex - start ) % currentSamplingLevel == 0 ) continue outer ; } newOffsets . setInt ( i * 4 , newEntriesOffset ) ; i ++ ; long start = existing . getPositionInSummary ( oldSummaryIndex ) ; long length = existing . getEndInSummary ( oldSummaryIndex ) - start ; newEntries . put ( newEntriesOffset , oldEntries , start , length ) ; newEntriesOffset += length ; } assert newEntriesOffset == newEntriesLength ; return new IndexSummary ( partitioner , newOffsets , newKeyCount , newEntries , newEntriesLength , existing . getMaxNumberOfEntries ( ) , minIndexInterval , newSamplingLevel ) ; }
Downsamples an existing index summary to a new sampling level.
@param existing an existing IndexSummary
@param newSamplingLevel the target level for the new IndexSummary.  This must be less than the current sampling
level for `existing`.
@param partitioner the partitioner used for the index summary
@return a new IndexSummary