hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
/**
   * Compute the estimated number of bloom filter comparisons to be performed on each file group.
   */
Compute the estimated number of bloom filter comparisons to be performed on each file group.
Compute the estimated number of bloom filter comparisons to be performed on each file group .
private Map<String, Long> computeComparisonsPerFileGroup(final Map<String, Long> recordsPerPartition,
                                                           final Map<String, List<BloomIndexFileInfo>> partitionToFileInfo,
                                                           JavaPairRDD<String, String> partitionRecordKeyPairRDD) {
    Map<String, Long> fileToComparisons;
    if (config.getBloomIndexPruneByRanges()) {
      fileToComparisons = explodeRecordRDDWithFileComparisons(partitionToFileInfo, partitionRecordKeyPairRDD)
          .mapToPair(t -> t).countByKey();
    } else {
      fileToComparisons = new HashMap<>();
      partitionToFileInfo.forEach((key, value) -> {
        for (BloomIndexFileInfo fileInfo : value) {
          fileToComparisons.put(fileInfo.getFileId(), recordsPerPartition.get(key));
        }
      });
    }
    return fileToComparisons;
  }
private Map < String , Long > computeComparisonsPerFileGroup ( final Map < String , Long > recordsPerPartition , final Map < String , List < BloomIndexFileInfo > > partitionToFileInfo , JavaPairRDD < String , String > partitionRecordKeyPairRDD ) { Map < String , Long > fileToComparisons ; if ( config . getBloomIndexPruneByRanges ( ) ) { fileToComparisons = explodeRecordRDDWithFileComparisons ( partitionToFileInfo , partitionRecordKeyPairRDD ) . mapToPair ( t -> t ) . countByKey ( ) ; } else { fileToComparisons = new HashMap < > ( ) ; partitionToFileInfo . forEach ( ( key , value ) -> { for ( BloomIndexFileInfo fileInfo : value ) { fileToComparisons . put ( fileInfo . getFileId ( ) , recordsPerPartition . get ( key ) ) ; } } ) ; } return fileToComparisons ; }
Compute the estimated number of bloom filter comparisons to be performed on each file group.
Compute the estimated number of bloom filter comparisons to be performed on each file group .
// we will just try exploding the input and then count to determine comparisons // FIX(vc): Only do sampling here and extrapolate? // each file needs to be compared against all the records coming into the partition