hudi-client/src/main/java/org/apache/hudi/index/bloom/HoodieBloomIndex.java
/**
   * Load all involved files as <Partition, filename> pair RDD.
   */
Load all involved files as  pair RDD.
Load all involved files as pair RDD .
List<Tuple2<String, BloomIndexFileInfo>> loadInvolvedFiles(List<String> partitions, final JavaSparkContext jsc,
                                                             final HoodieTable hoodieTable) {
    List<Pair<String, String>> partitionPathFileIDList = getLatestBaseFilesForAllPartitions(partitions, jsc, hoodieTable).stream()
        .map(pair -> Pair.of(pair.getKey(), pair.getValue().getFileId()))
        .collect(toList());
    if (config.getBloomIndexPruneByRanges()) {
      jsc.setJobDescription("Obtain key ranges for file slices (range pruning=on)");
      return jsc.parallelize(partitionPathFileIDList, Math.max(partitionPathFileIDList.size(), 1)).mapToPair(pf -> {
        try {
          HoodieRangeInfoHandle<T> rangeInfoHandle = new HoodieRangeInfoHandle<T>(config, hoodieTable, pf);
          String[] minMaxKeys = rangeInfoHandle.getMinMaxKeys();
          return new Tuple2<>(pf.getKey(), new BloomIndexFileInfo(pf.getValue(), minMaxKeys[0], minMaxKeys[1]));
        } catch (MetadataNotFoundException me) {
          LOG.warn("Unable to find range metadata in file :" + pf);
          return new Tuple2<>(pf.getKey(), new BloomIndexFileInfo(pf.getValue()));
        }
      }).collect();
    } else {
      return partitionPathFileIDList.stream()
          .map(pf -> new Tuple2<>(pf.getKey(), new BloomIndexFileInfo(pf.getValue()))).collect(toList());
    }
  }
List < Tuple2 < String , BloomIndexFileInfo > > loadInvolvedFiles ( List < String > partitions , final JavaSparkContext jsc , final HoodieTable hoodieTable ) { List < Pair < String , String > > partitionPathFileIDList = getLatestBaseFilesForAllPartitions ( partitions , jsc , hoodieTable ) . stream ( ) . map ( pair -> Pair . of ( pair . getKey ( ) , pair . getValue ( ) . getFileId ( ) ) ) . collect ( toList ( ) ) ; if ( config . getBloomIndexPruneByRanges ( ) ) { jsc . setJobDescription ( "Obtain key ranges for file slices (range pruning=on)" ) ; return jsc . parallelize ( partitionPathFileIDList , Math . max ( partitionPathFileIDList . size ( ) , 1 ) ) . mapToPair ( pf -> { try { HoodieRangeInfoHandle < T > rangeInfoHandle = new HoodieRangeInfoHandle < T > ( config , hoodieTable , pf ) ; String [ ] minMaxKeys = rangeInfoHandle . getMinMaxKeys ( ) ; return new Tuple2 < > ( pf . getKey ( ) , new BloomIndexFileInfo ( pf . getValue ( ) , minMaxKeys [ 0 ] , minMaxKeys [ 1 ] ) ) ; } catch ( MetadataNotFoundException me ) { LOG . warn ( "Unable to find range metadata in file :" + pf ) ; return new Tuple2 < > ( pf . getKey ( ) , new BloomIndexFileInfo ( pf . getValue ( ) ) ) ; } } ) . collect ( ) ; } else { return partitionPathFileIDList . stream ( ) . map ( pf -> new Tuple2 < > ( pf . getKey ( ) , new BloomIndexFileInfo ( pf . getValue ( ) ) ) ) . collect ( toList ( ) ) ; } }
Load all involved files as <Partition, filename> pair RDD.
Load all involved files as <Partition filename > pair RDD .
// Obtain the latest data files from all the partitions. // also obtain file ranges, if range pruning is enabled