hudi-client/src/main/java/org/apache/hudi/table/action/clean/CleanActionExecutor.java
/**
   * Generates List of files to be cleaned.
   *
   * @param jsc JavaSparkContext
   * @return Cleaner Plan
   */
Generates List of files to be cleaned.
@param jsc JavaSparkContext
@return Cleaner Plan
Generates List of files to be cleaned . @param jsc JavaSparkContext @return Cleaner Plan
HoodieCleanerPlan requestClean(JavaSparkContext jsc) {
    try {
      CleanPlanner<?> planner = new CleanPlanner<>(table, config);
      Option<HoodieInstant> earliestInstant = planner.getEarliestCommitToRetain();
      List<String> partitionsToClean = planner.getPartitionPathsToClean(earliestInstant);
      if (partitionsToClean.isEmpty()) {
        LOG.info("Nothing to clean here. It is already clean");
        return HoodieCleanerPlan.newBuilder().setPolicy(HoodieCleaningPolicy.KEEP_LATEST_COMMITS.name()).build();
      }
      LOG.info("Total Partitions to clean : " + partitionsToClean.size() + ", with policy " + config.getCleanerPolicy());
      int cleanerParallelism = Math.min(partitionsToClean.size(), config.getCleanerParallelism());
      LOG.info("Using cleanerParallelism: " + cleanerParallelism);
      jsc.setJobGroup(this.getClass().getSimpleName(), "Generates list of file slices to be cleaned");
      Map<String, List<HoodieCleanFileInfo>> cleanOps = jsc
          .parallelize(partitionsToClean, cleanerParallelism)
          .map(partitionPathToClean -> Pair.of(partitionPathToClean, planner.getDeletePaths(partitionPathToClean)))
          .collect().stream()
          .collect(Collectors.toMap(Pair::getKey, y -> CleanerUtils.convertToHoodieCleanFileInfoList(y.getValue())));
      return new HoodieCleanerPlan(earliestInstant
          .map(x -> new HoodieActionInstant(x.getTimestamp(), x.getAction(), x.getState().name())).orElse(null),
          config.getCleanerPolicy().name(), CollectionUtils.createImmutableMap(),
          CleanPlanner.LATEST_CLEAN_PLAN_VERSION, cleanOps);
    } catch (IOException e) {
      throw new HoodieIOException("Failed to schedule clean operation", e);
    }
  }
HoodieCleanerPlan requestClean ( JavaSparkContext jsc ) { try { CleanPlanner < ? > planner = new CleanPlanner < > ( table , config ) ; Option < HoodieInstant > earliestInstant = planner . getEarliestCommitToRetain ( ) ; List < String > partitionsToClean = planner . getPartitionPathsToClean ( earliestInstant ) ; if ( partitionsToClean . isEmpty ( ) ) { LOG . info ( "Nothing to clean here. It is already clean" ) ; return HoodieCleanerPlan . newBuilder ( ) . setPolicy ( HoodieCleaningPolicy . KEEP_LATEST_COMMITS . name ( ) ) . build ( ) ; } LOG . info ( "Total Partitions to clean : " + partitionsToClean . size ( ) + ", with policy " + config . getCleanerPolicy ( ) ) ; int cleanerParallelism = Math . min ( partitionsToClean . size ( ) , config . getCleanerParallelism ( ) ) ; LOG . info ( "Using cleanerParallelism: " + cleanerParallelism ) ; jsc . setJobGroup ( this . getClass ( ) . getSimpleName ( ) , "Generates list of file slices to be cleaned" ) ; Map < String , List < HoodieCleanFileInfo > > cleanOps = jsc . parallelize ( partitionsToClean , cleanerParallelism ) . map ( partitionPathToClean -> Pair . of ( partitionPathToClean , planner . getDeletePaths ( partitionPathToClean ) ) ) . collect ( ) . stream ( ) . collect ( Collectors . toMap ( Pair :: getKey , y -> CleanerUtils . convertToHoodieCleanFileInfoList ( y . getValue ( ) ) ) ) ; return new HoodieCleanerPlan ( earliestInstant . map ( x -> new HoodieActionInstant ( x . getTimestamp ( ) , x . getAction ( ) , x . getState ( ) . name ( ) ) ) . orElse ( null ) , config . getCleanerPolicy ( ) . name ( ) , CollectionUtils . createImmutableMap ( ) , CleanPlanner . LATEST_CLEAN_PLAN_VERSION , cleanOps ) ; } catch ( IOException e ) { throw new HoodieIOException ( "Failed to schedule clean operation" , e ) ; } }
Generates List of files to be cleaned.
Generates List of files to be cleaned .