hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieInputFormatUtils.java
/**
   * Takes in a list of filesStatus and a list of table metadatas. Groups the files status list
   * based on given table metadata.
   * @param fileStatuses
   * @param fileExtension
   * @param metaClientList
   * @return
   * @throws IOException
   */
Takes in a list of filesStatus and a list of table metadatas. Groups the files status list
based on given table metadata.
Takes in a list of filesStatus and a list of table metadatas . Groups the files status list based on given table metadata .
public static Map<HoodieTableMetaClient, List<FileStatus>> groupFileStatusForSnapshotPaths(
      FileStatus[] fileStatuses, String fileExtension, Collection<HoodieTableMetaClient> metaClientList) {
    Map<HoodieTableMetaClient, List<FileStatus>> grouped = new HashMap<>();
    HoodieTableMetaClient metadata = null;
    for (FileStatus status : fileStatuses) {
      Path inputPath = status.getPath();
      if (!inputPath.getName().endsWith(fileExtension)) {
        continue;
      }
      if ((metadata == null) || (!inputPath.toString().contains(metadata.getBasePath()))) {
        for (HoodieTableMetaClient metaClient : metaClientList) {
          if (inputPath.toString().contains(metaClient.getBasePath())) {
            metadata = metaClient;
            if (!grouped.containsKey(metadata)) {
              grouped.put(metadata, new ArrayList<>());
            }
            break;
          }
        }
      }
      grouped.get(metadata).add(status);
    }
    return grouped;
  }
public static Map < HoodieTableMetaClient , List < FileStatus > > groupFileStatusForSnapshotPaths ( FileStatus [ ] fileStatuses , String fileExtension , Collection < HoodieTableMetaClient > metaClientList ) { Map < HoodieTableMetaClient , List < FileStatus > > grouped = new HashMap < > ( ) ; HoodieTableMetaClient metadata = null ; for ( FileStatus status : fileStatuses ) { Path inputPath = status . getPath ( ) ; if ( ! inputPath . getName ( ) . endsWith ( fileExtension ) ) { continue ; } if ( ( metadata == null ) || ( ! inputPath . toString ( ) . contains ( metadata . getBasePath ( ) ) ) ) { for ( HoodieTableMetaClient metaClient : metaClientList ) { if ( inputPath . toString ( ) . contains ( metaClient . getBasePath ( ) ) ) { metadata = metaClient ; if ( ! grouped . containsKey ( metadata ) ) { grouped . put ( metadata , new ArrayList < > ( ) ) ; } break ; } } } grouped . get ( metadata ) . add ( status ) ; } return grouped ; }
Takes in a list of filesStatus and a list of table metadatas.
Takes in a list of filesStatus and a list of table metadatas .
// This assumes the paths for different tables are grouped together //FIXME(vc): skip non data files for now. This wont be needed once log file name start // with "."