fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
/**
   * Populates disk/volume ID metadata inside FileDescriptors given a list of
   * BlockLocations. The FileDescriptors are passed as a Map of parent directory
   * (partition location) to list of files (FileDescriptors) under that directory.
   */
Populates disk/volume ID metadata inside FileDescriptors given a list of
BlockLocations. The FileDescriptors are passed as a Map of parent directory
(partition location) to list of files (FileDescriptors) under that directory.
Populates disk / volume ID metadata inside FileDescriptors given a list of BlockLocations . The FileDescriptors are passed as a Map of parent directory ( partition location ) to list of files ( FileDescriptors ) under that directory .
private void loadDiskIds(List<BlockLocation> blockLocations,
      Map<String, List<FileDescriptor>> fileDescriptors) {
    BlockStorageLocation[] locations = null;
    try {
      locations = DFS.getFileBlockStorageLocations(blockLocations);
    } catch (IOException e) {
      LOG.error("Couldn't determine block storage locations:\n" + e.getMessage());
      return;
    }
    if (locations == null || locations.length == 0) {
      LOG.warn("Attempted to get block locations but the call returned nulls");
      return;
    }
    if (locations.length != blockLocations.size()) {
      LOG.error("Number of block locations not equal to number of blocks: "
          + "#locations=" + Long.toString(locations.length)
          + " #blocks=" + Long.toString(blockLocations.size()));
      return;
    }
    int locationsIdx = 0;
    int unknownDiskIdCount = 0;
    for (String parentPath: fileDescriptors.keySet()) {
      for (FileDescriptor fileDescriptor: fileDescriptors.get(parentPath)) {
        for (THdfsFileBlock blockMd: fileDescriptor.getFileBlocks()) {
          VolumeId[] volumeIds = locations[locationsIdx++].getVolumeIds();
          int[] diskIds = new int[volumeIds.length];
          for (int i = 0; i < volumeIds.length; ++i) {
            diskIds[i] = getDiskId(volumeIds[i]);
            if (diskIds[i] < 0) ++unknownDiskIdCount;
          }
          FileBlock.setDiskIds(diskIds, blockMd);
        }
      }
      if (unknownDiskIdCount > 0) {
        LOG.warn("unknown disk id count " + unknownDiskIdCount);
      }
    }
  }
private void loadDiskIds ( List < BlockLocation > blockLocations , Map < String , List < FileDescriptor > > fileDescriptors ) { BlockStorageLocation [ ] locations = null ; try { locations = DFS . getFileBlockStorageLocations ( blockLocations ) ; } catch ( IOException e ) { LOG . error ( "Couldn't determine block storage locations:\n" + e . getMessage ( ) ) ; return ; } if ( locations == null || locations . length == 0 ) { LOG . warn ( "Attempted to get block locations but the call returned nulls" ) ; return ; } if ( locations . length != blockLocations . size ( ) ) { LOG . error ( "Number of block locations not equal to number of blocks: " + "#locations=" + Long . toString ( locations . length ) + " #blocks=" + Long . toString ( blockLocations . size ( ) ) ) ; return ; } int locationsIdx = 0 ; int unknownDiskIdCount = 0 ; for ( String parentPath : fileDescriptors . keySet ( ) ) { for ( FileDescriptor fileDescriptor : fileDescriptors . get ( parentPath ) ) { for ( THdfsFileBlock blockMd : fileDescriptor . getFileBlocks ( ) ) { VolumeId [ ] volumeIds = locations [ locationsIdx ++ ] . getVolumeIds ( ) ; int [ ] diskIds = new int [ volumeIds . length ] ; for ( int i = 0 ; i < volumeIds . length ; ++ i ) { diskIds [ i ] = getDiskId ( volumeIds [ i ] ) ; if ( diskIds [ i ] < 0 ) ++ unknownDiskIdCount ; } FileBlock . setDiskIds ( diskIds , blockMd ) ; } } if ( unknownDiskIdCount > 0 ) { LOG . warn ( "unknown disk id count " + unknownDiskIdCount ) ; } } }
Populates disk/volume ID metadata inside FileDescriptors given a list of
 BlockLocations.
Populates disk / volume ID metadata inside FileDescriptors given a list of BlockLocations .
// BlockStorageLocations for all the blocks // block described by blockMetadataList[i] is located at locations[i] // Get the BlockStorageLocations for all the blocks // blocks and locations don't match up // Convert opaque VolumeId to 0 based ids. // TODO: the diskId should be eventually retrievable from Hdfs when // the community agrees this API is useful.