fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
/**
   * Adds a new HdfsPartition to the internal partition list, populating with file format
   * information and file locations. If a partition contains no files, it's not added.
   * For unchanged files (indicated by unchanged mtime), reuses the FileDescriptor from
   * the oldFileDescMap. Otherwise, creates a new FileDescriptor for each modified or
   * new file and adds it to newFileDescsMap. Both old and newFileDescMap are Maps of
   * parent directory (partition location) to list of files (FileDescriptors) under that
   * directory.
   * Returns new partition or null, if none was added.
   *
   * @throws InvalidStorageDescriptorException
   *           if the supplied storage descriptor contains
   *           metadata that Impala can't understand.
   */
Adds a new HdfsPartition to the internal partition list, populating with file format
information and file locations. If a partition contains no files, it's not added.
For unchanged files (indicated by unchanged mtime), reuses the FileDescriptor from
the oldFileDescMap. Otherwise, creates a new FileDescriptor for each modified or
new file and adds it to newFileDescsMap. Both old and newFileDescMap are Maps of
parent directory (partition location) to list of files (FileDescriptors) under that
directory.
Returns new partition or null, if none was added.

@throws InvalidStorageDescriptorException
if the supplied storage descriptor contains
metadata that Impala can't understand.
Adds a new HdfsPartition to the internal partition list populating with file format information and file locations . If a partition contains no files it ' s not added . For unchanged files ( indicated by unchanged mtime ) reuses the FileDescriptor from the oldFileDescMap . Otherwise creates a new FileDescriptor for each modified or new file and adds it to newFileDescsMap . Both old and newFileDescMap are Maps of parent directory ( partition location ) to list of files ( FileDescriptors ) under that directory . Returns new partition or null if none was added . @throws InvalidStorageDescriptorException if the supplied storage descriptor contains metadata that Impala can ' t understand .
private HdfsPartition addPartition(StorageDescriptor storageDescriptor,
      org.apache.hadoop.hive.metastore.api.Partition msPartition,
      List<LiteralExpr> partitionKeyExprs,
      Map<String, List<FileDescriptor>> oldFileDescMap,
      Map<String, List<FileDescriptor>> newFileDescMap)
      throws IOException, InvalidStorageDescriptorException {
    HdfsStorageDescriptor fileFormatDescriptor =
        HdfsStorageDescriptor.fromStorageDescriptor(this.name_, storageDescriptor);
    Path partDirPath = new Path(storageDescriptor.getLocation());
    List<FileDescriptor> fileDescriptors = Lists.newArrayList();
    if (DFS.exists(partDirPath)) {
      for (FileStatus fileStatus: DFS.listStatus(partDirPath)) {
        String fileName = fileStatus.getPath().getName().toString();
        if (fileStatus.isDirectory() || FileSystemUtil.isHiddenFile(fileName) ||
            HdfsCompression.fromFileName(fileName) == HdfsCompression.LZO_INDEX) {
          continue;
        }
        String partitionDir = fileStatus.getPath().getParent().toString();
        FileDescriptor fd = null;
        if (oldFileDescMap != null && oldFileDescMap.get(partitionDir) != null) {
          for (FileDescriptor oldFileDesc: oldFileDescMap.get(partitionDir)) {
            if (oldFileDesc.getFileName().equals(fileName)) {
              fd = oldFileDesc;
              break;
            }
          }
        }
        if (fd == null || fd.getFileLength() != fileStatus.getLen()
            || fd.getModificationTime() != fileStatus.getModificationTime()) {
          fd = new FileDescriptor(fileName, fileStatus.getLen(),
              fileStatus.getModificationTime());
          List<FileDescriptor> fds = newFileDescMap.get(partitionDir);
          if (fds == null) {
            fds = Lists.newArrayList();
            newFileDescMap.put(partitionDir, fds);
          }
          fds.add(fd);
        }
        List<FileDescriptor> fds = fileDescMap_.get(partitionDir);
        if (fds == null) {
          fds = Lists.newArrayList();
          fileDescMap_.put(partitionDir, fds);
        }
        fds.add(fd);
        fileDescriptors.add(fd);
      }
      HdfsPartition partition = new HdfsPartition(this, msPartition, partitionKeyExprs,
          fileFormatDescriptor, fileDescriptors, getAvailableAccessLevel(partDirPath));
      partitions_.add(partition);
      numHdfsFiles_ += fileDescriptors.size();
      totalHdfsBytes_ += partition.getSize();
      return partition;
    } else {
      LOG.warn("Path " + partDirPath + " does not exist for partition. Ignoring.");
      return null;
    }
  }
private HdfsPartition addPartition ( StorageDescriptor storageDescriptor , org . apache . hadoop . hive . metastore . api . Partition msPartition , List < LiteralExpr > partitionKeyExprs , Map < String , List < FileDescriptor > > oldFileDescMap , Map < String , List < FileDescriptor > > newFileDescMap ) throws IOException , InvalidStorageDescriptorException { HdfsStorageDescriptor fileFormatDescriptor = HdfsStorageDescriptor . fromStorageDescriptor ( this . name_ , storageDescriptor ) ; Path partDirPath = new Path ( storageDescriptor . getLocation ( ) ) ; List < FileDescriptor > fileDescriptors = Lists . newArrayList ( ) ; if ( DFS . exists ( partDirPath ) ) { for ( FileStatus fileStatus : DFS . listStatus ( partDirPath ) ) { String fileName = fileStatus . getPath ( ) . getName ( ) . toString ( ) ; if ( fileStatus . isDirectory ( ) || FileSystemUtil . isHiddenFile ( fileName ) || HdfsCompression . fromFileName ( fileName ) == HdfsCompression . LZO_INDEX ) { continue ; } String partitionDir = fileStatus . getPath ( ) . getParent ( ) . toString ( ) ; FileDescriptor fd = null ; if ( oldFileDescMap != null && oldFileDescMap . get ( partitionDir ) != null ) { for ( FileDescriptor oldFileDesc : oldFileDescMap . get ( partitionDir ) ) { if ( oldFileDesc . getFileName ( ) . equals ( fileName ) ) { fd = oldFileDesc ; break ; } } } if ( fd == null || fd . getFileLength ( ) != fileStatus . getLen ( ) || fd . getModificationTime ( ) != fileStatus . getModificationTime ( ) ) { fd = new FileDescriptor ( fileName , fileStatus . getLen ( ) , fileStatus . getModificationTime ( ) ) ; List < FileDescriptor > fds = newFileDescMap . get ( partitionDir ) ; if ( fds == null ) { fds = Lists . newArrayList ( ) ; newFileDescMap . put ( partitionDir , fds ) ; } fds . add ( fd ) ; } List < FileDescriptor > fds = fileDescMap_ . get ( partitionDir ) ; if ( fds == null ) { fds = Lists . newArrayList ( ) ; fileDescMap_ . put ( partitionDir , fds ) ; } fds . add ( fd ) ; fileDescriptors . add ( fd ) ; } HdfsPartition partition = new HdfsPartition ( this , msPartition , partitionKeyExprs , fileFormatDescriptor , fileDescriptors , getAvailableAccessLevel ( partDirPath ) ) ; partitions_ . add ( partition ) ; numHdfsFiles_ += fileDescriptors . size ( ) ; totalHdfsBytes_ += partition . getSize ( ) ; return partition ; } else { LOG . warn ( "Path " + partDirPath + " does not exist for partition. Ignoring." ) ; return null ; } }
Adds a new HdfsPartition to the internal partition list, populating with file format
 information and file locations.
Adds a new HdfsPartition to the internal partition list populating with file format information and file locations .
// DistributedFilesystem does not have an API that takes in a timestamp and return // a list of files that has been added/changed since. Therefore, we are calling // DFS.listStatus() to list all the files. // Ignore directory, hidden file starting with . or _, and LZO index files // If a directory is erroneously created as a subdirectory of a partition dir // we should ignore it and move on. Hive will not recurse into directories. // Skip index files, these are read by the LZO scanner directly. // Search for a FileDescriptor with the same partition dir and file name. If one // is found, it will be chosen as a candidate to reuse. // Check if this FileDescriptor has been modified since last loading its block // location information. If it has not been changed, the previously loaded // value can be reused. // Create a new file descriptor, the block metadata will be populated by // loadBlockMd. // Add to the list of FileDescriptors for this partition.