fe/src/main/java/com/cloudera/impala/catalog/HdfsTable.java
/**
   * Create HdfsPartition objects corresponding to 'partitions'.
   *
   * If there are no partitions in the Hive metadata, a single partition is added with no
   * partition keys.
   *
   * For files that have not been changed, reuses file descriptors from oldFileDescMap.
   */
Create HdfsPartition objects corresponding to 'partitions'.
If there are no partitions in the Hive metadata, a single partition is added with no
partition keys.

For files that have not been changed, reuses file descriptors from oldFileDescMap.
Create HdfsPartition objects corresponding to ' partitions ' . If there are no partitions in the Hive metadata a single partition is added with no partition keys . For files that have not been changed reuses file descriptors from oldFileDescMap .
private void loadPartitions(
      List<org.apache.hadoop.hive.metastore.api.Partition> msPartitions,
      org.apache.hadoop.hive.metastore.api.Table msTbl,
      Map<String, List<FileDescriptor>> oldFileDescMap) throws IOException,
      CatalogException {
    partitions_.clear();
    hdfsBaseDir_ = msTbl.getSd().getLocation();
    Map<String, List<FileDescriptor>> fileDescsToLoad = Maps.newHashMap();
    addDefaultPartition(msTbl.getSd());
    if (msTbl.getPartitionKeysSize() == 0) {
      Preconditions.checkArgument(msPartitions == null || msPartitions.isEmpty());
      addPartition(msTbl.getSd(), null,
          new ArrayList<LiteralExpr>(), oldFileDescMap, fileDescsToLoad);
      Path location = new Path(hdfsBaseDir_);
      if (DFS.exists(location)) {
        accessLevel_ = getAvailableAccessLevel(location);
      }
    } else {
      Set<String>[] uniquePartitionKeys = new HashSet[numClusteringCols_];
      long[] numNullKeys = new long[numClusteringCols_];
      for (int i = 0; i < numClusteringCols_; ++i) {
        uniquePartitionKeys[i] = new HashSet<String>();
        numNullKeys[i] = 0;
      }
      for (org.apache.hadoop.hive.metastore.api.Partition msPartition: msPartitions) {
        List<LiteralExpr> keyValues = Lists.newArrayList();
        int i = 0;
        for (String partitionKey: msPartition.getValues()) {
          uniquePartitionKeys[i].add(partitionKey);
          if (partitionKey.equals(nullPartitionKeyValue_)) {
            keyValues.add(new NullLiteral());
            ++numNullKeys[i];
          } else {
            ColumnType type = colsByPos_.get(keyValues.size()).getType();
            try {
              Expr expr = LiteralExpr.create(partitionKey, type);
              expr = expr.castTo(type);
              keyValues.add((LiteralExpr) expr);
            } catch (Exception ex) {
              LOG.warn("Failed to create literal expression of type: " + type, ex);
              throw new InvalidStorageDescriptorException(ex);
            }
          }
          ++i;
        }
        HdfsPartition partition = addPartition(msPartition.getSd(), msPartition,
            keyValues, oldFileDescMap, fileDescsToLoad);
        if (partition == null) continue;
        if (msPartition.getParameters() != null) {
          partition.setNumRows(getRowCount(msPartition.getParameters()));
        }
        if (!TAccessLevelUtil.impliesWriteAccess(partition.getAccessLevel())) {
          accessLevel_ = TAccessLevel.READ_ONLY;
        }
      }
      for (int i = 0; i < numClusteringCols_; ++i) {
        ColumnStats stats = colsByPos_.get(i).getStats();
        stats.setNumNulls(numNullKeys[i]);
        stats.setNumDistinctValues(uniquePartitionKeys[i].size());
        LOG.debug("#col=" + Integer.toString(i) + " stats=" + stats.toString());
      }
    }
    if (fileDescsToLoad.size() > 0) {
      loadBlockMd(fileDescsToLoad);
    }
  }
private void loadPartitions ( List < org . apache . hadoop . hive . metastore . api . Partition > msPartitions , org . apache . hadoop . hive . metastore . api . Table msTbl , Map < String , List < FileDescriptor > > oldFileDescMap ) throws IOException , CatalogException { partitions_ . clear ( ) ; hdfsBaseDir_ = msTbl . getSd ( ) . getLocation ( ) ; Map < String , List < FileDescriptor > > fileDescsToLoad = Maps . newHashMap ( ) ; addDefaultPartition ( msTbl . getSd ( ) ) ; if ( msTbl . getPartitionKeysSize ( ) == 0 ) { Preconditions . checkArgument ( msPartitions == null || msPartitions . isEmpty ( ) ) ; addPartition ( msTbl . getSd ( ) , null , new ArrayList < LiteralExpr > ( ) , oldFileDescMap , fileDescsToLoad ) ; Path location = new Path ( hdfsBaseDir_ ) ; if ( DFS . exists ( location ) ) { accessLevel_ = getAvailableAccessLevel ( location ) ; } } else { Set < String > [ ] uniquePartitionKeys = new HashSet [ numClusteringCols_ ] ; long [ ] numNullKeys = new long [ numClusteringCols_ ] ; for ( int i = 0 ; i < numClusteringCols_ ; ++ i ) { uniquePartitionKeys [ i ] = new HashSet < String > ( ) ; numNullKeys [ i ] = 0 ; } for ( org . apache . hadoop . hive . metastore . api . Partition msPartition : msPartitions ) { List < LiteralExpr > keyValues = Lists . newArrayList ( ) ; int i = 0 ; for ( String partitionKey : msPartition . getValues ( ) ) { uniquePartitionKeys [ i ] . add ( partitionKey ) ; if ( partitionKey . equals ( nullPartitionKeyValue_ ) ) { keyValues . add ( new NullLiteral ( ) ) ; ++ numNullKeys [ i ] ; } else { ColumnType type = colsByPos_ . get ( keyValues . size ( ) ) . getType ( ) ; try { Expr expr = LiteralExpr . create ( partitionKey , type ) ; expr = expr . castTo ( type ) ; keyValues . add ( ( LiteralExpr ) expr ) ; } catch ( Exception ex ) { LOG . warn ( "Failed to create literal expression of type: " + type , ex ) ; throw new InvalidStorageDescriptorException ( ex ) ; } } ++ i ; } HdfsPartition partition = addPartition ( msPartition . getSd ( ) , msPartition , keyValues , oldFileDescMap , fileDescsToLoad ) ; if ( partition == null ) continue ; if ( msPartition . getParameters ( ) != null ) { partition . setNumRows ( getRowCount ( msPartition . getParameters ( ) ) ) ; } if ( ! TAccessLevelUtil . impliesWriteAccess ( partition . getAccessLevel ( ) ) ) { accessLevel_ = TAccessLevel . READ_ONLY ; } } for ( int i = 0 ; i < numClusteringCols_ ; ++ i ) { ColumnStats stats = colsByPos_ . get ( i ) . getStats ( ) ; stats . setNumNulls ( numNullKeys [ i ] ) ; stats . setNumDistinctValues ( uniquePartitionKeys [ i ] . size ( ) ) ; LOG . debug ( "#col=" + Integer . toString ( i ) + " stats=" + stats . toString ( ) ) ; } } if ( fileDescsToLoad . size ( ) > 0 ) { loadBlockMd ( fileDescsToLoad ) ; } }
Create HdfsPartition objects corresponding to 'partitions'.
Create HdfsPartition objects corresponding to ' partitions ' .
// Map of parent path to a list of new/modified FileDescriptors. FileDescriptors // in this Map will have their block location information (re)loaded. This is used // to speedup the incremental refresh of a table's metadata by skipping unmodified, // previously loaded FileDescriptors. // INSERT statements need to refer to this if they try to write to new partitions // Scans don't refer to this because by definition all partitions they refer to // exist. // This table has no partition key, which means it has no declared partitions. // We model partitions slightly differently to Hive - every file must exist in a // partition, so add a single partition with no keys which will get all the // files in the table's root directory. // keep track of distinct partition key values and how many nulls there are // load key values // Deal with Hive's special NULL partition key. // Force the literal to be of type declared in the metadata. // If the partition is null, its HDFS path does not exist, and it was not added to // this table's partition list. Skip the partition. // TODO: READ_ONLY isn't exactly correct because the it's possible the // partition does not have READ permissions either. When we start checking // whether we can READ from a table, this should be updated to set the // table's access level to the "lowest" effective level across all // partitions. That is, if one partition has READ_ONLY and another has // WRITE_ONLY the table's access level should be NONE. // update col stats for partition key cols