core/src/main/java/org/apache/carbondata/core/metadata/PartitionMapFileStore.java
/**
   * Clean up invalid data after drop partition in all segments of table
   * @param table
   * @param currentPartitions Current partitions of table
   * @param forceDelete Whether it should be deleted force or check the time for an hour creation
   *                    to delete data.
   * @throws IOException
   */
Clean up invalid data after drop partition in all segments of table
@param table
@param currentPartitions Current partitions of table
@param forceDelete Whether it should be deleted force or check the time for an hour creation
to delete data.
Clean up invalid data after drop partition in all segments of table @param table @param currentPartitions Current partitions of table @param forceDelete Whether it should be deleted force or check the time for an hour creation to delete data .
public void cleanSegments(
      CarbonTable table,
      List<String> currentPartitions,
      boolean forceDelete) throws IOException {
    SegmentStatusManager ssm = new SegmentStatusManager(table.getAbsoluteTableIdentifier());
    CarbonTablePath carbonTablePath = CarbonStorePath
        .getCarbonTablePath(table.getAbsoluteTableIdentifier().getTablePath(),
            table.getAbsoluteTableIdentifier().getCarbonTableIdentifier());
    LoadMetadataDetails[] details = ssm.readLoadMetadata(table.getMetaDataFilepath());
    List<String> segmentsNeedToBeDeleted = new ArrayList<>();
    for (LoadMetadataDetails segment : details) {
      if (segment.getSegmentStatus() == SegmentStatus.SUCCESS
          || segment.getSegmentStatus() == SegmentStatus.LOAD_PARTIAL_SUCCESS) {
        List<String> toBeDeletedIndexFiles = new ArrayList<>();
        List<String> toBeDeletedDataFiles = new ArrayList<>();
        String segmentPath = carbonTablePath.getCarbonDataDirectoryPath("0", segment.getLoadName());
        String partitionFilePath = getPartitionFilePath(segmentPath);
        if (partitionFilePath != null) {
          PartitionMapper partitionMapper = readPartitionMap(partitionFilePath);
          if (partitionMapper.partitionMap.size() == 0) {
            segmentsNeedToBeDeleted.add(segment.getLoadName());
            continue;
          }
          DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter();
          SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore();
          indexFileStore.readAllIIndexOfSegment(segmentPath);
          Set<String> indexFilesFromSegment = indexFileStore.getCarbonIndexMap().keySet();
          for (String indexFile : indexFilesFromSegment) {
            List<String> indexPartitions = partitionMapper.partitionMap.get(indexFile);
            if (indexPartitions == null || !currentPartitions.containsAll(indexPartitions)) {
              Long fileTimestamp = CarbonUpdateUtil.getTimeStampAsLong(indexFile
                  .substring(indexFile.lastIndexOf(CarbonCommonConstants.HYPHEN) + 1,
                      indexFile.length() - CarbonTablePath.INDEX_FILE_EXT.length()));
              if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimestamp) || forceDelete) {
                toBeDeletedIndexFiles.add(indexFile);
                byte[] fileData = indexFileStore.getFileData(indexFile);
                List<DataFileFooter> indexInfo =
                    fileFooterConverter.getIndexInfo(segmentPath + "/" + indexFile, fileData);
                for (DataFileFooter footer : indexInfo) {
                  toBeDeletedDataFiles.add(footer.getBlockInfo().getTableBlockInfo().getFilePath());
                }
              }
            }
          }
          if (toBeDeletedIndexFiles.size() > 0) {
            indexFilesFromSegment.removeAll(toBeDeletedIndexFiles);
            new CarbonIndexFileMergeWriter().mergeCarbonIndexFilesOfSegment(segmentPath,
                new ArrayList<String>(indexFilesFromSegment));
            for (String dataFile : toBeDeletedDataFiles) {
              FileFactory.deleteFile(dataFile, FileFactory.getFileType(dataFile));
            }
          }
          CarbonFile[] partitionFiles = getPartitionFiles(segmentPath);
          CarbonFile currentPartitionFile = FileFactory.getCarbonFile(partitionFilePath);
          if (partitionFiles != null) {
            for (CarbonFile partitionFile : partitionFiles) {
              if (!currentPartitionFile.getName().equalsIgnoreCase(partitionFile.getName())) {
                long fileTimeStamp = Long.parseLong(partitionFile.getName().substring(0,
                    partitionFile.getName().length() - CarbonTablePath.PARTITION_MAP_EXT.length()));
                if (CarbonUpdateUtil.isMaxQueryTimeoutExceeded(fileTimeStamp) || forceDelete) {
                  partitionFile.delete();
                }
              }
            }
          }
          partitionMapper = readPartitionMap(partitionFilePath);
          if (partitionMapper != null) {
            if (partitionMapper.partitionMap.size() == 0) {
              currentPartitionFile.delete();
            }
          }
        }
      }
    }
    if (segmentsNeedToBeDeleted.size() > 0) {
      try {
        SegmentStatusManager.updateDeletionStatus(
            table.getAbsoluteTableIdentifier(),
            segmentsNeedToBeDeleted,
            table.getMetaDataFilepath());
      } catch (Exception e) {
        throw new IOException(e);
      }
    }
  }
public void cleanSegments ( CarbonTable table , List < String > currentPartitions , boolean forceDelete ) throws IOException { SegmentStatusManager ssm = new SegmentStatusManager ( table . getAbsoluteTableIdentifier ( ) ) ; CarbonTablePath carbonTablePath = CarbonStorePath . getCarbonTablePath ( table . getAbsoluteTableIdentifier ( ) . getTablePath ( ) , table . getAbsoluteTableIdentifier ( ) . getCarbonTableIdentifier ( ) ) ; LoadMetadataDetails [ ] details = ssm . readLoadMetadata ( table . getMetaDataFilepath ( ) ) ; List < String > segmentsNeedToBeDeleted = new ArrayList < > ( ) ; for ( LoadMetadataDetails segment : details ) { if ( segment . getSegmentStatus ( ) == SegmentStatus . SUCCESS || segment . getSegmentStatus ( ) == SegmentStatus . LOAD_PARTIAL_SUCCESS ) { List < String > toBeDeletedIndexFiles = new ArrayList < > ( ) ; List < String > toBeDeletedDataFiles = new ArrayList < > ( ) ; String segmentPath = carbonTablePath . getCarbonDataDirectoryPath ( "0" , segment . getLoadName ( ) ) ; String partitionFilePath = getPartitionFilePath ( segmentPath ) ; if ( partitionFilePath != null ) { PartitionMapper partitionMapper = readPartitionMap ( partitionFilePath ) ; if ( partitionMapper . partitionMap . size ( ) == 0 ) { segmentsNeedToBeDeleted . add ( segment . getLoadName ( ) ) ; continue ; } DataFileFooterConverter fileFooterConverter = new DataFileFooterConverter ( ) ; SegmentIndexFileStore indexFileStore = new SegmentIndexFileStore ( ) ; indexFileStore . readAllIIndexOfSegment ( segmentPath ) ; Set < String > indexFilesFromSegment = indexFileStore . getCarbonIndexMap ( ) . keySet ( ) ; for ( String indexFile : indexFilesFromSegment ) { List < String > indexPartitions = partitionMapper . partitionMap . get ( indexFile ) ; if ( indexPartitions == null || ! currentPartitions . containsAll ( indexPartitions ) ) { Long fileTimestamp = CarbonUpdateUtil . getTimeStampAsLong ( indexFile . substring ( indexFile . lastIndexOf ( CarbonCommonConstants . HYPHEN ) + 1 , indexFile . length ( ) - CarbonTablePath . INDEX_FILE_EXT . length ( ) ) ) ; if ( CarbonUpdateUtil . isMaxQueryTimeoutExceeded ( fileTimestamp ) || forceDelete ) { toBeDeletedIndexFiles . add ( indexFile ) ; byte [ ] fileData = indexFileStore . getFileData ( indexFile ) ; List < DataFileFooter > indexInfo = fileFooterConverter . getIndexInfo ( segmentPath + "/" + indexFile , fileData ) ; for ( DataFileFooter footer : indexInfo ) { toBeDeletedDataFiles . add ( footer . getBlockInfo ( ) . getTableBlockInfo ( ) . getFilePath ( ) ) ; } } } } if ( toBeDeletedIndexFiles . size ( ) > 0 ) { indexFilesFromSegment . removeAll ( toBeDeletedIndexFiles ) ; new CarbonIndexFileMergeWriter ( ) . mergeCarbonIndexFilesOfSegment ( segmentPath , new ArrayList < String > ( indexFilesFromSegment ) ) ; for ( String dataFile : toBeDeletedDataFiles ) { FileFactory . deleteFile ( dataFile , FileFactory . getFileType ( dataFile ) ) ; } } CarbonFile [ ] partitionFiles = getPartitionFiles ( segmentPath ) ; CarbonFile currentPartitionFile = FileFactory . getCarbonFile ( partitionFilePath ) ; if ( partitionFiles != null ) { for ( CarbonFile partitionFile : partitionFiles ) { if ( ! currentPartitionFile . getName ( ) . equalsIgnoreCase ( partitionFile . getName ( ) ) ) { long fileTimeStamp = Long . parseLong ( partitionFile . getName ( ) . substring ( 0 , partitionFile . getName ( ) . length ( ) - CarbonTablePath . PARTITION_MAP_EXT . length ( ) ) ) ; if ( CarbonUpdateUtil . isMaxQueryTimeoutExceeded ( fileTimeStamp ) || forceDelete ) { partitionFile . delete ( ) ; } } } } partitionMapper = readPartitionMap ( partitionFilePath ) ; if ( partitionMapper != null ) { if ( partitionMapper . partitionMap . size ( ) == 0 ) { currentPartitionFile . delete ( ) ; } } } } } if ( segmentsNeedToBeDeleted . size ( ) > 0 ) { try { SegmentStatusManager . updateDeletionStatus ( table . getAbsoluteTableIdentifier ( ) , segmentsNeedToBeDeleted , table . getMetaDataFilepath ( ) ) ; } catch ( Exception e ) { throw new IOException ( e ) ; } } }
Clean up invalid data after drop partition in all segments of table
 @param table
 @param currentPartitions Current partitions of table
 @param forceDelete Whether it should be deleted force or check the time for an hour creation
                    to delete data.
Clean up invalid data after drop partition in all segments of table @param table @param currentPartitions Current partitions of table @param forceDelete Whether it should be deleted force or check the time for an hour creation to delete data .
// scan through each segment. // if this segment is valid then only we will go for deletion of related // dropped partition files. if the segment is mark for delete or compacted then any way // it will get deleted. // take the list of files from this segment. // There is no partition information, it means all partitions are dropped. // So segment need to be marked as delete. // Check the partition information in the partiton mapper // Add the corresponding carbondata files to the delete list. // Delete all old partition files // delete partition map if there is no partition files exist // If any segments that are required to delete // Mark the segments as delete.