static FileStatus[] pruneFileListBySize(long maxFileSize, FileStatus[] origList, FileSystem hdfs,
      Path inputPath) {
    LOG.info("Pruning orig list  of size " + origList.length + " for source" + inputPath.toUri());

    long fileSize = 0L;
    List<FileStatus> prunedFileList = new ArrayList<FileStatus>();

    Set<String> toBeRemovedJobId = new HashSet<String>();
    for (int i = 0; i < origList.length; i++) {
      fileSize = origList[i].getLen();

      // check if hbase can store this file if yes, consider it for processing
      if (fileSize <= maxFileSize) {
        prunedFileList.add(origList[i]);
      } else {
        Path hugeFile = origList[i].getPath();
        LOG.info("In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize "
            + maxFileSize + " for " + hugeFile.toUri());

        // note the job id so that we can remove the other file (job conf or job history)
        toBeRemovedJobId.add(getJobIdFromPath(hugeFile));
      }
    }
    if (prunedFileList.size() == 0) {
      LOG.info("Found no files worth processing. Returning 0 sized array");
      return new FileStatus[0];
    }

    String jobId = null;
    ListIterator<FileStatus> it = prunedFileList.listIterator();
    while (it.hasNext()) {
      if (toBeRemovedJobId.size() == 0) {
        // no files to remove
        break;
      }
      Path curFile = it.next().getPath();
      jobId = getJobIdFromPath(curFile);
      if (toBeRemovedJobId.contains(jobId)) {
        LOG.info("Removing from prunedList " + curFile.toUri());
        it.remove();
        /*
         * removing the job id from the hash set since there would be only
         * one file with this job id in the prunedList, the other file with
         * this job id was huge and was already moved out
         */
        toBeRemovedJobId.remove(jobId);
      }
    }
    return prunedFileList.toArray(new FileStatus[prunedFileList.size()]);
  }
static FileStatus [ ] pruneFileListBySize ( long maxFileSize , FileStatus [ ] origList , FileSystem hdfs , Path inputPath ) { LOG . info ( "Pruning orig list  of size " + origList . length + " for source" + inputPath . toUri ( ) ) ; long fileSize = 0L ; List < FileStatus > prunedFileList = new ArrayList < FileStatus > ( ) ; Set < String > toBeRemovedJobId = new HashSet < String > ( ) ; for ( int i = 0 ; i < origList . length ; i ++ ) { fileSize = origList [ i ] . getLen ( ) ; if ( fileSize <= maxFileSize ) { prunedFileList . add ( origList [ i ] ) ; } else { Path hugeFile = origList [ i ] . getPath ( ) ; LOG . info ( "In getListFilesToProcess filesize " + fileSize + " has exceeded maxFileSize " + maxFileSize + " for " + hugeFile . toUri ( ) ) ; toBeRemovedJobId . add ( getJobIdFromPath ( hugeFile ) ) ; } } if ( prunedFileList . size ( ) == 0 ) { LOG . info ( "Found no files worth processing. Returning 0 sized array" ) ; return new FileStatus [ 0 ] ; } String jobId = null ; ListIterator < FileStatus > it = prunedFileList . listIterator ( ) ; while ( it . hasNext ( ) ) { if ( toBeRemovedJobId . size ( ) == 0 ) { break ; } Path curFile = it . next ( ) . getPath ( ) ; jobId = getJobIdFromPath ( curFile ) ; if ( toBeRemovedJobId . contains ( jobId ) ) { LOG . info ( "Removing from prunedList " + curFile . toUri ( ) ) ; it . remove ( ) ; toBeRemovedJobId . remove ( jobId ) ; } } return prunedFileList . toArray ( new FileStatus [ prunedFileList . size ( ) ] ) ; }
prunes the given list/array of files based on their sizes

@param maxFileSize -max #bytes to be stored in an hbase cell
@param origList - input list of files to be processed
@param hdfs - filesystem to be looked at
@param inputPath - root dir of the path containing history files
@return - pruned array of FileStatus of files to be processed