public static boolean checkCPReblock(ExecutionContext ec,String varin) throws DMLRuntimeException {
  CacheableData<?> obj=ec.getCacheableData(varin);
  MatrixCharacteristics mc=ec.getMatrixCharacteristics(varin);
  long rows=mc.getRows();
  long cols=mc.getCols();
  long nnz=mc.getNonZeros();
  if (!ConfigurationManager.isDynamicRecompilation() || !OptimizerUtils.isHybridExecutionMode()) {
    return false;
  }
  MatrixFormatMetaData iimd=(MatrixFormatMetaData)obj.getMetaData();
  if (obj.getRDDHandle() != null && iimd.getInputInfo() != InputInfo.BinaryBlockInputInfo && iimd.getInputInfo() != InputInfo.BinaryCellInputInfo) {
    return false;
  }
  if (rows <= 0 || cols <= 0) {
    return false;
  }
  double sp=OptimizerUtils.getSparsity(rows,cols,nnz);
  double mem=MatrixBlock.estimateSizeInMemory(rows,cols,sp);
  if (!OptimizerUtils.isValidCPDimensions(rows,cols) || !OptimizerUtils.isValidCPMatrixSize(rows,cols,sp) || mem >= OptimizerUtils.getLocalMemBudget()) {
    return false;
  }
  long estFilesize=(long)(3.5 * mem);
  long cpThreshold=CP_REBLOCK_THRESHOLD_SIZE * OptimizerUtils.getParallelTextReadParallelism();
  return (estFilesize < cpThreshold);
}

public static boolean checkCPReblock ( ExecutionContext ec , String varin ) throws DMLRuntimeException { CacheableData < ? > obj = ec . getCacheableData ( varin ) ; MatrixCharacteristics mc = ec . getMatrixCharacteristics ( varin ) ; long rows = mc . getRows ( ) ; long cols = mc . getCols ( ) ; long nnz = mc . getNonZeros ( ) ; if ( ! ConfigurationManager . isDynamicRecompilation ( ) || ! OptimizerUtils . isHybridExecutionMode ( ) ) { return false ; } MatrixFormatMetaData iimd = ( MatrixFormatMetaData ) obj . getMetaData ( ) ; if ( obj . getRDDHandle ( ) != null && iimd . getInputInfo ( ) != InputInfo . BinaryBlockInputInfo && iimd . getInputInfo ( ) != InputInfo . BinaryCellInputInfo ) { return false ; } if ( rows <= 0 || cols <= 0 ) { return false ; } double sp = OptimizerUtils . getSparsity ( rows , cols , nnz ) ; double mem = MatrixBlock . estimateSizeInMemory ( rows , cols , sp ) ; if ( ! OptimizerUtils . isValidCPDimensions ( rows , cols ) || ! OptimizerUtils . isValidCPMatrixSize ( rows , cols , sp ) || mem >= OptimizerUtils . getLocalMemBudget ( ) ) { return false ; } long estFilesize = ( long ) ( 3.5 * mem ) ; long cpThreshold = CP_REBLOCK_THRESHOLD_SIZE * OptimizerUtils . getParallelTextReadParallelism ( ) ; return ( estFilesize < cpThreshold ) ; }
cp reblock check for spark instructions ; in contrast to mr , we can not rely on the input file sizes because inputs might be passed via rdds .
