src/main/java/org/apache/sysds/runtime/compress/cocode/PlanningCoCoder.java
/**
	 * Identify columns to code together. Uses a greedy approach that merges pairs of column groups into larger groups.
	 * Each phase of the greedy algorithm considers all combinations of pairs to merge.
	 * 
	 * TODO Find better faster ways of finding cocodes than brute force.
	 * 
	 * @param sizeEstimator  compressed size estimator
	 * @param numRowsWeight  number of rows weight
	 * @param singltonGroups planning co-coding groups
	 * @return A PlanningCoCodingGroup.
	 */
Identify columns to code together. Uses a greedy approach that merges pairs of column groups into larger groups.
Each phase of the greedy algorithm considers all combinations of pairs to merge.



@param sizeEstimator  compressed size estimator
@param numRowsWeight  number of rows weight
@param singltonGroups planning co-coding groups
@return A PlanningCoCodingGroup.
Identify columns to code together . Uses a greedy approach that merges pairs of column groups into larger groups . Each phase of the greedy algorithm considers all combinations of pairs to merge . @param sizeEstimator compressed size estimator @param numRowsWeight number of rows weight @param singltonGroups planning co - coding groups @return A PlanningCoCodingGroup .
private static PlanningCoCodingGroup[] findCocodesBruteForce(CompressedSizeEstimator estimator, int numRows,
		PlanningCoCodingGroup[] singletonGroups) {
		if(LOG.isTraceEnabled())
			LOG.trace("Cocoding: process " + singletonGroups.length);
		List<PlanningCoCodingGroup> workset = new ArrayList<>(Arrays.asList(singletonGroups));
		PlanningMemoTable memo = new PlanningMemoTable();
		boolean changed = true;
		while(changed && workset.size() > 1) {
			PlanningCoCodingGroup tmp = null;
			for(int i = 0; i < workset.size(); i++) {
				for(int j = i + 1; j < workset.size(); j++) {
					PlanningCoCodingGroup c1 = workset.get(i);
					PlanningCoCodingGroup c2 = workset.get(j);
					memo.incrStats(1, 0, 0);
					if(-Math.min(c1.getEstSize(), c2.getEstSize()) > memo.getOptChangeInSize())
						continue;
					PlanningCoCodingGroup c1c2 = memo.getOrCreate(c1, c2, estimator, numRows);
					if(tmp == null || c1c2.getChangeInSize() < tmp.getChangeInSize() ||
						(c1c2.getChangeInSize() == tmp.getChangeInSize() &&
							c1c2.getColIndices().length < tmp.getColIndices().length))
						tmp = c1c2;
				}
			}
			if(tmp != null && tmp.getChangeInSize() < 0) {
				workset.remove(tmp.getLeftGroup());
				workset.remove(tmp.getRightGroup());
				workset.add(tmp);
				memo.remove(tmp);
				if(LOG.isTraceEnabled()) {
					LOG.trace("--merge groups: " + Arrays.toString(tmp.getLeftGroup().getColIndices()) + " and "
						+ Arrays.toString(tmp.getRightGroup().getColIndices()));
				}
			}
			else {
				changed = false;
			}
		}
		if(LOG.isTraceEnabled())
			LOG.trace("--stats: " + Arrays.toString(memo.getStats()));
		return workset.toArray(new PlanningCoCodingGroup[0]);
	}
private static PlanningCoCodingGroup [ ] findCocodesBruteForce ( CompressedSizeEstimator estimator , int numRows , PlanningCoCodingGroup [ ] singletonGroups ) { if ( LOG . isTraceEnabled ( ) ) LOG . trace ( "Cocoding: process " + singletonGroups . length ) ; List < PlanningCoCodingGroup > workset = new ArrayList < > ( Arrays . asList ( singletonGroups ) ) ; PlanningMemoTable memo = new PlanningMemoTable ( ) ; boolean changed = true ; while ( changed && workset . size ( ) > 1 ) { PlanningCoCodingGroup tmp = null ; for ( int i = 0 ; i < workset . size ( ) ; i ++ ) { for ( int j = i + 1 ; j < workset . size ( ) ; j ++ ) { PlanningCoCodingGroup c1 = workset . get ( i ) ; PlanningCoCodingGroup c2 = workset . get ( j ) ; memo . incrStats ( 1 , 0 , 0 ) ; if ( - Math . min ( c1 . getEstSize ( ) , c2 . getEstSize ( ) ) > memo . getOptChangeInSize ( ) ) continue ; PlanningCoCodingGroup c1c2 = memo . getOrCreate ( c1 , c2 , estimator , numRows ) ; if ( tmp == null || c1c2 . getChangeInSize ( ) < tmp . getChangeInSize ( ) || ( c1c2 . getChangeInSize ( ) == tmp . getChangeInSize ( ) && c1c2 . getColIndices ( ) . length < tmp . getColIndices ( ) . length ) ) tmp = c1c2 ; } } if ( tmp != null && tmp . getChangeInSize ( ) < 0 ) { workset . remove ( tmp . getLeftGroup ( ) ) ; workset . remove ( tmp . getRightGroup ( ) ) ; workset . add ( tmp ) ; memo . remove ( tmp ) ; if ( LOG . isTraceEnabled ( ) ) { LOG . trace ( "--merge groups: " + Arrays . toString ( tmp . getLeftGroup ( ) . getColIndices ( ) ) + " and " + Arrays . toString ( tmp . getRightGroup ( ) . getColIndices ( ) ) ) ; } } else { changed = false ; } } if ( LOG . isTraceEnabled ( ) ) LOG . trace ( "--stats: " + Arrays . toString ( memo . getStats ( ) ) ) ; return workset . toArray ( new PlanningCoCodingGroup [ 0 ] ) ; }
Identify columns to code together.
Identify columns to code together .
// establish memo table for extracted column groups // process merging iterations until no more change // find best merge, incl memoization // pruning filter: skip dominated candidates // memoization or newly created group (incl bitmap extraction) // keep best merged group only // modify working set