@Override
	public void operate(OpContext context) throws ProcessingException {
		final TableDataSource table = (TableDataSource)context.get(tableInput);
		int ipaTidx = super.getColumnIndex(table, SystemTierType.IPATarget.getName());
		if(ipaTidx < 0) {
			throw new ProcessingException(null, "Table has no " + SystemTierType.IPATarget.getName() + " column.");
		}
		int ipaAidx = super.getColumnIndex(table, SystemTierType.IPAActual.getName());
		if(ipaAidx < 0) {
			throw new ProcessingException(null, "Table has no " + SystemTierType.IPAActual.getName() + " column.");
		}
		// group by session if info is available
		// TODO make this an option
		int sessionIdx = super.getColumnIndex(table, "Session");
		boolean ignoreDiacritics = isIgnoreDiacritics();
		if(context.containsKey("ignoreDiacritics")) {
			ignoreDiacritics = (boolean)context.get("ignoreDiacritics");
		}
		final Map<GroupKey, IpaTernaryTree<List<IPATranscript>>> tokenCounts =
				new LinkedHashMap<>();
		for(int row = 0; row < table.getRowCount(); row++) {
			checkCanceled();
			final Object groupVal =
					(sessionIdx >= 0 ? table.getValueAt(row, sessionIdx) : "*");
			final GroupKey groupKey = new GroupKey(groupVal, ignoreDiacritics);
			IpaTernaryTree<List<IPATranscript>> tokenCount =
					tokenCounts.get(groupKey);
			if(tokenCount == null) {
				tokenCount = new IpaTernaryTree<>();
				tokenCounts.put(groupKey, tokenCount);
			}
			IPATranscript ipaT =
					IPATranscript.class.cast(table.getValueAt(row, ipaTidx));
			IPATranscript ipaA =
					IPATranscript.class.cast(table.getValueAt(row, ipaAidx));
			if(ignoreDiacritics) {
				ipaT = ipaT.removePunctuation().stripDiacritics();
				ipaA = ipaA.removePunctuation().stripDiacritics();
			}
			List<IPATranscript> productions = tokenCount.get(ipaT);
			if(productions == null) {
				productions = new ArrayList<>();
				tokenCount.put(ipaT, productions);
			}
			productions.add(ipaA);
		}
		final DefaultTableDataSource outputTable = new DefaultTableDataSource();
		for(GroupKey groupKey:tokenCounts.keySet()) {
			final Object[] rowData = new Object[7];
			// produce table
			int numRepatedWords = 0;
			int numAllCorrect = 0;
			int numOneOrMoreCorrect = 0;
			int numSameError = 0;
			int numDifferentErrors = 0;
			float sumOfAvgDistances = 0;
			final IpaTernaryTree<List<IPATranscript>> tokenCount = tokenCounts.get(groupKey);
			final List<IPATranscript> repeatedTokens =
					tokenCount.keySet().stream()
						.filter( (ipa) -> tokenCount.get(ipa).size() > 1 )
						.collect(Collectors.toList());
			numRepatedWords = repeatedTokens.size();
			for(IPATranscript ipa:repeatedTokens) {
				checkCanceled();
				int numCorrect = 0;
				final List<IPATranscript> productions = tokenCount.get(ipa);
				final Set<IPATranscript> distinctProductions = new LinkedHashSet<>(productions);
				for(IPATranscript production:tokenCount.get(ipa)) {
					if(TableUtils.checkEquals(ipa, production, false, ignoreDiacritics)) {
						++numCorrect;
					}
				}
				if(numCorrect == productions.size()) {
					++numAllCorrect;
				} else {
					if(numCorrect > 0 && numCorrect < productions.size()) {
						++numOneOrMoreCorrect;
					}
					distinctProductions.remove(ipa);
					if(distinctProductions.size() == 1) {
						++numSameError;
					} else {
						++numDifferentErrors;
					}
					int totalDistance = 0;
					for(IPATranscript production:productions) {
						totalDistance += LevenshteinDistance.distance(ipa, production);
					}
					float avg = ((float)totalDistance)/((float)productions.size());
					sumOfAvgDistances += avg;
				}
			}
			// append row to table
			rowData[0] = groupKey.key;
			rowData[1] = numRepatedWords;
			rowData[2] = numAllCorrect;
			rowData[3] = numOneOrMoreCorrect;
			rowData[4] = numSameError;
			rowData[5] = numDifferentErrors;
			rowData[6] = sumOfAvgDistances / numRepatedWords;
			outputTable.addRow(rowData);
		}
		outputTable.setColumnTitle(0, "Session");
		outputTable.setColumnTitle(1, "# Repeated IPA Target");
		outputTable.setColumnTitle(2, "# All Correct");
		outputTable.setColumnTitle(3, "# One or More Correct");
		outputTable.setColumnTitle(4, "# Same Error");
		outputTable.setColumnTitle(5, "# Different Errors");
		outputTable.setColumnTitle(6, "Avg Distance");
		context.put(tableOutput, outputTable);
	}
@Override
	public void operate(OpContext context) throws ProcessingException {
		final TableDataSource table = (TableDataSource)context.get(tableInput);
		int ipaTidx = super.getColumnIndex(table, SystemTierType.IPATarget.getName());
		if(ipaTidx < 0) {
			throw new ProcessingException(null, "Table has no " + SystemTierType.IPATarget.getName() + " column.");
		}
		int ipaAidx = super.getColumnIndex(table, SystemTierType.IPAActual.getName());
		if(ipaAidx < 0) {
			throw new ProcessingException(null, "Table has no " + SystemTierType.IPAActual.getName() + " column.");
		}
	
	
		int sessionIdx = super.getColumnIndex(table, "Session");
		boolean ignoreDiacritics = isIgnoreDiacritics();
		if(context.containsKey("ignoreDiacritics")) {
			ignoreDiacritics = (boolean)context.get("ignoreDiacritics");
		}
		final Map<GroupKey, IpaTernaryTree<List<IPATranscript>>> tokenCounts =
				new LinkedHashMap<>();
		for(int row = 0; row < table.getRowCount(); row++) {
			checkCanceled();
			final Object groupVal =
					(sessionIdx >= 0 ? table.getValueAt(row, sessionIdx) : "*");
			final GroupKey groupKey = new GroupKey(groupVal, ignoreDiacritics);
			IpaTernaryTree<List<IPATranscript>> tokenCount =
					tokenCounts.get(groupKey);
			if(tokenCount == null) {
				tokenCount = new IpaTernaryTree<>();
				tokenCounts.put(groupKey, tokenCount);
			}
			IPATranscript ipaT =
					IPATranscript.class.cast(table.getValueAt(row, ipaTidx));
			IPATranscript ipaA =
					IPATranscript.class.cast(table.getValueAt(row, ipaAidx));
			if(ignoreDiacritics) {
				ipaT = ipaT.removePunctuation().stripDiacritics();
				ipaA = ipaA.removePunctuation().stripDiacritics();
			}
			List<IPATranscript> productions = tokenCount.get(ipaT);
			if(productions == null) {
				productions = new ArrayList<>();
				tokenCount.put(ipaT, productions);
			}
			productions.add(ipaA);
		}
		final DefaultTableDataSource outputTable = new DefaultTableDataSource();
		for(GroupKey groupKey:tokenCounts.keySet()) {
			final Object[] rowData = new Object[7];
		
			int numRepatedWords = 0;
			int numAllCorrect = 0;
			int numOneOrMoreCorrect = 0;
			int numSameError = 0;
			int numDifferentErrors = 0;
			float sumOfAvgDistances = 0;
			final IpaTernaryTree<List<IPATranscript>> tokenCount = tokenCounts.get(groupKey);
			final List<IPATranscript> repeatedTokens =
					tokenCount.keySet().stream()
						.filter( (ipa) -> tokenCount.get(ipa).size() > 1 )
						.collect(Collectors.toList());
			numRepatedWords = repeatedTokens.size();
			for(IPATranscript ipa:repeatedTokens) {
				checkCanceled();
				int numCorrect = 0;
				final List<IPATranscript> productions = tokenCount.get(ipa);
				final Set<IPATranscript> distinctProductions = new LinkedHashSet<>(productions);
				for(IPATranscript production:tokenCount.get(ipa)) {
					if(TableUtils.checkEquals(ipa, production, false, ignoreDiacritics)) {
						++numCorrect;
					}
				}
				if(numCorrect == productions.size()) {
					++numAllCorrect;
				} else {
					if(numCorrect > 0 && numCorrect < productions.size()) {
						++numOneOrMoreCorrect;
					}
					distinctProductions.remove(ipa);
					if(distinctProductions.size() == 1) {
						++numSameError;
					} else {
						++numDifferentErrors;
					}
					int totalDistance = 0;
					for(IPATranscript production:productions) {
						totalDistance += LevenshteinDistance.distance(ipa, production);
					}
					float avg = ((float)totalDistance)/((float)productions.size());
					sumOfAvgDistances += avg;
				}
			}
		
			rowData[0] = groupKey.key;
			rowData[1] = numRepatedWords;
			rowData[2] = numAllCorrect;
			rowData[3] = numOneOrMoreCorrect;
			rowData[4] = numSameError;
			rowData[5] = numDifferentErrors;
			rowData[6] = sumOfAvgDistances / numRepatedWords;
			outputTable.addRow(rowData);
		}
		outputTable.setColumnTitle(0, "Session");
		outputTable.setColumnTitle(1, "# Repeated IPA Target");
		outputTable.setColumnTitle(2, "# All Correct");
		outputTable.setColumnTitle(3, "# One or More Correct");
		outputTable.setColumnTitle(4, "# Same Error");
		outputTable.setColumnTitle(5, "# Different Errors");
		outputTable.setColumnTitle(6, "Avg Distance");
		context.put(tableOutput, outputTable);
	}
@override public void operate(opcontext context) throws processingexception { final tabledatasource table = (tabledatasource)context.get(tableinput); int ipatidx = super.getcolumnindex(table, systemtiertype.ipatarget.getname()); if(ipatidx < 0) { throw new processingexception(null, "table has no " + systemtiertype.ipatarget.getname() + " column."); } int ipaaidx = super.getcolumnindex(table, systemtiertype.ipaactual.getname()); if(ipaaidx < 0) { throw new processingexception(null, "table has no " + systemtiertype.ipaactual.getname() + " column."); } int sessionidx = super.getcolumnindex(table, "session"); boolean ignorediacritics = isignorediacritics(); if(context.containskey("ignorediacritics")) { ignorediacritics = (boolean)context.get("ignorediacritics"); } final map<groupkey, ipaternarytree<list<ipatranscript>>> tokencounts = new linkedhashmap<>(); for(int row = 0; row < table.getrowcount(); row++) { checkcanceled(); final object groupval = (sessionidx >= 0 ? table.getvalueat(row, sessionidx) : "*"); final groupkey groupkey = new groupkey(groupval, ignorediacritics); ipaternarytree<list<ipatranscript>> tokencount = tokencounts.get(groupkey); if(tokencount == null) { tokencount = new ipaternarytree<>(); tokencounts.put(groupkey, tokencount); } ipatranscript ipat = ipatranscript.class.cast(table.getvalueat(row, ipatidx)); ipatranscript ipaa = ipatranscript.class.cast(table.getvalueat(row, ipaaidx)); if(ignorediacritics) { ipat = ipat.removepunctuation().stripdiacritics(); ipaa = ipaa.removepunctuation().stripdiacritics(); } list<ipatranscript> productions = tokencount.get(ipat); if(productions == null) { productions = new arraylist<>(); tokencount.put(ipat, productions); } productions.add(ipaa); } final defaulttabledatasource outputtable = new defaulttabledatasource(); for(groupkey groupkey:tokencounts.keyset()) { final object[] rowdata = new object[7]; int numrepatedwords = 0; int numallcorrect = 0; int numoneormorecorrect = 0; int numsameerror = 0; int numdifferenterrors = 0; float sumofavgdistances = 0; final ipaternarytree<list<ipatranscript>> tokencount = tokencounts.get(groupkey); final list<ipatranscript> repeatedtokens = tokencount.keyset().stream() .filter( (ipa) -> tokencount.get(ipa).size() > 1 ) .collect(collectors.tolist()); numrepatedwords = repeatedtokens.size(); for(ipatranscript ipa:repeatedtokens) { checkcanceled(); int numcorrect = 0; final list<ipatranscript> productions = tokencount.get(ipa); final set<ipatranscript> distinctproductions = new linkedhashset<>(productions); for(ipatranscript production:tokencount.get(ipa)) { if(tableutils.checkequals(ipa, production, false, ignorediacritics)) { ++numcorrect; } } if(numcorrect == productions.size()) { ++numallcorrect; } else { if(numcorrect > 0 && numcorrect < productions.size()) { ++numoneormorecorrect; } distinctproductions.remove(ipa); if(distinctproductions.size() == 1) { ++numsameerror; } else { ++numdifferenterrors; } int totaldistance = 0; for(ipatranscript production:productions) { totaldistance += levenshteindistance.distance(ipa, production); } float avg = ((float)totaldistance)/((float)productions.size()); sumofavgdistances += avg; } } rowdata[0] = groupkey.key; rowdata[1] = numrepatedwords; rowdata[2] = numallcorrect; rowdata[3] = numoneormorecorrect; rowdata[4] = numsameerror; rowdata[5] = numdifferenterrors; rowdata[6] = sumofavgdistances / numrepatedwords; outputtable.addrow(rowdata); } outputtable.setcolumntitle(0, "session"); outputtable.setcolumntitle(1, "# repeated ipa target"); outputtable.setcolumntitle(2, "# all correct"); outputtable.setcolumntitle(3, "# one or more correct"); outputtable.setcolumntitle(4, "# same error"); outputtable.setcolumntitle(5, "# different errors"); outputtable.setcolumntitle(6, "avg distance"); context.put(tableoutput, outputtable); }