private static List<CoreLabel> toSequence(String sourceToken,List<String> targetTokens,int outputIndex){
  StringBuilder sb=new StringBuilder();
  for (  String s : targetTokens) {
    if (sb.length() > 0)     sb.append(WHITESPACE);
    sb.append(s);
  }
  String target=sb.toString();
  int[] t2sGrid=alignStrings(sourceToken,target);
  assert t2sGrid.length == target.length();
  List<CoreLabel> sequence=new ArrayList<>(target.length());
  int[] s2t=new int[sourceToken.length()];
  Arrays.fill(s2t,-1);
  int parentIndex=0;
  int charIndex=0;
  for (int i=0; i < t2sGrid.length; ++i) {
    String tChar=String.valueOf(target.charAt(i));
    if (tChar.equals(WHITESPACE)) {
      ++parentIndex;
      charIndex=-1;
    }
    String parentToken=parentIndex >= targetTokens.size() ? "#NoNe#" : targetTokens.get(parentIndex);
    int sIndex=t2sGrid[i];
    if (sIndex < 0) {
      sequence.add(createDatum(tChar,Operation.Delete.toString(),i,parentToken,charIndex));
    }
 else {
      String sChar=String.valueOf(sourceToken.charAt(sIndex));
      assert sIndex < s2t.length;
      s2t[sIndex]=i;
      if (tChar.equals(sChar) || TokenUtils.isPunctuation(sChar)) {
        sequence.add(createDatum(tChar,Operation.None.toString(),i,parentToken,charIndex));
      }
 else       if (tChar.equals(sChar.toLowerCase())) {
        sequence.add(createDatum(tChar,Operation.ToUpper.toString(),i,parentToken,charIndex));
      }
 else {
        sequence.add(createDatum(tChar,Operation.None.toString(),i,parentToken,charIndex));
      }
    }
    ++charIndex;
  }
  for (int i=0; i < s2t.length; ++i) {
    if (s2t[i] >= 0)     continue;
    int j=i + 1;
    while (j < s2t.length && s2t[j] < 0)     ++j;
    int p=i > 0 ? s2t[i - 1] : -1;
    int q=j < s2t.length ? s2t[j] : -1;
    String pLabel=p > 0 ? sequence.get(p).get(CoreAnnotations.GoldAnswerAnnotation.class) : null;
    String qLabel=q > 0 ? sequence.get(q).get(CoreAnnotations.GoldAnswerAnnotation.class) : null;
    Operation pOperation=null;
    Operation qOperation=null;
    try {
      pOperation=pLabel == null ? null : Operation.valueOf(pLabel);
    }
 catch (    Exception e) {
    }
    try {
      qOperation=qLabel == null ? null : Operation.valueOf(qLabel);
    }
 catch (    Exception e) {
    }
    if (pOperation != null && pOperation == Operation.None) {
      String span=sourceToken.substring(i,j);
      if (!TokenUtils.isPunctuation(span)) {
        String label=Operation.InsertAfter.toString() + OP_DELIM + span;
        sequence.get(p).set(CoreAnnotations.GoldAnswerAnnotation.class,label);
      }
    }
 else     if (qOperation == Operation.None) {
      String span=sourceToken.substring(i,j);
      if (!TokenUtils.isPunctuation(span)) {
        String label=Operation.InsertBefore.toString() + OP_DELIM + span;
        sequence.get(q).set(CoreAnnotations.GoldAnswerAnnotation.class,label);
      }
    }
 else {
      if (Pattern.compile("\u00AD").matcher(sourceToken).find()) {
      }
 else {
        System.err.printf("WARNING: Unmanageable span (%s): %s -> %s%n",sourceToken.substring(i,j),sourceToken,target);
      }
    }
  }
  return sequence;
}

private static List < CoreLabel > toSequence ( String sourceToken , List < String > targetTokens , int outputIndex ) { StringBuilder sb = new StringBuilder ( ) ; for ( String s : targetTokens ) { if ( sb . length ( ) > 0 ) sb . append ( WHITESPACE ) ; sb . append ( s ) ; } String target = sb . toString ( ) ; int [ ] t2sGrid = alignStrings ( sourceToken , target ) ; assert t2sGrid . length == target . length ( ) ; List < CoreLabel > sequence = new ArrayList < > ( target . length ( ) ) ; int [ ] s2t = new int [ sourceToken . length ( ) ] ; Arrays . fill ( s2t , - 1 ) ; int parentIndex = 0 ; int charIndex = 0 ; for ( int i = 0 ; i < t2sGrid . length ; ++ i ) { String tChar = String . valueOf ( target . charAt ( i ) ) ; if ( tChar . equals ( WHITESPACE ) ) { ++ parentIndex ; charIndex = - 1 ; } String parentToken = parentIndex >= targetTokens . size ( ) ? "#NoNe#" : targetTokens . get ( parentIndex ) ; int sIndex = t2sGrid [ i ] ; if ( sIndex < 0 ) { sequence . add ( createDatum ( tChar , Operation . Delete . toString ( ) , i , parentToken , charIndex ) ) ; } else { String sChar = String . valueOf ( sourceToken . charAt ( sIndex ) ) ; assert sIndex < s2t . length ; s2t [ sIndex ] = i ; if ( tChar . equals ( sChar ) || TokenUtils . isPunctuation ( sChar ) ) { sequence . add ( createDatum ( tChar , Operation . None . toString ( ) , i , parentToken , charIndex ) ) ; } else if ( tChar . equals ( sChar . toLowerCase ( ) ) ) { sequence . add ( createDatum ( tChar , Operation . ToUpper . toString ( ) , i , parentToken , charIndex ) ) ; } else { sequence . add ( createDatum ( tChar , Operation . None . toString ( ) , i , parentToken , charIndex ) ) ; } } ++ charIndex ; } for ( int i = 0 ; i < s2t . length ; ++ i ) { if ( s2t [ i ] >= 0 ) continue ; int j = i + 1 ; while ( j < s2t . length && s2t [ j ] < 0 ) ++ j ; int p = i > 0 ? s2t [ i - 1 ] : - 1 ; int q = j < s2t . length ? s2t [ j ] : - 1 ; String pLabel = p > 0 ? sequence . get ( p ) . get ( CoreAnnotations . GoldAnswerAnnotation . class ) : null ; String qLabel = q > 0 ? sequence . get ( q ) . get ( CoreAnnotations . GoldAnswerAnnotation . class ) : null ; Operation pOperation = null ; Operation qOperation = null ; try { pOperation = pLabel == null ? null : Operation . valueOf ( pLabel ) ; } catch ( Exception e ) { } try { qOperation = qLabel == null ? null : Operation . valueOf ( qLabel ) ; } catch ( Exception e ) { } if ( pOperation != null && pOperation == Operation . None ) { String span = sourceToken . substring ( i , j ) ; if ( ! TokenUtils . isPunctuation ( span ) ) { String label = Operation . InsertAfter . toString ( ) + OP_DELIM + span ; sequence . get ( p ) . set ( CoreAnnotations . GoldAnswerAnnotation . class , label ) ; } } else if ( qOperation == Operation . None ) { String span = sourceToken . substring ( i , j ) ; if ( ! TokenUtils . isPunctuation ( span ) ) { String label = Operation . InsertBefore . toString ( ) + OP_DELIM + span ; sequence . get ( q ) . set ( CoreAnnotations . GoldAnswerAnnotation . class , label ) ; } } else { if ( Pattern . compile ( "­" ) . matcher ( sourceToken ) . find ( ) ) { } else { System . err . printf ( "WARNING: Unmanageable span (%s): %s -> %s%n" , sourceToken . substring ( i , j ) , sourceToken , target ) ; } } } return sequence ; }
create the observed label sequence from a training example . note : this method ignores punctuation normalization , which causes a huge blow - up in the number of replace and insert operations . most pre - processors convert punctuation to ascii equivalents , which is probably acceptable as output for most users .
