public Pair<int[][][], int[]> documentToDataAndLabels(List<IN> document) {

    int docSize = document.size();
    // first index is position in the document also the index of the
    // clique/factor table
    // second index is the number of elements in the clique/window these
    // features are for (starting with last element)
    // third index is position of the feature in the array that holds them
    // element in data[j][k][m] is the index of the mth feature occurring in
    // position k of the jth clique
    int[][][] data = new int[docSize][windowSize][];
    // index is the position in the document
    // element in labels[j] is the index of the correct label (if it exists) at
    // position j of document
    int[] labels = new int[docSize];

    if (flags.useReverse) {
      Collections.reverse(document);
    }

    // System.err.println("docSize:"+docSize);
    for (int j = 0; j < docSize; j++) {
      CRFDatum<List<String>, CRFLabel> d = makeDatum(document, j, featureFactory);

      List<List<String>> features = d.asFeatures();
      for (int k = 0, fSize = features.size(); k < fSize; k++) {
        Collection<String> cliqueFeatures = features.get(k);
        data[j][k] = new int[cliqueFeatures.size()];
        int m = 0;
        for (String feature : cliqueFeatures) {
          int index = featureIndex.indexOf(feature);
          if (index >= 0) {
            data[j][k][m] = index;
            m++;
          } else {
            // this is where we end up when we do feature threshold cutoffs
          }
        }
        // Reduce memory use when some features were cut out by threshold
        if (m < data[j][k].length) {
          int[] f = new int[m];
          System.arraycopy(data[j][k], 0, f, 0, m);
          data[j][k] = f;
        }
      }

      IN wi = document.get(j);
      labels[j] = classIndex.indexOf(wi.get(AnswerAnnotation.class));
    }

    if (flags.useReverse) {
      Collections.reverse(document);
    }

    // System.err.println("numClasses: "+classIndex.size()+" "+classIndex);
    // System.err.println("numDocuments: 1");
    // System.err.println("numDatums: "+data.length);
    // System.err.println("numFeatures: "+featureIndex.size());

    return new Pair<int[][][], int[]>(data, labels);
  }
public Pair < int [ ] [ ] [ ] , int [ ] > documentToDataAndLabels ( List < IN > document ) { int docSize = document . size ( ) ; int [ ] [ ] [ ] data = new int [ docSize ] [ windowSize ] [ ] ; int [ ] labels = new int [ docSize ] ; if ( flags . useReverse ) { Collections . reverse ( document ) ; } for ( int j = 0 ; j < docSize ; j ++ ) { CRFDatum < List < String > , CRFLabel > d = makeDatum ( document , j , featureFactory ) ; List < List < String > > features = d . asFeatures ( ) ; for ( int k = 0 , fSize = features . size ( ) ; k < fSize ; k ++ ) { Collection < String > cliqueFeatures = features . get ( k ) ; data [ j ] [ k ] = new int [ cliqueFeatures . size ( ) ] ; int m = 0 ; for ( String feature : cliqueFeatures ) { int index = featureIndex . indexOf ( feature ) ; if ( index >= 0 ) { data [ j ] [ k ] [ m ] = index ; m ++ ; } else { } } if ( m < data [ j ] [ k ] . length ) { int [ ] f = new int [ m ] ; System . arraycopy ( data [ j ] [ k ] , 0 , f , 0 , m ) ; data [ j ] [ k ] = f ; } } IN wi = document . get ( j ) ; labels [ j ] = classIndex . indexOf ( wi . get ( AnswerAnnotation . class ) ) ; } if ( flags . useReverse ) { Collections . reverse ( document ) ; } return new Pair < int [ ] [ ] [ ] , int [ ] > ( data , labels ) ; }
Convert a document List into arrays storing the data features and labels.

@param document
Training documents
@return A Pair, where the first element is an int[][][] representing the
data and the second element is an int[] representing the labels