private static DataSet loadG(Reader reader,double sparseRatio,int vectorLength,boolean classification) throws IOException {
  StringBuilder processBuffer=new StringBuilder(20);
  StringBuilder charBuffer=new StringBuilder(1024);
  char[] buffer=new char[1024];
  List<SparseVector> sparceVecs=new ArrayList<SparseVector>();
  List<Double> labelVals=new DoubleList();
  Map<Double,Integer> possibleCats=new HashMap<Double,Integer>();
  int maxLen=1;
  STATE state=STATE.INITIAL;
  int position=0;
  SparseVector tempVec=new SparseVector(1,1);
  int indexProcessing=-1;
  while (true) {
    while (charBuffer.length() - position <= 1) {
      charBuffer.delete(0,position);
      position=0;
      int read=reader.read(buffer);
      if (read < 0)       break;
      charBuffer.append(buffer,0,read);
    }
    if (charBuffer.length() - position == 0) {
      if (state == STATE.LABEL) {
        double label=Double.parseDouble(processBuffer.toString());
        if (!possibleCats.containsKey(label) && classification)         possibleCats.put(label,possibleCats.size());
        labelVals.add(label);
        sparceVecs.add(new SparseVector(maxLen,0));
      }
 else       if (state == STATE.WHITESPACE_AFTER_LABEL) {
        sparceVecs.add(new SparseVector(maxLen,0));
      }
 else       if (state == STATE.FEATURE_VALUE || state == STATE.WHITESPACE_AFTER_FEATURE) {
        double value=StringUtils.parseDouble(processBuffer,0,processBuffer.length());
        processBuffer.delete(0,processBuffer.length());
        maxLen=Math.max(maxLen,indexProcessing + 1);
        tempVec.setLength(maxLen);
        if (value != 0)         tempVec.set(indexProcessing,value);
        sparceVecs.add(tempVec.clone());
      }
 else       if (state == STATE.NEWLINE) {
        break;
      }
 else       throw new RuntimeException();
      break;
    }
    char ch=charBuffer.charAt(position);
switch (state) {
case INITIAL:
      state=STATE.LABEL;
    break;
case LABEL:
  if (Character.isDigit(ch) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+') {
    processBuffer.append(ch);
    position++;
  }
 else   if (Character.isWhitespace(ch)) {
    double label=Double.parseDouble(processBuffer.toString());
    if (!possibleCats.containsKey(label) && classification)     possibleCats.put(label,possibleCats.size());
    labelVals.add(label);
    processBuffer.delete(0,processBuffer.length());
    if (ch == '\n' || ch == '\r') {
      tempVec.zeroOut();
      sparceVecs.add(new SparseVector(maxLen,0));
      state=STATE.NEWLINE;
    }
 else {
      tempVec.zeroOut();
      state=STATE.WHITESPACE_AFTER_LABEL;
    }
  }
 else   throw new RuntimeException("Invalid LIBSVM file");
break;
case WHITESPACE_AFTER_LABEL:
if (Character.isDigit(ch)) {
state=STATE.FEATURE_INDEX;
}
 else if (Character.isWhitespace(ch)) {
if (ch == '\n' || ch == '\r') {
  tempVec.zeroOut();
  sparceVecs.add(new SparseVector(maxLen,0));
  state=STATE.NEWLINE;
}
 else position++;
}
 else throw new RuntimeException();
break;
case FEATURE_INDEX:
if (Character.isDigit(ch)) {
processBuffer.append(ch);
position++;
}
 else if (ch == ':') {
indexProcessing=StringUtils.parseInt(processBuffer,0,processBuffer.length()) - 1;
processBuffer.delete(0,processBuffer.length());
state=STATE.FEATURE_VALUE;
position++;
}
 else throw new RuntimeException();
break;
case FEATURE_VALUE:
if (Character.isDigit(ch) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+') {
processBuffer.append(ch);
position++;
}
 else {
double value=StringUtils.parseDouble(processBuffer,0,processBuffer.length());
processBuffer.delete(0,processBuffer.length());
maxLen=Math.max(maxLen,indexProcessing + 1);
tempVec.setLength(maxLen);
if (value != 0) tempVec.set(indexProcessing,value);
if (Character.isWhitespace(ch)) state=STATE.WHITESPACE_AFTER_FEATURE;
 else throw new RuntimeException();
}
break;
case WHITESPACE_AFTER_FEATURE:
if (Character.isDigit(ch)) state=STATE.FEATURE_INDEX;
 else if (Character.isWhitespace(ch)) {
if (ch == '\n' || ch == '\r') {
sparceVecs.add(tempVec.clone());
tempVec.zeroOut();
state=STATE.NEWLINE;
}
 else position++;
}
break;
case NEWLINE:
if (ch == '\n' || ch == '\r') position++;
 else {
state=STATE.LABEL;
}
break;
}
}
if (vectorLength > 0) if (maxLen > vectorLength) throw new RuntimeException("Length given was " + vectorLength + ", but observed length was "+ maxLen);
 else maxLen=vectorLength;
if (classification) {
CategoricalData predicting=new CategoricalData(possibleCats.size());
List<Double> allCatKeys=new DoubleList(possibleCats.keySet());
Collections.sort(allCatKeys);
for (int i=0; i < allCatKeys.size(); i++) possibleCats.put(allCatKeys.get(i),i);
ClassificationDataSet cds=new ClassificationDataSet(maxLen,new CategoricalData[0],predicting);
for (int i=0; i < labelVals.size(); i++) {
SparseVector vec=sparceVecs.get(i);
vec.setLength(maxLen);
cds.addDataPoint(vec,new int[0],possibleCats.get(labelVals.get(i)));
}
cds.applyTransform(new DenseSparceTransform(sparseRatio));
return cds;
}
 else {
RegressionDataSet rds=new RegressionDataSet(maxLen,new CategoricalData[0]);
for (int i=0; i < sparceVecs.size(); i++) {
SparseVector sv=sparceVecs.get(i);
sv.setLength(maxLen);
rds.addDataPoint(sv,new int[0],labelVals.get(i));
}
rds.applyTransform(new DenseSparceTransform(sparseRatio));
return rds;
}
}

private static DataSet loadG ( Reader reader , double sparseRatio , int vectorLength , boolean classification ) throws IOException { StringBuilder processBuffer = new StringBuilder ( 20 ) ; StringBuilder charBuffer = new StringBuilder ( 1024 ) ; char [ ] buffer = new char [ 1024 ] ; List < SparseVector > sparceVecs = new ArrayList < SparseVector > ( ) ; List < Double > labelVals = new DoubleList ( ) ; Map < Double , Integer > possibleCats = new HashMap < Double , Integer > ( ) ; int maxLen = 1 ; STATE state = STATE . INITIAL ; int position = 0 ; SparseVector tempVec = new SparseVector ( 1 , 1 ) ; int indexProcessing = - 1 ; while ( true ) { while ( charBuffer . length ( ) - position <= 1 ) { charBuffer . delete ( 0 , position ) ; position = 0 ; int read = reader . read ( buffer ) ; if ( read < 0 ) break ; charBuffer . append ( buffer , 0 , read ) ; } if ( charBuffer . length ( ) - position == 0 ) { if ( state == STATE . LABEL ) { double label = Double . parseDouble ( processBuffer . toString ( ) ) ; if ( ! possibleCats . containsKey ( label ) && classification ) possibleCats . put ( label , possibleCats . size ( ) ) ; labelVals . add ( label ) ; sparceVecs . add ( new SparseVector ( maxLen , 0 ) ) ; } else if ( state == STATE . WHITESPACE_AFTER_LABEL ) { sparceVecs . add ( new SparseVector ( maxLen , 0 ) ) ; } else if ( state == STATE . FEATURE_VALUE || state == STATE . WHITESPACE_AFTER_FEATURE ) { double value = StringUtils . parseDouble ( processBuffer , 0 , processBuffer . length ( ) ) ; processBuffer . delete ( 0 , processBuffer . length ( ) ) ; maxLen = Math . max ( maxLen , indexProcessing + 1 ) ; tempVec . setLength ( maxLen ) ; if ( value != 0 ) tempVec . set ( indexProcessing , value ) ; sparceVecs . add ( tempVec . clone ( ) ) ; } else if ( state == STATE . NEWLINE ) { break ; } else throw new RuntimeException ( ) ; break ; } char ch = charBuffer . charAt ( position ) ; switch ( state ) { case INITIAL : state = STATE . LABEL ; break ; case LABEL : if ( Character . isDigit ( ch ) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+' ) { processBuffer . append ( ch ) ; position ++ ; } else if ( Character . isWhitespace ( ch ) ) { double label = Double . parseDouble ( processBuffer . toString ( ) ) ; if ( ! possibleCats . containsKey ( label ) && classification ) possibleCats . put ( label , possibleCats . size ( ) ) ; labelVals . add ( label ) ; processBuffer . delete ( 0 , processBuffer . length ( ) ) ; if ( ch == '\n' || ch == '\r' ) { tempVec . zeroOut ( ) ; sparceVecs . add ( new SparseVector ( maxLen , 0 ) ) ; state = STATE . NEWLINE ; } else { tempVec . zeroOut ( ) ; state = STATE . WHITESPACE_AFTER_LABEL ; } } else throw new RuntimeException ( "Invalid LIBSVM file" ) ; break ; case WHITESPACE_AFTER_LABEL : if ( Character . isDigit ( ch ) ) { state = STATE . FEATURE_INDEX ; } else if ( Character . isWhitespace ( ch ) ) { if ( ch == '\n' || ch == '\r' ) { tempVec . zeroOut ( ) ; sparceVecs . add ( new SparseVector ( maxLen , 0 ) ) ; state = STATE . NEWLINE ; } else position ++ ; } else throw new RuntimeException ( ) ; break ; case FEATURE_INDEX : if ( Character . isDigit ( ch ) ) { processBuffer . append ( ch ) ; position ++ ; } else if ( ch == ':' ) { indexProcessing = StringUtils . parseInt ( processBuffer , 0 , processBuffer . length ( ) ) - 1 ; processBuffer . delete ( 0 , processBuffer . length ( ) ) ; state = STATE . FEATURE_VALUE ; position ++ ; } else throw new RuntimeException ( ) ; break ; case FEATURE_VALUE : if ( Character . isDigit ( ch ) || ch == '.' || ch == 'E' || ch == 'e' || ch == '-' || ch == '+' ) { processBuffer . append ( ch ) ; position ++ ; } else { double value = StringUtils . parseDouble ( processBuffer , 0 , processBuffer . length ( ) ) ; processBuffer . delete ( 0 , processBuffer . length ( ) ) ; maxLen = Math . max ( maxLen , indexProcessing + 1 ) ; tempVec . setLength ( maxLen ) ; if ( value != 0 ) tempVec . set ( indexProcessing , value ) ; if ( Character . isWhitespace ( ch ) ) state = STATE . WHITESPACE_AFTER_FEATURE ; else throw new RuntimeException ( ) ; } break ; case WHITESPACE_AFTER_FEATURE : if ( Character . isDigit ( ch ) ) state = STATE . FEATURE_INDEX ; else if ( Character . isWhitespace ( ch ) ) { if ( ch == '\n' || ch == '\r' ) { sparceVecs . add ( tempVec . clone ( ) ) ; tempVec . zeroOut ( ) ; state = STATE . NEWLINE ; } else position ++ ; } break ; case NEWLINE : if ( ch == '\n' || ch == '\r' ) position ++ ; else { state = STATE . LABEL ; } break ; } } if ( vectorLength > 0 ) if ( maxLen > vectorLength ) throw new RuntimeException ( "Length given was " + vectorLength + ", but observed length was " + maxLen ) ; else maxLen = vectorLength ; if ( classification ) { CategoricalData predicting = new CategoricalData ( possibleCats . size ( ) ) ; List < Double > allCatKeys = new DoubleList ( possibleCats . keySet ( ) ) ; Collections . sort ( allCatKeys ) ; for ( int i = 0 ; i < allCatKeys . size ( ) ; i ++ ) possibleCats . put ( allCatKeys . get ( i ) , i ) ; ClassificationDataSet cds = new ClassificationDataSet ( maxLen , new CategoricalData [ 0 ] , predicting ) ; for ( int i = 0 ; i < labelVals . size ( ) ; i ++ ) { SparseVector vec = sparceVecs . get ( i ) ; vec . setLength ( maxLen ) ; cds . addDataPoint ( vec , new int [ 0 ] , possibleCats . get ( labelVals . get ( i ) ) ) ; } cds . applyTransform ( new DenseSparceTransform ( sparseRatio ) ) ; return cds ; } else { RegressionDataSet rds = new RegressionDataSet ( maxLen , new CategoricalData [ 0 ] ) ; for ( int i = 0 ; i < sparceVecs . size ( ) ; i ++ ) { SparseVector sv = sparceVecs . get ( i ) ; sv . setLength ( maxLen ) ; rds . addDataPoint ( sv , new int [ 0 ] , labelVals . get ( i ) ) ; } rds . applyTransform ( new DenseSparceTransform ( sparseRatio ) ) ; return rds ; } }
generic loader for both classification and regression interpretations .
