public static List<Writable> sampleInvalidFromColumn(int numToSample, String columnName, Schema schema,
                    JavaRDD<List<Writable>> data, boolean ignoreMissing) {
        //First: filter out all valid entries, to leave only invalid entries
        int colIdx = schema.getIndexOfColumn(columnName);
        JavaRDD<Writable> ithColumn = data.map(new SelectColumnFunction(colIdx));

        ColumnMetaData meta = schema.getMetaData(columnName);

        JavaRDD<Writable> invalid = ithColumn.filter(new FilterWritablesBySchemaFunction(meta, false, ignoreMissing));

        return invalid.takeSample(false, numToSample);
    }
public static List < Writable > sampleInvalidFromColumn ( int numToSample , String columnName , Schema schema , JavaRDD < List < Writable > > data , boolean ignoreMissing ) { int colIdx = schema . getIndexOfColumn ( columnName ) ; JavaRDD < Writable > ithColumn = data . map ( new SelectColumnFunction ( colIdx ) ) ; ColumnMetaData meta = schema . getMetaData ( columnName ) ; JavaRDD < Writable > invalid = ithColumn . filter ( new FilterWritablesBySchemaFunction ( meta , false , ignoreMissing ) ) ; return invalid . takeSample ( false , numToSample ) ; }
Randomly sample a set of invalid values from a specified column.
Values are considered invalid according to the Schema / ColumnMetaData

@param numToSample    Maximum number of invalid values to sample
@param columnName     Same of the column from which to sample invalid values
@param schema         Data schema
@param data           Data
@param ignoreMissing  If true: ignore missing values (NullWritable or empty/null string) when sampling. If false: include missing values in sampling
@return               List of invalid examples