private DataSet doDelimitedFile(final Reader dataSource, final boolean createMDFromFile) throws IOException {
        if (dataSource == null) {
            throw new IllegalArgumentException("dataSource is null");
        }
        final DefaultDataSet ds = new DefaultDataSet(getPzMetaData(), this);
        try (BufferedReader br = new BufferedReader(dataSource)) {
            // gather the conversion properties
            ds.setPZConvertProps(ParserUtils.loadConvertProperties());

            boolean processedFirst = false;
            /** loop through each line in the file */
            String line = null;
            int estimatedColCount = FPConstants.SPLITLINE_SIZE_INIT;
            while ((line = fetchNextRecord(br, getQualifier(), getDelimiter())) != null) {
                // check to see if the user has elected to skip the first record
                if (!processedFirst && isIgnoreFirstRecord()) {
                    processedFirst = true;
                    continue;
                } else if (!processedFirst && createMDFromFile) {
                    processedFirst = true;
                    setPzMetaData(ParserUtils.getPZMetaDataFromFile(line, delimiter, qualifier, this, isAddSuffixToDuplicateColumnNames()));
                    ds.setMetaData(getPzMetaData());
                    continue;
                }
                // column values

                // check number of Qualifier, if ODD number --> Incorrect!!!
                if (oddNumberOfQualifier(line, getQualifier())) {
                    addError(ds, "Odd number of Qualifier characters", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    continue;
                }

                List<String> columns = ParserUtils.splitLine(line, getDelimiter(), getQualifier(), estimatedColCount, isPreserveLeadingWhitespace(),
                        isPreserveTrailingWhitespace());
                final String mdkey = ParserUtils.getCMDKeyForDelimitedFile(getPzMetaData(), columns);
                final List<ColumnMetaData> metaData = ParserUtils.getColumnMetaData(mdkey, getPzMetaData());
                final int columnCount = metaData.size();
                estimatedColCount = columnCount;

                if (columns.size() > columnCount) {
                    // Incorrect record length on line log the error. Line
                    // will not be included in the dataset log the error
                    if (isIgnoreExtraColumns()) {
                        // user has chosen to ignore the fact that we have too many columns in the data from
                        // what the mapping has described. sublist the array to remove un-needed columns
                        columns = columns.subList(0, columnCount);
                        addError(ds, "Flatpack truncated line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);
                    } else {
                        addError(ds, "Too many columns expected: " + columnCount + " Flatpack got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                } else if (columns.size() < columnCount) {
                    if (isHandlingShortLines()) {
                        // We can pad this line out
                        while (columns.size() < columnCount) {
                            columns.add("");
                        }

                        // log a warning
                        addError(ds, "Flatpack padded line to correct number of columns", lineCount, 1, isStoreRawDataToDataError() ? line : null);

                    } else {
                        addError(ds, "Too few columns expected: " + columnCount + " only got: " + columns.size(), lineCount, 2,
                                isStoreRawDataToDataError() ? line : null);
                        continue;
                    }
                }

                final Row row = new Row();
                row.setMdkey(mdkey.equals(FPConstants.DETAIL_ID) ? null : mdkey); // try
                // to limit the memory use
                row.setCols(columns);
                row.setRowNumber(lineCount);
                if (isFlagEmptyRows()) {
                    // user has elected to have the parser flag rows that are empty
                    row.setEmpty(ParserUtils.isListElementsEmpty(columns));
                }
                if (isStoreRawDataToDataSet()) {
                    // user told the parser to keep a copy of the raw data in the row
                    // WARNING potential for high memory usage here
                    row.setRawData(line);
                }

                // add the row to the array
                ds.addRow(row);

            }
        } finally {
            closeReaders();
        }
        return ds;
    }
private DataSet doDelimitedFile ( final Reader dataSource , final boolean createMDFromFile ) throws IOException { if ( dataSource == null ) { throw new IllegalArgumentException ( "dataSource is null" ) ; } final DefaultDataSet ds = new DefaultDataSet ( getPzMetaData ( ) , this ) ; try ( BufferedReader br = new BufferedReader ( dataSource ) ) { ds . setPZConvertProps ( ParserUtils . loadConvertProperties ( ) ) ; boolean processedFirst = false ; String line = null ; int estimatedColCount = FPConstants . SPLITLINE_SIZE_INIT ; while ( ( line = fetchNextRecord ( br , getQualifier ( ) , getDelimiter ( ) ) ) != null ) { if ( ! processedFirst && isIgnoreFirstRecord ( ) ) { processedFirst = true ; continue ; } else if ( ! processedFirst && createMDFromFile ) { processedFirst = true ; setPzMetaData ( ParserUtils . getPZMetaDataFromFile ( line , delimiter , qualifier , this , isAddSuffixToDuplicateColumnNames ( ) ) ) ; ds . setMetaData ( getPzMetaData ( ) ) ; continue ; } if ( oddNumberOfQualifier ( line , getQualifier ( ) ) ) { addError ( ds , "Odd number of Qualifier characters" , lineCount , 1 , isStoreRawDataToDataError ( ) ? line : null ) ; continue ; } List < String > columns = ParserUtils . splitLine ( line , getDelimiter ( ) , getQualifier ( ) , estimatedColCount , isPreserveLeadingWhitespace ( ) , isPreserveTrailingWhitespace ( ) ) ; final String mdkey = ParserUtils . getCMDKeyForDelimitedFile ( getPzMetaData ( ) , columns ) ; final List < ColumnMetaData > metaData = ParserUtils . getColumnMetaData ( mdkey , getPzMetaData ( ) ) ; final int columnCount = metaData . size ( ) ; estimatedColCount = columnCount ; if ( columns . size ( ) > columnCount ) { if ( isIgnoreExtraColumns ( ) ) { columns = columns . subList ( 0 , columnCount ) ; addError ( ds , "Flatpack truncated line to correct number of columns" , lineCount , 1 , isStoreRawDataToDataError ( ) ? line : null ) ; } else { addError ( ds , "Too many columns expected: " + columnCount + " Flatpack got: " + columns . size ( ) , lineCount , 2 , isStoreRawDataToDataError ( ) ? line : null ) ; continue ; } } else if ( columns . size ( ) < columnCount ) { if ( isHandlingShortLines ( ) ) { while ( columns . size ( ) < columnCount ) { columns . add ( "" ) ; } addError ( ds , "Flatpack padded line to correct number of columns" , lineCount , 1 , isStoreRawDataToDataError ( ) ? line : null ) ; } else { addError ( ds , "Too few columns expected: " + columnCount + " only got: " + columns . size ( ) , lineCount , 2 , isStoreRawDataToDataError ( ) ? line : null ) ; continue ; } } final Row row = new Row ( ) ; row . setMdkey ( mdkey . equals ( FPConstants . DETAIL_ID ) ? null : mdkey ) ; row . setCols ( columns ) ; row . setRowNumber ( lineCount ) ; if ( isFlagEmptyRows ( ) ) { row . setEmpty ( ParserUtils . isListElementsEmpty ( columns ) ) ; } if ( isStoreRawDataToDataSet ( ) ) { row . setRawData ( line ) ; } ds . addRow ( row ) ; } } finally { closeReaders ( ) ; } return ds ; }
/*
This is the new version of doDelimitedFile using InputStream instead of
File. This is more flexible especially it is working with WebStart.

puts together the dataset for a DELIMITED file. This is used for PZ XML
mappings, and SQL table mappings