lib/airparselib.rb
# Processes csv file passed with optional rules class to produce result and error output
#
# @opts format [Hash] the format type, `:text` or `:html`
# @param [Hash] opts the options to process a csv file
# @option opts [String] :file CSV filename to parse
# @option opts [String] :output output filename
# @option opts [String] :errors errors filename
# @option opts [String] :rules rules Class name (use camelcase, filename containg the specified class
#                              should be named after the Class, but lowercase; all rules files must be
#                              located within lib/ folder), default: 'AirParseDefaultRules'
# @return [Boolean] true if processed with success, false otherwise
Processes csv file passed with optional rules class to produce result and error output
Processes csv file passed with optional rules class to produce result and error output
def process(opts)
    mtk_rules_file = opts[:rules]
    mtk_rules_file_sym = mtk_rules_file.to_sym
    mtk_rules_file_sym_klass = Kernel.const_get(mtk_rules_file_sym) 
    parser = mtk_rules_file_sym_klass.new 
    [opts[:errors], opts[:output]].each do |e|
      if File.exist?(e)
        puts 'removing file: ' + e
        FileUtils.rm(e)
      end
    end
    File.open(opts[:output], 'w') { |f| f.write(parser.output_data_header) }
    File.open(opts[:errors], 'w') { |f| f.write(parser.errors_data_header) }
    ct = 0 
    @mtkfailreasons = [] 
    @air_model = [] 
    CSV.foreach(opts[:file]) do |values|
      ct += 1
      if ct == 1
        @air_model = values 
      else 
        if opts[:verbose]
          puts "\nprocessing:"
          pp values
        end
        validation = { correct: true, reason: [] } 
        @mtkfailreasons = [] 
        if values.length != @air_model.length
          validation = { correct: false, reason: [] }
          @mtkfailreasons << ('incorrect number of columns: ' << values.length.to_s << ' instead of ' << @air_model.length.to_s)
        end
        data_output = []
        error_output = []
        if validation[:correct]
          data_output = []
          error_output = []
          cti = 0 
          values.each do |value|
            value = '' if value.nil? 
            dynamic_validator_mehod_name = 'is_valid_' + @air_model[cti]
            puts 'calling validator: ' + dynamic_validator_mehod_name if opts[:verbose]
            if parser.respond_to?(dynamic_validator_mehod_name.to_sym) 
              validation = parser.send(dynamic_validator_mehod_name, value)
            else 
              validation = { correct: true, reason: [] }
            end
            puts 'row validation results: ' + validation.to_json if opts[:verbose]
            unless validation[:correct] 
              if validation[:reason] 
                @mtkfailreasons << ('field: ' << @air_model[cti] << '  value: [' + value + ']' << '  errors: ' << validation[:reason].to_s)
              end
            end
            ==================
            if validation[:correct] 
              dynamic_processor_mehod_name = 'process_' + @air_model[cti]
              if parser.respond_to?(dynamic_processor_mehod_name.to_sym) 
                puts 'calling processor: ' + dynamic_processor_mehod_name if opts[:verbose]
                data_output_result = parser.send(dynamic_processor_mehod_name, value)
                puts 'processed result: ' + data_output_result.to_s if opts[:verbose]
                data_output_result.each { |e| data_output << e }
              else 
                data_output << value
              end
            end 
            error_output << value
            cti += 1 
          end 
        end 
        if @mtkfailreasons == [] 
          File.open(opts[:output], 'a') { |f| f.write(data_output.to_csv) }
        else 
          puts
          puts 'row ' + ct.to_s + ' VALIDATION FAILED'
          puts @mtkfailreasons
          puts
          File.open(opts[:errors], 'a') { |f| f.write(error_output.to_csv) }
        end
      end
      # end 
    end 
    @mtkfailreasons == []
  end
def process ( opts ) mtk_rules_file = opts [ :rules ] mtk_rules_file_sym = mtk_rules_file . to_sym mtk_rules_file_sym_klass = Kernel . const_get ( mtk_rules_file_sym ) parser = mtk_rules_file_sym_klass . new [ opts [ :errors ] , opts [ :output ] ] . each do | e | if File . exist? ( e ) puts 'removing file: ' + e FileUtils . rm ( e ) end end File . open ( opts [ :output ] , 'w' ) { | f | f . write ( parser . output_data_header ) } File . open ( opts [ :errors ] , 'w' ) { | f | f . write ( parser . errors_data_header ) } ct = 0 @mtkfailreasons = [ ] @air_model = [ ] CSV . foreach ( opts [ :file ] ) do | values | ct += 1 if ct == 1 @air_model = values else if opts [ :verbose ] puts "\nprocessing:" pp values end validation = { correct : true , reason : [ ] } @mtkfailreasons = [ ] if values . length != @air_model . length validation = { correct : false , reason : [ ] } @mtkfailreasons << ( 'incorrect number of columns: ' << values . length . to_s << ' instead of ' << @air_model . length . to_s ) end data_output = [ ] error_output = [ ] if validation [ :correct ] data_output = [ ] error_output = [ ] cti = 0 values . each do | value | value = '' if value . nil? dynamic_validator_mehod_name = 'is_valid_' + @air_model [ cti ] puts 'calling validator: ' + dynamic_validator_mehod_name if opts [ :verbose ] if parser . respond_to? ( dynamic_validator_mehod_name . to_sym ) validation = parser . send ( dynamic_validator_mehod_name , value ) else validation = { correct : true , reason : [ ] } end puts 'row validation results: ' + validation . to_json if opts [ :verbose ] unless validation [ :correct ] if validation [ :reason ] @mtkfailreasons << ( 'field: ' << @air_model [ cti ] << '  value: [' + value + ']' << '  errors: ' << validation [ :reason ] . to_s ) end end if validation [ :correct ] dynamic_processor_mehod_name = 'process_' + @air_model [ cti ] if parser . respond_to? ( dynamic_processor_mehod_name . to_sym ) puts 'calling processor: ' + dynamic_processor_mehod_name if opts [ :verbose ] data_output_result = parser . send ( dynamic_processor_mehod_name , value ) puts 'processed result: ' + data_output_result . to_s if opts [ :verbose ] data_output_result . each { | e | data_output << e } else data_output << value end end error_output << value cti += 1 end end if @mtkfailreasons == [ ] File . open ( opts [ :output ] , 'a' ) { | f | f . write ( data_output . to_csv ) } else puts puts 'row ' + ct . to_s + ' VALIDATION FAILED' puts @mtkfailreasons puts File . open ( opts [ :errors ] , 'a' ) { | f | f . write ( error_output . to_csv ) } end end end @mtkfailreasons == [ ] end
Processes csv file passed with optional rules class to produce result and error output
Processes csv file passed with optional rules class to produce result and error output
# create specific AirParse instance from rules file specified on commandline # some meta hacks # to use Class with a name passed as string # from this moment, parser is a normal class instance with our custom rules loaded # prepare output files ## if they exist, remove output files before processing ## write headers to output files, an output/error csv headers are specified in custom rules file/class # processed line counter # an array with all cumulated errors found in a processed line # an array with input file column names # open(opts[:file]) do |csv| # csv.each_line do |line| # old 3 lines, before CSV.foreach, so parse method is obsolete now # parse current line # values = parser.parse(line) # read line 1 with csv header to model array; # we assume, that the first line of the csv file contains header with proper field names # the names must be matched by validator and processors defined in AirParseClasifyAirlineCodes # read parsed column names from header into model # for all remaining lines # business rules validation # we assume correct validation results # an array with all cumulated errors found in a processed line # rule: numbers of data fields should eq to number of header/model variables # current field counter # process each value in a line # change nil to empty string, we don't like nils in output csv # step 1. input data validation # ============================= # a validator method name is constructed from processed values' field name # call a proper validator method for each value # check if custom validator is defined in AirParseClassifyAirlineCodes class # if no custom validator defined, we must assume correct result, so that the field/line will be output unchanged # process field validation results # if validation failed # if exists, add detailed validation result description to row validation results array # step 2. if validation correct, process the data # =============================================== # if validation succeeded # a processor method name is constructed from processed values' field name # call a proper processor method for each value # check if custom processor is defined in AirParseClassifyAirlineCodes class # call custom processor # if no custom processor defined, we return field value unchanged # data processing # let's move to the next field to be processed # values.each # if validation correct # no errors during validation/processing # all fields in a line have been processed, so append the result line to result file # error(s) found # append bad csv line to the error file # TODO: if output errors desc to result file # File.open('errors.csv', 'a') {|f| f.write("row "+ct.to_s+' VALIDATION FAILED'+"\n"+@mtkfailreasons.to_s+"\n\n") } # end # csv.each_line do |line| # do csv # return true if no fail reasons
true if processed with success, false otherwise
true if processed with success false otherwise
Boolean
opts
:rules rules Class name (use camelcase, filename containg the specified class should be named after the Class, but lowercase; all rules files must be located within lib/ folder), default: 'AirParseDefaultRules'
: rules rules Class name ( use camelcase filename containg the specified class should be named after the Class but lowercase ; all rules files must be located within lib / folder ) default : ' AirParseDefaultRules '
<not_specific>
opts the options to process a csv file
opts the options to process a csv file
opts
format [Hash] the format type, `:text` or `:html`
format [ Hash ] the format type ` : text ` or ` : html `