app/importers/data_transformers/parseable_csv_string.rb
# Encode as UTF-8 and handle invalid character encodings
Encode as UTF-8 and handle invalid character encodings
Encode as UTF - 8 and handle invalid character encodings
def encode_as_utf8(raw_file_contents)
    @log.puts '#encode_as_utf8...'
    failed_encoding_count = 0
    replaced_encoding_count = 0
    failed_encodings = []
    encoding_fallback = Proc.new do |invalid|
      converted_value = map_invalid_encoding(invalid)
      if converted_value.nil?
        failed_encodings << invalid
        failed_encoding_count = failed_encoding_count + 1
        ''
      else
        replaced_encoding_count = replaced_encoding_count + 1
        converted_value
      end
    end
    encoded = raw_file_contents.encode('UTF-8', 'binary', fallback: encoding_fallback)
    if replaced_encoding_count > 0
      @log.puts "  converted #{replaced_encoding_count} known invalid character encodings"
    end
    if failed_encoding_count > 0
      @log.puts "  failed to convert #{failed_encoding_count} unexpected invalid character encodings for #{failed_encodings.uniq}"
    end
    encoded
  end
def encode_as_utf8 ( raw_file_contents ) @log . puts '#encode_as_utf8...' failed_encoding_count = 0 replaced_encoding_count = 0 failed_encodings = [ ] encoding_fallback = Proc . new do | invalid | converted_value = map_invalid_encoding ( invalid ) if converted_value . nil? failed_encodings << invalid failed_encoding_count = failed_encoding_count + 1 '' else replaced_encoding_count = replaced_encoding_count + 1 converted_value end end encoded = raw_file_contents . encode ( 'UTF-8' , 'binary' , fallback : encoding_fallback ) if replaced_encoding_count > 0 @log . puts "  converted #{replaced_encoding_count} known invalid character encodings" end if failed_encoding_count > 0 @log . puts "  failed to convert #{failed_encoding_count} unexpected invalid character encodings for #{failed_encodings.uniq}" end encoded end
Encode as UTF-8 and handle invalid character encodings
Encode as UTF - 8 and handle invalid character encodings
# track encoding failures and handle some # do the encoding
raw_file_contents
<not_specific>