lib/importer/lib/cartodb-importer/importer.rb
# core import method
#
# Has 3 expansion points
#
# * decompression (eg zip, gzip, bz)
# * preprocessing (transpose data into a format recognised by loader, eg. gpx)
# * loader (loads the data into postgres, eg csv, shp)
#
core import method
Has 3 expansion points

decompression
preprocessing (transpose data into a format recognised by loader, eg. gpx)
loader (loads the data into postgres, eg csv, shp)
core import method Has 3 expansion points decompression preprocessing ( transpose data into a format recognised by loader eg . gpx ) loader ( loads the data into postgres eg csv shp )
def import!
      begin
        fs = File.size(@path)
        if fs.to_i == 0
          @data_import.set_error_code(1005)
          @data_import.log_error("File contains no information, check it locally" )
          raise "File contains no information, check it locally"
        elsif @remaining_quota < (0.3*fs)
          disk_quota_overspend = (File.size(@path) - @remaining_quota).to_int
          @data_import.set_error_code(8001)
          @data_import.log_error("#{disk_quota_overspend / 1024}KB more space is required" )
          raise CartoDB::QuotaExceeded, "#{disk_quota_overspend / 1024}KB more space is required"
        end
        errors = Array.new
        suggested = @suggested_name.nil? ? get_valid_name(File.basename(@original_name,@ext).tr('.','_').downcase.sanitize) : @suggested_name
        import_data = [{
          :ext            => @ext,
          :path           => @path,
          :suggested_name => suggested
        }]
        @entries << @path
        decompressor  = decompressor_for(@ext)
        import_data   = decompressor.process! if decompressor
        @data_import.log_update('file unzipped') if import_data && decompressor
        @data_import.reload
        processed_imports = Array.new
        import_data.each do |data|
          @entries << data[:path]
          @working_data = data
          @working_data[:suggested_name] = get_valid_name(@working_data[:suggested_name])
          preprocessor = preprocessor_for(data.fetch(:ext))
          @data_import.refresh
          if preprocessor
            begin
              out = preprocessor.process!
              if out == false
                processed_imports << data
              else
                out.each { |d| processed_imports << d }
                @data_import.log_update('file preprocessed')
              end
            rescue
              @data_import.reload
              errors << OpenStruct.new(
                description:  @data_import.get_error_text[:title],
                stack:        @data_import.get_error_text[:what_about],
                code:         @data_import.error_code
              )
            end
          else
            processed_imports << data
          end
        end
        payloads = Array.new
        processed_imports.each do |data|
          @entries << data[:path]
          @working_data = data
          @working_data[:suggested_name] = @suggested_name.nil? ? get_valid_name(@working_data[:suggested_name]) : get_valid_name(@suggested_name)
          @data_path = data[:path]
          loader = loader_for(data.fetch(:ext))
          if !loader
            @data_import.log_update("no importer for this type of data, #{@ext}")
            @data_import.set_error_code(1002)
          else
            begin
              out = loader.process!
              out.each { |d| payloads << d }
              @data_import.log_update("#{data[:ext]} successfully loaded")
            rescue => e
              @data_import.reload
              errors << OpenStruct.new(
                description:  @data_import.get_error_text[:title],
                stack:        @data_import.log_json,
                code:         @data_import.error_code
              )
            end
          end
        end
        @data_import.raise_error_if_over_quota payloads.length
        @data_import.refresh
        if payloads.length > 0
          @data_import.tables_created_count = payloads.size
          @data_import.table_names = payloads.map(&:name).join(',')
          @data_import.log_update("#{payloads.size} tables imported")
        else
          @data_import.failed
        end
        @data_import.save
        return [payloads, errors]
      rescue => e
        puts e
        @data_import.refresh
        drop_created_tables(payloads.map(&:name)) unless payloads.blank?
        raise e
      ensure
        @db_connection.disconnect
        cleanup_disk
        if @import_from_file.is_a?(File) && File.file?(@import_from_file.path)
          File.unlink(@import_from_file)
        elsif @import_from_file.is_a? Tempfile
          @import_from_file.unlink
        end
        @data_import.save if @data_import
      end
    end
def import! begin fs = File . size ( @path ) if fs . to_i == 0 @data_import . set_error_code ( 1005 ) @data_import . log_error ( "File contains no information, check it locally" ) raise "File contains no information, check it locally" elsif @remaining_quota < ( 0.3 * fs ) disk_quota_overspend = ( File . size ( @path ) - @remaining_quota ) . to_int @data_import . set_error_code ( 8001 ) @data_import . log_error ( "#{disk_quota_overspend / 1024}KB more space is required" ) raise CartoDB :: QuotaExceeded , "#{disk_quota_overspend / 1024}KB more space is required" end errors = Array . new suggested = @suggested_name . nil? ? get_valid_name ( File . basename ( @original_name , @ext ) . tr ( '.' , '_' ) . downcase . sanitize ) : @suggested_name import_data = [ { :ext => @ext , :path => @path , :suggested_name => suggested } ] @entries << @path decompressor = decompressor_for ( @ext ) import_data = decompressor . process! if decompressor @data_import . log_update ( 'file unzipped' ) if import_data && decompressor @data_import . reload processed_imports = Array . new import_data . each do | data | @entries << data [ :path ] @working_data = data @working_data [ :suggested_name ] = get_valid_name ( @working_data [ :suggested_name ] ) preprocessor = preprocessor_for ( data . fetch ( :ext ) ) @data_import . refresh if preprocessor begin out = preprocessor . process! if out == false processed_imports << data else out . each { | d | processed_imports << d } @data_import . log_update ( 'file preprocessed' ) end rescue @data_import . reload errors << OpenStruct . new ( description : @data_import . get_error_text [ :title ] , stack : @data_import . get_error_text [ :what_about ] , code : @data_import . error_code ) end else processed_imports << data end end payloads = Array . new processed_imports . each do | data | @entries << data [ :path ] @working_data = data @working_data [ :suggested_name ] = @suggested_name . nil? ? get_valid_name ( @working_data [ :suggested_name ] ) : get_valid_name ( @suggested_name ) @data_path = data [ :path ] loader = loader_for ( data . fetch ( :ext ) ) if ! loader @data_import . log_update ( "no importer for this type of data, #{@ext}" ) @data_import . set_error_code ( 1002 ) else begin out = loader . process! out . each { | d | payloads << d } @data_import . log_update ( "#{data[:ext]} successfully loaded" ) rescue => e @data_import . reload errors << OpenStruct . new ( description : @data_import . get_error_text [ :title ] , stack : @data_import . log_json , code : @data_import . error_code ) end end end @data_import . raise_error_if_over_quota payloads . length @data_import . refresh if payloads . length > 0 @data_import . tables_created_count = payloads . size @data_import . table_names = payloads . map ( & :name ) . join ( ',' ) @data_import . log_update ( "#{payloads.size} tables imported" ) else @data_import . failed end @data_import . save return [ payloads , errors ] rescue => e puts e @data_import . refresh drop_created_tables ( payloads . map ( & :name ) ) unless payloads . blank? raise e ensure @db_connection . disconnect cleanup_disk if @import_from_file . is_a? ( File ) && File . file? ( @import_from_file . path ) File . unlink ( @import_from_file ) elsif @import_from_file . is_a? Tempfile @import_from_file . unlink end @data_import . save if @data_import end end
core import method
Has 3 expansion points
core import method Has 3 expansion points
# All imports start with only one file, so 'import_data' is an array of # on length = 1 # A record of all file paths for cleanup # TODO: A Zip -> KMZ/Zip will fail because it wont know  # how to go back and do the Decompressor stage again # Preprocess data and update self with results # preprocessors are expected to return a hash datastructure # Return raw data if preprocessor returns false # For example: we don't want to run JSON preprocessor # on GEOJSON files # Load data in # re-check suggested_name in the case that it has been taken by another in this import # Check if user is over table quota, raise the appropiate error # Flag the data import as failed when no files were imported, # save imported table names otherwise