lib/gooddata/models/model.rb
# Uploads multiple data sets using batch upload interface
# @param data [String|Array] Input data
# @param project_blueprint [ProjectBlueprint] Project blueprint
# @param options [Hash] Additional options
# @return [Hash] Batch upload result
Uploads multiple data sets using batch upload interface
Uploads multiple data sets using batch upload interface
def upload_multiple_data(data, project_blueprint, options = { :client => GoodData.connection, :project => GoodData.project })
        client, project = GoodData.get_client_and_project(options)
        project ||= GoodData.project
        manifest = {
          'dataSetSLIManifestList' => data.map do |d|
            mode = d[:options] && d[:options][:mode] ? d[:options][:mode] : options[:mode] || 'FULL'
            GoodData::Model::ToManifest.dataset_to_manifest(project_blueprint, d[:dataset], mode)
          end
        }
        csv_headers = []
        dir = Dir.mktmpdir
        begin
          Zip::File.open("#{dir}/upload.zip", Zip::File::CREATE) do |zip|
            zip.get_output_stream('upload_info.json') { |f| f.puts JSON.pretty_generate(manifest) }
            data.zip(manifest['dataSetSLIManifestList']).each do |item|
              path = item[0][:data]
              path = item[0][:data].path if item[0][:data].respond_to? :path
              inline_data = !path.is_a?(String)
              csv_header = nil
              filename = item[1]['dataSetSLIManifest']['file']
              if inline_data
                csv_header = path.first
                zip.get_output_stream(filename) do |f|
                  path.each do |row|
                    f.puts row.to_csv
                  end
                end
              else
                csv_header = File.open(path, &:gets).chomp.split(',')
                zip.add(filename, path)
              end
              csv_headers << csv_header
            end
          end
          client.upload_to_user_webdav("#{dir}/upload.zip", :directory => File.basename(dir), :client => options[:client], :project => options[:project])
        ensure
          FileUtils.rm_rf dir
        end
        csv_headers.flatten!
        pull = { 'pullIntegration' => File.basename(dir) }
        link = project.md.links('etl')['pull2']
        task = client.post(link, pull, :info_message => 'Starting the data load from user storage to dataset.')
        res = client.poll_on_response(task['pull2Task']['links']['poll'], :info_message => 'Getting status of the dataload task.') do |body|
          body['wTaskStatus']['status'] == 'RUNNING' || body['wTaskStatus']['status'] == 'PREPARED'
        end
        if res['wTaskStatus']['status'] == 'ERROR'
          s = StringIO.new
          messages = res['wTaskStatus']['messages'] || []
          messages.each do |msg|
            GoodData.logger.error(JSON.pretty_generate(msg))
          end
          begin
            client.download_from_user_webdav(File.basename(dir) + '/upload_status.json', s, :client => client, :project => project)
          rescue => e
            raise "Unable to download upload_status.json from remote server, reason: #{e.message}"
          end
          js = MultiJson.load(s.string)
          manifests = manifest['dataSetSLIManifestList'].map do |m|
            m['dataSetSLIManifest']
          end
          parts = manifests.map do |m|
            m['parts']
          end
          manifest_cols = parts.flatten.map { |c| c['columnName'] }
          manifest_extra = manifest_cols - csv_headers
          csv_extra = csv_headers - manifest_cols
          error_message = begin
            js['error']['message'] % js['error']['parameters']
          rescue NoMethodError, ArgumentError
            ''
          end
          m = "Load failed with error '#{error_message}'.\n"
          m += "Columns that should be there (manifest) but aren't in uploaded csv: #{manifest_extra}\n" unless manifest_extra.empty?
          m += "Columns that are in csv but shouldn't be there (manifest): #{csv_extra}\n" unless csv_extra.empty?
          m += "Columns in the uploaded csv: #{csv_headers}\n"
          m += "Columns in the manifest: #{manifest_cols}\n"
          m += "Original message:\n#{JSON.pretty_generate(js)}\n"
          m += "Manifest used for uploading:\n#{JSON.pretty_generate(manifest)}"
          fail m
        end
        res
      end
def upload_multiple_data ( data , project_blueprint , options = { :client => GoodData . connection , :project => GoodData . project } ) client , project = GoodData . get_client_and_project ( options ) project ||= GoodData . project manifest = { 'dataSetSLIManifestList' => data . map do | d | mode = d [ :options ] && d [ :options ] [ :mode ] ? d [ :options ] [ :mode ] : options [ :mode ] || 'FULL' GoodData :: Model :: ToManifest . dataset_to_manifest ( project_blueprint , d [ :dataset ] , mode ) end } csv_headers = [ ] dir = Dir . mktmpdir begin Zip :: File . open ( "#{dir}/upload.zip" , Zip :: File :: CREATE ) do | zip | zip . get_output_stream ( 'upload_info.json' ) { | f | f . puts JSON . pretty_generate ( manifest ) } data . zip ( manifest [ 'dataSetSLIManifestList' ] ) . each do | item | path = item [ 0 ] [ :data ] path = item [ 0 ] [ :data ] . path if item [ 0 ] [ :data ] . respond_to? :path inline_data = ! path . is_a? ( String ) csv_header = nil filename = item [ 1 ] [ 'dataSetSLIManifest' ] [ 'file' ] if inline_data csv_header = path . first zip . get_output_stream ( filename ) do | f | path . each do | row | f . puts row . to_csv end end else csv_header = File . open ( path , & :gets ) . chomp . split ( ',' ) zip . add ( filename , path ) end csv_headers << csv_header end end client . upload_to_user_webdav ( "#{dir}/upload.zip" , :directory => File . basename ( dir ) , :client => options [ :client ] , :project => options [ :project ] ) ensure FileUtils . rm_rf dir end csv_headers . flatten! pull = { 'pullIntegration' => File . basename ( dir ) } link = project . md . links ( 'etl' ) [ 'pull2' ] task = client . post ( link , pull , :info_message => 'Starting the data load from user storage to dataset.' ) res = client . poll_on_response ( task [ 'pull2Task' ] [ 'links' ] [ 'poll' ] , :info_message => 'Getting status of the dataload task.' ) do | body | body [ 'wTaskStatus' ] [ 'status' ] == 'RUNNING' || body [ 'wTaskStatus' ] [ 'status' ] == 'PREPARED' end if res [ 'wTaskStatus' ] [ 'status' ] == 'ERROR' s = StringIO . new messages = res [ 'wTaskStatus' ] [ 'messages' ] || [ ] messages . each do | msg | GoodData . logger . error ( JSON . pretty_generate ( msg ) ) end begin client . download_from_user_webdav ( File . basename ( dir ) + '/upload_status.json' , s , :client => client , :project => project ) rescue => e raise "Unable to download upload_status.json from remote server, reason: #{e.message}" end js = MultiJson . load ( s . string ) manifests = manifest [ 'dataSetSLIManifestList' ] . map do | m | m [ 'dataSetSLIManifest' ] end parts = manifests . map do | m | m [ 'parts' ] end manifest_cols = parts . flatten . map { | c | c [ 'columnName' ] } manifest_extra = manifest_cols - csv_headers csv_extra = csv_headers - manifest_cols error_message = begin js [ 'error' ] [ 'message' ] % js [ 'error' ] [ 'parameters' ] rescue NoMethodError , ArgumentError '' end m = "Load failed with error '#{error_message}'.\n" m += "Columns that should be there (manifest) but aren't in uploaded csv: #{manifest_extra}\n" unless manifest_extra . empty? m += "Columns that are in csv but shouldn't be there (manifest): #{csv_extra}\n" unless csv_extra . empty? m += "Columns in the uploaded csv: #{csv_headers}\n" m += "Columns in the manifest: #{manifest_cols}\n" m += "Original message:\n#{JSON.pretty_generate(js)}\n" m += "Manifest used for uploading:\n#{JSON.pretty_generate(manifest)}" fail m end res end
Uploads multiple data sets using batch upload interface
Uploads multiple data sets using batch upload interface
# create a temporary zip file # TODO: make sure schema columns match CSV column names # upload it # kick the load # TODO: List uploaded datasets # extract some human readable error message from the webdav file
Batch upload result
Batch upload result
Hash
data
None
project_blueprint
None
options
None
connection
project