def download(self):
    """Method which downloads submission to local directory."""
    # Structure of the download directory:
    # submission_dir=LOCAL_SUBMISSIONS_DIR/submission_id
    # submission_dir/s.ext   <-- archived submission
    # submission_dir/extracted      <-- extracted submission

    # Check whether submission is already there
    if self.extracted_submission_dir:
      return
    self.submission_dir = os.path.join(LOCAL_SUBMISSIONS_DIR,
                                       self.submission_id)
    if (os.path.isdir(self.submission_dir)
        and os.path.isdir(os.path.join(self.submission_dir, 'extracted'))):
      # submission already there, just re-read metadata
      self.extracted_submission_dir = os.path.join(self.submission_dir,
                                                   'extracted')
      with open(os.path.join(self.extracted_submission_dir, 'metadata.json'),
                'r') as f:
        meta_json = json.load(f)
      self.container_name = str(meta_json[METADATA_CONTAINER])
      self.entry_point = str(meta_json[METADATA_ENTRY_POINT])
      return
    # figure out submission location in the Cloud and determine extractor
    submission_cloud_path = os.path.join('gs://', self.storage_bucket,
                                         self.submission.path)
    extract_command_tmpl = None
    extension = None
    for k, v in iteritems(EXTRACT_COMMAND):
      if submission_cloud_path.endswith(k):
        extension = k
        extract_command_tmpl = v
        break
    if not extract_command_tmpl:
      raise WorkerError('Unsupported submission extension')
    # download archive
    try:
      os.makedirs(self.submission_dir)
      tmp_extract_dir = os.path.join(self.submission_dir, 'tmp')
      os.makedirs(tmp_extract_dir)
      download_path = os.path.join(self.submission_dir, 's' + extension)
      try:
        logging.info('Downloading submission from %s to %s',
                     submission_cloud_path, download_path)
        shell_call(['gsutil', 'cp', submission_cloud_path, download_path])
      except subprocess.CalledProcessError as e:
        raise WorkerError('Can''t copy submission locally', e)
      # extract archive
      try:
        shell_call(extract_command_tmpl,
                   src=download_path, dst=tmp_extract_dir)
      except subprocess.CalledProcessError as e:
        # proceed even if extraction returned non zero error code,
        # sometimes it's just warning
        logging.warning('Submission extraction returned non-zero error code. '
                        'It may be just a warning, continuing execution. '
                        'Error: %s', e)
      try:
        make_directory_writable(tmp_extract_dir)
      except subprocess.CalledProcessError as e:
        raise WorkerError('Can''t make submission directory writable', e)
      # determine root of the submission
      tmp_root_dir = tmp_extract_dir
      root_dir_content = [d for d in os.listdir(tmp_root_dir)
                          if d != '__MACOSX']
      if (len(root_dir_content) == 1
          and os.path.isdir(os.path.join(tmp_root_dir, root_dir_content[0]))):
        tmp_root_dir = os.path.join(tmp_root_dir, root_dir_content[0])
      # move files to extract subdirectory
      self.extracted_submission_dir = os.path.join(self.submission_dir,
                                                   'extracted')
      try:
        shell_call(['mv', os.path.join(tmp_root_dir),
                    self.extracted_submission_dir])
      except subprocess.CalledProcessError as e:
        raise WorkerError('Can''t move submission files', e)
      # read metadata file
      try:
        with open(os.path.join(self.extracted_submission_dir, 'metadata.json'),
                  'r') as f:
          meta_json = json.load(f)
      except IOError as e:
        raise WorkerError(
            'Can''t read metadata.json for submission "{0}"'.format(
                self.submission_id),
            e)
      try:
        self.container_name = str(meta_json[METADATA_CONTAINER])
        self.entry_point = str(meta_json[METADATA_ENTRY_POINT])
        type_from_meta = METADATA_JSON_TYPE_TO_TYPE[meta_json[METADATA_TYPE]]
      except KeyError as e:
        raise WorkerError('Invalid metadata.json file', e)
      if type_from_meta != self.type:
        raise WorkerError('Inconsistent submission type in metadata: '
                          + type_from_meta + ' vs ' + self.type)
    except WorkerError as e:
      self.extracted_submission_dir = None
      sudo_remove_dirtree(self.submission_dir)
      raise
def download ( self ) : if self . extracted_submission_dir : return self . submission_dir = os . path . join ( LOCAL_SUBMISSIONS_DIR , self . submission_id ) if ( os . path . isdir ( self . submission_dir ) and os . path . isdir ( os . path . join ( self . submission_dir , 'extracted' ) ) ) : self . extracted_submission_dir = os . path . join ( self . submission_dir , 'extracted' ) with open ( os . path . join ( self . extracted_submission_dir , 'metadata.json' ) , 'r' ) as f : meta_json = json . load ( f ) self . container_name = str ( meta_json [ METADATA_CONTAINER ] ) self . entry_point = str ( meta_json [ METADATA_ENTRY_POINT ] ) return submission_cloud_path = os . path . join ( 'gs://' , self . storage_bucket , self . submission . path ) extract_command_tmpl = None extension = None for k , v in iteritems ( EXTRACT_COMMAND ) : if submission_cloud_path . endswith ( k ) : extension = k extract_command_tmpl = v break if not extract_command_tmpl : raise WorkerError ( 'Unsupported submission extension' ) try : os . makedirs ( self . submission_dir ) tmp_extract_dir = os . path . join ( self . submission_dir , 'tmp' ) os . makedirs ( tmp_extract_dir ) download_path = os . path . join ( self . submission_dir , 's' + extension ) try : logging . info ( 'Downloading submission from %s to %s' , submission_cloud_path , download_path ) shell_call ( [ 'gsutil' , 'cp' , submission_cloud_path , download_path ] ) except subprocess . CalledProcessError as e : raise WorkerError ( 'Can' 't copy submission locally' , e ) try : shell_call ( extract_command_tmpl , src = download_path , dst = tmp_extract_dir ) except subprocess . CalledProcessError as e : logging . warning ( 'Submission extraction returned non-zero error code. ' 'It may be just a warning, continuing execution. ' 'Error: %s' , e ) try : make_directory_writable ( tmp_extract_dir ) except subprocess . CalledProcessError as e : raise WorkerError ( 'Can' 't make submission directory writable' , e ) tmp_root_dir = tmp_extract_dir root_dir_content = [ d for d in os . listdir ( tmp_root_dir ) if d != '__MACOSX' ] if ( len ( root_dir_content ) == 1 and os . path . isdir ( os . path . join ( tmp_root_dir , root_dir_content [ 0 ] ) ) ) : tmp_root_dir = os . path . join ( tmp_root_dir , root_dir_content [ 0 ] ) self . extracted_submission_dir = os . path . join ( self . submission_dir , 'extracted' ) try : shell_call ( [ 'mv' , os . path . join ( tmp_root_dir ) , self . extracted_submission_dir ] ) except subprocess . CalledProcessError as e : raise WorkerError ( 'Can' 't move submission files' , e ) try : with open ( os . path . join ( self . extracted_submission_dir , 'metadata.json' ) , 'r' ) as f : meta_json = json . load ( f ) except IOError as e : raise WorkerError ( 'Can' 't read metadata.json for submission "{0}"' . format ( self . submission_id ) , e ) try : self . container_name = str ( meta_json [ METADATA_CONTAINER ] ) self . entry_point = str ( meta_json [ METADATA_ENTRY_POINT ] ) type_from_meta = METADATA_JSON_TYPE_TO_TYPE [ meta_json [ METADATA_TYPE ] ] except KeyError as e : raise WorkerError ( 'Invalid metadata.json file' , e ) if type_from_meta != self . type : raise WorkerError ( 'Inconsistent submission type in metadata: ' + type_from_meta + ' vs ' + self . type ) except WorkerError as e : self . extracted_submission_dir = None sudo_remove_dirtree ( self . submission_dir ) raise
Method which downloads submission to local directory.