streamcorpus_pipeline/_s3_storage.py
def get_chunk(self, bucket_name, key_path):
        '''return Chunk object full of records
        bucket_name may be None'''
        bucket = get_bucket(self.config, bucket_name=bucket_name)
        key = bucket.get_key(key_path)
        if key is None:
            raise FailedExtraction('Key "%s" does not exist.' % key_path)

        fh = StringIO()
        key.get_contents_to_file(fh)
        data = fh.getvalue()
        if not data:
            raise FailedExtraction('%s: no data (does the key exist?)'
                                   % key.key)

        chunk_type, compression, encryption = parse_file_extensions(key_path)
        if encryption == 'gpg':
            if not self.gpg_decryption_key_path:
                raise FailedExtraction('%s ends with ".gpg" but gpg_decryption_key_path=%s'
                                       % (key.key, self.gpg_decryption_key_path))

        _errors = []
        if compression or encryption:
            _errors, data = decrypt_and_uncompress(
                data, 
                self.gpg_decryption_key_path,
                tmp_dir=self.config.get('tmp_dir_path'),
                compression=compression,
                )
            if not data:
                msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' \
                    + ' downloaded, errors: {2}' \
                        .format(key_path, len(data), '\n'.join(_errors))
                logger.error(msg)
                raise FailedExtraction(msg)
            logger.info( '\n'.join(_errors) )

        if not self.config['compare_md5_in_file_name']:
            logger.warn('not checking md5 in file name, consider setting '
                        'from_s3_chunks:compare_md5_in_file_name')
        else:
            logger.info('Verifying md5 for "%s"...' % key.key)

            # The regex hammer.
            m = re.search('([a-z0-9]{32})(?:\.|$)', key.key)
            if m is None:
                raise FailedExtraction(
                    'Could not extract md5 from key "%s". '
                    'Perhaps you should disable compare_md5_in_file_name?'
                    % key.key)

            i_content_md5 = m.group(1)
            #import pdb; pdb.set_trace()
            verify_md5(i_content_md5, data, other_errors=_errors)
        return self._decode(data)
def get_chunk(self, bucket_name, key_path):
        '''return Chunk object full of records
        bucket_name may be None'''
        bucket = get_bucket(self.config, bucket_name=bucket_name)
        key = bucket.get_key(key_path)
        if key is None:
            raise FailedExtraction('Key "%s" does not exist.' % key_path)

        fh = StringIO()
        key.get_contents_to_file(fh)
        data = fh.getvalue()
        if not data:
            raise FailedExtraction('%s: no data (does the key exist?)'
                                   % key.key)

        chunk_type, compression, encryption = parse_file_extensions(key_path)
        if encryption == 'gpg':
            if not self.gpg_decryption_key_path:
                raise FailedExtraction('%s ends with ".gpg" but gpg_decryption_key_path=%s'
                                       % (key.key, self.gpg_decryption_key_path))

        _errors = []
        if compression or encryption:
            _errors, data = decrypt_and_uncompress(
                data, 
                self.gpg_decryption_key_path,
                tmp_dir=self.config.get('tmp_dir_path'),
                compression=compression,
                )
            if not data:
                msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' \
                    + ' downloaded, errors: {2}' \
                        .format(key_path, len(data), '\n'.join(_errors))
                logger.error(msg)
                raise FailedExtraction(msg)
            logger.info( '\n'.join(_errors) )

        if not self.config['compare_md5_in_file_name']:
            logger.warn('not checking md5 in file name, consider setting '
                        'from_s3_chunks:compare_md5_in_file_name')
        else:
            logger.info('Verifying md5 for "%s"...' % key.key)

            # The regex hammer.
            m = re.search('([a-z0-9]{32})(?:\.|$)', key.key)
            if m is None:
                raise FailedExtraction(
                    'Could not extract md5 from key "%s". '
                    'Perhaps you should disable compare_md5_in_file_name?'
                    % key.key)

            i_content_md5 = m.group(1)
            #import pdb; pdb.set_trace()
            verify_md5(i_content_md5, data, other_errors=_errors)
        return self._decode(data)
def get_chunk ( self , bucket_name , key_path ) : bucket = get_bucket ( self . config , bucket_name = bucket_name ) key = bucket . get_key ( key_path ) if key is None : raise FailedExtraction ( 'Key "%s" does not exist.' % key_path ) fh = StringIO ( ) key . get_contents_to_file ( fh ) data = fh . getvalue ( ) if not data : raise FailedExtraction ( '%s: no data (does the key exist?)' % key . key ) chunk_type , compression , encryption = parse_file_extensions ( key_path ) if encryption == 'gpg' : if not self . gpg_decryption_key_path : raise FailedExtraction ( '%s ends with ".gpg" but gpg_decryption_key_path=%s' % ( key . key , self . gpg_decryption_key_path ) ) _errors = [ ] if compression or encryption : _errors , data = decrypt_and_uncompress ( data , self . gpg_decryption_key_path , tmp_dir = self . config . get ( 'tmp_dir_path' ) , compression = compression , ) if not data : msg = 'decrypt_and_uncompress got no data for {0!r}, from {1} bytes' + ' downloaded, errors: {2}' . format ( key_path , len ( data ) , '\n' . join ( _errors ) ) logger . error ( msg ) raise FailedExtraction ( msg ) logger . info ( '\n' . join ( _errors ) ) if not self . config [ 'compare_md5_in_file_name' ] : logger . warn ( 'not checking md5 in file name, consider setting ' 'from_s3_chunks:compare_md5_in_file_name' ) else : logger . info ( 'Verifying md5 for "%s"...' % key . key ) # The regex hammer. m = re . search ( '([a-z0-9]{32})(?:\.|$)' , key . key ) if m is None : raise FailedExtraction ( 'Could not extract md5 from key "%s". ' 'Perhaps you should disable compare_md5_in_file_name?' % key . key ) i_content_md5 = m . group ( 1 ) #import pdb; pdb.set_trace() verify_md5 ( i_content_md5 , data , other_errors = _errors ) return self . _decode ( data )
return Chunk object full of records
        bucket_name may be None
return Chunk object full of records bucket_name may be None