/// Downloads a source hosted on an S3 bucket.
///
/// # Directly thrown errors
/// - [`DownloadError::Io`]
/// - [`DownloadError::Canceled`]
Downloads a source hosted on an S3 bucket.
Downloads a source hosted on an S3 bucket .
pub async fn download_source(
        &self,
        file_source: S3RemoteDif,
        destination: PathBuf,
    ) -> Result<DownloadStatus, DownloadError> {
        let key = file_source.key();
        let bucket = file_source.bucket();
        log::debug!("Fetching from s3: {} (from {})", &key, &bucket);
        let source_key = &file_source.source.source_key;
        let client = self.get_s3_client(source_key);
        let request = client.get_object(rusoto_s3::GetObjectRequest {
            key: key.clone(),
            bucket: bucket.clone(),
            ..Default::default()
        });
        let source = RemoteDif::from(file_source);
        let request = tokio::time::timeout(self.connect_timeout, request);
        let request = super::measure_download_time(source.source_metric_key(), request);
        let response = match request.await {
            Ok(Ok(response)) => response,
            Ok(Err(err)) => {
                log::debug!("Skipping response from s3://{}/{}: {}", bucket, &key, err);
                // Do note that it's possible for Amazon to return different status codes when a
                // file is missing. 403 is returned if the `ListBucket` permission isn't available,
                // which means that a 403 returned below may actually be for a missing file.
                // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html
                match &err {
                    RusotoError::Service(_) => return Ok(DownloadStatus::NotFound),
                    RusotoError::Unknown(response) if response.status == StatusCode::FORBIDDEN => {
                        return Err(DownloadError::Permissions)
                    }
                    RusotoError::Unknown(response) if response.status.is_client_error() => {
                        return Ok(DownloadStatus::NotFound)
                    }
                    _ => return Err(err.into()),
                }
            }
            Err(_) => {
                // Timed out
                return Err(DownloadError::Canceled);
            }
        };
        let stream = match response.body {
            Some(body) => body.map_err(DownloadError::Io),
            None => {
                log::debug!("Empty response from s3:{}{}", bucket, &key);
                return Ok(DownloadStatus::NotFound);
            }
        };
        let content_length = response
            .content_length
            .and_then(|cl| u32::try_from(cl).ok());
        let timeout = content_length.map(|cl| content_length_timeout(cl, self.streaming_timeout));
        super::download_stream(source, stream, destination, timeout).await
    }
pub async fn download_source ( & self , file_source : S3RemoteDif , destination : PathBuf , ) -> Result < DownloadStatus , DownloadError > { let key = file_source . key ( ) ; let bucket = file_source . bucket ( ) ; log :: debug ! ( " " key bucket ) ; let source_key = & file_source . source . source_key ; let client = self . get_s3_client ( source_key ) ; let request = client . get_object ( rusoto_s3 :: GetObjectRequest { key : key . clone ( ) , bucket : bucket . clone ( ) , .. Default :: default ( ) } ) ; let source = RemoteDif :: from ( file_source ) ; let request = tokio :: time :: timeout ( self . connect_timeout , request ) ; let request = super :: measure_download_time ( source . source_metric_key ( ) , request ) ; let response = match request . await { Ok ( Ok ( response ) ) => response , Ok ( Err ( err ) ) => { log :: debug ! ( " " bucket key err ) ; // Do note that it's possible for Amazon to return different status codes when a // file is missing. 403 is returned if the `ListBucket` permission isn't available, // which means that a 403 returned below may actually be for a missing file. // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html match & err { RusotoError :: Service ( _ ) => return Ok ( DownloadStatus :: NotFound ) , RusotoError :: Unknown ( response ) if response . status == StatusCode :: FORBIDDEN => { return Err ( DownloadError :: Permissions ) } RusotoError :: Unknown ( response ) if response . status . is_client_error ( ) => { return Ok ( DownloadStatus :: NotFound ) } _ => return Err ( err . into ( ) ) , } } Err ( _ ) => { // Timed out return Err ( DownloadError :: Canceled ) ; } } ; let stream = match response . body { Some ( body ) => body . map_err ( DownloadError :: Io ) , None => { log :: debug ! ( " " bucket key ) ; return Ok ( DownloadStatus :: NotFound ) ; } } ; let content_length = response . content_length . and_then ( | cl | u32 :: try_from ( cl ) . ok ( ) ) ; let timeout = content_length . map ( | cl | content_length_timeout ( cl , self . streaming_timeout ) ) ; super :: download_stream ( source , stream , destination , timeout ) . await }
Downloads a source hosted on an S3 bucket.
Downloads a source hosted on an S3 bucket .
&self
file_source
S3RemoteDif
destination
PathBuf
directly thrown errors
None