/// Read a whole chunk directly from *backend*.
/// The fetched chunk could be compressed or not by different compressors.
/// It depends on `cki` how to describe the chunk data.
/// Moreover, chunk data from backend can be validated per as to nydus configuration.
/// Above is not redundant with blob cache's validation given IO path backend -> blobcache
Read a whole chunk directly from *backend*.
The fetched chunk could be compressed or not by different compressors.
It depends on `cki` how to describe the chunk data.
Moreover, chunk data from backend can be validated per as to nydus configuration.
Above is not redundant with blob cache's validation given IO path backend -> blobcache
Read a whole chunk directly from * backend * . The fetched chunk could be compressed or not by different compressors . It depends on ` cki ` how to describe the chunk data . Moreover chunk data from backend can be validated per as to nydus configuration . Above is not redundant with blob cache ' s validation given IO path backend - > blobcache
fn read_backend_chunk<F>(
        &self,
        blob_id: &str,
        cki: &dyn RafsChunkInfo,
        chunk: &mut [u8],
        cacher: F,
    ) -> Result<usize>
    where
        F: FnOnce(&[u8], &[u8]) -> Result<()>,
        Self: Sized,
    {
        let offset = cki.compress_offset();
        let mut d;
        let raw_chunk = if cki.is_compressed() {
            // Need to put compressed data into a temporary buffer so as to perform decompression.
            // TODO: Use a buffer pool for lower latency?
            //
            // gzip is special that it doesn't carry compress_size, instead, we can read as much
            // as chunk_decompress_size compressed data per chunk, decompress as much as necessary to fill in chunk
            // that has the original uncompressed data size.
            let c_size = if self.compressor() != compress::Algorithm::GZip {
                cki.compress_size() as usize
            } else {
                // Per man(1) gzip
                // The worst case expansion is a few bytes for the gzip file header,
                // plus 5 bytes every 32K block, or an expansion ratio of 0.015% for
                // large files.
                //
                // Per http://www.zlib.org/rfc-gzip.html#header-trailer
                // Each member has the following structure:
                // +---+---+---+---+---+---+---+---+---+---+
                // |ID1|ID2|CM |FLG|     MTIME     |XFL|OS | (more-->)
                // +---+---+---+---+---+---+---+---+---+---+
                // (if FLG.FEXTRA set)
                // +---+---+=================================+
                // | XLEN  |...XLEN bytes of "extra field"...| (more-->)
                // +---+---+=================================+
                // (if FLG.FNAME set)
                // +=========================================+
                // |...original file name, zero-terminated...| (more-->)
                // +=========================================+
                // (if FLG.FCOMMENT set)
                // +===================================+
                // |...file comment, zero-terminated...| (more-->)
                // +===================================+
                // (if FLG.FHCRC set)
                // +---+---+
                // | CRC16 |
                // +---+---+
                // +=======================+
                // |...compressed blocks...| (more-->)
                // +=======================+
                //   0   1   2   3   4   5   6   7
                // +---+---+---+---+---+---+---+---+
                // |     CRC32     |     ISIZE     |
                // +---+---+---+---+---+---+---+---+
                // gzip head+footer is at least 10+8 bytes, stargz header doesn't include any flags
                // so it's 18 bytes. Let's read at least 128 bytes more, to allow the decompressor to
                // find out end of the gzip stream.
                //
                // Ideally we should introduce a streaming cache for stargz that maintains internal
                // chunks and expose stream APIs.
                let size = chunk.len() + 10 + 8 + 5 + (chunk.len() / (16 << 10)) * 5 + 128;
                cmp::min(
                    size as u64,
                    self.blob_size(blob_id)? - cki.compress_offset(),
                ) as usize
            };
            d = alloc_buf(c_size);
            d.as_mut_slice()
        } else {
            // We have this unsafe assignment as it can directly store data into call's buffer.
            unsafe { slice::from_raw_parts_mut(chunk.as_mut_ptr(), chunk.len()) }
        };
        self.backend().read(blob_id, raw_chunk, offset)?;
        // Try to validate data just fetched from backend inside.
        self.process_raw_chunk(
            cki,
            raw_chunk,
            None,
            chunk,
            cki.is_compressed(),
            self.need_validate(),
        )
        .map_err(|e| eio!(format!("fail to read from backend: {}", e)))?;
        cacher(raw_chunk, chunk)?;
        Ok(chunk.len())
    }
fn read_backend_chunk < F > ( & self , blob_id : & str , cki : & dyn RafsChunkInfo , chunk : & mut [ u8 ] , cacher : F , ) -> Result < usize > where F : FnOnce ( & [ u8 ] , & [ u8 ] ) -> Result < ( ) > , Self : Sized , { let offset = cki . compress_offset ( ) ; let mut d ; let raw_chunk = if cki . is_compressed ( ) { // Need to put compressed data into a temporary buffer so as to perform decompression. // TODO: Use a buffer pool for lower latency? // // gzip is special that it doesn't carry compress_size, instead, we can read as much // as chunk_decompress_size compressed data per chunk, decompress as much as necessary to fill in chunk // that has the original uncompressed data size. let c_size = if self . compressor ( ) != compress :: Algorithm :: GZip { cki . compress_size ( ) as usize } else { // Per man(1) gzip // The worst case expansion is a few bytes for the gzip file header, // plus 5 bytes every 32K block, or an expansion ratio of 0.015% for // large files. // // Per http://www.zlib.org/rfc-gzip.html#header-trailer // Each member has the following structure: // +---+---+---+---+---+---+---+---+---+---+ // |ID1|ID2|CM |FLG|     MTIME     |XFL|OS | (more-->) // +---+---+---+---+---+---+---+---+---+---+ // (if FLG.FEXTRA set) // +---+---+=================================+ // | XLEN  |...XLEN bytes of "extra field"...| (more-->) // +---+---+=================================+ // (if FLG.FNAME set) // +=========================================+ // |...original file name, zero-terminated...| (more-->) // +=========================================+ // (if FLG.FCOMMENT set) // +===================================+ // |...file comment, zero-terminated...| (more-->) // +===================================+ // (if FLG.FHCRC set) // +---+---+ // | CRC16 | // +---+---+ // +=======================+ // |...compressed blocks...| (more-->) // +=======================+ //   0   1   2   3   4   5   6   7 // +---+---+---+---+---+---+---+---+ // |     CRC32     |     ISIZE     | // +---+---+---+---+---+---+---+---+ // gzip head+footer is at least 10+8 bytes, stargz header doesn't include any flags // so it's 18 bytes. Let's read at least 128 bytes more, to allow the decompressor to // find out end of the gzip stream. // // Ideally we should introduce a streaming cache for stargz that maintains internal // chunks and expose stream APIs. let size = chunk . len ( ) + 10 + 8 + 5 + ( chunk . len ( ) / ( 16 << 10 ) ) * 5 + 128 ; cmp :: min ( size as u64 , self . blob_size ( blob_id ) ? - cki . compress_offset ( ) , ) as usize } ; d = alloc_buf ( c_size ) ; d . as_mut_slice ( ) } else { // We have this unsafe assignment as it can directly store data into call's buffer. unsafe { slice :: from_raw_parts_mut ( chunk . as_mut_ptr ( ) , chunk . len ( ) ) } } ; self . backend ( ) . read ( blob_id , raw_chunk , offset ) ? ; // Try to validate data just fetched from backend inside. self . process_raw_chunk ( cki , raw_chunk , None , chunk , cki . is_compressed ( ) , self . need_validate ( ) , ) . map_err ( | e | eio ! ( format ( " " e ) ) ) ? ; cacher ( raw_chunk , chunk ) ? ; Ok ( chunk . len ( ) ) }
Read a whole chunk directly from *backend*.
Read a whole chunk directly from * backend * .
&self
blob_id
&str
cki
&dyn RafsChunkInfo
chunk
&mut [u8]
cacher
F