tensorflow_io_gcs_filesystem/core/gcs_filesystem.cc
// A helper function to actually read the data from GCS.
A helper function to actually read the data from GCS.
A helper function to actually read the data from GCS .
static int64_t LoadBufferFromGCS(
    const std::string& path, size_t offset, size_t buffer_size, char* buffer,
    tf_gcs_filesystem::GCSFileSystemImplementation* gcs_file,
    TF_Status* status) {
  std::string bucket, object;
  ParseGCSPath(path, false, &bucket, &object, status);
  if (TF_GetCode(status) != TF_OK) return -1;
  auto stream = gcs_file->gcs_client.ReadObject(
      bucket, object, gcs::ReadRange(offset, offset + buffer_size));
  TF_SetStatusFromGCSStatus(stream.status(), status);
  if ((TF_GetCode(status) != TF_OK) &&
      (TF_GetCode(status) != TF_OUT_OF_RANGE)) {
    return -1;
  }
  int64_t read;
  auto content_length = stream.headers().find("content-length");
  if (content_length == stream.headers().end()) {
    read = 0;
  } else if (!absl::SimpleAtoi(content_length->second, &read)) {
    TF_SetStatus(status, TF_UNKNOWN, "Could not get content-length header");
    return -1;
  }
  TF_SetStatus(status, TF_OK, "");
  TF_VLog(1, "Successful read of %s @ %u of size: %u", path.c_str(), offset,
          read);
  stream.read(buffer, read);
  read = stream.gcount();
  if (read < buffer_size) {
    tf_gcs_filesystem::GcsFileSystemStat stat;
    if (gcs_file->stat_cache->Lookup(path, &stat)) {
      if (offset + read < stat.base.length) {
        TF_SetStatus(status, TF_INTERNAL,
                     absl::StrCat("File contents are inconsistent for file: ",
                                  path, " @ ", offset)
                         .c_str());
      }
      TF_VLog(2, "Successful integrity check for: %s @ %u", path.c_str(),
              offset);
    }
  }
  return read;
}
static int64_t LoadBufferFromGCS ( const std :: string & path , size_t offset , size_t buffer_size , char * buffer , tf_gcs_filesystem :: GCSFileSystemImplementation * gcs_file , TF_Status * status ) { std :: string bucket , object ; ParseGCSPath ( path , false , & bucket , & object , status ) ; if ( TF_GetCode ( status ) != TF_OK ) return -1 ; auto stream = gcs_file -> gcs_client . ReadObject ( bucket , object , gcs :: ReadRange ( offset , offset + buffer_size ) ) ; TF_SetStatusFromGCSStatus ( stream . status ( ) , status ) ; if ( ( TF_GetCode ( status ) != TF_OK ) && ( TF_GetCode ( status ) != TF_OUT_OF_RANGE ) ) { return -1 ; } int64_t read ; auto content_length = stream . headers ( ) . find ( " " ) ; if ( content_length == stream . headers ( ) . end ( ) ) { read = 0 ; } else if ( ! absl :: SimpleAtoi ( content_length -> second , & read ) ) { TF_SetStatus ( status , TF_UNKNOWN , " " ) ; return -1 ; } TF_SetStatus ( status , TF_OK , " " ) ; TF_VLog ( 1 , " " , path . c_str ( ) , offset , read ) ; stream . read ( buffer , read ) ; read = stream . gcount ( ) ; if ( read < buffer_size ) { tf_gcs_filesystem :: GcsFileSystemStat stat ; if ( gcs_file -> stat_cache -> Lookup ( path , & stat ) ) { if ( offset + read < stat . base . length ) { TF_SetStatus ( status , TF_INTERNAL , absl :: StrCat ( " " , path , " " , offset ) . c_str ( ) ) ; } TF_VLog ( 2 , " " , path . c_str ( ) , offset ) ; } } return read ; }
A helper function to actually read the data from GCS.
A helper function to actually read the data from GCS .
// When we read a file with offset that is bigger than the actual file size. // GCS will return an empty header (e.g no `content-length` header). In this // case, we will set read to `0` and continue. // `TF_OUT_OF_RANGE` isn't considered as an error. So we clear it here. // Check stat cache to see if we encountered an interrupted read.