tensorflow_io_gcs_filesystem/core/gcs_filesystem.cc
// TODO(vnvo2409): Use partial reponse for better performance.
// TODO(vnvo2409): We could do some cleanups like `return TF_SetStatus`.
// TODO(vnvo2409): Refactor the filesystem implementation when
// https://github.com/googleapis/google-cloud-cpp/issues/4482 is done.
Use partial reponse for better performance.
TODO(vnvo2409): We could do some cleanups like `return TF_SetStatus`.
Use partial reponse for better performance . TODO ( vnvo2409 ) : We could do some cleanups like ` return TF_SetStatus ` .
GCSFileSystemImplementation::GCSFileSystemImplementation(
    google::cloud::storage::Client&& gcs_client)
    : gcs_client(gcs_client), block_cache_lock() {
  const char* append_mode = std::getenv(kAppendMode);
  compose = (append_mode != nullptr) && (!strcmp(kComposeAppend, append_mode));
  uint64_t value;
  block_size = kDefaultBlockSize;
  size_t max_bytes = kDefaultMaxCacheSize;
  uint64_t max_staleness = kDefaultMaxStaleness;
  if (absl::SimpleAtoi(std::getenv(kBlockSize), &value)) {
    block_size = value * 1024 * 1024;
  }
  if (absl::SimpleAtoi(std::getenv(kMaxCacheSize), &value)) {
    max_bytes = static_cast<size_t>(value * 1024 * 1024);
  }
  if (absl::SimpleAtoi(std::getenv(kMaxStaleness), &value)) {
    max_staleness = value;
  }
  TF_VLog(1, "GCS cache max size = %u ; block size = %u ; max staleness = %u",
          max_bytes, block_size, max_staleness);
  file_block_cache = std::make_unique<RamFileBlockCache>(
      block_size, max_bytes, max_staleness,
      [this](const std::string& filename, size_t offset, size_t buffer_size,
             char* buffer, TF_Status* status) {
        return LoadBufferFromGCS(filename, offset, buffer_size, buffer, this,
                                 status);
      });
  uint64_t stat_cache_max_age = kStatCacheDefaultMaxAge;
  size_t stat_cache_max_entries = kStatCacheDefaultMaxEntries;
  if (absl::SimpleAtoi(std::getenv(kStatCacheMaxAge), &value)) {
    stat_cache_max_age = value;
  }
  if (absl::SimpleAtoi(std::getenv(kStatCacheMaxEntries), &value)) {
    stat_cache_max_entries = static_cast<size_t>(value);
  }
  stat_cache = std::make_unique<ExpiringLRUCache<GcsFileSystemStat>>(
      stat_cache_max_age, stat_cache_max_entries);
}
GCSFileSystemImplementation :: GCSFileSystemImplementation ( google :: cloud :: storage :: Client && gcs_client ) : gcs_client ( gcs_client ) , block_cache_lock ( ) { const char * append_mode = std :: getenv ( kAppendMode ) ; compose = ( append_mode != nullptr ) && ( ! strcmp ( kComposeAppend , append_mode ) ) ; uint64_t value ; block_size = kDefaultBlockSize ; size_t max_bytes = kDefaultMaxCacheSize ; uint64_t max_staleness = kDefaultMaxStaleness ; if ( absl :: SimpleAtoi ( std :: getenv ( kBlockSize ) , & value ) ) { block_size = value * 1024 * 1024 ; } if ( absl :: SimpleAtoi ( std :: getenv ( kMaxCacheSize ) , & value ) ) { max_bytes = static_cast < size_t > ( value * 1024 * 1024 ) ; } if ( absl :: SimpleAtoi ( std :: getenv ( kMaxStaleness ) , & value ) ) { max_staleness = value ; } TF_VLog ( 1 , " " , max_bytes , block_size , max_staleness ) ; file_block_cache = std :: make_unique < RamFileBlockCache > ( block_size , max_bytes , max_staleness , [ this ] ( const std :: string & filename , size_t offset , size_t buffer_size , char * buffer , TF_Status * status ) { return LoadBufferFromGCS ( filename , offset , buffer_size , buffer , this , status ) ; } ) ; uint64_t stat_cache_max_age = kStatCacheDefaultMaxAge ; size_t stat_cache_max_entries = kStatCacheDefaultMaxEntries ; if ( absl :: SimpleAtoi ( std :: getenv ( kStatCacheMaxAge ) , & value ) ) { stat_cache_max_age = value ; } if ( absl :: SimpleAtoi ( std :: getenv ( kStatCacheMaxEntries ) , & value ) ) { stat_cache_max_entries = static_cast < size_t > ( value ) ; } stat_cache = std :: make_unique < ExpiringLRUCache < GcsFileSystemStat > > ( stat_cache_max_age , stat_cache_max_entries ) ; }
TODO(vnvo2409): Use partial reponse for better performance.
TODO ( vnvo2409 ) : Use partial reponse for better performance .
// Apply the overrides for the block size (MB), max bytes (MB), and max // staleness (seconds) if provided.