title: ray save tensorflow model: OSError: Unable to create file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
body: <p>I use official example to train the mnist dataset, but when I save tf model, it occurred an error, the python code:</p>
  <pre><code># -*- coding: utf-8 -*-
  import argparse
  import json
  import os
  
  import numpy as np
  import tensorflow as tf
  from tensorflow.keras.callbacks import Callback
  
  import ray.train as train
  from ray.train import Trainer
  
  
  class TrainReportCallback(Callback):
      def on_epoch_end(self, epoch, logs=None):
          train.report(**logs)
  
  
  def mnist_dataset(batch_size):
      (x_train, y_train), _ = tf.keras.datasets.mnist.load_data()
      x_train = x_train / np.float32(255)
      y_train = y_train.astype(np.int64)
      train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train)).shuffle(60000).repeat().batch(batch_size)
      return train_dataset
  
  
  def build_and_compile_cnn_model(config):
      import tensorflow as tf
  
      learning_rate = config.get(&quot;lr&quot;, 0.001)
      model = tf.keras.Sequential(
          [
              tf.keras.Input(shape=(28, 28)),
              tf.keras.layers.Reshape(target_shape=(28, 28, 1)),
              tf.keras.layers.Conv2D(32, 3, activation=&quot;relu&quot;),
              tf.keras.layers.Flatten(),
              tf.keras.layers.Dense(128, activation=&quot;relu&quot;),
              tf.keras.layers.Dense(10)
          ]
      )
      model.compile(
          loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
          optimizer=tf.keras.optimizers.SGD(learning_rate=learning_rate),
          metrics=[&quot;accuracy&quot;]
      )
      return model
  
  
  def train_func(config):
      per_work_batch_size = config.get(&quot;batch_size&quot;, 64)
      epochs = config.get(&quot;epochs&quot;, 3)
      steps_per_epoch = config.get(&quot;steps_per_epoch&quot;, 70)
  
      tf_config = json.loads(os.environ[&quot;TF_CONFIG&quot;])
      num_works = len(tf_config[&quot;cluster&quot;][&quot;worker&quot;])
  
      strategy = tf.distribute.MultiWorkerMirroredStrategy()
  
      global_batch_size = per_work_batch_size * num_works
      multi_worker_dataset = mnist_dataset(global_batch_size)
  
      with strategy.scope():
          multi_worker_model = build_and_compile_cnn_model(config)
  
      history = multi_worker_model.fit(
          multi_worker_dataset,
          epochs=epochs,
          steps_per_epoch=steps_per_epoch,
          callbacks=[TrainReportCallback()]
      )
      multi_worker_model.save(&quot;/multi_worker_model&quot;, save_format=&quot;h5&quot;)
      # tf.saved_model.save(multi_worker_model, &quot;/multi_worker_model&quot;)
  
      result = history.history
      return result
  
  
  def train_tf_mnist(num_workers=2, use_gpu=False, epochs=5):
      trainer = Trainer(backend=&quot;tensorflow&quot;, num_workers=num_workers, use_gpu=use_gpu)
      trainer.start()
      result = trainer.run(
          train_func=train_func,
          config={&quot;lr&quot;: 1e-3, &quot;batch_size&quot;: 64, &quot;epochs&quot;: epochs}
      )
      trainer.shutdown()
      print(f&quot;result: {result}&quot;)
  
  
  if __name__ == '__main__':
      parser = argparse.ArgumentParser()
      parser.add_argument(&quot;--address&quot;, required=False, type=str, help=&quot;The address to use for ray&quot;)
      parser.add_argument(&quot;--num_workers&quot;, &quot;-n&quot;, type=int, default=4, help=&quot;Sets number of workers for training&quot;)
      parser.add_argument(&quot;--use_gpu&quot;, action=&quot;store_true&quot;, default=False, help=&quot;Enable GPU training&quot;)
      parser.add_argument(&quot;--epochs&quot;, type=int, default=3, help=&quot;Number of epochs to train for&quot;)
      parser.add_argument(&quot;--smoke-test&quot;, action=&quot;store_true&quot;, default=False, help=&quot;Finish quickly for testing&quot;)
  
      args, _ = parser.parse_known_args()
  
      import ray
      if args.smoke_test:
          ray.init(num_cpus=2)
          train_tf_mnist()
      else:
          ray.init(address=&quot;auto&quot;)
          train_tf_mnist(
              num_workers=args.num_workers,
              use_gpu=args.use_gpu,
              epochs=args.epochs
          )
  
  </code></pre>
  <p>The error:</p>
  <pre><code>Traceback (most recent call last):
    File &quot;train_test.py&quot;, line 117, in &lt;module&gt;
      epochs=args.epochs
    File &quot;train_test.py&quot;, line 88, in train_tf_mnist
      config={&quot;lr&quot;: 1e-3, &quot;batch_size&quot;: 64, &quot;epochs&quot;: epochs}
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/trainer.py&quot;, line 356, in run
      for intermediate_result in iterator:
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/trainer.py&quot;, line 753, in __next__
      self._finish_training
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/trainer.py&quot;, line 713, in _run_with_error_handling
      return func()
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/trainer.py&quot;, line 824, in _finish_training
      return self._backend_executor.finish_training()
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/utils.py&quot;, line 168, in &lt;lambda&gt;
      return lambda *args, **kwargs: ray.get(actor_method.remote(*args, **kwargs))
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/_private/client_mode_hook.py&quot;, line 104, in wrapper
      return getattr(ray, func.__name__)(*args, **kwargs)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/util/client/api.py&quot;, line 44, in get
      return self.worker.get(vals, timeout=timeout)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/util/client/worker.py&quot;, line 438, in get
      res = self._get(to_get, op_timeout)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/util/client/worker.py&quot;, line 466, in _get
      raise err
  types.RayTaskError(OSError): ray::BackendExecutor.finish_training() (pid=5557, ip=10.0.5.145, repr=&lt;ray.train.backend.BackendExecutor object at 0x7f5ad4cc5490&gt;)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/backend.py&quot;, line 498, in finish_training
      results = self.get_with_failure_handling(futures)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/backend.py&quot;, line 517, in get_with_failure_handling
      success = check_for_failure(remote_values)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/utils.py&quot;, line 50, in check_for_failure
      ray.get(object_ref)
  ray.exceptions.RayTaskError(OSError): ray::BaseWorkerMixin._BaseWorkerMixin__execute() (pid=6012, ip=10.0.5.145, repr=&lt;ray.train.worker_group.BaseWorkerMixin object at 0x7f2330318bd0&gt;)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/worker_group.py&quot;, line 26, in __execute
      return func(*args, **kwargs)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/backend.py&quot;, line 489, in end_training
      output = session.finish()
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/session.py&quot;, line 118, in finish
      func_output = self.training_thread.join()
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/utils.py&quot;, line 96, in join
      raise self.exc
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/utils.py&quot;, line 89, in run
      self.ret = self._target(*self._args, **self._kwargs)
    File &quot;/mnt/softwares/hvd_env/lib/python3.7/site-packages/ray/train/utils.py&quot;, line 138, in &lt;lambda&gt;
      return lambda: train_func(config)
    File &quot;train_test.py&quot;, line 76, in train_func
    File &quot;/mnt/softwares/hvd_env/lib64/python3.7/site-packages/tensorflow/python/keras/engine/training.py&quot;, line 2112, in save
      signatures, options, save_traces)
    File &quot;/mnt/softwares/hvd_env/lib64/python3.7/site-packages/tensorflow/python/keras/saving/save.py&quot;, line 147, in save_model
      model, filepath, overwrite, include_optimizer)
    File &quot;/mnt/softwares/hvd_env/lib64/python3.7/site-packages/tensorflow/python/keras/saving/hdf5_format.py&quot;, line 103, in save_model_to_hdf5
      f = h5py.File(filepath, mode='w')
    File &quot;/mnt/softwares/hvd_env/lib64/python3.7/site-packages/h5py/_hl/files.py&quot;, line 427, in __init__
      swmr=swmr)
    File &quot;/mnt/softwares/hvd_env/lib64/python3.7/site-packages/h5py/_hl/files.py&quot;, line 196, in make_fid
      fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl)
    File &quot;h5py/_objects.pyx&quot;, line 54, in h5py._objects.with_phil.wrapper
    File &quot;h5py/_objects.pyx&quot;, line 55, in h5py._objects.with_phil.wrapper
    File &quot;h5py/h5f.pyx&quot;, line 116, in h5py.h5f.create
  OSError: Unable to create file (unable to lock file, errno = 11, error message = 'Resource temporarily unavailable')
  </code></pre>
  <p>The verion:</p>
  <pre><code>ray 1.13.0
  h5py 3.1.0
  tensorflow 2.5.0
  </code></pre>