torch/lib/c10d/ProcessGroupGloo.cpp
// Constructs an AlgorithmEntry instance, except for the algorithm
// itself. It allocates the temporary input/output tensors necessary
// to have a fixed address to pass to the Gloo algorithms. The
// AlgorithmEntry is lazily allocated and reused for collective calls
// with the same signature.
//
// Construction of the Gloo algorithm itself it delayed until a thread
// picks up the work, because it performs I/O and can fail. Any I/O
// failure must be signaled through the Work future.
//
Constructs an AlgorithmEntry instance, except for the algorithm
itself. It allocates the temporary input/output tensors necessary
to have a fixed address to pass to the Gloo algorithms. The
AlgorithmEntry is lazily allocated and reused for collective calls
with the same signature.

Construction of the Gloo algorithm itself it delayed until a thread
picks up the work, because it performs I/O and can fail. Any I/O
failure must be signaled through the Work future.
Constructs an AlgorithmEntry instance except for the algorithm itself . It allocates the temporary input / output tensors necessary to have a fixed address to pass to the Gloo algorithms . The AlgorithmEntry is lazily allocated and reused for collective calls with the same signature . Construction of the Gloo algorithm itself it delayed until a thread picks up the work because it performs I / O and can fail . Any I / O failure must be signaled through the Work future .
EntryType ProcessGroupGloo::construct(const AlgorithmKey& key) {
  at::DeviceGuard deviceGuard;
  auto entry = std::unique_ptr<AlgorithmEntry>(new AlgorithmEntry);
  entry->key = key;
  auto& srcSizes = key.srcSizes;
  entry->src.resize(srcSizes.size());
  for (size_t i = 0; i < srcSizes.size(); i++) {
    deviceGuard.set_index(key.type->is_cuda() ? key.devices[i] : -1);
    entry->src[i] = key.type->tensor(srcSizes[i]);
  }
  if (key.type->is_cuda()) {
    entry->streams.resize(key.devices.size());
    entry->events.resize(key.devices.size());
    for (size_t i = 0; i < key.devices.size(); i++) {
      deviceGuard.set_index(key.devices[i]);
      entry->streams[i] = CUDAStream::create();
      entry->events[i] = CUDAEvent::create();
    }
  }
  return entry;
}
EntryType ProcessGroupGloo :: construct ( const AlgorithmKey & key ) { at :: DeviceGuard deviceGuard ; auto entry = std :: unique_ptr < AlgorithmEntry > ( new AlgorithmEntry ) ; entry -> key = key ; auto & srcSizes = key . srcSizes ; entry -> src . resize ( srcSizes . size ( ) ) ; for ( size_t i = 0 ; i < srcSizes . size ( ) ; i ++ ) { deviceGuard . set_index ( key . type -> is_cuda ( ) ? key . devices [ i ] : -1 ) ; entry -> src [ i ] = key . type -> tensor ( srcSizes [ i ] ) ; } if ( key . type -> is_cuda ( ) ) { entry -> streams . resize ( key . devices . size ( ) ) ; entry -> events . resize ( key . devices . size ( ) ) ; for ( size_t i = 0 ; i < key . devices . size ( ) ; i ++ ) { deviceGuard . set_index ( key . devices [ i ] ) ; entry -> streams [ i ] = CUDAStream :: create ( ) ; entry -> events [ i ] = CUDAEvent :: create ( ) ; } } return entry ; }
Constructs an AlgorithmEntry instance, except for the algorithm
itself.
Constructs an AlgorithmEntry instance except for the algorithm itself .
// Allocate source tensors for this entry // If these are CUDA tensors, create streams and events