torch/lib/c10d/ProcessGroupNCCL.cpp
// Helper function that checks the input and output tensors for validity
Helper function that checks the input and output tensors for validity
Helper function that checks the input and output tensors for validity
void ProcessGroupNCCL::tensorCheckHelper(
    const std::vector<at::Tensor>& input,
    const std::vector<at::Tensor>& output,
    int outputOverInput) {
  if (input.size() != output.size()) {
    throw std::runtime_error(
        "Input tensor sequence should have the same "
        "number of tensors as the output tensor sequence");
  }
  if (input.size() == 0) {
    throw std::runtime_error("The number of input tensors should not be zero");
  }
  if (input.size() > static_cast<size_t>(thcState_->numDevices)) {
    throw std::runtime_error(
        "The number of input tensors is larger than "
        "the number of available GPUs");
  }
  std::unordered_set<int> usedDevices;
  usedDevices.reserve(input.size());
  auto inputNumElement = input[0].numel();
  auto elementType = input[0].type().scalarType();
  for (size_t i = 0; i < input.size(); ++i) {
    if (!(input[i].type().is_cuda() && !input[i].type().is_sparse() &&
          output[i].type().is_cuda() && !output[i].type().is_sparse())) {
      throw std::runtime_error(
          "Only CUDA dense tensor is supported for NCCL "
          "collective operations");
    }
    if (input[i].type().scalarType() != elementType ||
        output[i].type().scalarType() != elementType) {
      throw std::runtime_error(
          "Expecting all GPU tensors to have identical "
          "type");
    }
    if (input[i].numel() != inputNumElement) {
      throw std::runtime_error(
          "Expecting all input tensors to have identical "
          "number of elements");
    }
    if (output[i].numel() != inputNumElement * outputOverInput) {
      throw std::runtime_error(
          "The number of elements of output tensor does "
          "not match the number of elements of the input "
          "tensor");
    }
    if (!input[i].is_contiguous() || !output[i].is_contiguous()) {
      throw std::runtime_error("Expecting all GPU tensors to be contiguous");
    }
    bool inserted;
    std::tie(std::ignore, inserted) = usedDevices.insert(input[i].get_device());
    if (!inserted) {
      throw std::runtime_error("Expecting inputs on different GPU devices");
    }
    if (input[i].get_device() != output[i].get_device()) {
      throw std::runtime_error(
          "Expecting input and output tensors to be on "
          "the same device");
    }
  }
}
void ProcessGroupNCCL :: tensorCheckHelper ( const std :: vector < at :: Tensor > & input , const std :: vector < at :: Tensor > & output , int outputOverInput ) { if ( input . size ( ) != output . size ( ) ) { throw std :: runtime_error ( " " " " ) ; } if ( input . size ( ) == 0 ) { throw std :: runtime_error ( " " ) ; } if ( input . size ( ) > static_cast < size_t > ( thcState_ -> numDevices ) ) { throw std :: runtime_error ( " " " " ) ; } std :: unordered_set < int > usedDevices ; usedDevices . reserve ( input . size ( ) ) ; auto inputNumElement = input [ 0 ] . numel ( ) ; auto elementType = input [ 0 ] . type ( ) . scalarType ( ) ; for ( size_t i = 0 ; i < input . size ( ) ; ++ i ) { if ( ! ( input [ i ] . type ( ) . is_cuda ( ) && ! input [ i ] . type ( ) . is_sparse ( ) && output [ i ] . type ( ) . is_cuda ( ) && ! output [ i ] . type ( ) . is_sparse ( ) ) ) { throw std :: runtime_error ( " " " " ) ; } if ( input [ i ] . type ( ) . scalarType ( ) != elementType || output [ i ] . type ( ) . scalarType ( ) != elementType ) { throw std :: runtime_error ( " " " " ) ; } if ( input [ i ] . numel ( ) != inputNumElement ) { throw std :: runtime_error ( " " " " ) ; } if ( output [ i ] . numel ( ) != inputNumElement * outputOverInput ) { throw std :: runtime_error ( " " " " " " ) ; } if ( ! input [ i ] . is_contiguous ( ) || ! output [ i ] . is_contiguous ( ) ) { throw std :: runtime_error ( " " ) ; } bool inserted ; std :: tie ( std :: ignore , inserted ) = usedDevices . insert ( input [ i ] . get_device ( ) ) ; if ( ! inserted ) { throw std :: runtime_error ( " " ) ; } if ( input [ i ] . get_device ( ) != output [ i ] . get_device ( ) ) { throw std :: runtime_error ( " " " " ) ; } } }
Helper function that checks the input and output tensors for validity
Helper function that checks the input and output tensors for validity
// To make sure each tensor is on separate devices //  Check to make sure it's a GPU dense tensor // Check the tensor type is identical // Check the input tensor size is identical // Check the output tensor size equals to input tensor size // Contiguous verification // Device verification, if the insertion didn't take place // Now check the output device