onnxruntime/core/framework/utils.cc
// copies outputs across devices only if required
copies outputs across devices only if required
copies outputs across devices only if required
static common::Status CopyOutputsAcrossDevices(const SessionState& session_state,
                                               const std::vector<MLValue>& fetches,
                                               std::vector<MLValue>& user_fetches,
                                               bool& needed_copy,
                                               std::vector<FeedsFetchesManager::MLValueCopyInfo>* copiers) {
  needed_copy = false;
  auto num_outputs = fetches.size();
  if (copiers) {
    copiers->resize(num_outputs);
  }
  auto& execution_providers = session_state.GetExecutionProviders();
  const auto* cpu_execution_provider = execution_providers.Get(onnxruntime::kCpuExecutionProvider);
  for (size_t idx = 0; idx < num_outputs; ++idx) {
    auto& fetched_mlvalue = fetches[idx];
    if (!fetched_mlvalue.IsTensor()) {
      user_fetches[idx] = fetched_mlvalue;
      continue;
    }
    auto& fetched_tensor = fetched_mlvalue.Get<Tensor>();
    auto& fetched_tensor_location = fetched_tensor.Location();
    auto* p_fetched_provider = execution_providers.Get(fetched_tensor_location);
    if (!p_fetched_provider) {
      p_fetched_provider = cpu_execution_provider;
    }
    auto fetched_provider_type = p_fetched_provider->Type();
    auto& output_mlvalue = user_fetches[idx];
    const IExecutionProvider* p_output_provider = nullptr;
    if (output_mlvalue.IsAllocated()) {
      Tensor* p_output_tensor = output_mlvalue.GetMutable<Tensor>();
      p_output_provider = execution_providers.Get(p_output_tensor->Location());
    }
    if (!p_output_provider) {
      p_output_provider = cpu_execution_provider;
    }
    auto output_provider_type = p_output_provider->Type();
    if (fetched_provider_type == output_provider_type ||
        (p_output_provider == cpu_execution_provider && fetched_tensor_location.mem_type == OrtMemTypeCPUOutput)) {
      user_fetches[idx] = fetched_mlvalue;
      continue;
    }
    needed_copy = true;
    auto* p_copy_provider = (fetched_provider_type != onnxruntime::kCpuExecutionProvider)
                                ? p_fetched_provider
                                : p_output_provider;
    const int device_id = 0;  
    FeedsFetchesManager::MLValueCopyInfo copy_info{device_id, p_output_provider, p_copy_provider};
    ORT_RETURN_IF_ERROR(CopyMLValue(copy_info, fetched_mlvalue, output_mlvalue));
    if (copiers) {
      (*copiers)[idx] = std::move(copy_info);
    }
  }
  return Status::OK();
}
static common :: Status CopyOutputsAcrossDevices ( const SessionState & session_state , const std :: vector < MLValue > & fetches , std :: vector < MLValue > & user_fetches , bool & needed_copy , std :: vector < FeedsFetchesManager :: MLValueCopyInfo > * copiers ) { needed_copy = false ; auto num_outputs = fetches . size ( ) ; if ( copiers ) { copiers -> resize ( num_outputs ) ; } auto & execution_providers = session_state . GetExecutionProviders ( ) ; const auto * cpu_execution_provider = execution_providers . Get ( onnxruntime :: kCpuExecutionProvider ) ; for ( size_t idx = 0 ; idx < num_outputs ; ++ idx ) { auto & fetched_mlvalue = fetches [ idx ] ; if ( ! fetched_mlvalue . IsTensor ( ) ) { user_fetches [ idx ] = fetched_mlvalue ; continue ; } auto & fetched_tensor = fetched_mlvalue . Get < Tensor > ( ) ; auto & fetched_tensor_location = fetched_tensor . Location ( ) ; auto * p_fetched_provider = execution_providers . Get ( fetched_tensor_location ) ; if ( ! p_fetched_provider ) { p_fetched_provider = cpu_execution_provider ; } auto fetched_provider_type = p_fetched_provider -> Type ( ) ; auto & output_mlvalue = user_fetches [ idx ] ; const IExecutionProvider * p_output_provider = nullptr ; if ( output_mlvalue . IsAllocated ( ) ) { Tensor * p_output_tensor = output_mlvalue . GetMutable < Tensor > ( ) ; p_output_provider = execution_providers . Get ( p_output_tensor -> Location ( ) ) ; } if ( ! p_output_provider ) { p_output_provider = cpu_execution_provider ; } auto output_provider_type = p_output_provider -> Type ( ) ; if ( fetched_provider_type == output_provider_type || ( p_output_provider == cpu_execution_provider && fetched_tensor_location . mem_type == OrtMemTypeCPUOutput ) ) { user_fetches [ idx ] = fetched_mlvalue ; continue ; } needed_copy = true ; auto * p_copy_provider = ( fetched_provider_type != onnxruntime :: kCpuExecutionProvider ) ? p_fetched_provider : p_output_provider ; const int device_id = 0 ; FeedsFetchesManager :: MLValueCopyInfo copy_info { device_id , p_output_provider , p_copy_provider } ; ORT_RETURN_IF_ERROR ( CopyMLValue ( copy_info , fetched_mlvalue , output_mlvalue ) ) ; if ( copiers ) { ( * copiers ) [ idx ] = std :: move ( copy_info ) ; } } return Status :: OK ( ) ; }
copies outputs across devices only if required
copies outputs across devices only if required
// resize so we have default values and only need to update an entry if there's a device copy required. // CPU execution provider is always registered so this is not null // TODO: As per comment in the copy input code, make this configurable.