onnxruntime/core/framework/utils.cc
// execute graph and update feeds_fetches_manager with cached copy info if cache_copy_info is true
execute graph and update feeds_fetches_manager with cached copy info if cache_copy_info is true
execute graph and update feeds_fetches_manager with cached copy info if cache_copy_info is true
common::Status ExecuteGraph(const SessionState& session_state,
                            FeedsFetchesManager& feeds_fetches_manager,
                            const std::vector<MLValue>& feeds,
                            std::vector<MLValue>& fetches,
                            const std::unordered_map<size_t, IExecutor::CustomAllocator>& fetch_allocators,
                            bool sequential_execution,
                            const bool& terminate_flag,
                            const logging::Logger& logger,
                            bool cache_copy_info) {
  const auto& feeds_fetches_info = feeds_fetches_manager.GetFeedsFetchesInfo();
  auto device_copy_checks = feeds_fetches_manager.GetDeviceCopyChecks();
  ORT_ENFORCE(device_copy_checks.status == DeviceCopyCheck::Unknown);
  std::unique_ptr<IExecutor> p_exec;
  if (sequential_execution) {
    p_exec = std::unique_ptr<IExecutor>(new SequentialExecutor(terminate_flag));
  } else {
    p_exec = std::unique_ptr<IExecutor>(new ParallelExecutor(session_state, terminate_flag));
  }
  if (CheckExecutionProviders(session_state.GetExecutionProviders()) == DeviceCopyCheck::NoCopy) {
    device_copy_checks.input_copy_needed = DeviceCopyCheck::NoCopy;
    device_copy_checks.output_copy_needed = DeviceCopyCheck::NoCopy;
    ORT_RETURN_IF_ERROR(p_exec->Execute(session_state,
                                        feeds_fetches_info.feeds_mlvalue_idxs, feeds,
                                        feeds_fetches_info.fetches_mlvalue_idxs, fetches, fetch_allocators, logger));
  } else {
    bool copy_needed = false;
    const std::vector<MLValue>* p_feeds = &feeds;
    std::vector<MLValue>* p_fetches = &fetches;
    std::vector<MLValue> device_feeds;
    std::vector<MLValue> device_fetches;
    auto* copiers = cache_copy_info ? &feeds_fetches_manager.GetMutableFeedsDeviceCopiers() : nullptr;
    ORT_RETURN_IF_ERROR(CopyInputsAcrossDevices(session_state,
                                                feeds_fetches_info.feed_names, feeds, device_feeds,
                                                copy_needed, copiers));
    if (copy_needed) {
      p_feeds = &device_feeds;
    }
    device_copy_checks.input_copy_needed = copy_needed ? DeviceCopyCheck::Copy
                                                       : DeviceCopyCheck::NoCopy;
    if (fetches.empty()) {
      fetches.resize(feeds_fetches_info.output_names.size());
    }
    auto* use_provided_fetch_flags =
        cache_copy_info ? &feeds_fetches_manager.GetMutableCanUseFetchDuringExecutionFlags()
                        : nullptr;
    ORT_RETURN_IF_ERROR(SetupFetchesForExecute(session_state, feeds_fetches_info.output_names,
                                               fetches, device_fetches,
                                               use_provided_fetch_flags));
    p_fetches = &device_fetches;
    ORT_RETURN_IF_ERROR(p_exec->Execute(session_state,
                                        feeds_fetches_info.feeds_mlvalue_idxs, *p_feeds,
                                        feeds_fetches_info.fetches_mlvalue_idxs, *p_fetches, fetch_allocators,
                                        logger));
    copiers = cache_copy_info ? &feeds_fetches_manager.GetMutableFetchesDeviceCopiers() : nullptr;
    ORT_RETURN_IF_ERROR(CopyOutputsAcrossDevices(session_state, *p_fetches, fetches, copy_needed, copiers));
    device_copy_checks.output_copy_needed = copy_needed ? DeviceCopyCheck::Copy : DeviceCopyCheck::NoCopy;
  }
  if (cache_copy_info) {
    feeds_fetches_manager.SetDeviceCopyChecks(device_copy_checks);
  }
  return Status::OK();
}
common :: Status ExecuteGraph ( const SessionState & session_state , FeedsFetchesManager & feeds_fetches_manager , const std :: vector < MLValue > & feeds , std :: vector < MLValue > & fetches , const std :: unordered_map < size_t , IExecutor :: CustomAllocator > & fetch_allocators , bool sequential_execution , const bool & terminate_flag , const logging :: Logger & logger , bool cache_copy_info ) { const auto & feeds_fetches_info = feeds_fetches_manager . GetFeedsFetchesInfo ( ) ; auto device_copy_checks = feeds_fetches_manager . GetDeviceCopyChecks ( ) ; ORT_ENFORCE ( device_copy_checks . status == DeviceCopyCheck :: Unknown ) ; std :: unique_ptr < IExecutor > p_exec ; if ( sequential_execution ) { p_exec = std :: unique_ptr < IExecutor > ( new SequentialExecutor ( terminate_flag ) ) ; } else { p_exec = std :: unique_ptr < IExecutor > ( new ParallelExecutor ( session_state , terminate_flag ) ) ; } if ( CheckExecutionProviders ( session_state . GetExecutionProviders ( ) ) == DeviceCopyCheck :: NoCopy ) { device_copy_checks . input_copy_needed = DeviceCopyCheck :: NoCopy ; device_copy_checks . output_copy_needed = DeviceCopyCheck :: NoCopy ; ORT_RETURN_IF_ERROR ( p_exec -> Execute ( session_state , feeds_fetches_info . feeds_mlvalue_idxs , feeds , feeds_fetches_info . fetches_mlvalue_idxs , fetches , fetch_allocators , logger ) ) ; } else { bool copy_needed = false ; const std :: vector < MLValue > * p_feeds = & feeds ; std :: vector < MLValue > * p_fetches = & fetches ; std :: vector < MLValue > device_feeds ; std :: vector < MLValue > device_fetches ; auto * copiers = cache_copy_info ? & feeds_fetches_manager . GetMutableFeedsDeviceCopiers ( ) : nullptr ; ORT_RETURN_IF_ERROR ( CopyInputsAcrossDevices ( session_state , feeds_fetches_info . feed_names , feeds , device_feeds , copy_needed , copiers ) ) ; if ( copy_needed ) { p_feeds = & device_feeds ; } device_copy_checks . input_copy_needed = copy_needed ? DeviceCopyCheck :: Copy : DeviceCopyCheck :: NoCopy ; if ( fetches . empty ( ) ) { fetches . resize ( feeds_fetches_info . output_names . size ( ) ) ; } auto * use_provided_fetch_flags = cache_copy_info ? & feeds_fetches_manager . GetMutableCanUseFetchDuringExecutionFlags ( ) : nullptr ; ORT_RETURN_IF_ERROR ( SetupFetchesForExecute ( session_state , feeds_fetches_info . output_names , fetches , device_fetches , use_provided_fetch_flags ) ) ; p_fetches = & device_fetches ; ORT_RETURN_IF_ERROR ( p_exec -> Execute ( session_state , feeds_fetches_info . feeds_mlvalue_idxs , * p_feeds , feeds_fetches_info . fetches_mlvalue_idxs , * p_fetches , fetch_allocators , logger ) ) ; copiers = cache_copy_info ? & feeds_fetches_manager . GetMutableFetchesDeviceCopiers ( ) : nullptr ; ORT_RETURN_IF_ERROR ( CopyOutputsAcrossDevices ( session_state , * p_fetches , fetches , copy_needed , copiers ) ) ; device_copy_checks . output_copy_needed = copy_needed ? DeviceCopyCheck :: Copy : DeviceCopyCheck :: NoCopy ; } if ( cache_copy_info ) { feeds_fetches_manager . SetDeviceCopyChecks ( device_copy_checks ) ; } return Status :: OK ( ) ; }
execute graph and update feeds_fetches_manager with cached copy info if cache_copy_info is true
execute graph and update feeds_fetches_manager with cached copy info if cache_copy_info is true
// see if we can skip copies due to the types of execution providers available // no device copies are needed so simple execute // Copy inputs // setup fetches. // save the result of all the checks and use cached info next time