onnxruntime/core/providers/ngraph/ngraph_custom_op.cc
//This method gets called in critical path of execution: Optimize
This method gets called in critical path of execution: Optimize
This method gets called in critical path of execution : Optimize
void NGRAPHCustomOp::Initialize(const ONNXRunTimeTensor* input_tensors, const size_t& num_inputs) const {
  LOGS_DEFAULT(INFO) << "nGraph compiling customOp: " << name_;
  std::string uniq_input_shape;
  uniq_input_shape.reserve(4 * sizeof(int64_t) * num_inputs + num_inputs);
  for (size_t i = 0; i < num_inputs; i++) {
    const auto& ndim = input_tensors[i].ndim;
    uniq_input_shape.append(reinterpret_cast<const char*>(&ndim), sizeof(ndim));
    uniq_input_shape.append(reinterpret_cast<const char*>(input_tensors[i].shape), ndim * sizeof(int64_t));
  }
  auto it = ng_exe_map_.insert({uniq_input_shape, nullptr});  
  if (!it.second) {
    ng_curr_exe_ = it.first->second;
    return;
  } else {
    auto graph_proto = model_proto_.mutable_graph();
    for (size_t i = 0; i < num_inputs; i++) {
      auto g_in_shape = graph_proto->mutable_input(i)->mutable_type()->mutable_tensor_type()->mutable_shape();
      g_in_shape->clear_dim();
      for (size_t dim = 0; dim < input_tensors[i].ndim; dim++) {
        g_in_shape->add_dim()->set_dim_value(input_tensors[i].shape[dim]);
      }
    }
    std::istringstream model_stream{model_proto_.SerializeAsString()};
    std::shared_ptr<ngraph::Function> ng_function;
    try {
      ng_function = ngraph::onnx_import::import_onnx_model(model_stream);
    } catch (const std::exception& exp) {
      LOGS_DEFAULT(FATAL) << "[" << name_ << "] "
                          << "Exception while converting onnx to nGraph: " << std::string(exp.what());
      throw;
    } catch (...) {
      LOGS_DEFAULT(FATAL) << "[" << name_ << "] "
                          << "Unknown exception while converting onnx to nGraph";
      throw;
    }
    for (auto& result : ng_function->get_results()) {
      result->set_needs_default_layout(true);
    }
    try {
      ng_curr_exe_ = ng_backend_->compile(ng_function);
    } catch (const std::exception& exp) {
      LOGS_DEFAULT(FATAL) << "Exception while compiling nGraph Op: " << name_ << std::string(exp.what());
    } catch (...) {
      LOGS_DEFAULT(FATAL) << "Unknown exception while compiling nGraph Op: " << name_;
    }
    it.first->second = ng_curr_exe_;
  }
}
void NGRAPHCustomOp :: Initialize ( const ONNXRunTimeTensor * input_tensors , const size_t & num_inputs ) const { LOGS_DEFAULT ( INFO ) << " " << name_ ; std :: string uniq_input_shape ; uniq_input_shape . reserve ( 4 * sizeof ( int64_t ) * num_inputs + num_inputs ) ; for ( size_t i = 0 ; i < num_inputs ; i ++ ) { const auto & ndim = input_tensors [ i ] . ndim ; uniq_input_shape . append ( reinterpret_cast < const char * > ( & ndim ) , sizeof ( ndim ) ) ; uniq_input_shape . append ( reinterpret_cast < const char * > ( input_tensors [ i ] . shape ) , ndim * sizeof ( int64_t ) ) ; } auto it = ng_exe_map_ . insert ( { uniq_input_shape , nullptr } ) ; if ( ! it . second ) { ng_curr_exe_ = it . first -> second ; return ; } else { auto graph_proto = model_proto_ . mutable_graph ( ) ; for ( size_t i = 0 ; i < num_inputs ; i ++ ) { auto g_in_shape = graph_proto -> mutable_input ( i ) -> mutable_type ( ) -> mutable_tensor_type ( ) -> mutable_shape ( ) ; g_in_shape -> clear_dim ( ) ; for ( size_t dim = 0 ; dim < input_tensors [ i ] . ndim ; dim ++ ) { g_in_shape -> add_dim ( ) -> set_dim_value ( input_tensors [ i ] . shape [ dim ] ) ; } } std :: istringstream model_stream { model_proto_ . SerializeAsString ( ) } ; std :: shared_ptr < ngraph :: Function > ng_function ; try { ng_function = ngraph :: onnx_import :: import_onnx_model ( model_stream ) ; } catch ( const std :: exception & exp ) { LOGS_DEFAULT ( FATAL ) << " " << name_ << " " << " " << std :: string ( exp . what ( ) ) ; throw ; } catch ( ... ) { LOGS_DEFAULT ( FATAL ) << " " << name_ << " " << " " ; throw ; } for ( auto & result : ng_function -> get_results ( ) ) { result -> set_needs_default_layout ( true ) ; } try { ng_curr_exe_ = ng_backend_ -> compile ( ng_function ) ; } catch ( const std :: exception & exp ) { LOGS_DEFAULT ( FATAL ) << " " << name_ << std :: string ( exp . what ( ) ) ; } catch ( ... ) { LOGS_DEFAULT ( FATAL ) << " " << name_ ; } it . first -> second = ng_curr_exe_ ; } }
This method gets called in critical path of execution: Optimize
This method gets called in critical path of execution : Optimize
//Key for ng_exe_map //Optimizing for general case of 4D tensors //TODO: Limit the size of map with configurable size. //ng_exe with current shape already exists // Clear previous shapes if any and set new input shapes // Finally compile nGraph with backend.