caffe2/opt/onnxifi_transformer.cc
// Cutting off the runnable part and replace with ONNXIFI ops. Asssume the nets
// were topologically sorted
Cutting off the runnable part and replace with ONNXIFI ops. Asssume the nets
were topologically sorted
Cutting off the runnable part and replace with ONNXIFI ops . Asssume the nets were topologically sorted
void OnnxifiTransformer::Transform(
    Workspace* ws,
    NetDef* pred_net,
    const std::unordered_map<std::string, TensorShape>& input_shape_hints) {
  CAFFE_ENFORCE(ws);
  auto shape_hints_ordered =
      SsaRewriteAndMapNames(ws, pred_net, input_shape_hints);
  Workspace mapped_ws(ws, input_mapping_);
  auto shape_hints = InferShapes(&mapped_ws, pred_net, &shape_hints_ordered);
  CAFFE_ENFORCE(pred_net, "Predict net cannot be nullptr");
  onnx::OnnxExporter exporter(nullptr, true);
  onnxifi_library* backend = lib_;
  onnxBackendID backend_id = backend_ids_[0];
  auto supports =
      [&exporter, &shape_hints, backend, backend_id](
          const caffe2::OperatorDef& op) {
        const OpSchema* schema = OpSchemaRegistry::Schema(op.type());
        if (!schema || schema->onnx_schema().empty()) {
          LOG(INFO) << "Cannot export c2 op " << op.type()
                    << " to onnx as there is no corresponding ONNX schema.";
          return false;
        }
        ::ONNX_NAMESPACE::ModelProto onnx_model;
        FillModelInfo(&onnx_model);
        auto results = exporter.Caffe2OpToOnnxNodes(op, shape_hints);
        for (const auto& n : results.first) {
          onnx_model.mutable_graph()->add_node()->CopyFrom(n);
        }
        std::string onnx_model_str;
        onnx_model.SerializeToString(&onnx_model_str);
        auto ret = backend->onnxGetBackendCompatibility(
            backend_id, onnx_model_str.size(), onnx_model_str.c_str());
        if (ret != ONNXIFI_STATUS_SUCCESS) {
          LOG(INFO) << "Don't support onnx for " << op.type() << " c2 op ("
                    << ret << ")";
          return false;
        } else {
          return true;
        }
      };
  onnx::OnnxExporter exporter2(nullptr, true);
  auto trt_converter = [this, &mapped_ws, &shape_hints, &exporter2](
                           const caffe2::NetDef& net) mutable {
    return SubnetToOnnxifiOp(net, &mapped_ws, &exporter2, &shape_hints);
  };
  NetDef net_opt = opt::OptimizeForBackend(*pred_net, supports, trt_converter);
  net_opt.mutable_device_option()->CopyFrom(pred_net->device_option());
  pred_net->Swap(&net_opt);
}
void OnnxifiTransformer :: Transform ( Workspace * ws , NetDef * pred_net , const std :: unordered_map < std :: string , TensorShape > & input_shape_hints ) { CAFFE_ENFORCE ( ws ) ; auto shape_hints_ordered = SsaRewriteAndMapNames ( ws , pred_net , input_shape_hints ) ; Workspace mapped_ws ( ws , input_mapping_ ) ; auto shape_hints = InferShapes ( & mapped_ws , pred_net , & shape_hints_ordered ) ; CAFFE_ENFORCE ( pred_net , " " ) ; onnx :: OnnxExporter exporter ( nullptr , true ) ; onnxifi_library * backend = lib_ ; onnxBackendID backend_id = backend_ids_ [ 0 ] ; auto supports = [ & exporter , & shape_hints , backend , backend_id ] ( const caffe2 :: OperatorDef & op ) { const OpSchema * schema = OpSchemaRegistry :: Schema ( op . type ( ) ) ; if ( ! schema || schema -> onnx_schema ( ) . empty ( ) ) { LOG ( INFO ) << " " << op . type ( ) << " " ; return false ; } :: ONNX_NAMESPACE :: ModelProto onnx_model ; FillModelInfo ( & onnx_model ) ; auto results = exporter . Caffe2OpToOnnxNodes ( op , shape_hints ) ; for ( const auto & n : results . first ) { onnx_model . mutable_graph ( ) -> add_node ( ) -> CopyFrom ( n ) ; } std :: string onnx_model_str ; onnx_model . SerializeToString ( & onnx_model_str ) ; auto ret = backend -> onnxGetBackendCompatibility ( backend_id , onnx_model_str . size ( ) , onnx_model_str . c_str ( ) ) ; if ( ret != ONNXIFI_STATUS_SUCCESS ) { LOG ( INFO ) << " " << op . type ( ) << " " << ret << " " ; return false ; } else { return true ; } } ; onnx :: OnnxExporter exporter2 ( nullptr , true ) ; auto trt_converter = [ this , & mapped_ws , & shape_hints , & exporter2 ] ( const caffe2 :: NetDef & net ) mutable { return SubnetToOnnxifiOp ( net , & mapped_ws , & exporter2 , & shape_hints ) ; } ; NetDef net_opt = opt :: OptimizeForBackend ( * pred_net , supports , trt_converter ) ; net_opt . mutable_device_option ( ) -> CopyFrom ( pred_net -> device_option ( ) ) ; pred_net -> Swap ( & net_opt ) ; }
Cutting off the runnable part and replace with ONNXIFI ops.
Cutting off the runnable part and replace with ONNXIFI ops .
// function to tell whether the ONNXIFI backend supports a given C2 op or not // TODO: choose backend id // NB: this might not be a hard constraint as we can just export C2 // domain specific ops to ONNX // function to convert runnbale subgraph into a trt op. Note that to keep the // interface clean, we do the double conversion from C2 op to Onnx ops here // but it should be OK as the cost is really small. We also need to keep the // same exporter throughout the process to avoid duplicated dummy name // generation // Need to figure out a proper place to handle device option