[Benchmark] Add GPU OpenVIVO Option in benchmark (#786)

* add paddle_trt in benchmark

* update benchmark in device

* update benchmark

* update result doc

* fixed for CI

* update python api_docs

* update index.rst

* add runtime cpp examples

* deal with comments

* Update infer_paddle_tensorrt.py

* Add runtime quick start

* deal with comments

* fixed reused_input_tensors&&reused_output_tensors

* fixed docs

* fixed headpose typo

* fixed typo

* refactor yolov5

* update model infer

* refactor pybind for yolov5

* rm origin yolov5

* fixed bugs

* rm cuda preprocess

* fixed bugs

* fixed bugs

* fixed bug

* fixed bug

* fix pybind

* rm useless code

* add convert_and_permute

* fixed bugs

* fixed im_info for bs_predict

* fixed bug

* add bs_predict for yolov5

* Add runtime test and batch eval

* deal with comments

* fixed bug

* update testcase

* fixed batch eval bug

* fixed preprocess bug

* refactor yolov7

* add yolov7 testcase

* rm resize_after_load and add is_scale_up

* fixed bug

* set multi_label true

* optimize rvm preprocess

* optimizer rvm postprocess

* fixed bug

* deal with comments

* fixed bugs

* add gpu ov for benchmark

Co-authored-by: Jason <[email protected]>
Co-authored-by: Jason <[email protected]>

Author: wjj19950828

benchmark/benchmark_ppcls.py

@@ -75,6 +75,11 @@ def build_option(args):
             option.use_ort_backend()
         elif backend == "paddle":
             option.use_paddle_backend()
+        elif backend == "ov":
+            option.use_openvino_backend()
+            option.set_openvino_device(name="GPU")
+            # change name and shape for models
+            option.set_openvino_shape_info({"x": [1, 3, 224, 224]})
         elif backend in ["trt", "paddle_trt"]:
             option.use_trt_backend()
             if backend == "paddle_trt":

benchmark/benchmark_ppdet.py

@@ -75,6 +75,17 @@ def build_option(args):
             option.use_ort_backend()
         elif backend == "paddle":
             option.use_paddle_backend()
+        elif backend == "ov":
+            option.use_openvino_backend()
+            # Using GPU and CPU heterogeneous execution mode
+            option.set_openvino_device("HETERO:GPU,CPU")
+            # change name and shape for models
+            option.set_openvino_shape_info({
+                "image": [1, 3, 320, 320],
+                "scale_factor": [1, 2]
+            })
+            # Set CPU up operator
+            option.set_openvino_cpu_operators(["MulticlassNms"])
         elif backend in ["trt", "paddle_trt"]:
             option.use_trt_backend()
             if backend == "paddle_trt":

benchmark/benchmark_ppseg.py

@@ -75,6 +75,11 @@ def build_option(args):
             option.use_ort_backend()
         elif backend == "paddle":
             option.use_paddle_backend()
+        elif backend == "ov":
+            option.use_openvino_backend()
+            option.set_openvino_device(name="GPU")  # use gpu
+            # change name and shape for models
+            option.set_openvino_shape_info({"x": [1, 3, 512, 512]})
         elif backend in ["trt", "paddle_trt"]:
             option.use_trt_backend()
             if backend == "paddle_trt":

benchmark/benchmark_yolo.py

@@ -75,6 +75,11 @@ def build_option(args):
             option.use_ort_backend()
         elif backend == "paddle":
             option.use_paddle_backend()
+        elif backend == "ov":
+            option.use_openvino_backend()
+            option.set_openvino_device(name="GPU")
+            # change name and shape for models
+            option.set_openvino_shape_info({"images": [1, 3, 640, 640]})
         elif backend in ["trt", "paddle_trt"]:
             option.use_trt_backend()
             if backend == "paddle_trt":

examples/vision/detection/paddledetection/cpp/infer_ppyoloe.cc

@@ -176,7 +176,7 @@ bool OpenVINOBackend::InitFromPaddle(const std::string& model_file,
   }
 
   ov::AnyMap properties;
-  if (option_.cpu_thread_num > 0) {
+  if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
     properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
   }
   if (option_.device == "CPU") {
@@ -306,7 +306,7 @@ bool OpenVINOBackend::InitFromOnnx(const std::string& model_file,
   }
 
   ov::AnyMap properties;
-  if (option_.cpu_thread_num > 0) {
+  if (option_.device == "CPU" && option_.cpu_thread_num > 0) {
     properties["INFERENCE_NUM_THREADS"] = option_.cpu_thread_num;
   }
   if (option_.device == "CPU") {