add demo module

demo/csrc/cpp/detector.cxx

+#include "mmdeploy/detector.hpp"
+
+#include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "utils/argparse.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "detector_output.jpg", "Output image path");
+
+DEFINE_double(det_thr, .5, "Detection score threshold");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+  mmdeploy::Profiler profiler("/tmp/profile.bin");
+  mmdeploy::Context context;
+  context.Add(mmdeploy::Device(FLAGS_device));
+  context.Add(profiler);
+
+  // construct a detector instance
+  mmdeploy::Detector detector(mmdeploy::Model{ARGS_model}, context);
+
+  // warmup
+  for (int i = 0; i < 20; ++i) {
+    detector.Apply(img);
+  }
+
+  // apply the detector, the result is an array-like class holding references to
+  // `mmdeploy_detection_t`, will be released automatically on destruction
+  mmdeploy::Detector::Result dets = detector.Apply(img);
+
+  // visualize
+  utils::Visualize v;
+  auto sess = v.get_session(img);
+  int count = 0;
+  for (const mmdeploy_detection_t& det : dets) {
+    if (det.score > FLAGS_det_thr) {  // filter bboxes
+      sess.add_det(det.bbox, det.label_id, det.score, det.mask, count++);
+    }
+  }
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
+
+  return 0;
+}

demo/csrc/cpp/pose_detector.cxx

+
+#include "mmdeploy/pose_detector.hpp"
+
+#include <opencv2/imgcodecs.hpp>
+#include <opencv2/imgproc.hpp>
+#include <string>
+
+int main(int argc, char *argv[]) {
+  if (argc != 4) {
+    fprintf(stderr, "usage:\n  pose_detection device_name model_path image_path\n");
+    return 1;
+  }
+  auto device_name = argv[1];
+  auto model_path = argv[2];
+  auto image_path = argv[3];
+  cv::Mat img = cv::imread(image_path);
+  if (!img.data) {
+    fprintf(stderr, "failed to load image: %s\n", image_path);
+    return 1;
+  }
+
+  using namespace mmdeploy;
+
+  mmdeploy::Profiler profiler("/tmp/profile.bin");
+  mmdeploy::Context context;
+  context.Add(mmdeploy::Device(device_name));
+  context.Add(profiler);
+
+  PoseDetector detector{Model(model_path), context};
+
+  // warmup
+  for (int i = 0; i < 20; ++i) {
+    detector.Apply(img);
+  }
+
+  auto res = detector.Apply(img);
+
+  for (int i = 0; i < res[0].length; i++) {
+    cv::circle(img, {(int)res[0].point[i].x, (int)res[0].point[i].y}, 1, {0, 255, 0}, 2);
+  }
+  cv::imwrite("output_pose.png", img);
+
+  return 0;
+}

demo/csrc/cpp/pose_tracker.cxx

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#include "mmdeploy/pose_tracker.hpp"
+
+#include "utils/argparse.h"
+#include "utils/mediaio.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(det_model, "Object detection model path");
+DEFINE_ARG_string(pose_model, "Pose estimation model path");
+DEFINE_ARG_string(input, "Input video path or camera index");
+
+DEFINE_string(device, "cpu", "Device name, e.g. \"cpu\", \"cuda\"");
+DEFINE_string(output, "", "Output video path or format string");
+
+DEFINE_int32(output_size, 0, "Long-edge of output frames");
+DEFINE_int32(flip, 0, "Set to 1 for flipping the input horizontally");
+DEFINE_int32(show, 1, "Delay passed to `cv::waitKey` when using `cv::imshow`; -1: disable");
+
+DEFINE_string(skeleton, "coco",
+              R"(Path to skeleton data or name of predefined skeletons: "coco", "coco-wholebody", "coco-wholebody-hand")");
+DEFINE_string(background, "default",
+              R"(Output background, "default": original image, "black": black background)");
+
+#include "pose_tracker_params.h"
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  // create pose tracker pipeline
+  mmdeploy::PoseTracker tracker(mmdeploy::Model(ARGS_det_model), mmdeploy::Model(ARGS_pose_model),
+                                mmdeploy::Device{FLAGS_device});
+
+  mmdeploy::PoseTracker::Params params;
+  // initialize tracker params with program arguments
+  InitTrackerParams(params);
+
+  // create a tracker state for each video
+  mmdeploy::PoseTracker::State state = tracker.CreateState(params);
+
+  utils::mediaio::Input input(ARGS_input, FLAGS_flip);
+  utils::mediaio::Output output(FLAGS_output, FLAGS_show);
+
+  utils::Visualize v(FLAGS_output_size);
+  v.set_background(FLAGS_background);
+  v.set_skeleton(utils::Skeleton::get(FLAGS_skeleton));
+
+  for (const cv::Mat& frame : input) {
+    // apply the pipeline with the tracker state and video frame; the result is an array-like class
+    // holding references to `mmdeploy_pose_tracker_target_t`, will be released automatically on
+    // destruction
+    mmdeploy::PoseTracker::Result result = tracker.Apply(state, frame);
+
+    // visualize results
+    auto sess = v.get_session(frame);
+    for (const mmdeploy_pose_tracker_target_t& target : result) {
+      sess.add_pose(target.keypoints, target.scores, target.keypoint_count, FLAGS_pose_kpt_thr);
+    }
+
+    // write to output stream
+    if (!output.write(sess.get())) {
+      // user requested exit by pressing ESC
+      break;
+    }
+  }
+
+  return 0;
+}

demo/csrc/cpp/pose_tracker_params.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+DEFINE_int32(det_interval, 1, "Detection interval");
+DEFINE_int32(det_label, 0, "Detection label use for pose estimation");
+DEFINE_double(det_thr, 0.5, "Detection score threshold");
+DEFINE_double(det_min_bbox_size, -1, "Detection minimum bbox size");
+DEFINE_double(det_nms_thr, .7,
+              "NMS IOU threshold for merging detected bboxes and bboxes from tracked targets");
+
+DEFINE_int32(pose_max_num_bboxes, -1, "Max number of bboxes used for pose estimation per frame");
+DEFINE_double(pose_kpt_thr, .5, "Threshold for visible key-points");
+DEFINE_int32(pose_min_keypoints, -1,
+             "Min number of key-points for valid poses, -1 indicates ceil(n_kpts/2)");
+DEFINE_double(pose_bbox_scale, 1.25, "Scale for expanding key-points to bbox");
+DEFINE_double(
+    pose_min_bbox_size, -1,
+    "Min pose bbox size, tracks with bbox size smaller than the threshold will be dropped");
+DEFINE_double(pose_nms_thr, 0.5,
+              "NMS OKS/IOU threshold for suppressing overlapped poses, useful when multiple pose "
+              "estimations collapse to the same target");
+
+DEFINE_double(track_iou_thr, 0.4, "IOU threshold for associating missing tracks");
+DEFINE_int32(track_max_missing, 10,
+             "Max number of missing frames before a missing tracks is removed");
+
+void InitTrackerParams(mmdeploy::PoseTracker::Params& params) {
+  params->det_interval = FLAGS_det_interval;
+  params->det_label = FLAGS_det_label;
+  params->det_thr = FLAGS_det_thr;
+  params->det_min_bbox_size = FLAGS_det_min_bbox_size;
+  params->pose_max_num_bboxes = FLAGS_pose_max_num_bboxes;
+  params->pose_kpt_thr = FLAGS_pose_kpt_thr;
+  params->pose_min_keypoints = FLAGS_pose_min_keypoints;
+  params->pose_bbox_scale = FLAGS_pose_bbox_scale;
+  params->pose_min_bbox_size = FLAGS_pose_min_bbox_size;
+  params->pose_nms_thr = FLAGS_pose_nms_thr;
+  params->track_iou_thr = FLAGS_track_iou_thr;
+  params->track_max_missing = FLAGS_track_max_missing;
+}

demo/csrc/cpp/restorer.cxx

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#include "mmdeploy/restorer.hpp"
+
+#include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "opencv2/imgproc/imgproc.hpp"
+#include "utils/argparse.h"
+
+DEFINE_ARG_string(model, "Super-resolution model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "restorer_output.jpg", "Output image path");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+
+  mmdeploy::Profiler profiler("/tmp/profile.bin");
+  mmdeploy::Context context;
+  context.Add(mmdeploy::Device(FLAGS_device));
+  context.Add(profiler);
+
+  // construct a restorer instance
+  mmdeploy::Restorer restorer{mmdeploy::Model{ARGS_model}, context};
+
+  // warmup
+  for (int i = 0; i < 20; ++i) {
+    restorer.Apply(img);
+  }
+
+  // apply restorer to the image
+  mmdeploy::Restorer::Result result = restorer.Apply(img);
+
+  // convert to BGR
+  cv::Mat upsampled(result->height, result->width, CV_8UC3, result->data);
+  cv::cvtColor(upsampled, upsampled, cv::COLOR_RGB2BGR);
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, upsampled);
+  }
+
+  return 0;
+}

demo/csrc/cpp/rotated_detector.cxx

+
+#include "mmdeploy/rotated_detector.hpp"
+
+#include "utils/argparse.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "rotated_detector_output.jpg", "Output image path");
+
+DEFINE_double(det_thr, 0.1, "Detection score threshold");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+
+  mmdeploy::Profiler profiler("/tmp/profile.bin");
+  mmdeploy::Context context;
+  context.Add(mmdeploy::Device(FLAGS_device));
+  context.Add(profiler);
+
+  // construct a detector instance
+  mmdeploy::RotatedDetector detector(mmdeploy::Model{ARGS_model}, context);
+
+  // warmup
+  for (int i = 0; i < 20; ++i) {
+    detector.Apply(img);
+  }
+
+  // apply the detector, the result is an array-like class holding references to
+  // `mmdeploy_rotated_detection_t`, will be released automatically on destruction
+  mmdeploy::RotatedDetector::Result dets = detector.Apply(img);
+
+  // visualize results
+  utils::Visualize v;
+  auto sess = v.get_session(img);
+  for (const mmdeploy_rotated_detection_t& det : dets) {
+    if (det.score > FLAGS_det_thr) {
+      sess.add_rotated_det(det.rbbox, det.label_id, det.score);
+    }
+  }
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
+
+  return 0;
+}

demo/csrc/cpp/segmentor.cxx

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#include "mmdeploy/segmentor.hpp"
+
+#include <string>
+#include <vector>
+
+#include "utils/argparse.h"
+#include "utils/mediaio.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "segmentor_output.jpg", "Output image path");
+DEFINE_string(palette, "cityscapes",
+              R"(Path to palette data or name of predefined palettes: "cityscapes")");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+
+  mmdeploy::Profiler profiler("/tmp/profile.bin");
+  mmdeploy::Context context;
+  context.Add(mmdeploy::Device(FLAGS_device));
+  context.Add(profiler);
+  mmdeploy::Segmentor segmentor{mmdeploy::Model{ARGS_model}, context};
+
+  // warmup
+  for (int i = 0; i < 20; ++i) {
+    segmentor.Apply(img);
+  }
+
+  // apply the detector, the result is an array-like class holding a reference to
+  // `mmdeploy_segmentation_t`, will be released automatically on destruction
+  mmdeploy::Segmentor::Result seg = segmentor.Apply(img);
+
+  // visualize
+  utils::Visualize v;
+  v.set_palette(utils::Palette::get(FLAGS_palette));
+  auto sess = v.get_session(img);
+  sess.add_mask(seg->height, seg->width, seg->classes, seg->mask, seg->score);
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
+
+  return 0;
+}

demo/csrc/cpp/text_det_recog.cxx

+
+#include "mmdeploy/archive/json_archive.h"
+#include "mmdeploy/core/utils/formatter.h"
+#include "mmdeploy/core/value.h"
+#include "mmdeploy/pipeline.hpp"
+#include "opencv2/imgcodecs.hpp"
+
+const auto config_json = R"(
+{
+  "type": "Pipeline",
+  "input": "img",
+  "output": ["dets", "texts"],
+  "tasks": [
+    {
+      "type": "Inference",
+      "input": "img",
+      "output": "dets",
+      "params": { "model": "text_detection" }
+    },
+    {
+      "type": "Pipeline",
+      "input": ["bboxes=*dets", "imgs=+img"],
+      "tasks": [
+        {
+          "type": "Task",
+          "module": "WarpBbox",
+          "scheduler": "thread_pool",
+          "input": ["imgs", "bboxes"],
+          "output": "patches"
+        },
+        {
+          "type": "Inference",
+          "input": "patches",
+          "output": "texts",
+          "params": { "model": "text_recognition" }
+        }
+      ],
+      "output": "*texts"
+    }
+  ]
+}
+)"_json;
+
+using namespace mmdeploy;
+
+int main(int argc, char* argv[]) {
+  if (argc != 5) {
+    fprintf(stderr,
+            "usage:\n\ttext_det_recog device_name det_model_path reg_model_path image_path\n");
+    return -1;
+  }
+
+  auto device_name = argv[1];
+  auto det_model_path = argv[2];
+  auto reg_model_path = argv[3];
+  auto image_path = argv[4];
+
+  cv::Mat mat = cv::imread(image_path);
+  if (!mat.data) {
+    fprintf(stderr, "failed to open image %s\n", image_path);
+    return -1;
+  }
+
+  auto config = from_json<Value>(config_json);
+
+  Context context(Device(device_name, 0));
+
+  auto thread_pool = Scheduler::ThreadPool(4);
+  auto infer_thread = Scheduler::Thread();
+  context.Add("thread_pool", thread_pool);
+  context.Add("infer_thread", infer_thread);
+  context.Add("text_detection", Model(det_model_path));
+  context.Add("text_recognition", Model(reg_model_path));
+
+  Pipeline pipeline(config, context);
+
+  auto output = pipeline.Apply(mat);
+
+  // MMDEPLOY_INFO("output:\n{}", output);
+
+  return 0;
+}

demo/csrc/cpp/text_ocr.cxx

+
+#include <string>
+
+#include "mmdeploy/text_detector.hpp"
+#include "mmdeploy/text_recognizer.hpp"
+#include "utils/argparse.h"
+#include "utils/mediaio.h"
+#include "utils/visualize.h"
+
+DEFINE_ARG_string(det_model, "Text detection model path");
+DEFINE_ARG_string(reg_model, "Text recognition model path");
+DEFINE_ARG_string(image, "Input image path");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+DEFINE_string(output, "text_ocr_output.jpg", "Output image path");
+
+using mmdeploy::TextDetector;
+using mmdeploy::TextRecognizer;
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  cv::Mat img = cv::imread(ARGS_image);
+  if (img.empty()) {
+    fprintf(stderr, "failed to load image: %s\n", ARGS_image.c_str());
+    return -1;
+  }
+
+  mmdeploy::Device device(FLAGS_device);
+  TextDetector detector{mmdeploy::Model(ARGS_det_model), device};
+  TextRecognizer recognizer{mmdeploy::Model(ARGS_reg_model), device};
+
+  // apply the detector, the result is an array-like class holding references to
+  // `mmdeploy_text_detection_t`, will be released automatically on destruction
+  TextDetector::Result bboxes = detector.Apply(img);
+
+  // apply recognizer, if no bboxes are provided, full image will be used; the result is an
+  // array-like class holding references to `mmdeploy_text_recognition_t`, will be released
+  // automatically on destruction
+  TextRecognizer::Result texts = recognizer.Apply(img, {bboxes.begin(), bboxes.size()});
+
+  // visualize results
+  utils::Visualize v;
+  auto sess = v.get_session(img);
+  for (size_t i = 0; i < bboxes.size(); ++i) {
+    mmdeploy_text_detection_t& bbox = bboxes[i];
+    mmdeploy_text_recognition_t& text = texts[i];
+    sess.add_text_det(bbox.bbox, bbox.score, text.text, text.length, i);
+  }
+
+  if (!FLAGS_output.empty()) {
+    cv::imwrite(FLAGS_output, sess.get());
+  }
+
+  return 0;
+}

demo/csrc/cpp/utils/argparse.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_ARGPARSE_H
+#define MMDEPLOY_ARGPARSE_H
+
+#include <algorithm>
+#include <cstring>
+#include <iomanip>
+#include <iostream>
+#include <map>
+#include <string>
+#include <vector>
+
+#define DEFINE_int32(name, init, msg) _MMDEPLOY_DEFINE_FLAG(int32_t, name, init, msg)
+#define DEFINE_double(name, init, msg) _MMDEPLOY_DEFINE_FLAG(double, name, init, msg)
+#define DEFINE_string(name, init, msg) _MMDEPLOY_DEFINE_FLAG(std::string, name, init, msg)
+
+#define DEFINE_ARG_int32(name, msg) _MMDEPLOY_DEFINE_ARG(int32_t, name, msg)
+#define DEFINE_ARG_double(name, msg) _MMDEPLOY_DEFINE_ARG(double, name, msg)
+#define DEFINE_ARG_string(name, msg) _MMDEPLOY_DEFINE_ARG(std::string, name, msg)
+
+namespace utils {
+
+class ArgParse {
+ public:
+  template <typename T>
+  static T Register(const std::string& type, const std::string& name, T init,
+                    const std::string& msg, void* ptr) {
+    instance()._Register(type, name, msg, true, ptr);
+    return init;
+  }
+
+  template <typename T>
+  static T Register(const std::string& type, const std::string& name, const std::string& msg,
+                    void* ptr) {
+    instance()._Register(type, name, msg, false, ptr);
+    return {};
+  }
+
+  static bool ParseArguments(int argc, char* argv[]) {
+    if (!instance()._Parse(argc, argv)) {
+      ShowUsageWithFlags(argv[0]);
+      return false;
+    }
+    return true;
+  }
+
+  static void ShowUsageWithFlags(const char* argv0) { instance()._ShowUsageWithFlags(argv0); }
+
+ private:
+  static ArgParse& instance() {
+    static ArgParse inst;
+    return inst;
+  }
+
+  struct Info {
+    std::string name;
+    std::string type;
+    std::string msg;
+    bool is_flag;
+    void* ptr;
+  };
+
+  void _Register(std::string type, const std::string& name, const std::string& msg, bool is_flag,
+                 void* ptr) {
+    if (type == "std::string") {
+      type = "string";
+    } else if (type == "int32_t") {
+      type = "int32";
+    }
+    infos_.push_back({name, type, msg, is_flag, ptr});
+  }
+
+  bool _Parse(int argc, char* argv[]) {
+    int arg_idx{-1};
+    std::vector<std::string> args(infos_.size());
+    std::vector<int> used(infos_.size());
+    for (int i = 1; i < argc; ++i) {
+      if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) {
+        return false;
+      }
+      if (argv[i][0] == '-' && argv[i][1] == '-') {
+        // parse flag key-value pair (--x=y or --x y)
+        int eq{-1};
+        for (int k = 2; argv[i][k]; ++k) {
+          if (argv[i][k] == '=') {
+            eq = k;
+            break;
+          }
+        }
+        std::string key;
+        std::string val;
+        if (eq >= 0) {
+          key = std::string(argv[i] + 2, argv[i] + eq);
+          val = std::string(argv[i] + eq + 1);
+        } else {
+          key = std::string(argv[i] + 2);
+          if (i < argc - 1) {
+            val = argv[++i];
+          }
+        }
+        bool found{};
+        for (int j = 0; j < infos_.size(); ++j) {
+          auto& flag = infos_[j];
+          if (key == flag.name) {
+            args[j] = val;
+            found = used[j] = 1;
+            break;
+          }
+        }
+        if (!found) {
+          std::cout << "error: unknown option: " << key << std::endl;
+          return false;
+        }
+      } else {
+        for (arg_idx++; arg_idx < infos_.size(); ++arg_idx) {
+          if (!infos_[arg_idx].is_flag) {
+            args[arg_idx] = argv[i];
+            used[arg_idx] = 1;
+            break;
+          }
+        }
+        if (arg_idx == infos_.size()) {
+          std::cout << "error: unknown argument: " << argv[i] << std::endl;
+          return false;
+        }
+      }
+    }
+    std::vector<std::string> missing;
+    for (arg_idx++; arg_idx < infos_.size(); ++arg_idx) {
+      if (!infos_[arg_idx].is_flag) {
+        missing.push_back(infos_[arg_idx].name);
+      }
+    }
+    if (!missing.empty()) {
+      std::cout << "error: the following arguments are required:";
+      for (int i = 0; i < missing.size(); ++i) {
+        std::cout << " " << missing[i];
+        if (i != missing.size() - 1) {
+          std::cout << ",";
+        }
+      }
+      std::cout << "\n";
+      return false;
+    }
+
+    for (int i = 0; i < infos_.size(); ++i) {
+      if (used[i]) {
+        try {
+          parse_str(infos_[i], args[i]);
+        } catch (...) {
+          std::cout << "error: failed to parse " << infos_[i].name << ": " << args[i] << std::endl;
+          return false;
+        }
+      }
+    }
+
+    return true;
+  }
+
+  static void parse_str(Info& info, const std::string& str) {
+    if (info.type == "int32") {
+      *static_cast<int32_t*>(info.ptr) = std::stoi(str);
+    } else if (info.type == "double") {
+      *static_cast<double*>(info.ptr) = std::stod(str);
+    } else if (info.type == "string") {
+      *static_cast<std::string*>(info.ptr) = str;
+    } else {
+      // pass
+    }
+  }
+
+  static std::string get_default_str(const Info& info) {
+    if (info.type == "int32") {
+      return std::to_string(*static_cast<int32_t*>(info.ptr));
+    } else if (info.type == "double") {
+      std::ostringstream os;
+      os << std::setprecision(3) << *static_cast<double*>(info.ptr);
+      return os.str();
+    } else if (info.type == "string") {
+      return "\"" + *(static_cast<std::string*>(info.ptr)) + "\"";
+    } else {
+      return "<unknown type>";
+    }
+  }
+
+  void _ShowUsageWithFlags(const char* argv0) const {
+    ShowUsage(argv0);
+    static constexpr const auto kLineLength = 80;
+    std::cout << std::endl;
+    int max_name_length = 0;
+    for (const auto& info : infos_) {
+      max_name_length = std::max(max_name_length, (int)info.name.length());
+    }
+    max_name_length += 4;
+    auto name_col_size = max_name_length + 1;
+    auto msg_col_size = kLineLength - name_col_size;
+    std::cout << "required arguments:\n";
+    ShowFlags(name_col_size, msg_col_size, false);
+    std::cout << std::endl;
+    std::cout << "optional arguments:\n";
+    ShowFlags(name_col_size, msg_col_size, true);
+  }
+
+  void ShowFlags(int name_col_size, int msg_col_size, bool is_flag) const {
+    for (const auto& info : infos_) {
+      if (info.is_flag != is_flag) {
+        continue;
+      }
+      std::string name = "  ";
+      if (info.is_flag) {
+        name.append("--");
+      }
+      name.append(info.name);
+      while (name.length() < name_col_size) {
+        name.append(" ");
+      }
+      std::cout << name;
+      std::string msg = info.msg;
+      while (msg.length() > msg_col_size) {  // insert line-breaks when msg is too long
+        auto pos = msg.rend() - std::find(std::make_reverse_iterator(msg.begin() + msg_col_size),
+                                          msg.rend(), ' ');
+        std::cout << msg.substr(0, pos - 1) << std::endl;
+        std::cout << std::string(name_col_size, ' ');
+        msg = msg.substr(pos);
+      }
+      std::cout << msg;
+      std::string type;
+      type.append("[").append(info.type);
+      if (info.is_flag) {
+        type.append(" = ").append(get_default_str(info));
+      }
+      type.append("]");
+      if (msg.length() + type.length() + 1 > msg_col_size) {
+        std::cout << std::endl << std::string(name_col_size, ' ') << type;
+      } else {
+        std::cout << " " << type;
+      }
+      std::cout << std::endl;
+    }
+  }
+
+  void ShowUsage(const char* argv0) const {
+    for (auto p = argv0; *p; ++p) {
+      if (*p == '/' || *p == '\'') {
+        argv0 = p + 1;
+      }
+    }
+    std::cout << "Usage: " << argv0 << " [options]";
+    for (const auto& info : infos_) {
+      if (!info.is_flag) {
+        std::cout << " " << info.name;
+      }
+    }
+    std::cout << std::endl;
+  }
+
+ private:
+  std::vector<Info> infos_;
+};
+
+inline bool ParseArguments(int argc, char* argv[]) { return ArgParse::ParseArguments(argc, argv); }
+
+}  // namespace utils
+
+#define _MMDEPLOY_DEFINE_FLAG(type, name, init, msg) \
+  type FLAGS_##name = ::utils::ArgParse::Register(#type, #name, type(init), msg, &FLAGS_##name)
+
+#define _MMDEPLOY_DEFINE_ARG(type, name, msg) \
+  type ARGS_##name = ::utils::ArgParse::Register<type>(#type, #name, msg, &ARGS_##name)
+
+#endif  // MMDEPLOY_ARGPARSE_H

demo/csrc/cpp/utils/mediaio.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_MEDIAIO_H
+#define MMDEPLOY_MEDIAIO_H
+
+#include <fstream>
+#include <set>
+
+#include "opencv2/highgui/highgui.hpp"
+#include "opencv2/imgcodecs/imgcodecs.hpp"
+#include "opencv2/videoio/videoio.hpp"
+
+namespace utils {
+namespace mediaio {
+
+enum class MediaType { kUnknown, kImage, kVideo, kImageList, kWebcam, kFmtStr, kDisable };
+
+namespace detail {
+
+static std::string get_extension(const std::string& path) {
+  std::string ext;
+  for (auto i = (int)path.size() - 1; i >= 0; --i) {
+    if (path[i] == '.') {
+      ext.push_back(path[i]);
+      for (++i; i < path.size(); ++i) {
+        ext.push_back((char)std::tolower((unsigned char)path[i]));
+      }
+      return ext;
+    }
+  }
+  return {};
+}
+
+int ext2fourcc(const std::string& ext) {
+  auto get_fourcc = [](const char* s) { return cv::VideoWriter::fourcc(s[0], s[1], s[2], s[3]); };
+  static std::map<std::string, int> ext2fourcc{
+      {".mp4", get_fourcc("mp4v")},
+      {".avi", get_fourcc("DIVX")},
+      {".mkv", get_fourcc("X264")},
+      {".wmv", get_fourcc("WMV3")},
+  };
+  auto it = ext2fourcc.find(ext);
+  if (it != ext2fourcc.end()) {
+    return it->second;
+  }
+  return get_fourcc("DIVX");
+}
+
+static bool is_video(const std::string& ext) {
+  static const std::set<std::string> es{".mp4", ".avi", ".mkv", ".webm", ".mov", ".mpg", ".wmv"};
+  return es.count(ext);
+}
+
+static bool is_list(const std::string& ext) {
+  static const std::set<std::string> es{".txt"};
+  return es.count(ext);
+}
+
+static bool is_image(const std::string& ext) {
+  static const std::set<std::string> es{".jpg", ".jpeg", ".png", ".tif", ".tiff",
+                                        ".bmp", ".ppm",  ".pgm", ".webp"};
+  return es.count(ext);
+}
+
+static bool is_fmtstr(const std::string& str) {
+  for (const auto& c : str) {
+    if (c == '%') {
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace detail
+
+class Input;
+
+class InputIterator {
+ public:
+  using iterator_category = std::input_iterator_tag;
+  using difference_type = std::ptrdiff_t;
+  using reference = cv::Mat&;
+  using value_type = reference;
+  using pointer = void;
+
+ public:
+  InputIterator() = default;
+  explicit InputIterator(Input& input) : input_(&input) { next(); }
+  InputIterator& operator++() {
+    next();
+    return *this;
+  }
+  reference operator*() { return frame_; }
+  friend bool operator==(const InputIterator& a, const InputIterator& b) {
+    return &a == &b || a.is_end() == b.is_end();
+  }
+  friend bool operator!=(const InputIterator& a, const InputIterator& b) { return !(a == b); }
+
+ private:
+  void next();
+  bool is_end() const noexcept { return frame_.data != nullptr; }
+
+ private:
+  cv::Mat frame_;
+  Input* input_{};
+};
+
+class BatchInputIterator {
+ public:
+  using iterator_category = std::input_iterator_tag;
+  using difference_type = std::ptrdiff_t;
+  using reference = std::vector<cv::Mat>&;
+  using value_type = reference;
+  using pointer = void;
+
+ public:
+  BatchInputIterator() = default;
+  BatchInputIterator(InputIterator iter, InputIterator end, size_t batch_size)
+      : iter_(std::move(iter)), end_(std::move(end)), batch_size_(batch_size) {
+    next();
+  }
+
+  BatchInputIterator& operator++() {
+    next();
+    return *this;
+  }
+
+  reference operator*() { return data_; }
+
+  friend bool operator==(const BatchInputIterator& a, const BatchInputIterator& b) {
+    return &a == &b || a.is_end() == b.is_end();
+  }
+
+  friend bool operator!=(const BatchInputIterator& a, const BatchInputIterator& b) {
+    return !(a == b);
+  }
+
+ private:
+  void next() {
+    data_.clear();
+    for (size_t i = 0; i < batch_size_ && iter_ != end_; ++i, ++iter_) {
+      data_.push_back(*iter_);
+    }
+  }
+
+  bool is_end() const { return data_.empty(); }
+
+ private:
+  InputIterator iter_;
+  InputIterator end_;
+  size_t batch_size_{1};
+  std::vector<cv::Mat> data_;
+};
+
+class Input {
+ public:
+  explicit Input(const std::string& path, bool flip = false, MediaType type = MediaType::kUnknown)
+      : path_(path), flip_(flip), type_(type) {
+    if (type_ == MediaType::kUnknown) {
+      auto ext = detail::get_extension(path);
+      if (detail::is_image(ext)) {
+        type_ = MediaType::kImage;
+      } else if (detail::is_video(ext)) {
+        type_ = MediaType::kVideo;
+      } else if (path.size() == 1 && std::isdigit((unsigned char)path[0])) {
+        type_ = MediaType::kWebcam;
+      } else if (detail::is_list(ext) || try_image_list(path)) {
+        type_ = MediaType::kImageList;
+      } else if (try_image(path)) {
+        type_ = MediaType::kImage;
+      } else if (try_video(path)) {
+        type_ = MediaType::kVideo;
+      } else {
+        std::cout << "unknown file type: " << path << "\n";
+      }
+    }
+    if (type_ != MediaType::kUnknown) {
+      if (type_ == MediaType::kVideo) {
+        cap_.open(path_);
+        if (!cap_.isOpened()) {
+          std::cerr << "failed to open video file: " << path_ << "\n";
+        }
+      } else if (type_ == MediaType::kWebcam) {
+        cap_.open(std::stoi(path_));
+        if (!cap_.isOpened()) {
+          std::cerr << "failed to open camera index: " << path_ << "\n";
+        }
+        type_ = MediaType::kVideo;
+      } else if (type_ == MediaType::kImage) {
+        items_ = {path_};
+        type_ = MediaType::kImageList;
+      } else if (type_ == MediaType::kImageList) {
+        if (items_.empty()) {
+          items_ = load_image_list(path);
+        }
+      }
+    }
+  }
+  InputIterator begin() { return InputIterator(*this); }
+  InputIterator end() { return {}; }  // NOLINT
+
+  cv::Mat read() {
+    cv::Mat img;
+    if (type_ == MediaType::kVideo) {
+      cap_ >> img;
+    } else if (type_ == MediaType::kImageList) {
+      while (!img.data && index_ < items_.size()) {
+        auto path = items_[index_++];
+        img = cv::imread(path);
+        if (!img.data) {
+          std::cerr << "failed to load image: " << path << "\n";
+        }
+      }
+    }
+    if (flip_ && !img.empty()) {
+      cv::flip(img, img, 1);
+    }
+    return img;
+  }
+
+  class Batch {
+   public:
+    Batch(Input& input, size_t batch_size) : input_(&input), batch_size_(batch_size) {}
+    BatchInputIterator begin() { return {input_->begin(), input_->end(), batch_size_}; }
+    BatchInputIterator end() { return {}; }  // NOLINT
+
+   private:
+    Input* input_{};
+    size_t batch_size_{1};
+  };
+
+  Batch batch(size_t batch_size) { return {*this, batch_size}; }
+
+ private:
+  static bool try_image(const std::string& path) { return cv::imread(path).data; }
+
+  static bool try_video(const std::string& path) { return cv::VideoCapture(path).isOpened(); }
+
+  static std::vector<std::string> load_image_list(const std::string& path, size_t max_bytes = 0) {
+    std::ifstream ifs(path);
+    ifs.seekg(0, std::ifstream::end);
+    auto size = ifs.tellg();
+    ifs.seekg(0, std::ifstream::beg);
+    if (max_bytes && size > max_bytes) {
+      return {};
+    }
+    auto strip = [](std::string& s) {
+      while (!s.empty() && std::isspace((unsigned char)s.back())) {
+        s.pop_back();
+      }
+    };
+    std::vector<std::string> ret;
+    std::string line;
+    while (std::getline(ifs, line)) {
+      strip(line);
+      if (!line.empty()) {
+        ret.push_back(std::move(line));
+      }
+    }
+    return ret;
+  }
+
+  bool try_image_list(const std::string& path) {
+    auto items = load_image_list(path, 1 << 20);
+    size_t count = 0;
+    for (const auto& item : items) {
+      if (detail::is_image(detail::get_extension(item)) && ++count > items.size() / 10) {
+        items_ = std::move(items);
+        return true;
+      }
+    }
+    return false;
+  }
+
+ private:
+  std::string path_;
+  bool flip_{};
+  MediaType type_{MediaType::kUnknown};
+  std::vector<std::string> items_;
+  cv::VideoCapture cap_;
+  size_t index_{};
+};
+
+inline void InputIterator::next() {
+  assert(input_);
+  frame_ = input_->read();
+}
+
+class Output;
+
+class OutputIterator {
+ public:
+  using iterator_category = std::output_iterator_tag;
+  using difference_type = std::ptrdiff_t;
+  using reference = void;
+  using value_type = void;
+  using pointer = void;
+
+ public:
+  explicit OutputIterator(Output& output) : output_(&output) {}
+
+  OutputIterator& operator=(const cv::Mat& frame);
+
+  OutputIterator& operator*() { return *this; }
+  OutputIterator& operator++() { return *this; }
+  OutputIterator& operator++(int) { return *this; }  // NOLINT
+
+ private:
+  Output* output_{};
+};
+
+class Output {
+ public:
+  explicit Output(const std::string& path, int show, MediaType type = MediaType::kUnknown)
+      : path_(path), type_(type), show_(show) {
+    ext_ = detail::get_extension(path);
+    if (type_ == MediaType::kUnknown) {
+      if (path_.empty()) {
+        type_ = MediaType::kDisable;
+      } else if (detail::is_image(ext_)) {
+        if (detail::is_fmtstr(path)) {
+          type_ = MediaType::kFmtStr;
+        } else {
+          type_ = MediaType::kImage;
+        }
+      } else if (detail::is_video(ext_)) {
+        type_ = MediaType::kVideo;
+      } else {
+        std::cout << "unknown file type: " << path << "\n";
+      }
+    }
+  }
+
+  bool write(const cv::Mat& frame) {
+    bool exit = false;
+    switch (type_) {
+      case MediaType::kDisable:
+        break;
+      case MediaType::kImage:
+        cv::imwrite(path_, frame);
+        break;
+      case MediaType::kFmtStr: {
+        char buf[256];
+        snprintf(buf, sizeof(buf), path_.c_str(), frame_id_);
+        cv::imwrite(buf, frame);
+        break;
+      }
+      case MediaType::kVideo:
+        write_video(frame);
+        break;
+      default:
+        std::cout << "unsupported output media type\n";
+        assert(0);
+    }
+    if (show_ >= 0) {
+      cv::imshow("", frame);
+      exit = cv::waitKey(show_) == 27;  // ESC
+    }
+    ++frame_id_;
+    return !exit;
+  }
+
+  OutputIterator inserter() { return OutputIterator{*this}; }
+
+ private:
+  void write_video(const cv::Mat& frame) {
+    if (!video_.isOpened()) {
+      open_video(frame.size());
+    }
+    video_ << frame;
+  }
+
+  void open_video(const cv::Size& size) { video_.open(path_, detail::ext2fourcc(ext_), 30, size); }
+
+ private:
+  std::string path_;
+  std::string ext_;
+  MediaType type_{MediaType::kUnknown};
+  int show_{1};
+  size_t frame_id_{0};
+  cv::VideoWriter video_;
+};
+
+OutputIterator& OutputIterator::operator=(const cv::Mat& frame) {
+  assert(output_);
+  output_->write(frame);
+  return *this;
+}
+
+}  // namespace mediaio
+}  // namespace utils
+
+#endif  // MMDEPLOY_MEDIAIO_H

demo/csrc/cpp/utils/palette.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_PALETTE_H
+#define MMDEPLOY_PALETTE_H
+
+#include <fstream>
+#include <opencv2/core/core.hpp>
+#include <random>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace utils {
+
+struct Palette {
+  std::vector<cv::Vec3b> data;
+  static Palette get(const std::string& path);
+  static Palette get(int n);
+};
+
+inline Palette Palette::get(const std::string& path) {
+  if (path == "coco") {
+    Palette p{{
+        {220, 20, 60},   {119, 11, 32},   {0, 0, 142},     {0, 0, 230},     {106, 0, 228},
+        {0, 60, 100},    {0, 80, 100},    {0, 0, 70},      {0, 0, 192},     {250, 170, 30},
+        {100, 170, 30},  {220, 220, 0},   {175, 116, 175}, {250, 0, 30},    {165, 42, 42},
+        {255, 77, 255},  {0, 226, 252},   {182, 182, 255}, {0, 82, 0},      {120, 166, 157},
+        {110, 76, 0},    {174, 57, 255},  {199, 100, 0},   {72, 0, 118},    {255, 179, 240},
+        {0, 125, 92},    {209, 0, 151},   {188, 208, 182}, {0, 220, 176},   {255, 99, 164},
+        {92, 0, 73},     {133, 129, 255}, {78, 180, 255},  {0, 228, 0},     {174, 255, 243},
+        {45, 89, 255},   {134, 134, 103}, {145, 148, 174}, {255, 208, 186}, {197, 226, 255},
+        {171, 134, 1},   {109, 63, 54},   {207, 138, 255}, {151, 0, 95},    {9, 80, 61},
+        {84, 105, 51},   {74, 65, 105},   {166, 196, 102}, {208, 195, 210}, {255, 109, 65},
+        {0, 143, 149},   {179, 0, 194},   {209, 99, 106},  {5, 121, 0},     {227, 255, 205},
+        {147, 186, 208}, {153, 69, 1},    {3, 95, 161},    {163, 255, 0},   {119, 0, 170},
+        {0, 182, 199},   {0, 165, 120},   {183, 130, 88},  {95, 32, 0},     {130, 114, 135},
+        {110, 129, 133}, {166, 74, 118},  {219, 142, 185}, {79, 210, 114},  {178, 90, 62},
+        {65, 70, 15},    {127, 167, 115}, {59, 105, 106},  {142, 108, 45},  {196, 172, 0},
+        {95, 54, 80},    {128, 76, 255},  {201, 57, 1},    {246, 0, 122},   {191, 162, 208},
+    }};
+    for (auto& x : p.data) {
+      std::swap(x[0], x[2]);
+    }
+    return p;
+  } else if (path == "cityscapes") {
+    Palette p{{
+        {128, 64, 128},  {244, 35, 232}, {70, 70, 70},  {102, 102, 156}, {190, 153, 153},
+        {153, 153, 153}, {250, 170, 30}, {220, 220, 0}, {107, 142, 35},  {152, 251, 152},
+        {70, 130, 180},  {220, 20, 60},  {255, 0, 0},   {0, 0, 142},     {0, 0, 70},
+        {0, 60, 100},    {0, 80, 100},   {0, 0, 230},   {119, 11, 32},
+    }};
+    for (auto& x : p.data) {
+      std::swap(x[0], x[2]);
+    }
+    return p;
+  }
+  std::ifstream ifs(path);
+  if (!ifs.is_open()) {
+    std::cout << "error: failed to open palette data file: " << path << "\n";
+    std::abort();
+  }
+  Palette p;
+  int n = 0;
+  ifs >> n;
+  for (int i = 0; i < n; ++i) {
+    cv::Vec3b x{};
+    ifs >> x[0] >> x[1] >> x[2];
+    p.data.push_back(x);
+  }
+  return p;
+}
+
+inline Palette Palette::get(int n) {
+  std::vector<cv::Point3f> samples(n * 100);
+  std::vector<int> indices(samples.size());
+  std::iota(indices.begin(), indices.end(), 0);
+  std::mt19937 gen;  // NOLINT
+  std::uniform_int_distribution<ushort> uniform_dist(0, 255);
+  for (auto& x : samples) {
+    x = {(float)uniform_dist(gen), (float)uniform_dist(gen), (float)uniform_dist(gen)};
+  }
+  std::vector<cv::Point3f> centers(n);
+  cv::Mat c_mat(centers, false);
+  cv::Mat s_mat(samples, false);
+  c_mat = c_mat.reshape(1, {n, 3});  // CV_32FC3 -> CV_32FC1 for cv::kmeans output
+  cv::kmeans(s_mat, n, indices, cv::TermCriteria(cv::TermCriteria::Type::COUNT, 10, 0), 1,
+             cv::KMEANS_PP_CENTERS, c_mat);
+  Palette p;
+  for (const auto& c : centers) {
+    p.data.emplace_back((uchar)c.x, (uchar)c.y, (uchar)c.z);
+  }
+  return p;
+}
+
+}  // namespace utils
+
+#endif  // MMDEPLOY_PALETTE_H

demo/csrc/cpp/utils/skeleton.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_SKELETON_H
+#define MMDEPLOY_SKELETON_H
+
+#include <fstream>
+#include <opencv2/core/core.hpp>
+#include <string>
+#include <utility>
+#include <vector>
+
+namespace utils {
+
+struct Skeleton {
+  std::vector<std::pair<int, int>> links;
+  std::vector<cv::Scalar> palette;
+  std::vector<int> link_colors;
+  std::vector<int> point_colors;
+  static Skeleton get(const std::string& path);
+};
+
+const Skeleton& gSkeletonCoco() {
+  static const Skeleton inst{
+      {
+          {15, 13}, {13, 11}, {16, 14}, {14, 12}, {11, 12}, {5, 11}, {6, 12},
+          {5, 6},   {5, 7},   {6, 8},   {7, 9},   {8, 10},  {1, 2},  {0, 1},
+          {0, 2},   {1, 3},   {2, 4},   {3, 5},   {4, 6},
+      },
+      {
+          {255, 128, 0},   {255, 153, 51},  {255, 178, 102}, {230, 230, 0},   {255, 153, 255},
+          {153, 204, 255}, {255, 102, 255}, {255, 51, 255},  {102, 178, 255}, {51, 153, 255},
+          {255, 153, 153}, {255, 102, 102}, {255, 51, 51},   {153, 255, 153}, {102, 255, 102},
+          {51, 255, 51},   {0, 255, 0},     {0, 0, 255},     {255, 0, 0},     {255, 255, 255},
+      },
+      {0, 0, 0, 0, 7, 7, 7, 9, 9, 9, 9, 9, 16, 16, 16, 16, 16, 16, 16},
+      {16, 16, 16, 16, 16, 9, 9, 9, 9, 9, 9, 0, 0, 0, 0, 0, 0},
+  };
+  return inst;
+}
+
+const Skeleton& gSkeletonCocoWholeBody() {
+  static const Skeleton inst{
+      {
+          {15, 13},   {13, 11},   {16, 14},   {14, 12},   {11, 12},   {5, 11},    {6, 12},
+          {5, 6},     {5, 7},     {6, 8},     {7, 9},     {8, 10},    {1, 2},     {0, 1},
+          {0, 2},     {1, 3},     {2, 4},     {3, 5},     {4, 6},     {15, 17},   {15, 18},
+          {15, 19},   {16, 20},   {16, 21},   {16, 22},   {91, 92},   {92, 93},   {93, 94},
+          {94, 95},   {91, 96},   {96, 97},   {97, 98},   {98, 99},   {91, 100},  {100, 101},
+          {101, 102}, {102, 103}, {91, 104},  {104, 105}, {105, 106}, {106, 107}, {91, 108},
+          {108, 109}, {109, 110}, {110, 111}, {112, 113}, {113, 114}, {114, 115}, {115, 116},
+          {112, 117}, {117, 118}, {118, 119}, {119, 120}, {112, 121}, {121, 122}, {122, 123},
+          {123, 124}, {112, 125}, {125, 126}, {126, 127}, {127, 128}, {112, 129}, {129, 130},
+          {130, 131}, {131, 132},
+      },
+      {
+          {51, 153, 255},
+          {0, 255, 0},
+          {255, 128, 0},
+          {255, 255, 255},
+          {255, 153, 255},
+          {102, 178, 255},
+          {255, 51, 51},
+      },
+      {1, 1, 2, 2, 0, 0, 0, 0, 1, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1,
+       2, 2, 2, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 1, 1, 1,
+       1, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 1, 1, 1, 1},
+      {0, 0, 0, 0, 0, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3,
+       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+       3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6,
+       1, 1, 1, 1, 3, 2, 2, 2, 2, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 6, 6, 1, 1, 1, 1},
+  };
+  return inst;
+}
+
+const Skeleton& gSkeletonCocoWholeBodyHand() {
+  static const Skeleton inst{
+      {
+          {0, 1},  {1, 2},   {2, 3},   {3, 4},
+          {0, 5},  {5, 6},   {6, 7},   {7, 8},
+          {0, 9},  {9, 10},  {10, 11}, {11, 12},
+          {0, 13}, {13, 14}, {14, 15}, {15, 16},
+          {0, 17}, {17, 18}, {18, 19}, {19, 20},
+      },
+      {
+          {255, 255, 255}, {255, 128, 0}, {255, 153, 255},
+          {102, 178, 255}, {255, 51, 51}, {0, 255, 0},
+      },
+      {1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5,},
+      {0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5,},
+  };
+  return inst;
+}
+
+// n_links
+// u0, v0, u1, v1, ..., un-1, vn-1
+// n_palette
+// b0, g0, r0, ..., bn-1, gn-1, rn-1
+// n_link_color
+// i0, i1, ..., in-1
+// n_point_color
+// j0, j1, ..., jn-1
+inline Skeleton Skeleton::get(const std::string& path) {
+  if (path == "coco") {
+    return gSkeletonCoco();
+  } else if (path == "coco-wholebody") {
+    return gSkeletonCocoWholeBody();
+  } else if (path == "coco-wholebody-hand") {
+    return gSkeletonCocoWholeBodyHand();
+  }
+  std::ifstream ifs(path);
+  if (!ifs.is_open()) {
+    std::cout << "error: failed to open skeleton data file: " << path << "\n";
+    std::abort();
+  }
+  Skeleton skel;
+  int n = 0;
+  ifs >> n;
+  for (int i = 0; i < n; ++i) {
+    int u{}, v{};
+    ifs >> u >> v;
+    skel.links.emplace_back(u, v);
+  }
+  ifs >> n;
+  for (int i = 0; i < n; ++i) {
+    int b{}, g{}, r{};
+    ifs >> b >> g >> r;
+    skel.palette.emplace_back(b, g, r);
+  }
+  ifs >> n;
+  for (int i = 0; i < n; ++i) {
+    int x{};
+    ifs >> x;
+    skel.link_colors.push_back(x);
+  }
+  ifs >> n;
+  for (int i = 0; i < n; ++i) {
+    int x{};
+    ifs >> x;
+    skel.point_colors.push_back(x);
+  }
+  return skel;
+}
+
+}  // namespace utils
+
+#endif  // MMDEPLOY_SKELETON_H

demo/csrc/cpp/utils/visualize.h

+// Copyright (c) OpenMMLab. All rights reserved.
+
+#ifndef MMDEPLOY_VISUALIZE_H
+#define MMDEPLOY_VISUALIZE_H
+
+#include <algorithm>
+#include <iomanip>
+#include <numeric>
+#include <vector>
+
+#include "opencv2/highgui/highgui.hpp"
+#include "opencv2/imgproc/imgproc.hpp"
+#include "palette.h"
+#include "skeleton.h"
+
+namespace utils {
+
+class Visualize {
+ public:
+  class Session {
+   public:
+    explicit Session(Visualize& v, const cv::Mat& frame) : v_(v) {
+      if (v_.size_) {
+        scale_ = (float)v_.size_ / (float)std::max(frame.cols, frame.rows);
+      }
+      cv::Mat img;
+      if (v.background_ == "black") {
+        img = cv::Mat::zeros(frame.size(), CV_8UC3);
+      } else {
+        img = frame;
+        if (img.channels() == 1) {
+          cv::cvtColor(img, img, cv::COLOR_GRAY2BGR);
+        }
+      }
+      if (scale_ != 1) {
+        cv::resize(img, img, {}, scale_, scale_);
+      } else if (img.data == frame.data) {
+        img = img.clone();
+      }
+      img_ = std::move(img);
+    }
+
+    void add_label(int label_id, float score, int index) {
+      printf("label: %d, label_id: %d, score: %.4f\n", index, label_id, score);
+      auto size = .5f * static_cast<float>(img_.rows + img_.cols);
+      offset_ += add_text(to_text(label_id, score), {1, (float)offset_}, size) + 2;
+    }
+
+    int add_text(const std::string& text, const cv::Point2f& origin, float size) {
+      static constexpr const int font_face = cv::FONT_HERSHEY_SIMPLEX;
+      static constexpr const int thickness = 1;
+      static constexpr const auto max_font_scale = .5f;
+      static constexpr const auto min_font_scale = .25f;
+      float font_scale{};
+      if (size < 20) {
+        font_scale = min_font_scale;
+      } else if (size > 200) {
+        font_scale = max_font_scale;
+      } else {
+        font_scale = min_font_scale + (size - 20) / (200 - 20) * (max_font_scale - min_font_scale);
+      }
+      int baseline{};
+      auto text_size = cv::getTextSize(text, font_face, font_scale, thickness, &baseline);
+      cv::Rect rect(origin + cv::Point2f(0, text_size.height + 2 * thickness),
+                    origin + cv::Point2f(text_size.width, 0));
+      rect &= cv::Rect({}, img_.size());
+      if (rect.area() > 0) {
+        img_(rect) *= .35f;
+        cv::putText(img_, text, origin + cv::Point2f(0, text_size.height), font_face, font_scale,
+                    cv::Scalar::all(255), thickness, cv::LINE_AA);
+      }
+      return text_size.height;
+    }
+
+    static std::string to_text(int label_id, float score) {
+      std::stringstream ss;
+      ss << label_id << ": " << std::fixed << std::setprecision(1) << score * 100;
+      return ss.str();
+    }
+
+    template <typename Mask>
+    void add_det(const mmdeploy_rect_t& rect, int label_id, float score, const Mask* mask,
+                 int index) {
+      printf("bbox %d, left=%.2f, top=%.2f, right=%.2f, bottom=%.2f, label=%d, score=%.4f\n", index,
+             rect.left, rect.top, rect.right, rect.bottom, label_id, score);
+      if (mask) {
+        fprintf(stdout, "mask %d, height=%d, width=%d\n", index, mask->height, mask->width);
+        int x0 = 0, y0 = 0, img_h=img_.size().height, img_w =img_.size().width ;
+        if (img_h != (int)mask->height || img_w != (int)mask->width ) { // maskrcnn
+          x0 = (int)std::max(std::floor(rect.left) - 1, 0.f);
+          y0 = (int)std::max(std::floor(rect.top) - 1, 0.f);
+        }
+        add_instance_mask({x0, y0}, rand(), mask->data, mask->height, mask->width);
+      }
+      add_bbox(rect, label_id, score);
+    }
+
+    void add_instance_mask(const cv::Point& origin, int color_id, const char* mask_data, int mask_h,
+                           int mask_w, float alpha = .5f) {
+      auto color = v_.palette_.data[color_id % v_.palette_.data.size()];
+      auto x_end = std::min(origin.x + mask_w, img_.cols);
+      auto y_end = std::min(origin.y + mask_h, img_.rows);
+      auto img_data = img_.ptr<cv::Vec3b>();
+      for (int i = origin.y; i < y_end; ++i) {
+        for (int j = origin.x; j < x_end; ++j) {
+          if (mask_data[(i - origin.y) * mask_w + (j - origin.x)]) {
+            img_data[i * img_.cols + j] = img_data[i * img_.cols + j] * (1 - alpha) + color * alpha;
+          }
+        }
+      }
+    }
+
+    void add_bbox(mmdeploy_rect_t rect, int label_id, float score) {
+      rect.left *= scale_;
+      rect.right *= scale_;
+      rect.top *= scale_;
+      rect.bottom *= scale_;
+      if (label_id >= 0 && score > 0) {
+        auto area = std::max(0.f, (rect.right - rect.left) * (rect.bottom - rect.top));
+        add_text(to_text(label_id, score), {rect.left, rect.top}, std::sqrt(area));
+      }
+      cv::rectangle(img_, cv::Point2f(rect.left, rect.top), cv::Point2f(rect.right, rect.bottom),
+                    cv::Scalar(0, 255, 0));
+    }
+
+    void add_text_det(mmdeploy_point_t bbox[4], float score, const char* text, size_t text_size,
+                      int index) {
+      printf("bbox[%d]: (%.2f, %.2f), (%.2f, %.2f), (%.2f, %.2f), (%.2f, %.2f), %.2f\n", index,  //
+             bbox[0].x, bbox[0].y,                                                               //
+             bbox[1].x, bbox[1].y,                                                               //
+             bbox[2].x, bbox[2].y,                                                               //
+             bbox[3].x, bbox[3].y, score);
+      std::vector<cv::Point> poly_points;
+      cv::Point2f center{};
+      for (int i = 0; i < 4; ++i) {
+        poly_points.emplace_back(bbox[i].x * scale_, bbox[i].y * scale_);
+        center += cv::Point2f(poly_points.back());
+      }
+      cv::polylines(img_, poly_points, true, cv::Scalar{0, 255, 0}, 1, cv::LINE_AA);
+      if (text) {
+        auto area = cv::contourArea(poly_points);
+        fprintf(stdout, "text[%d]: %s\n", index, text);
+        add_text(std::string(text, text + text_size), center / 4, std::sqrt(area));
+      }
+    }
+
+    void add_rotated_det(const float bbox[5], int label_id, float score) {
+      float xc = bbox[0] * scale_;
+      float yc = bbox[1] * scale_;
+      float w = bbox[2] * scale_;
+      float h = bbox[3] * scale_;
+      float ag = bbox[4];
+      float wx = w / 2 * std::cos(ag);
+      float wy = w / 2 * std::sin(ag);
+      float hx = -h / 2 * std::sin(ag);
+      float hy = h / 2 * std::cos(ag);
+      cv::Point2f p1{xc - wx - hx, yc - wy - hy};
+      cv::Point2f p2{xc + wx - hx, yc + wy - hy};
+      cv::Point2f p3{xc + wx + hx, yc + wy + hy};
+      cv::Point2f p4{xc - wx + hx, yc - wy + hy};
+      cv::Point2f c = .25f * (p1 + p2 + p3 + p4);
+      cv::drawContours(
+          img_,
+          std::vector<std::vector<cv::Point>>{{p1 * scale_, p2 * scale_, p3 * scale_, p4 * scale_}},
+          -1, {0, 255, 0}, 2, cv::LINE_AA);
+      add_text(to_text(label_id, score), c, std::sqrt(w * h));
+    }
+
+    void add_mask(int height, int width, int n_classes, const int* mask, const float* score) {
+      cv::Mat color_mask = cv::Mat::zeros(height, width, CV_8UC3);
+      auto n_pix = color_mask.total();
+
+      // compute top 1 idx if score (CHW) is available
+      cv::Mat_<int> top;
+      if (!mask && score) {
+        top = cv::Mat_<int>::zeros(height, width);
+        for (auto c = 1; c < n_classes; ++c) {
+          top.forEach([&](int& x, const int* idx) {
+            auto offset = idx[0] * width + idx[1];
+            if (score[c * n_pix + offset] > score[x * n_pix + offset]) {
+              x = c;
+            }
+          });
+        }
+        mask = top.ptr<int>();
+      }
+
+      if (mask) {
+        // palette look-up
+        color_mask.forEach<cv::Vec3b>([&](cv::Vec3b& x, const int* idx) {
+          auto& palette = v_.palette_.data;
+          x = palette[mask[idx[0] * width + idx[1]] % palette.size()];
+        });
+
+        if (color_mask.size() != img_.size()) {
+          cv::resize(color_mask, color_mask, img_.size());
+        }
+
+        // blend mask and background image
+        cv::addWeighted(img_, .5, color_mask, .5, 0., img_);
+      }
+    }
+
+    void add_pose(const mmdeploy_point_t* pts, const float* scores, int32_t pts_size, double thr) {
+      auto& skel = v_.skeleton_;
+      if (skel.point_colors.size() != pts_size) {
+        std::cout << "error: mismatched number of keypoints: " << skel.point_colors.size() << " vs "
+                  << pts_size << ", skip pose visualization.\n";
+        return;
+      }
+      std::vector<int> used(pts_size);
+      std::vector<int> is_end_point(pts_size);
+      for (size_t i = 0; i < skel.links.size(); ++i) {
+        auto u = skel.links[i].first;
+        auto v = skel.links[i].second;
+        is_end_point[u] = is_end_point[v] = 1;
+        if (scores[u] > thr && scores[v] > thr) {
+          used[u] = used[v] = 1;
+          cv::Point2f p0(pts[u].x, pts[u].y);
+          cv::Point2f p1(pts[v].x, pts[v].y);
+          cv::line(img_, p0 * scale_, p1 * scale_, skel.palette[skel.link_colors[i]], 1,
+                   cv::LINE_AA);
+        }
+      }
+      for (size_t i = 0; i < pts_size; ++i) {
+        if (!is_end_point[i] && scores[i] > thr || used[i]) {
+          cv::Point2f p(pts[i].x, pts[i].y);
+          cv::circle(img_, p * scale_, 1, skel.palette[skel.point_colors[i]], 2, cv::LINE_AA);
+        }
+      }
+    }
+
+    cv::Mat get() { return img_; }
+
+   private:
+    Visualize& v_;
+    float scale_{1};
+    int offset_{1};
+    cv::Mat img_;
+  };
+
+  explicit Visualize(int size = 0) : size_(size) { palette_ = Palette::get(32); }
+
+  Session get_session(const cv::Mat& frame) { return Session(*this, frame); }
+
+  void set_skeleton(const Skeleton& skeleton) { skeleton_ = skeleton; }
+
+  void set_palette(const Palette& palette) { palette_ = palette; }
+
+  void set_background(const std::string& background) { background_ = background; }
+
+ private:
+  friend Session;
+  Skeleton skeleton_;
+  Palette palette_;
+  std::string background_;
+  int size_{};
+};
+
+}  // namespace utils
+
+#endif  // MMDEPLOY_VISUALIZE_H

demo/csrc/cpp/video_cls.cxx

+
+#include <map>
+#include <string>
+
+#include "mmdeploy/video_recognizer.hpp"
+#include "opencv2/videoio.hpp"
+#include "utils/argparse.h"
+
+void SampleFrames(const char* video_path, std::map<int, cv::Mat>& buffer,
+                  std::vector<mmdeploy::Mat>& clips, int clip_len, int frame_interval = 1,
+                  int num_clips = 1) {
+  cv::VideoCapture cap = cv::VideoCapture(video_path);
+  if (!cap.isOpened()) {
+    fprintf(stderr, "failed to load video: %s\n", video_path);
+    exit(1);
+  }
+
+  int num_frames = cap.get(cv::CAP_PROP_FRAME_COUNT);
+  printf("num_frames %d\n", num_frames);
+
+  int ori_clip_len = clip_len * frame_interval;
+  float avg_interval = (num_frames - ori_clip_len + 1.f) / num_clips;
+  std::vector<int> frame_inds;
+  for (int i = 0; i < num_clips; i++) {
+    int clip_offset = i * avg_interval + avg_interval / 2.0;
+    for (int j = 0; j < clip_len; j++) {
+      int ind = (j * frame_interval + clip_offset) % num_frames;
+      if (num_frames <= ori_clip_len - 1) {
+        ind = j % num_frames;
+      }
+      frame_inds.push_back(ind);
+    }
+  }
+
+  std::vector<int> unique_inds(frame_inds.begin(), frame_inds.end());
+  std::sort(unique_inds.begin(), unique_inds.end());
+  auto last = std::unique(unique_inds.begin(), unique_inds.end());
+  unique_inds.erase(last, unique_inds.end());
+
+  int ind = 0;
+  for (int i = 0; i < unique_inds.size(); i++) {
+    int tid = unique_inds[i];
+    cv::Mat frame;
+    while (ind < tid) {
+      cap.read(frame);
+      ind++;
+    }
+    cap.read(frame);
+    buffer[tid] = frame;
+    ind++;
+  }
+
+  clips.resize(frame_inds.size());
+  for (int i = 0; i < frame_inds.size(); i++) {
+    auto& img = buffer[frame_inds[i]];
+    clips[i] = img;
+  }
+}
+
+DEFINE_ARG_string(model, "Model path");
+DEFINE_ARG_string(video, "Input video path");
+DEFINE_ARG_int32(clip_len, "Clip length");
+DEFINE_ARG_int32(frame_interval, "Frame interval");
+DEFINE_ARG_int32(num_clips, "Number of clips");
+DEFINE_string(device, "cpu", R"(Device name, e.g. "cpu", "cuda")");
+
+int main(int argc, char* argv[]) {
+  if (!utils::ParseArguments(argc, argv)) {
+    return -1;
+  }
+
+  std::map<int, cv::Mat> buffer;
+  std::vector<mmdeploy::Mat> clips;
+  mmdeploy::VideoSampleInfo clip_info = {ARGS_clip_len, ARGS_num_clips};
+  SampleFrames(ARGS_video.c_str(), buffer, clips, ARGS_clip_len, ARGS_frame_interval,
+               ARGS_num_clips);
+
+  mmdeploy::Model model(ARGS_model);
+  mmdeploy::VideoRecognizer recognizer(model, mmdeploy::Device{FLAGS_device});
+
+  auto res = recognizer.Apply(clips, clip_info);
+
+  for (const auto& cls : res) {
+    fprintf(stderr, "label: %d, score: %.4f\n", cls.label_id, cls.score);
+  }
+
+  return 0;
+}

demo/demo_rewrite.py

+# Copyright (c) OpenMMLab. All rights reserved.
+import asyncio
+import os
+import shutil
+
+import torch
+from pyppeteer import launch
+from torchvision.models import resnet18
+
+from mmdeploy.core import FUNCTION_REWRITER, RewriterContext, patch_model
+from mmdeploy.utils import get_root_logger
+
+
+@FUNCTION_REWRITER.register_rewriter(
+    func_name='torchvision.models.ResNet._forward_impl')
+def forward_of_resnet(self, x):
+    """Rewrite the forward implementation of resnet.
+
+    Early return the feature map after two down-sampling steps.
+    """
+    x = self.conv1(x)
+    x = self.bn1(x)
+    x = self.relu(x)
+    x = self.maxpool(x)
+
+    x = self.layer1(x)
+    return x
+
+
+def rewrite_resnet18(original_path: str, rewritten_path: str):
+    # prepare inputs and original model
+    inputs = torch.rand(1, 3, 224, 224)
+    original_model = resnet18(pretrained=False)
+
+    # export original model
+    torch.onnx.export(original_model, inputs, original_path)
+
+    # patch model
+    patched_model = patch_model(original_model, cfg={}, backend='default')
+
+    # export rewritten onnx under a rewriter context manager
+    with RewriterContext(cfg={}, backend='default'), torch.no_grad():
+        torch.onnx.export(patched_model, inputs, rewritten_path)
+
+
+def screen_size():
+    """Get windows size through tkinter."""
+    import tkinter
+    tk = tkinter.Tk()
+    width = tk.winfo_screenwidth()
+    height = tk.winfo_screenheight()
+    tk.quit()
+    return width, height
+
+
+async def visualize(original_path: str, rewritten_path: str):
+    # launch a web browser
+    browser = await launch(headless=False, args=['--start-maximized'])
+    # create two new pages
+    page2 = await browser.newPage()
+    page1 = await browser.newPage()
+    # go to netron.app
+    width, height = screen_size()
+    await page1.setViewport({'width': width, 'height': height})
+    await page2.setViewport({'width': width, 'height': height})
+    await page1.goto('https://netron.app/')
+    await page2.goto('https://netron.app/')
+    await asyncio.sleep(2)
+
+    # open local two onnx files
+    mupinput1 = await page1.querySelector("input[type='file']")
+    mupinput2 = await page2.querySelector("input[type='file']")
+    await mupinput1.uploadFile(original_file_path)
+    await mupinput2.uploadFile(rewritten_file_path)
+    await asyncio.sleep(4)
+    for _ in range(6):
+        await page1.click('#zoom-out-button')
+        await asyncio.sleep(0.3)
+    await asyncio.sleep(1)
+    await page1.screenshot({'path': original_path.replace('.onnx', '.png')},
+                           clip={
+                               'x': width / 4,
+                               'y': 0,
+                               'width': width / 2,
+                               'height': height
+                           })
+    await page2.screenshot({'path': rewritten_path.replace('.onnx', '.png')},
+                           clip={
+                               'x': width / 4,
+                               'y': 0,
+                               'width': width / 2,
+                               'height': height
+                           })
+    await browser.close()
+
+
+if __name__ == '__main__':
+    tmp_dir = os.getcwd() + '/tmp'
+    if not os.path.exists(tmp_dir):
+        os.mkdir(tmp_dir)
+    original_file_path = os.path.join(tmp_dir, 'original.onnx')
+    rewritten_file_path = os.path.join(tmp_dir, 'rewritten.onnx')
+    logger = get_root_logger()
+    logger.info('Generating resnet18 and its rewritten model...')
+    rewrite_resnet18(original_file_path, rewritten_file_path)
+
+    logger.info('Visualizing models through netron...')
+    asyncio.get_event_loop().run_until_complete(
+        visualize(original_file_path, rewritten_file_path))
+    import mmcv
+    image1 = mmcv.imread(original_file_path.replace('.onnx', '.png'))
+    image2 = mmcv.imread(rewritten_file_path.replace('.onnx', '.png'))
+    mmcv.imshow(image1, win_name='original')
+    mmcv.imshow(image2, win_name='rewritten')
+    shutil.rmtree(tmp_dir)