how to deploy ressnet code

Makefile

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Makefile Example to deploy TVM modules.
+TVM_ROOT=$(shell cd ../..; pwd)
+DMLC_CORE=${TVM_ROOT}/3rdparty/dmlc-core
+ROCM_ROOT=/opt/dtk
+OPENCV_INCLUDE = $(shell pkg-config --cflags opencv)
+OPENCV_LIBS = $(shell pkg-config --libs opencv)
+
+PKG_CFLAGS = -std=c++17 -O2 -fPIC\
+	-I${TVM_ROOT}/include\
+	-I${DMLC_CORE}/include\
+	-I${ROCM_ROOT}/include\
+	-I${OPENCV_INCLUDE}\
+	-I${TVM_ROOT}/3rdparty/dlpack/include\
+	-DDMLC_USE_LOGGING_LIBRARY=\<tvm/runtime/logging.h\>
+	
+PKG_LDFLAGS = -L${TVM_ROOT}/build -ldl -pthread\
+	      -L${ROCM_ROOT}/lib -lamdhip64\
+	      -L${OPENCV_LIBS}\
+	      -L${ROCM_ROOT}/miopen/lib -lMIOpen\
+	      -L${ROCM_ROOT}/rocblas/lib -lrocblas
+
+
+.PHONY: clean all
+
+all: lib/libtvm_runtime_pack.o lib/resnet50-v2_deploy lib/resnet50-v2-7.so
+
+# Build rule for all in one TVM package library
+.PHONY: lib/libtvm_runtime_pack.o
+lib/libtvm_runtime_pack.o: tvm_runtime_pack.cc
+	@mkdir -p $(@D)
+	$(CXX) -c $(PKG_CFLAGS) -o $@  $^
+
+# Deploy using the all in one TVM package library
+.PHONY: lib/resnet50-v2_deploy
+lib/resnet50-v2_deploy: resnet50-v2_deploy.cc lib/resnet50-v2-7.so lib/libtvm_runtime_pack.o
+	@mkdir -p $(@D)
+	$(CXX) $(PKG_CFLAGS) -o $@  $^ $(PKG_LDFLAGS)
+
+clean:
+	rm -rf lib

prepare_test_libs.py

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Script to prepare test_addone.so"""
+import tvm
+import numpy as np
+from tvm import te
+from tvm import relay
+import os
+
+import onnx
+img_data = np.random.rand(1,3,224,224).astype("float32")/255
+input_name = "data"
+shape_dict = {input_name: img_data.shape}
+input_shape = img_data.shape
+print("input shape",img_data.shape)
+#mod, params = relay.frontend.from_onnx(onnx_model, shape_dict, dtype=dtype)
+model_path = "/resnet50-v2-7.onnx"
+onnx_model = onnx.load(model_path)
+#target = "rocm -libs=miopen,rocblas"
+np.random.seed(0)
+dtype = "float32"
+#img_data = np.random.rand(1,3,224,224).astype("float32")/255
+
+def prepare_test_libs(base_path):
+    #n = te.var("n")
+    #A = te.placeholder((n,), name="A")
+    #B = te.compute(A.shape, lambda *i: A(*i) + 1.0, name="B")
+    #s = te.create_schedule(B.op)
+    # Compile library as dynamic library
+    mod, params = relay.frontend.from_onnx(onnx_model, shape_dict, dtype=dtype)
+    fadd_dylib = tvm.build(mod, params, "rocm -libs=miopen,rocblas", name="addone")
+    dylib_path = os.path.join(base_path, "resnet50_addone_dll.so")
+    fadd_dylib.export_library(dylib_path)
+
+    # Compile library in system library mode
+    fadd_syslib = relay.build(mod, params, "rocm -libs=miopen,rocblas", name="add_resnet50")
+    syslib_path = os.path.join(base_path, "resnet50_addone_sys.o")
+    fadd_syslib.save(syslib_path)
+
+
+def prepare_graph_lib(base_path):
+    #x = relay.var("x", shape=(2, 2), dtype="float32")
+    #y = relay.var("y", shape=(2, 2), dtype="float32")
+    #params = {"y": np.ones((2, 2), dtype="float32")}
+    #mod = tvm.IRModule.from_expr(relay.Function([x, y], x + y))
+    mod, params = relay.frontend.from_onnx(onnx_model, shape_dict, dtype=dtype)
+    # build a module
+    #with tvm.transform.PassContext(opt_level=3):
+    compiled_lib = relay.build(mod, tvm.target.Target("rocm -libs=miopen,rocblas"), params=params)
+    # export it as a shared library
+    # If you are running cross compilation, you can also consider export
+    # to tar and invoke host compiler later.
+    dylib_path = os.path.join(base_path, "resnet50-v2-7.so")
+    compiled_lib.export_library(dylib_path)
+
+
+if __name__ == "__main__":
+    curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
+    #prepare_test_libs(os.path.join(curr_path, "lib"))
+    prepare_graph_lib(os.path.join(curr_path, "lib"))

resnet50-v2_deploy.cc

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \brief Example code on load and run TVM module.s
+ * \file cpp_deploy.cc
+ */
+#include <dlpack/dlpack.h>
+#include <tvm/runtime/module.h>
+#include <tvm/runtime/packed_func.h>
+#include <tvm/runtime/registry.h>
+#include <cstdio>
+#include <fstream>
+#include </usr/include/opencv2/opencv.hpp>
+#include </usr/include/opencv2/highgui/highgui.hpp>  
+#include </usr/include/opencv2/imgproc/imgproc.hpp> 
+#include <iostream>
+#include <typeinfo>
+#include <algorithm>
+
+
+using namespace cv;
+void Verify(tvm::runtime::Module mod, std::string fname) {
+  // Get the function from the module.
+  tvm::runtime::PackedFunc f = mod.GetFunction(fname);
+  ICHECK(f != nullptr);
+  // Allocate the DLPack data structures.
+  //
+  // Note that we use TVM runtime API to allocate the DLTensor in this example.
+  // TVM accept DLPack compatible DLTensors, so function can be invoked
+  // as long as we pass correct pointer to DLTensor array.
+  //
+  // For more information please refer to dlpack.
+  // One thing to notice is that DLPack contains alignment requirement for
+  // the data pointer and TVM takes advantage of that.
+  // If you plan to use your customized data container, please
+  // make sure the DLTensor you pass in meet the alignment requirement.
+  //
+  DLTensor* x;
+  DLTensor* y;
+  int ndim = 1;
+  int dtype_code = kDLFloat;
+  int dtype_bits = 32;
+  int dtype_lanes = 1;
+  int device_type = kDLCPU;
+  int device_id = 0;
+  int64_t shape[1] = {10};
+  TVMArrayAlloc(shape, ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &x);
+  TVMArrayAlloc(shape, ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &y);
+  for (int i = 0; i < shape[0]; ++i) {
+    static_cast<float*>(x->data)[i] = i;
+  }
+  // Invoke the function
+  // PackedFunc is a function that can be invoked via positional argument.
+  // The signature of the function is specified in tvm.build
+  f(x, y);
+  // Print out the output
+  for (int i = 0; i < shape[0]; ++i) {
+    ICHECK_EQ(static_cast<float*>(y->data)[i], i + 1.0f);
+  }
+  LOG(INFO) << "Finish verification...";
+  TVMArrayFree(x);
+  TVMArrayFree(y);
+}
+
+void DeploySingleOp() {
+  // Normally we can directly
+  tvm::runtime::Module mod_dylib = tvm::runtime::Module::LoadFromFile("lib/test_addone_dll.so");
+  LOG(INFO) << "Verify dynamic loading from test_addone_dll.so";
+  Verify(mod_dylib, "addone");
+  // For libraries that are directly packed as system lib and linked together with the app
+  // We can directly use GetSystemLib to get the system wide library.
+  LOG(INFO) << "Verify load function from system lib";
+  tvm::runtime::Module mod_syslib = (*tvm::runtime::Registry::Get("runtime.SystemLib"))();
+  Verify(mod_syslib, "addonesys");
+}
+
+void PreProcess(const Mat& image, Mat& image_blob)
+{
+	Mat input;
+	image.copyTo(input);
+
+	std::vector<Mat> channels, channel_p;
+	split(input, channels);
+	Mat R, G, B;
+	B = channels.at(0);
+	G = channels.at(1);
+	R = channels.at(2);
+
+	B = (B / 255. - 0.406) / 0.225;
+	G = (G / 255. - 0.456) / 0.224;
+	R = (R / 255. - 0.485) / 0.229;
+
+	channel_p.push_back(R);
+	channel_p.push_back(G);
+	channel_p.push_back(B);
+
+	Mat outt;
+	merge(channel_p, outt);
+	image_blob = outt;
+}
+
+void Mat_to_CHW(float *img_data, cv::Mat &frame)
+{
+    assert(img_data && !frame.empty());
+    unsigned int volChl = 224 * 224;
+
+    for(int c = 0; c < 3; ++c)
+    {
+        for (unsigned j = 0; j < volChl; ++j)
+            img_data[c*volChl + j] = static_cast<float>(float(frame.data[j * 3 + c])/255.0);
+    }
+
+}
+
+void DeployGraphExecutor() {
+  LOG(INFO) << "Running graph executor...";
+  // load in the library
+
+  DLDevice dev{kDLROCM, 0};
+  tvm::runtime::Module mod_factory = tvm::runtime::Module::LoadFromFile("lib/resnet50-v2-7.so");
+  
+  // create the graph executor module
+  using namespace std;
+  tvm::runtime::Module gmod = mod_factory.GetFunction("default")(dev);
+  cout<<"---------------"<<endl;
+
+  tvm::runtime::PackedFunc set_input = gmod.GetFunction("set_input");
+  tvm::runtime::PackedFunc get_output = gmod.GetFunction("get_output");
+  tvm::runtime::PackedFunc run = gmod.GetFunction("run");
+//  cv::Mat image = cv::imread("/home/linjq/ambulance.jpg");
+  cv::Mat image = cv::imread("./cow.jpg");
+  cv::Mat in_put;
+  cv::Mat img_in;
+  cv::resize(image, in_put, cv::Size(224, 224));
+  //cv::cvtColor(frame, in_put, cv::COLOR_BGR2RGB);  
+
+  float img_data[224*224*3];
+//  PreProcess(in_put, img_in);
+//  Mat_to_CHW(img_data, img_in);
+  Mat_to_CHW(img_data, in_put);
+  
+  DLTensor* y;
+  int out_ndim = 2;
+  int64_t out_shape[2] = {1, 1000};
+  int dtype_code = kDLFloat;
+  int dtype_bits = 32;
+  int dtype_lanes = 1;
+  int device_type = kDLROCM;
+  int device_id = 0;
+  TVMArrayAlloc(out_shape, out_ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &y);
+  
+
+
+  
+  DLTensor* x;
+  int ndim = 4;
+  int64_t shape[4] = {1, 3 ,224, 224};
+  TVMArrayAlloc(shape, ndim, dtype_code, dtype_bits, dtype_lanes, device_type, device_id, &x);
+  memcpy(x->data,&img_data,3*224*224*sizeof(float));
+  
+  // set the right input
+  set_input("data", x);
+  // run the code
+  run();
+  //get the output
+  
+  get_output(0, y);
+
+  float* result = new float[1000];
+  TVMArrayCopyToBytes(y, result, 1000 * sizeof(float));
+
+  float max_num = *max_element(result, result+1000);
+//  int max_num_index = max_element(result,result+1000)-result;
+  auto max_iter = std::max_element(result, result + 1000);
+  auto max_num_index = std::distance(result, max_iter);
+  cout<<"max_num:"<<max_num<<endl;
+  cout<<"max_iter:"<<max_iter<<endl;
+  cout<<"max_num_index:"<<max_num_index<<endl;
+}
+
+int main(void) {
+  //DeploySingleOp();
+  DeployGraphExecutor();
+  return 0;
+}

run_example.sh

+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+echo "Build the libraries.."
+mkdir -p lib
+make
+echo "Run the example"
+export LD_LIBRARY_PATH=../../build:${LD_LIBRARY_PATH}
+export DYLD_LIBRARY_PATH=../../build:${DYLD_LIBRARY_PATH}
+
+#echo "Run the deployment with all in one packed library..."
+#lib/cpp_deploy_pack
+#python prepare_test_libs.py
+#echo "Run the cpp deployment with all in normal library..."
+#lib/cpp_deploy_normal
+echo "Run the cpp deployment with all in normal library..."
+lib/resnet50-v2_deploy
+#echo "Run the python deployment with all in normal library..."
+#python3 python_deploy.py

tvm_runtime_pack.cc

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*!
+ * \brief This is an all in one TVM runtime file.
+ *
+ *   You only have to use this file to compile libtvm_runtime to
+ *   include in your project.
+ *
+ *  - Copy this file into your project which depends on tvm runtime.
+ *  - Compile with -std=c++17
+ *  - Add the following include path
+ *     - /path/to/tvm/include/
+ *     - /path/to/tvm/3rdparty/dmlc-core/include/
+ *     - /path/to/tvm/3rdparty/dlpack/include/
+ *   - Add -lpthread -ldl to the linked library.
+ *   - You are good to go.
+ *   - See the Makefile in the same folder for example.
+ *
+ *  The include files here are presented with relative path
+ *  You need to remember to change it to point to the right file.
+ *
+ */
+#define TVM_USE_LIBBACKTRACE 0
+#include "../../src/runtime/c_runtime_api.cc"
+#include "../../src/runtime/container.cc"
+#include "../../src/runtime/cpu_device_api.cc"
+#include "../../src/runtime/file_utils.cc"
+#include "../../src/runtime/library_module.cc"
+#include "../../src/runtime/logging.cc"
+#include "../../src/runtime/module.cc"
+#include "../../src/runtime/ndarray.cc"
+#include "../../src/runtime/object.cc"
+#include "../../src/runtime/registry.cc"
+#include "../../src/runtime/thread_pool.cc"
+#include "../../src/runtime/threading_backend.cc"
+#include "../../src/runtime/workspace_pool.cc"
+
+// NOTE: all the files after this are optional modules
+// that you can include remove, depending on how much feature you use.
+
+// Likely we only need to enable one of the following
+// If you use Module::Load, use dso_module
+// For system packed library, use system_lib_module
+#include "../../src/runtime/dso_library.cc"
+#include "../../src/runtime/system_library.cc"
+
+// Graph executor
+#include "../../src/runtime/graph_executor/graph_executor.cc"
+#include "../../src/runtime/graph_executor/graph_executor_factory.cc"
+
+// Uncomment the following lines to enable RPC
+// #include "../../src/runtime/rpc/rpc_session.cc"
+// #include "../../src/runtime/rpc/rpc_event_impl.cc"
+// #include "../../src/runtime/rpc/rpc_server_env.cc"
+
+// These macros enables the device API when uncommented.
+#define TVM_ROCM_RUNTIME 1
+#define TVM_USE_MIOPEN 1
+#define TVM_USE_ROCBLAS 1
+#define __HIP_PLATFORM_HCC__ 1
+#define TVM_METAL_RUNTIME 1
+#define TVM_OPENCL_RUNTIME 1
+
+// Uncomment the following lines to enable Metal
+// #include "../../src/runtime/metal/metal_device_api.mm"
+// #include "../../src/runtime/metal/metal_module.mm"
+
+// Uncomment the following lines to enable CUDA
+// #include "../../src/runtime/cuda/cuda_device_api.cc"
+// #include "../../src/runtime/cuda/cuda_module.cc"
+
+// Uncomment the following lines to enable ROCM
+#include "../../src/runtime/rocm/rocm_device_api.cc"
+#include "../../src/runtime/rocm/rocm_module.cc"
+#include "../../src/runtime/contrib/miopen/conv_forward.cc"
+#include "../../src/runtime/contrib/miopen/miopen_utils.cc"
+#include "../../src/runtime/contrib/rocblas/rocblas.cc"
+// Uncomment the following lines to enable OpenCL
+// #include "../../src/runtime/opencl/opencl_device_api.cc"
+// #include "../../src/runtime/opencl/opencl_module.cc"