Classifier.cpp

#include <Classifier.h>

#include <migraphx/onnx.hpp>
#include <migraphx/gpu/target.hpp>
#include <migraphx/quantization.hpp>
#include <hip/hip_runtime_api.h>
#include <Filesystem.h>
#include <SimpleLog.h>
#include <algorithm>
#include <CommonUtility.h>

namespace migraphxSamples
{

Classifier::Classifier() {}

Classifier::~Classifier() { configurationFile.release(); }

cv::Mat Classifier::Preprocess(const std::vector<cv::Mat>& srcImages)
{
    // 数据预处理
    std::vector<cv::Mat> image;
    for(int i = 0; i < srcImages.size(); ++i)
    {
        // BGR转换为RGB
        cv::Mat imgRGB;
        cv::cvtColor(srcImages[i], imgRGB, cv::COLOR_BGR2RGB);

        // 调整大小，使短边为256，保持长宽比
        cv::Mat shrink;
        float ratio = (float)256 / min(imgRGB.cols, imgRGB.rows);
        if(imgRGB.rows > imgRGB.cols)
        {
            cv::resize(imgRGB, shrink, cv::Size(256, int(ratio * imgRGB.rows)), 0, 0);
        }
        else
        {
            cv::resize(imgRGB, shrink, cv::Size(int(ratio * imgRGB.cols), 256), 0, 0);
        }

        // 裁剪中心窗口为224*224
        int start_x = shrink.cols / 2 - 224 / 2;
        int start_y = shrink.rows / 2 - 224 / 2;
        cv::Rect rect(start_x, start_y, 224, 224);
        cv::Mat images = shrink(rect);
        image.push_back(images);
    }

    // normalize并转换为NCHW
    cv::Mat inputBlob;
    Image2BlobParams image2BlobParams;
    image2BlobParams.scalefactor = cv::Scalar(1 / 58.395, 1 / 57.12, 1 / 57.375);
    image2BlobParams.mean        = cv::Scalar(123.675, 116.28, 103.53);
    image2BlobParams.swapRB      = false;
    blobFromImagesWithParams(image, inputBlob, image2BlobParams);
    return inputBlob;
}

ErrorCode
Classifier::Initialize(InitializationParameterOfClassifier initializationParameterOfClassifier)
{
    // 读取配置文件
    std::string configFilePath = initializationParameterOfClassifier.configFilePath;
    if(!Exists(configFilePath))
    {
        LOG_ERROR(stdout, "no configuration file!\n");
        return CONFIG_FILE_NOT_EXIST;
    }
    if(!configurationFile.open(configFilePath, cv::FileStorage::READ))
    {
        LOG_ERROR(stdout, "fail to open configuration file\n");
        return FAIL_TO_OPEN_CONFIG_FILE;
    }
    LOG_INFO(stdout, "succeed to open configuration file\n");

    // 获取配置文件参数
    cv::FileNode netNode  = configurationFile["Classifier"];
    std::string modelPath = (std::string)netNode["ModelPath"];
    useInt8               = (bool)(int)netNode["UseInt8"];
    useFP16               = (bool)(int)netNode["UseFP16"];
    useoffloadcopy        = (bool)(int)netNode["Useoffloadcopy"];

    // 加载模型
    if(!Exists(modelPath))
    {
        LOG_ERROR(stdout, "%s not exist!\n", modelPath.c_str());
        return MODEL_NOT_EXIST;
    }
    net = migraphx::parse_onnx(modelPath);
    LOG_INFO(stdout, "succeed to load model: %s\n", GetFileName(modelPath).c_str());

    // 获取模型输入/输出节点信息
    std::unordered_map<std::string, migraphx::shape> inputs  = net.get_inputs();
    std::unordered_map<std::string, migraphx::shape> outputs = net.get_outputs();
    inputName                                                = inputs.begin()->first;
    inputShape                                               = inputs.begin()->second;
    outputName                                               = outputs.begin()->first;
    outputShape                                              = outputs.begin()->second;
    int N                                                    = inputShape.lens()[0];
    int C                                                    = inputShape.lens()[1];
    int H                                                    = inputShape.lens()[2];
    int W                                                    = inputShape.lens()[3];
    inputSize                                                = cv::Size(W, H);

    // 设置模型为GPU模式
    migraphx::target gpuTarget = migraphx::gpu::target{};

    // 量化
    if(useInt8)
    {
        // 创建量化校准数据,建议使用测试集中的多张典型图像
        cv::Mat srcImage = cv::imread("../Resource/Images/ImageNet_test.jpg", 1);
        std::vector<cv::Mat> srcImages;
        for(int i = 0; i < inputShape.lens()[0]; ++i)
        {
            srcImages.push_back(srcImage);
        }

        // 数据预处理
        cv::Mat inputBlob = Preprocess(srcImages);
        std::unordered_map<std::string, migraphx::argument> inputData;
        inputData[inputName] = migraphx::argument{inputShape, (float*)inputBlob.data};
        std::vector<std::unordered_map<std::string, migraphx::argument>> calibrationData = {
            inputData};

        // INT8量化
        migraphx::quantize_int8(net, gpuTarget, calibrationData);
    }
    else if(useFP16)
    {
        migraphx::quantize_fp16(net);
    }

    // 编译模型
    migraphx::compile_options options;
    options.device_id = 0; // 设置GPU设备，默认为0号设备
    if(useoffloadcopy)
    {
        options.offload_copy = true;
    }
    else
    {
        options.offload_copy = false;
    }
    net.compile(gpuTarget, options);
    LOG_INFO(stdout, "succeed to compile model: %s\n", GetFileName(modelPath).c_str());

    // offloadcopy为false的时候，分配输入和输出内存
    if(!useoffloadcopy)
    {
        // 分配device输入内存
        inputBuffer_Device = nullptr;
        hipMalloc(&inputBuffer_Device, inputShape.bytes());
        programParameters[inputName] = migraphx::argument{inputShape, inputBuffer_Device};

        // 分配device和host输出内存
        outputBuffer_Device = nullptr;
        hipMalloc(&outputBuffer_Device, outputShape.bytes());
        programParameters[outputName] = migraphx::argument{outputShape, outputBuffer_Device};
        outputBuffer_Host             = nullptr; // host内存
        outputBuffer_Host             = malloc(outputShape.bytes());
    }

    // warm up
    if(useoffloadcopy)
    {
        std::unordered_map<std::string, migraphx::argument> inputData;
        inputData[inputName] = migraphx::argument{inputShape};
        net.eval(inputData);
    }
    else
    {
        migraphx::argument inputData = migraphx::argument{inputShape};
        hipMemcpy(inputBuffer_Device, inputData.data(), inputShape.bytes(), hipMemcpyHostToDevice);
        net.eval(programParameters);
    }

    // log
    LOG_INFO(stdout, "InputSize:%dx%d\n", inputSize.width, inputSize.height);
    LOG_INFO(stdout, "InputName:%s\n", inputName.c_str());
    LOG_INFO(stdout, "UseInt8:%d\n", (int)useInt8);
    LOG_INFO(stdout, "UseFP16:%d\n", (int)useFP16);
    LOG_INFO(stdout, "Useoffloadcopy:%d\n", (int)useoffloadcopy);

    return SUCCESS;
}

ErrorCode Classifier::Classify(const std::vector<cv::Mat>& srcImages,
                               std::vector<std::vector<ResultOfPrediction>>& predictions)
{
    if(srcImages.size() == 0 || srcImages[0].empty() || srcImages[0].depth() != CV_8U)
    {
        LOG_ERROR(stdout, "image error!\n");
        return IMAGE_ERROR;
    }

    // 数据预处理
    cv::Mat inputBlob = Preprocess(srcImages);

    // 当offload为true时，不需要内存拷贝
    if(useoffloadcopy)
    {
        std::unordered_map<std::string, migraphx::argument> inputData;
        inputData[inputName] = migraphx::argument{inputShape, (float*)inputBlob.data};

        // 推理
        std::vector<migraphx::argument> results = net.eval(inputData);

        // 获取输出节点的属性
        migraphx::argument result    = results[0];         // 获取第一个输出节点的数据
        migraphx::shape outputShapes = result.get_shape(); // 输出节点的shape
        std::vector<std::size_t> outputSize =
            outputShapes.lens();                      // 每一维大小，维度顺序为(N,C,H,W)
        int numberOfOutput = outputShapes.elements(); // 输出节点元素的个数
        float* logits      = (float*)result.data();   // 输出节点数据指针

        // 获取每张图像的预测结果
        int numberOfClasses = numberOfOutput / srcImages.size();
        for(int i = 0; i < srcImages.size(); ++i)
        {
            int startIndex = numberOfClasses * i;

            std::vector<float> logit;
            for(int j = 0; j < numberOfClasses; ++j)
            {
                logit.push_back(logits[startIndex + j]);
            }

            std::vector<ResultOfPrediction> resultOfPredictions;
            for(int j = 0; j < numberOfClasses; ++j)
            {
                ResultOfPrediction prediction;
                prediction.label      = j;
                prediction.confidence = logit[j];

                resultOfPredictions.push_back(prediction);
            }

            predictions.push_back(resultOfPredictions);
        }
    }
    else // 当offload为false时，需要内存拷贝
    {

        migraphx::argument inputData = migraphx::argument{inputShape, (float*)inputBlob.data};

        // 拷贝到device输入内存
        hipMemcpy(inputBuffer_Device, inputData.data(), inputShape.bytes(), hipMemcpyHostToDevice);

        // 推理
        std::vector<migraphx::argument> results = net.eval(programParameters);

        // 获取输出节点的属性
        migraphx::argument result    = results[0];         // 获取第一个输出节点的数据
        migraphx::shape outputShapes = result.get_shape(); // 输出节点的shape
        std::vector<std::size_t> outputSize =
            outputShapes.lens();                      // 每一维大小，维度顺序为(N,C,H,W)
        int numberOfOutput = outputShapes.elements(); // 输出节点元素的个数

        // 将device输出数据拷贝到分配好的host输出内存
        hipMemcpy(outputBuffer_Host,
                  outputBuffer_Device,
                  outputShapes.bytes(),
                  hipMemcpyDeviceToHost); // 直接使用事先分配好的输出内存拷贝

        // 获取每张图像的预测结果
        int numberOfClasses = numberOfOutput / srcImages.size();
        std::vector<float> logit;
        for(int i = 0; i < srcImages.size(); ++i)
        {
            int startIndex = numberOfClasses * i;
            for(int j = 0; j < numberOfClasses; ++j)
            {
                logit.push_back(((float*)outputBuffer_Host)[startIndex + j]);
            }

            std::vector<ResultOfPrediction> resultOfPredictions;
            for(int j = 0; j < numberOfClasses; ++j)
            {
                ResultOfPrediction prediction;
                prediction.label      = j;
                prediction.confidence = logit[j];

                resultOfPredictions.push_back(prediction);
            }

            predictions.push_back(resultOfPredictions);
        }

        // 释放
        hipFree(inputBuffer_Device);
        hipFree(outputBuffer_Device);
        free(outputBuffer_Host);
    }

    return SUCCESS;
}

} // namespace migraphxSamples